From 982d57c9ae06e28fa63715de5904edd01fcfdfd3 Mon Sep 17 00:00:00 2001 From: Jed Fox Date: Wed, 28 Jun 2023 07:03:52 -0400 Subject: [PATCH] Move the rest of the syncing protobuf code to the CRDT package (#1217) --- .prettierignore | 1 + packages/crdt/.gitignore | 2 + .../src/server/sync => crdt}/README.md | 23 ++--- packages/crdt/bin/generate-proto | 21 +++++ packages/crdt/package.json | 2 + .../server/sync => crdt/src/proto}/sync.proto | 0 packages/crdt/src/proto/sync_pb.d.ts | 84 +++++++++++-------- packages/loot-core/package.json | 1 - upcoming-release-notes/1217.md | 6 ++ yarn.lock | 2 +- 10 files changed, 88 insertions(+), 54 deletions(-) create mode 100644 .prettierignore create mode 100644 packages/crdt/.gitignore rename packages/{loot-core/src/server/sync => crdt}/README.md (57%) create mode 100755 packages/crdt/bin/generate-proto rename packages/{loot-core/src/server/sync => crdt/src/proto}/sync.proto (100%) create mode 100644 upcoming-release-notes/1217.md diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..2faa2ef87d --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +sync_pb.* diff --git a/packages/crdt/.gitignore b/packages/crdt/.gitignore new file mode 100644 index 0000000000..f1772b9098 --- /dev/null +++ b/packages/crdt/.gitignore @@ -0,0 +1,2 @@ +bin/protoc-gen-js +bin/protoc diff --git a/packages/loot-core/src/server/sync/README.md b/packages/crdt/README.md similarity index 57% rename from packages/loot-core/src/server/sync/README.md rename to packages/crdt/README.md index d9f4c17be7..bb046aa2f8 100644 --- a/packages/loot-core/src/server/sync/README.md +++ b/packages/crdt/README.md @@ -1,25 +1,18 @@ -# protobuf +# `@actual-app/crdt` + +This package contains the core CRDT logic that enables Actual’s syncing. It is shared between the client and server. We may or may not follow semver when updating this package; any usage of it outside Actual is undocumented and at your own risk. + +## protobuf We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network. -## Generating protobuf +### Generating protobuf The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler. -This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). +This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). You’ll also need to [download the latest version of `protoc-gen-js`](https://github.com/protocolbuffers/protobuf-javascript/releases/latest). For convenience, you can put both of these binaries in `./bin`. -Once installed, the protobuf can be generated by running the below commands. - -```bash -cd packages/loot-core - -protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \ - --ts_opt=esModuleInterop=true \ - --ts_out="src/server/sync/proto" \ - --js_out=import_style=commonjs,binary:src/server/sync/proto \ - --proto_path=src/server/sync \ - sync.proto -``` +Once installed, the protobuf can be generated by running `./bin/generate-proto`. However there is one very important thing to remember! The default output includes this near the top: diff --git a/packages/crdt/bin/generate-proto b/packages/crdt/bin/generate-proto new file mode 100755 index 0000000000..e539c56334 --- /dev/null +++ b/packages/crdt/bin/generate-proto @@ -0,0 +1,21 @@ +#!/bin/bash + +cd "$(dirname "$(dirname "$0")")" + +if ! [ -x "$(command -v protoc)" ]; then + echo 'Error: protoc is not installed. See the readme for installation instructions.' >&2 + exit 1 +fi + +export PATH="$PWD/bin:$PATH" + +protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \ + --ts_opt=esModuleInterop=true \ + --ts_out="src/proto" \ + --js_out=import_style=commonjs,binary:src/proto \ + --proto_path=src/proto \ + sync.proto + +../../node_modules/.bin/prettier --write src/proto/*.d.ts + +echo 'One more step! Find the `var global = ...` declaration in src/proto/sync_pb.js and change it to `var global = globalThis;`' diff --git a/packages/crdt/package.json b/packages/crdt/package.json index dbff3f1358..d6dd400c6e 100644 --- a/packages/crdt/package.json +++ b/packages/crdt/package.json @@ -10,6 +10,7 @@ ], "scripts": { "build:node": "tsc --p tsconfig.dist.json", + "proto:generate": "./bin/generate-proto", "build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/", "test": "jest -c jest.config.js" }, @@ -22,6 +23,7 @@ "@types/jest": "^27.5.0", "jest": "^27.0.0", "ts-jest": "^27.0.0", + "ts-protoc-gen": "^0.15.0", "typescript": "^5.0.2" } } diff --git a/packages/loot-core/src/server/sync/sync.proto b/packages/crdt/src/proto/sync.proto similarity index 100% rename from packages/loot-core/src/server/sync/sync.proto rename to packages/crdt/src/proto/sync.proto diff --git a/packages/crdt/src/proto/sync_pb.d.ts b/packages/crdt/src/proto/sync_pb.d.ts index 5a31387bf1..966b8a3385 100644 --- a/packages/crdt/src/proto/sync_pb.d.ts +++ b/packages/crdt/src/proto/sync_pb.d.ts @@ -20,11 +20,11 @@ export class EncryptedData extends jspb.Message { setData(value: Uint8Array | string): void; serializeBinary(): Uint8Array; - toObject(includeInstance?: boolean): EncryptedDataAsObject; + toObject(includeInstance?: boolean): EncryptedData.AsObject; static toObject( includeInstance: boolean, msg: EncryptedData, - ): EncryptedDataAsObject; + ): EncryptedData.AsObject; static extensions: { [key: number]: jspb.ExtensionFieldInfo }; static extensionsBinary: { [key: number]: jspb.ExtensionFieldBinaryInfo; @@ -40,11 +40,13 @@ export class EncryptedData extends jspb.Message { ): EncryptedData; } -export type EncryptedDataAsObject = { - iv: Uint8Array | string; - authtag: Uint8Array | string; - data: Uint8Array | string; -}; +export namespace EncryptedData { + export type AsObject = { + iv: Uint8Array | string; + authtag: Uint8Array | string; + data: Uint8Array | string; + }; +} export class Message extends jspb.Message { getDataset(): string; @@ -60,8 +62,8 @@ export class Message extends jspb.Message { setValue(value: string): void; serializeBinary(): Uint8Array; - toObject(includeInstance?: boolean): MessageAsObject; - static toObject(includeInstance: boolean, msg: Message): MessageAsObject; + toObject(includeInstance?: boolean): Message.AsObject; + static toObject(includeInstance: boolean, msg: Message): Message.AsObject; static extensions: { [key: number]: jspb.ExtensionFieldInfo }; static extensionsBinary: { [key: number]: jspb.ExtensionFieldBinaryInfo; @@ -77,12 +79,14 @@ export class Message extends jspb.Message { ): Message; } -export type MessageAsObject = { - dataset: string; - row: string; - column: string; - value: string; -}; +export namespace Message { + export type AsObject = { + dataset: string; + row: string; + column: string; + value: string; + }; +} export class MessageEnvelope extends jspb.Message { getTimestamp(): string; @@ -97,11 +101,11 @@ export class MessageEnvelope extends jspb.Message { setContent(value: Uint8Array | string): void; serializeBinary(): Uint8Array; - toObject(includeInstance?: boolean): MessageEnvelopeAsObject; + toObject(includeInstance?: boolean): MessageEnvelope.AsObject; static toObject( includeInstance: boolean, msg: MessageEnvelope, - ): MessageEnvelopeAsObject; + ): MessageEnvelope.AsObject; static extensions: { [key: number]: jspb.ExtensionFieldInfo }; static extensionsBinary: { [key: number]: jspb.ExtensionFieldBinaryInfo; @@ -117,11 +121,13 @@ export class MessageEnvelope extends jspb.Message { ): MessageEnvelope; } -export type MessageEnvelopeAsObject = { - timestamp: string; - isencrypted: boolean; - content: Uint8Array | string; -}; +export namespace MessageEnvelope { + export type AsObject = { + timestamp: string; + isencrypted: boolean; + content: Uint8Array | string; + }; +} export class SyncRequest extends jspb.Message { clearMessagesList(): void; @@ -142,11 +148,11 @@ export class SyncRequest extends jspb.Message { setSince(value: string): void; serializeBinary(): Uint8Array; - toObject(includeInstance?: boolean): SyncRequestAsObject; + toObject(includeInstance?: boolean): SyncRequest.AsObject; static toObject( includeInstance: boolean, msg: SyncRequest, - ): SyncRequestAsObject; + ): SyncRequest.AsObject; static extensions: { [key: number]: jspb.ExtensionFieldInfo }; static extensionsBinary: { [key: number]: jspb.ExtensionFieldBinaryInfo; @@ -162,13 +168,15 @@ export class SyncRequest extends jspb.Message { ): SyncRequest; } -export type SyncRequestAsObject = { - messagesList: Array; - fileid: string; - groupid: string; - keyid: string; - since: string; -}; +export namespace SyncRequest { + export type AsObject = { + messagesList: Array; + fileid: string; + groupid: string; + keyid: string; + since: string; + }; +} export class SyncResponse extends jspb.Message { clearMessagesList(): void; @@ -180,11 +188,11 @@ export class SyncResponse extends jspb.Message { setMerkle(value: string): void; serializeBinary(): Uint8Array; - toObject(includeInstance?: boolean): SyncResponseAsObject; + toObject(includeInstance?: boolean): SyncResponse.AsObject; static toObject( includeInstance: boolean, msg: SyncResponse, - ): SyncResponseAsObject; + ): SyncResponse.AsObject; static extensions: { [key: number]: jspb.ExtensionFieldInfo }; static extensionsBinary: { [key: number]: jspb.ExtensionFieldBinaryInfo; @@ -200,7 +208,9 @@ export class SyncResponse extends jspb.Message { ): SyncResponse; } -export type SyncResponseAsObject = { - messagesList: Array; - merkle: string; -}; +export namespace SyncResponse { + export type AsObject = { + messagesList: Array; + merkle: string; + }; +} diff --git a/packages/loot-core/package.json b/packages/loot-core/package.json index 14a1a16161..d32b9df5c9 100644 --- a/packages/loot-core/package.json +++ b/packages/loot-core/package.json @@ -72,7 +72,6 @@ "throttleit": "^1.0.0", "ts-jest": "^27.0.0", "ts-node": "^10.7.0", - "ts-protoc-gen": "^0.15.0", "typescript": "^4.6.4", "uuid": "3.3.2", "webpack": "^5.86.0", diff --git a/upcoming-release-notes/1217.md b/upcoming-release-notes/1217.md new file mode 100644 index 0000000000..137c23d64d --- /dev/null +++ b/upcoming-release-notes/1217.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [j-f1] +--- + +Move the rest of the syncing protobuf code to the CRDT package diff --git a/yarn.lock b/yarn.lock index 588e148eba..5759cfa18b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -32,6 +32,7 @@ __metadata: jest: ^27.0.0 murmurhash: ^0.0.2 ts-jest: ^27.0.0 + ts-protoc-gen: ^0.15.0 typescript: ^5.0.2 uuid: 3.3.2 languageName: unknown @@ -12380,7 +12381,6 @@ __metadata: throttleit: ^1.0.0 ts-jest: ^27.0.0 ts-node: ^10.7.0 - ts-protoc-gen: ^0.15.0 typescript: ^4.6.4 uuid: 3.3.2 webpack: ^5.86.0