feat: beta words import (#5)

packages/engine/src/cli/synsets.ts

@@ -49,11 +49,11 @@ async function fetchSynsets() {
     throw new Error('Cannot find the sheet: words');
   }
 
-  const sync = new GSheets2Git(
-    fileDatabase.multisynsets,
-    wordsSheet as WordsSheet,
-    false,
-  );
+  const sync = new GSheets2Git({
+    beta: process.env.ISV_BETA === 'true',
+    fs: fileDatabase.multisynsets,
+    gsheets: wordsSheet as WordsSheet,
+  });
 
   await sync.execute();
 }
@@ -68,4 +68,5 @@ export const builder: CommandBuilder<SynsetsArgv, any> = {
 
 export type SynsetsArgv = {
   subcommand: 'fetch' | 'repair';
+  beta: boolean;
 };

packages/engine/src/google/dto/WordsDTO.ts

@@ -1,4 +1,8 @@
-export type WordsDTO = {
+import type { ArrayMapped } from '@interslavic/database-engine-google';
+
+import type { amends, amendedBy } from '../../symbols';
+
+export type WordsDTO = ArrayMapped<{
   id: string | number;
   isv: string;
   addition: string;
@@ -25,4 +29,7 @@ export type WordsDTO = {
   frequency: string | number;
   intelligibility: string;
   using_example: string;
+}> & {
+  [amends]?: WordsDTO;
+  [amendedBy]?: WordsDTO;
 };

packages/engine/src/symbols/index.ts

@@ -0,0 +1,2 @@
+export const amends: unique symbol = Symbol('amends');
+export const amendedBy: unique symbol = Symbol('amendedBy');

packages/engine/src/sync/core/index.ts

@@ -0,0 +1,2 @@
+export * from './EntitySyncOperation';
+export * from './IdSyncOperation';

packages/engine/src/sync/index.ts

@@ -1 +1 @@
-export * from './GSheets2Git';
+export * from './words';

packages/engine/src/sync/words/GSheets2Git.ts

@@ -0,0 +1,77 @@
+import type { MultilingualSynsetRepository } from '@interslavic/database-engine-fs';
+
+import type { WordsDTO, WordsSheet } from '../../google';
+import { toMultiSynset } from '../../google';
+import { amends, amendedBy } from '../../symbols';
+import { IdSyncOperation } from '../core';
+
+export type GSheets2GitOptions = {
+  readonly beta: boolean;
+  readonly fs: MultilingualSynsetRepository;
+  readonly gsheets: WordsSheet;
+};
+
+export class GSheets2Git extends IdSyncOperation<number> {
+  private _gmap?: Map<number, WordsDTO>;
+  private readonly fs: MultilingualSynsetRepository;
+  private readonly gsheets: WordsSheet;
+  private readonly beta: boolean;
+
+  constructor(options: GSheets2GitOptions) {
+    super();
+
+    this.fs = options.fs;
+    this.gsheets = options.gsheets;
+    this.beta = options.beta;
+  }
+
+  protected async delete(id: number): Promise<void> {
+    // TODO: think about the future when lemma IDs and synset IDs diverge
+    await this.fs.deleteById(id);
+  }
+
+  protected async getAfterIds(): Promise<number[]> {
+    return this._grecords().then((r) => [...r.keys()]);
+  }
+
+  protected async getBeforeIds(): Promise<number[]> {
+    return this.fs.keys();
+  }
+
+  protected async insert(id: number): Promise<void> {
+    const dto = await this._grecords().then((r) => r.get(id));
+    const multisynset = toMultiSynset(dto!);
+    await this.fs.insert(multisynset);
+  }
+
+  protected async update(id: number): Promise<void> {
+    const dto = await this._grecords().then((r) => r.get(id));
+    const multisynset = toMultiSynset(dto!);
+    await this.fs.upsert(multisynset);
+  }
+
+  private async _grecords(): Promise<Map<number, WordsDTO>> {
+    if (!this._gmap) {
+      const dtos = await this.gsheets.getValues();
+      const stable = dtos.filter((dto: WordsDTO) => dto.id > 0);
+      const grecords = (this._gmap = new Map<number, WordsDTO>(
+        stable.map((dto: WordsDTO) => [Number(dto.id), dto]),
+      ));
+
+      if (this.beta) {
+        const beta = dtos.filter((dto: WordsDTO) => dto.id < 0);
+        for (const record of beta) {
+          const id = (record.id = -record.id);
+          const base = grecords.get(id) as WordsDTO;
+          if (base) {
+            base[amendedBy] = record;
+            record[amends] = base;
+          }
+          grecords.set(id, record);
+        }
+      }
+    }
+
+    return this._gmap;
+  }
+}

packages/engine/src/sync/words/index.ts

@@ -0,0 +1 @@
+export * from './GSheets2Git';

packages/google/src/index.ts

@@ -2,3 +2,4 @@ export * from './auth';
 export * from './drive';
 export * from './sheets';
 export * from './GoogleAPIs';
+export * from './utils';

packages/google/src/sheets/Sheet.ts

@@ -1,7 +1,7 @@
-import { isEqual, camelCase, upperFirst } from 'lodash';
+import { camelCase, upperFirst } from 'lodash';
 import type { sheets_v4 } from 'googleapis';
 
-import type { ArrayMapper } from '../utils/createArrayMapperClass';
+import type { ArrayMapped, ArrayMapper } from '../utils/createArrayMapperClass';
 import { createArrayMapperClass } from '../utils/createArrayMapperClass';
 
 import type { BatchExecutor } from './BatchExecutor';
@@ -23,7 +23,6 @@ export class Sheet<T extends SheetRecord = SheetRecord> {
   private readonly _api: sheets_v4.Sheets;
   private readonly _batch: BatchExecutor;
   private readonly _properties: sheets_v4.Schema$SheetProperties;
-  private _columnHeaders?: unknown[];
   private _arrayMapper?: ArrayMapper<T>;
 
   public readonly protectedRanges: sheets_v4.Schema$ProtectedRange[];
@@ -46,45 +45,35 @@ export class Sheet<T extends SheetRecord = SheetRecord> {
     return this._properties.title!;
   }
 
-  protected async ensureColumnHeaders() {
-    if (this._columnHeaders) {
-      return;
-    }
-
+  async getValues(): Promise<ArrayMapped<T>[]> {
     const res = await this._api.spreadsheets.values.get({
-      range: `${this.title}!1:1`,
+      range: this.title,
       spreadsheetId: this.spreadsheetId,
     });
 
-    this._columnHeaders = res.data.values![0];
-    const mapperClassName = `${upperFirst(camelCase(this.title))}Mapper`;
-    this._arrayMapper = createArrayMapperClass(
-      mapperClassName,
-      this._columnHeaders.map(String),
-    );
-  }
-
-  async getValues(options: Sheet$GetValuesOptions = {}): Promise<T[]> {
-    await this.ensureColumnHeaders();
-
-    const res = await this._api.spreadsheets.values.get({
-      range: options.range ? `${this.title}!${options.range}` : this.title,
-      spreadsheetId: this.spreadsheetId,
-    });
-
-    const Mapper = this._arrayMapper!;
     const values = res.data.values ?? [];
-    if (isEqual(values[0], this._columnHeaders)) {
-      values.shift();
-    }
-
-    return values.filter((row) => row[0] != null).map((row) => new Mapper(row));
+    const Mapper = this._ensureMapper(values);
+    values.shift();
+    // eslint-disable-next-line unicorn/no-array-callback-reference
+    return values.map(Mapper.mapFn) as ArrayMapped<T>[];
   }
 
   async flush(): Promise<void> {
     await this._batch.flush();
   }
 
+  private _ensureMapper([headers]: unknown[][]) {
+    if (!this._arrayMapper) {
+      const mapperClassName = `${upperFirst(camelCase(this.title))}Mapper`;
+      this._arrayMapper = createArrayMapperClass(
+        mapperClassName,
+        headers.map(String),
+      );
+    }
+
+    return this._arrayMapper;
+  }
+
   // async updateSameInLanguages(values: string[]) {
   //   const res = await this._api.spreadsheets.values.update({
   //     spreadsheetId: SHEET_IDs.new_interslavic_words_list,

packages/google/src/utils/createArrayMapperClass.test.ts

@@ -21,4 +21,11 @@ describe('createArrayMapperClass', () => {
     instance.b = 4;
     expect(array).toEqual([3, 4]);
   });
+
+  it('should support construction from an object', () => {
+    const DynamicClass = createArrayMapperClass('DynamicClass', ['a', 'b']);
+    const instance = new DynamicClass({ a: 4, b: 3, c: 5 });
+    const values = instance[DynamicClass.symbols.values];
+    expect(values).toEqual([4, 3]);
+  });
 });

packages/google/src/utils/createArrayMapperClass.ts

@@ -1,31 +1,66 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-export type ArrayMapper<R extends Record<string, any>> = new (
-  values: any[],
-) => ArrayMapped<R>;
+const _index = Symbol('index');
+const _values = Symbol('values');
+export const symbols = {
+  index: _index,
+  values: _values,
+} as const;
+
+export type ArrayMapper<R extends Record<string, any>> = (new (
+  values: any,
+  index?: number,
+) => ArrayMapped<R>) & {
+  readonly mapFn: <T extends Record<string, any>>(
+    values: any,
+    index?: number,
+  ) => ArrayMapped<T>;
+  readonly symbols: typeof symbols;
+};
 
 export type ArrayMapped<R extends Record<string, any>> = {
   [P in keyof R]: R[P];
-} & { _values: unknown[] };
+} & {
+  [_index]?: number;
+  [_values]: unknown[];
+};
 
 export function createArrayMapperClass<R extends Record<string, any>>(
   className: string,
   propertyNames: (keyof R)[],
 ): ArrayMapper<R> {
   const DynamicClass = {
     [className]: class {
-      // @ts-expect-error TS6138: Property '_values' is declared but its value is never read.
-      constructor(private readonly _values: unknown[]) {}
+      [_index]?: number;
+      [_values]: unknown[];
+
+      constructor(values: unknown, index?: number) {
+        this[_index] = index;
+        if (Array.isArray(values)) {
+          this[_values] = values;
+        } else {
+          const obj = values as R;
+          this[_values] = propertyNames.map((name) => obj[name]);
+        }
+      }
+
+      static mapFn(values: unknown, index?: number) {
+        return new DynamicClass(values, index);
+      }
     },
   }[className];
 
+  Object.defineProperty(DynamicClass, 'symbols', {
+    get: () => symbols,
+  });
+
   for (const [index, name] of propertyNames.entries()) {
     Object.defineProperty(DynamicClass.prototype, name, {
       get: function () {
-        return this._values[index];
+        return this[_values][index];
       },
       set: function (value) {
-        this._values[index] = value;
+        this[_values][index] = value;
       },
       enumerable: true,
     });

packages/google/src/utils/index.ts

@@ -0,0 +1,2 @@
+export { symbols as ArrayMapperSymbols } from './createArrayMapperClass';
+export type { ArrayMapped } from './createArrayMapperClass';