(core) updates from grist-core

Author: paulfitz

README.md

@@ -303,6 +303,7 @@ Grist can be configured in many ways. Here are the main environment variables it
 | GRIST_SESSION_DOMAIN               | if set, associates the cookie with the given domain - otherwise defaults to GRIST_DOMAIN                                                                                                                                                                                                                                                                      |
 | GRIST_SESSION_SECRET               | a key used to encode sessions                                                                                                                                                                                                                                                                                                                                 |
 | GRIST_SKIP_BUNDLED_WIDGETS         | if set, Grist will ignore any bundled widgets included via NPM packages.                                                                                                                                                                                                                                                                                      |
+| GRIST_SQLITE_MODE                  | if set to `wal`, use SQLite in [WAL mode](https://www.sqlite.org/wal.html), if set to `sync`, use SQLite with [SYNCHRONOUS=full](https://www.sqlite.org/pragma.html#pragma_synchronous)
 | GRIST_ANON_PLAYGROUND              | When set to `false` deny anonymous users access to the home page (but documents can still be shared to anonymous users). Defaults to `true`.                                                                                                                                                                                                                  |
 | GRIST_FORCE_LOGIN                  | Setting it to `true` is similar to setting `GRIST_ANON_PLAYGROUND: false` but it blocks any anonymous access (thus any document shared publicly actually requires the users to be authenticated before consulting them)                                                                                                                                       |
 | GRIST_SINGLE_ORG                   | set to an org "domain" to pin client to that org                                                                                                                                                                                                                                                                                                              |

app/gen-server/lib/DocApiForwarder.ts

@@ -59,6 +59,11 @@ export class DocApiForwarder {
     app.use('/api/docs/:docId/create-fork', withDoc);
     app.use('/api/docs/:docId/apply', withDoc);
     app.use('/api/docs/:docId/attachments', withDoc);
+    app.use('/api/docs/:docId/attachments/download', withDoc);
+    app.use('/api/docs/:docId/attachments/transferStatus', withDoc);
+    app.use('/api/docs/:docId/attachments/transferAll', withDoc);
+    app.use('/api/docs/:docId/attachments/store', withDoc);
+    app.use('/api/docs/:docId/attachments/stores', withDoc);
     app.use('/api/docs/:docId/snapshots', withDoc);
     app.use('/api/docs/:docId/usersForViewAs', withDoc);
     app.use('/api/docs/:docId/replace', withDoc);
@@ -74,10 +79,6 @@ export class DocApiForwarder {
     app.use('/api/docs/:docId/timing/start', withDoc);
     app.use('/api/docs/:docId/timing/stop', withDoc);
     app.use('/api/docs/:docId/forms/:vsId', withDoc);
-    app.use('/api/docs/:docId/attachments/transferStatus', withDoc);
-    app.use('/api/docs/:docId/attachments/transferAll', withDoc);
-    app.use('/api/docs/:docId/attachments/store', withDoc);
-    app.use('/api/docs/:docId/attachments/stores', withDoc);
 
     app.use('^/api/docs$', withoutDoc);
   }

app/server/lib/ActiveDoc.ts

@@ -90,7 +90,12 @@ import {Share} from 'app/gen-server/entity/Share';
 import {RecordWithStringId} from 'app/plugin/DocApiTypes';
 import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI, UIRowId} from 'app/plugin/GristAPI';
-import {Archive, ArchiveEntry, create_zip_archive} from 'app/server/lib/Archive';
+import {
+  Archive,
+  ArchiveEntry, CreatableArchiveFormats,
+  create_tar_archive,
+  create_zip_archive
+} from 'app/server/lib/Archive';
 import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {AssistanceContext} from 'app/common/AssistancePrompts';
 import {AuditEventAction} from 'app/server/lib/AuditEvent';
@@ -960,7 +965,8 @@ export class ActiveDoc extends EventEmitter {
     return data;
   }
 
-  public async getAttachmentsArchive(docSession: OptDocSession): Promise<Archive> {
+  public async getAttachmentsArchive(docSession: OptDocSession,
+                                     format: CreatableArchiveFormats = 'zip'): Promise<Archive> {
     if (
       !await this._granularAccess.canReadEverything(docSession) &&
       !await this.canDownload(docSession)
@@ -990,12 +996,20 @@ export class ActiveDoc extends EventEmitter {
         filesAdded.add(name);
         yield({
           name,
+          size: file.metadata.size,
           data: file.contentStream,
         });
       }
     }
 
-    return create_zip_archive({ store: true }, fileGenerator());
+    if (format == 'tar') {
+      return create_tar_archive(fileGenerator());
+    }
+    if (format == 'zip') {
+      return create_zip_archive({ store: true }, fileGenerator());
+    }
+    // Generally this won't happen, as long as the above is exhaustive over the type of `format`
+    throw new ApiError(`Unsupported archive format ${format}`, 400);
   }
 
   @ActiveDoc.keepDocOpen

app/server/lib/AppSettings.ts

@@ -72,6 +72,11 @@ export class AppSettings {
     } else if (query.defaultValue !== undefined) {
       this._value = query.defaultValue;
     }
+    if (query.acceptedValues && this._value) {
+      if (query.acceptedValues.every(v => v !== this._value)) {
+        throw new Error(`value is not accepted: ${this._value}`);
+      }
+    }
     return this;
   }
 
@@ -239,6 +244,8 @@ export interface AppSettingQuery {
    * When set to true, the value is obscured when printed.
    */
   censor?: boolean;
+
+  acceptedValues?: Array<JSONValue>;
 }
 
 export interface AppSettingQueryInt extends AppSettingQuery {

app/server/lib/Archive.ts

@@ -1,17 +1,25 @@
+import {StringUnion} from 'app/common/StringUnion';
 import {ZipArchiveEntry} from 'compress-commons';
 import stream from 'node:stream';
+import * as tar from 'tar-stream';
 import ZipStream, {ZipStreamOptions} from 'zip-stream';
 
 export interface ArchiveEntry {
   name: string;
+  size: number;
   data: stream.Readable | Buffer;
 }
 
 export interface Archive {
+  mimeType: string;
+  fileExtension: string;
   dataStream: stream.Readable;
   completed: Promise<void>;
 }
 
+export const CreatableArchiveFormats = StringUnion('zip', 'tar');
+export type CreatableArchiveFormats = typeof CreatableArchiveFormats.type;
+
 /**
  *
  * Creates a streamable zip archive, reading files on-demand from the entries iterator.
@@ -27,6 +35,8 @@ export async function create_zip_archive(
   const archive = new ZipStream(options);
 
   return {
+    mimeType: "application/zip",
+    fileExtension: "zip",
     dataStream: archive,
     // Caller is responsible for error handling/awaiting on this promise.
     completed: (async () => {
@@ -57,3 +67,39 @@ function addEntryToZipArchive(archive: ZipStream, file: ArchiveEntry): Promise<Z
     });
   });
 }
+
+/**
+ *
+ * Creates a streamable tar archive, reading files on-demand from the entries iterator.
+ * Entries are provided as an async iterable, to ensure the archive is constructed
+ * correctly. A generator can be used for convenience.
+ * @param {AsyncIterable<ArchiveEntry>} entries - Entries to add.
+ * @returns {Archive}
+ */
+export async function create_tar_archive(
+  entries: AsyncIterable<ArchiveEntry>
+): Promise<Archive> {
+  const archive = tar.pack();
+
+  return {
+    mimeType: "application/x-tar",
+    fileExtension: "tar",
+    dataStream: archive,
+    // Caller is responsible for error handling/awaiting on this promise.
+    completed: (async () => {
+      try {
+        for await (const entry of entries) {
+          const entryStream = archive.entry({ name: entry.name, size: entry.size });
+          await stream.promises.pipeline(entry.data, entryStream);
+        }
+        archive.finalize();
+      } catch (error) {
+        archive.destroy(error);
+      } finally {
+        // If the stream was destroyed with an error, this will re-throw the error we caught above.
+        // Without this, node will see the stream as having an uncaught error, and complain.
+        await stream.promises.finished(archive);
+      }
+    })()
+  };
+}

app/server/lib/DocApi.ts

@@ -45,6 +45,7 @@ import {
 } from 'app/plugin/TableOperationsImpl';
 import {ActiveDoc, colIdToRef as colIdToReference, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc";
 import {appSettings} from "app/server/lib/AppSettings";
+import {CreatableArchiveFormats} from 'app/server/lib/Archive';
 import {sendForCompletion} from 'app/server/lib/Assistance';
 import {getDocPoolIdFromDocInfo} from 'app/server/lib/AttachmentStore';
 import {
@@ -575,12 +576,19 @@ export class DocWorkerApi {
 
     // Responds with an archive of all attachment contents, with suitable Content-Type and Content-Disposition.
     this._app.get('/api/docs/:docId/attachments/download', canView, withDoc(async (activeDoc, req, res) => {
-      const archive = await activeDoc.getAttachmentsArchive(docSessionFromRequest(req));
+      const archiveFormatStr = optStringParam(req.query.format, 'format', {
+        allowed: CreatableArchiveFormats.values,
+        allowEmpty: true,
+      });
+
+      const archiveFormat = CreatableArchiveFormats.parse(archiveFormatStr) || 'zip';
+      const archive = await activeDoc.getAttachmentsArchive(docSessionFromRequest(req), archiveFormat);
       const docName = await this._getDownloadFilename(req, "Attachments", activeDoc.doc);
       res.status(200)
-        .type("application/zip")
+        .type(archive.mimeType)
         // Construct a content-disposition header of the form 'attachment; filename="NAME"'
-        .set('Content-Disposition', contentDisposition(`${docName}.zip`, {type: 'attachment'}))
+        .set('Content-Disposition',
+          contentDisposition(`${docName}.${archive.fileExtension}`, {type: 'attachment'}))
         // Avoid storing because this could be huge.
         .set('Cache-Control', 'no-store');
 

app/server/lib/DocStorage.ts

@@ -15,6 +15,7 @@ import * as marshal from 'app/common/marshal';
 import * as schema from 'app/common/schema';
 import {SingleCell} from 'app/common/TableData';
 import {GristObjCode} from "app/plugin/GristData";
+import {appSettings} from 'app/server/lib/AppSettings';
 import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
 import {combineExpr, ExpandedQuery} from 'app/server/lib/ExpandedQuery';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
@@ -48,6 +49,16 @@ export const ATTACHMENTS_EXPIRY_DAYS = 7;
 // Cleanup expired attachments every hour (also happens when shutting down).
 export const REMOVE_UNUSED_ATTACHMENTS_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 1000};
 
+/**
+ * Check what way we want to access SQLite files.
+ */
+export function getSqliteMode() {
+  return appSettings.section('features')
+    .section('sqlite').flag('mode').readString({
+      envVar: 'GRIST_SQLITE_MODE',
+      acceptedValues: ['wal', 'sync'],
+    }) as 'wal'|'sync'|undefined;
+}
 
 export class DocStorage implements ISQLiteDB, OnDemandStorage {
 
@@ -711,16 +722,25 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
     // a database being corrupted if the computer it is running on crashes.
     // TODO: Switch setting to FULL, but don't wait for SQLite transactions to finish before
     // returning responses to the user. Instead send error messages on unexpected errors.
-    return this._getDB().exec(
-      // "PRAGMA wal_autochceckpoint = 1000;" +
-      // "PRAGMA page_size           = 4096;" +
-      // "PRAGMA journal_size_limit  = 0;" +
-      // "PRAGMA journal_mode        = WAL;" +
-      // "PRAGMA auto_vacuum         = 0;" +
-      // "PRAGMA synchronous         = NORMAL"
-      "PRAGMA synchronous         = OFF;" +
-      "PRAGMA trusted_schema      = OFF;"  // mitigation suggested by https://www.sqlite.org/security.html#untrusted_sqlite_database_files
-    );
+    const settings = [
+      'PRAGMA trusted_schema = OFF;',  // mitigation suggested by https://www.sqlite.org/security.html#untrusted_sqlite_database_files
+    ];
+    const sqliteMode = getSqliteMode();
+    if (sqliteMode === undefined) {
+      // Historically, Grist has used this setting.
+      settings.push('PRAGMA synchronous = OFF;');
+    } else if (sqliteMode === 'sync') {
+      // This is a safer, but potentially slower, setting for general use.
+      settings.push('PRAGMA synchronous = FULL;');
+    } else if (sqliteMode === 'wal') {
+      // This is a good modern setting for servers, but awkward
+      // on a Desktop for users who interact with their documents
+      // directly as files on the file system. With WAL, at any
+      // time, changes may be stored in a companion file rather
+      // than the .grist file.
+      settings.push('PRAGMA journal_mode = WAL;');
+    }
+    return this._getDB().exec(settings.join('\n'));
   }
 
   /**

app/server/lib/DocStorageManager.ts

@@ -8,8 +8,10 @@ import {DocEntry, DocEntryTag} from 'app/common/DocListAPI';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentUsage} from 'app/common/DocUsage';
 import * as gutil from 'app/common/gutil';
+import {backupUsingBestConnection} from 'app/server/lib/backupSqliteDatabase';
 import {Comm} from 'app/server/lib/Comm';
 import * as docUtils from 'app/server/lib/docUtils';
+import {GristServer} from 'app/server/lib/GristServer';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {IShell} from 'app/server/lib/IShell';
 import log from 'app/server/lib/log';
@@ -36,7 +38,7 @@ export class DocStorageManager implements IDocStorageManager {
    * The file watcher is created if the optComm argument is given.
    */
   constructor(private _docsRoot: string, private _samplesRoot?: string,
-              private _comm?: Comm, shell?: IShell) {
+              private _comm?: Comm, shell?: IShell, private _gristServer?: GristServer) {
     // If we have a way to communicate with clients, watch the docsRoot for changes.
     this._shell = shell ?? {
       trashItem() { throw new Error('Unable to move document to trash'); },
@@ -55,6 +57,10 @@ export class DocStorageManager implements IDocStorageManager {
     return path.resolve(this._docsRoot, docName);
   }
 
+  public getSQLiteDB(docName: string) {
+    return this._gristServer?.getDocManager().getSQLiteDB(docName);
+  }
+
   /**
    * Returns the path to the given sample document.
    */
@@ -86,8 +92,10 @@ export class DocStorageManager implements IDocStorageManager {
 
   public async prepareFork(srcDocName: string, destDocName: string): Promise<string> {
     // This is implemented only to support old tests.
-    await fse.copy(this.getPath(srcDocName), this.getPath(destDocName));
-    return this.getPath(destDocName);
+    const output = this.getPath(destDocName);
+    return this._safeCopy(srcDocName, {
+      output,
+    });
   }
 
   /**
@@ -175,9 +183,8 @@ export class DocStorageManager implements IDocStorageManager {
       );
     }).tap((numberedBakPathPrefix: string) => { // do the copying, but return bakPath anyway
       finalBakPath = numberedBakPathPrefix + ext;
-      const docPath = this.getPath(docName);
       log.info(`Backing up ${docName} to ${finalBakPath}`);
-      return docUtils.copyFile(docPath, finalBakPath);
+      return this._safeCopy(docName, { output: finalBakPath });
     }).then(() => {
       log.debug("DocStorageManager: Backup made successfully at: %s", finalBakPath);
       return finalBakPath;
@@ -235,11 +242,9 @@ export class DocStorageManager implements IDocStorageManager {
   }
 
   public async getCopy(docName: string): Promise<string> {
-    const srcPath = this.getPath(docName);
-    const postfix = uuidv4();
-    const tmpPath = `${srcPath}-${postfix}`;
-    await docUtils.copyFile(srcPath, tmpPath);
-    return tmpPath;
+    return this._safeCopy(docName, {
+      postfix: uuidv4(),
+    });
   }
 
   public async getSnapshots(docName: string, skipMetadataCache?: boolean): Promise<DocSnapshots> {
@@ -316,6 +321,16 @@ export class DocStorageManager implements IDocStorageManager {
       this._comm.broadcastMessage('docListAction', { [actionType]: [data] });
     }
   }
+
+  private async _safeCopy(docName: string, options: {
+    postfix?: string,
+    output?: string,
+  }): Promise<string> {
+    return backupUsingBestConnection(this, docName, {
+      ...options,
+      log: (err) => log.error("DocStorageManager: copy failed for %s: %s", docName, String(err)),
+    });
+  }
 }
 
 /**

app/server/lib/FlexServer.ts

@@ -44,6 +44,7 @@ import {create} from 'app/server/lib/create';
 import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
 import {addDocApiRoutes} from 'app/server/lib/DocApi';
 import {DocManager} from 'app/server/lib/DocManager';
+import {getSqliteMode} from 'app/server/lib/DocStorage';
 import {DocWorker} from 'app/server/lib/DocWorker';
 import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@@ -1367,6 +1368,9 @@ export class FlexServer implements GristServer {
     if (!this._docManager) { this.addCleanup(); }
     await this.addLoginMiddleware();
     this.addComm();
+    // Check SQLite mode so it shows up in initial configuration readout
+    // (even though we don't need it until opening documents).
+    getSqliteMode();
 
     await this.create.configure?.();
 
@@ -1392,7 +1396,7 @@ export class FlexServer implements GristServer {
     } else {
       const samples = getAppPathTo(this.appRoot, 'public_samples');
       const storageManager = await this.create.createLocalDocStorageManager(
-        this.docsRoot, samples, this._comm);
+        this.docsRoot, samples, this._comm, undefined, this);
       this._storageManager = storageManager;
     }