Merge pull request #720 from MatrixAI/feature-vaults-review

General vaults review and fixes

src/git/http.ts

@@ -5,6 +5,7 @@ import type {
   ObjectIdList,
 } from './types';
 import type { EncryptedFS } from 'encryptedfs';
+import type { PackObjectsResult } from 'isomorphic-git';
 import { Buffer } from 'buffer';
 import git from 'isomorphic-git';
 import * as gitUtils from './utils';
@@ -398,14 +399,28 @@ async function* generatePackData({
   objectIds: Array<ObjectId>;
   chunkSize?: number;
 }): AsyncGenerator<Buffer, void, void> {
-  const packFile = await git.packObjects({
-    fs: efs,
-    dir,
-    gitdir: gitDir,
-    oids: objectIds,
-  });
-  if (packFile.packfile == null) utils.never('failed to create packFile data');
-  let packFileBuffer = Buffer.from(packFile.packfile.buffer);
+  let packFile: PackObjectsResult;
+  // In case of errors we don't want to throw them. This will result in the error being thrown into `isometric-git`
+  // when it consumes the response. It handles this by logging out the error which we don't want to happen.
+  try {
+    packFile = await git.packObjects({
+      fs: efs,
+      dir,
+      gitdir: gitDir,
+      oids: objectIds,
+    });
+  } catch {
+    // Return without sending any data
+    return;
+  }
+  // Pack file will only be undefined if it was written to disk instead
+  if (packFile.packfile == null) return;
+  // Convert to a buffer without copy, so we can process it
+  let packFileBuffer = Buffer.from(
+    packFile.packfile.buffer,
+    0,
+    packFile.packfile.byteLength,
+  );
 
   // Streaming the packFile as chunks of the length specified by the `chunkSize`.
   // Each line is formatted as a `PKT-LINE`

src/git/utils.ts

@@ -8,6 +8,7 @@ import type {
   RequestType,
 } from './types';
 import type { EncryptedFS } from 'encryptedfs';
+import path from 'path';
 import git from 'isomorphic-git';
 import { requestTypes } from './types';
 import * as utils from '../utils';
@@ -230,6 +231,35 @@ async function listObjects({
   return [...commits, ...trees, ...blobs, ...tags];
 }
 
+const objectsDirName = 'objects';
+const excludedDirs = ['pack', 'info'];
+
+/**
+ * Walks the filesystem to list out all git objects in the objects directory
+ */
+async function listObjectsAll({
+  fs,
+  gitDir,
+}: {
+  fs: EncryptedFS;
+  gitDir: string;
+}) {
+  const objectsDirPath = path.join(gitDir, objectsDirName);
+  const objectSet: Set<string> = new Set();
+  const objectDirs = await fs.promises.readdir(objectsDirPath);
+  for (const objectDir of objectDirs) {
+    if (typeof objectDir !== 'string') utils.never();
+    if (excludedDirs.includes(objectDir)) continue;
+    const objectIds = await fs.promises.readdir(
+      path.join(objectsDirPath, objectDir),
+    );
+    for (const objectId of objectIds) {
+      objectSet.add(objectDir + objectId);
+    }
+  }
+  return [...objectSet];
+}
+
 /**
  * Parses a want/has line from ref negotiation phase.
  */
@@ -309,6 +339,7 @@ export {
   listReferencesGenerator,
   referenceCapability,
   listObjects,
+  listObjectsAll,
   parseRequestLine,
   isObjectId,
   assertObjectId,