feat: add rawLeaves option to allow for downgradable CID (#105)

storacha · Dec 16, 2021 · 88940df · 88940df
1 parent 76e84dd
commit 88940df
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 9 deletions.
diff --git a/src/pack/blob.ts b/src/pack/blob.ts
@@ -8,15 +8,16 @@ import { MemoryBlockStore } from '../blockstore/memory'
 import { pack } from './index'
 import type { PackProperties } from './index'
 
-export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) {
+export async function packToBlob ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, rawLeaves }: PackProperties) {
   const blockstore = userBlockstore ? userBlockstore : new MemoryBlockStore()
   const { root, out } = await pack({
     input,
     blockstore,
     hasher,
     maxChunkSize,
     maxChildrenPerNode,
-    wrapWithDirectory
+    wrapWithDirectory,
+    rawLeaves
   })
 
   const carParts = await all(out)

diff --git a/src/pack/fs.ts b/src/pack/fs.ts
@@ -12,7 +12,7 @@ export interface PackToFsProperties extends PackProperties {
   output?: string
 }
 
-export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToFsProperties) {
+export async function packToFs ({ input, output, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, rawLeaves }: PackToFsProperties) {
   const blockstore = userBlockstore ? userBlockstore : new FsBlockStore()
   const location = output || `${os.tmpdir()}/${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}`
   const writable = fs.createWriteStream(location)
@@ -24,7 +24,8 @@ export async function packToFs ({ input, output, blockstore: userBlockstore, has
     hasher,
     maxChunkSize,
     maxChildrenPerNode,
-    wrapWithDirectory
+    wrapWithDirectory,
+    rawLeaves
   })
 
   if (!userBlockstore) {

diff --git a/src/pack/index.ts b/src/pack/index.ts
@@ -18,10 +18,14 @@ export interface PackProperties {
   maxChunkSize?: number,
   maxChildrenPerNode?: number,
   wrapWithDirectory?: boolean,
-  hasher?: MultihashHasher
+  hasher?: MultihashHasher,
+  /**
+   * Use raw codec for leaf nodes. Default: true.
+   */
+  rawLeaves?: boolean
 }
 
-export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackProperties) {
+export async function pack ({ input, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, rawLeaves }: PackProperties) {
   if (!input || (Array.isArray(input) && !input.length)) {
     throw new Error('missing input file(s)')
   }
@@ -36,7 +40,8 @@ export async function pack ({ input, blockstore: userBlockstore, hasher, maxChun
       hasher: hasher || unixfsImporterOptionsDefault.hasher,
       maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize,
       maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode,
-      wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory
+      wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory,
+      rawLeaves: rawLeaves == null ? unixfsImporterOptionsDefault.rawLeaves : rawLeaves
     })
   ))
 

diff --git a/src/pack/stream.ts b/src/pack/stream.ts
@@ -21,7 +21,7 @@ export interface PackToStreamProperties extends PackProperties {
 }
 
 // Node version of toCar with Node Stream Writable
-export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory }: PackToStreamProperties) {
+export async function packToStream ({ input, writable, blockstore: userBlockstore, hasher, maxChunkSize, maxChildrenPerNode, wrapWithDirectory, rawLeaves }: PackToStreamProperties) {
   if (!input || (Array.isArray(input) && !input.length)) {
     throw new Error('given input could not be parsed correctly')
   }
@@ -38,7 +38,8 @@ export async function packToStream ({ input, writable, blockstore: userBlockstor
       hasher: hasher || unixfsImporterOptionsDefault.hasher,
       maxChunkSize: maxChunkSize || unixfsImporterOptionsDefault.maxChunkSize,
       maxChildrenPerNode: maxChildrenPerNode || unixfsImporterOptionsDefault.maxChildrenPerNode,
-      wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory
+      wrapWithDirectory: wrapWithDirectory === false ? false : unixfsImporterOptionsDefault.wrapWithDirectory,
+      rawLeaves: rawLeaves == null ? unixfsImporterOptionsDefault.rawLeaves : rawLeaves
     })
   ))
 

diff --git a/test/pack/index.browser.test.ts b/test/pack/index.browser.test.ts
@@ -77,6 +77,18 @@ describe('pack', () => {
         expect(spy.callCount).to.eql(0)
         await blockstore.close()
       })
+
+      it('can create a DAG with non-raw leaf nodes allowing downgradable CID', async () => {
+        const { root } = await packToBlob({
+          input: [new Uint8Array([21, 31])],
+          blockstore: new Blockstore(),
+          rawLeaves: false,
+          wrapWithDirectory: false
+        })
+
+        expect(() => root.toV0()).to.not.throw()
+        expect(root.toV0().toString()).to.eql('QmNUCKvjKRFeHZR2wyYM5cPEbEB969hz2zowTYvwGrQXP2')
+      })
     })
   })
 })
diff --git a/test/pack/index.node.test.ts b/test/pack/index.node.test.ts
@@ -229,6 +229,18 @@ describe('pack', () => {
         }
         throw new Error('pack should throw error with empty input')
       })
+
+      it('can create a DAG with non-raw leaf nodes allowing downgradable CID', async () => {
+        const { root } = await pack({
+          input: [new Uint8Array([21, 31])],
+          blockstore: new Blockstore(),
+          rawLeaves: false,
+          wrapWithDirectory: false
+        })
+
+        expect(() => root.toV0()).to.not.throw()
+        expect(root.toV0().toString()).to.eql('QmNUCKvjKRFeHZR2wyYM5cPEbEB969hz2zowTYvwGrQXP2')
+      })
     })
   })
 })