Skip to content

Commit

Permalink
format stuff
Browse files Browse the repository at this point in the history
  • Loading branch information
Dhruv Soni authored and Dhruv Soni committed Jun 11, 2024
1 parent 4d8bd87 commit 096c7af
Show file tree
Hide file tree
Showing 3 changed files with 196 additions and 191 deletions.
272 changes: 154 additions & 118 deletions packages/encrypted-blockstore/src/encrypt-helpers.ts
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,32 @@
import { sha256 } from 'multiformats/hashes/sha2'
import { CID } from 'multiformats'
import { encode, decode, create as mfCreate } from 'multiformats/block'
import type { MultihashHasher, ToString } from 'multiformats'
import { sha256 } from "multiformats/hashes/sha2";
import { CID } from "multiformats";
import { encode, decode, create as mfCreate } from "multiformats/block";
import type { MultihashHasher, ToString } from "multiformats";

import type { CarReader } from '@ipld/car'
import * as dagcbor from '@ipld/dag-cbor'
import type { CarReader } from "@ipld/car";
import * as dagcbor from "@ipld/dag-cbor";

import { MemoryBlockstore } from '@web3-storage/pail/block'
import { MemoryBlockstore } from "@web3-storage/pail/block";

// @ts-ignorex
import { bf } from "prolly-trees/utils";
// @ts-ignore
import { bf } from 'prolly-trees/utils'
import { nocache as cache } from "prolly-trees/cache";
// @ts-ignore
import { nocache as cache } from 'prolly-trees/cache'
// @ts-ignore
import { create, load } from 'prolly-trees/cid-set'
import { create, load } from "prolly-trees/cid-set";

import { encodeCarFiles } from './loader-helpers'
import { makeCodec } from './encrypt-codec.js'
import type { AnyBlock, CarMakeable, AnyLink, AnyDecodedBlock, CryptoOpts } from './types'
import { encodeCarFile } from "./loader-helpers";
import { makeCodec } from "./encrypt-codec.js";
import type {
AnyBlock,
CarMakeable,
AnyLink,
AnyDecodedBlock,
CryptoOpts,
} from "./types";

function makeEncDec(crypto: any, randomBytes: (size: number) => Uint8Array) {
const codec = makeCodec(crypto, randomBytes)
const codec = makeCodec(crypto, randomBytes);

const encrypt = async function* ({
get,
Expand All @@ -29,130 +35,153 @@ function makeEncDec(crypto: any, randomBytes: (size: number) => Uint8Array) {
key,
cache,
chunker,
root
root,
}: {
get: (cid: AnyLink) => Promise<AnyBlock | undefined>
key: ArrayBuffer
cids: AnyLink[]
hasher: MultihashHasher<number>
chunker: (bytes: Uint8Array) => AsyncGenerator<Uint8Array>
cache: (cid: AnyLink) => Promise<AnyBlock>
root: AnyLink
get: (cid: AnyLink) => Promise<AnyBlock | undefined>;
key: ArrayBuffer;
cids: AnyLink[];
hasher: MultihashHasher<number>;
chunker: (bytes: Uint8Array) => AsyncGenerator<Uint8Array>;
cache: (cid: AnyLink) => Promise<AnyBlock>;
root: AnyLink;
}): AsyncGenerator<any, void, unknown> {
const set = new Set<ToString<AnyLink>>()
let eroot
const set = new Set<ToString<AnyLink>>();
let eroot;
for (const cid of cids) {
const unencrypted = await get(cid)
if (!unencrypted) throw new Error('missing cid: ' + cid.toString())
const encrypted = await codec.encrypt({ ...unencrypted, key })
const block = await encode({ ...encrypted, codec, hasher })
yield block
set.add(block.cid.toString())
if (unencrypted.cid.equals(root)) eroot = block.cid
const unencrypted = await get(cid);
if (!unencrypted) throw new Error("missing cid: " + cid.toString());
const encrypted = await codec.encrypt({ ...unencrypted, key });
const block = await encode({ ...encrypted, codec, hasher });
yield block;
set.add(block.cid.toString());
if (unencrypted.cid.equals(root)) eroot = block.cid;
}
if (!eroot) throw new Error('cids does not include root')
const list = [...set].map(s => CID.parse(s))
let last
if (!eroot) throw new Error("cids does not include root");
const list = [...set].map((s) => CID.parse(s));
let last;
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) {
for await (const node of create({
list,
get,
cache,
chunker,
hasher,
codec: dagcbor,
})) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const block = (await node.block) as AnyBlock
yield block
last = block
const block = (await node.block) as AnyBlock;
yield block;
last = block;
}
if (!last) throw new Error('missing last block')
const head = [eroot, last.cid]
const block = await encode({ value: head, codec: dagcbor, hasher })
yield block
}
if (!last) throw new Error("missing last block");
const head = [eroot, last.cid];
const block = await encode({ value: head, codec: dagcbor, hasher });
yield block;
};

const decrypt = async function* ({
root,
get,
key,
cache,
chunker,
hasher
hasher,
}: {
root: AnyLink
get: (cid: AnyLink) => Promise<AnyBlock | undefined>
key: ArrayBuffer
cache: (cid: AnyLink) => Promise<AnyBlock>
chunker: (bytes: Uint8Array) => AsyncGenerator<Uint8Array>
hasher: MultihashHasher<number>
root: AnyLink;
get: (cid: AnyLink) => Promise<AnyBlock | undefined>;
key: ArrayBuffer;
cache: (cid: AnyLink) => Promise<AnyBlock>;
chunker: (bytes: Uint8Array) => AsyncGenerator<Uint8Array>;
hasher: MultihashHasher<number>;
}): AsyncGenerator<AnyBlock, void, undefined> {
const getWithDecode = async (cid: AnyLink) =>
get(cid).then(async block => {
if (!block) return
const decoded = await decode({ ...block, codec: dagcbor, hasher })
return decoded
})
get(cid).then(async (block) => {
if (!block) return;
const decoded = await decode({ ...block, codec: dagcbor, hasher });
return decoded;
});
const getWithDecrypt = async (cid: AnyLink) =>
get(cid).then(async block => {
if (!block) return
const decoded = await decode({ ...block, codec, hasher })
return decoded
})
const decodedRoot = await getWithDecode(root)
if (!decodedRoot) throw new Error('missing root')
if (!decodedRoot.bytes) throw new Error('missing bytes')
get(cid).then(async (block) => {
if (!block) return;
const decoded = await decode({ ...block, codec, hasher });
return decoded;
});
const decodedRoot = await getWithDecode(root);
if (!decodedRoot) throw new Error("missing root");
if (!decodedRoot.bytes) throw new Error("missing bytes");
const {
value: [eroot, tree]
} = decodedRoot as { value: [AnyLink, AnyLink] }
const rootBlock = (await get(eroot)) as AnyDecodedBlock
if (!rootBlock) throw new Error('missing root block')
value: [eroot, tree],
} = decodedRoot as { value: [AnyLink, AnyLink] };
const rootBlock = (await get(eroot)) as AnyDecodedBlock;
if (!rootBlock) throw new Error("missing root block");
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const cidset = await load({ cid: tree, get: getWithDecode, cache, chunker, codec, hasher })
const cidset = await load({
cid: tree,
get: getWithDecode,
cache,
chunker,
codec,
hasher,
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const { result: nodes } = (await cidset.getAllEntries()) as { result: { cid: CID }[] }
const { result: nodes } = (await cidset.getAllEntries()) as {
result: { cid: CID }[];
};
const unwrap = async (eblock: AnyDecodedBlock | undefined) => {
if (!eblock) throw new Error('missing block')
if (!eblock) throw new Error("missing block");
if (!eblock.value) {
eblock = (await decode({ ...eblock, codec, hasher })) as AnyDecodedBlock
eblock = (await decode({
...eblock,
codec,
hasher,
})) as AnyDecodedBlock;
}
const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => {
throw e
})
const block = await mfCreate({ cid, bytes, hasher, codec })
return block
}
const promises = []
const { bytes, cid } = await codec
.decrypt({ ...eblock, key })
.catch((e) => {
throw e;
});
const block = await mfCreate({ cid, bytes, hasher, codec });
return block;
};
const promises = [];
for (const { cid } of nodes) {
if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap))
if (!rootBlock.cid.equals(cid))
promises.push(getWithDecrypt(cid).then(unwrap));
}
yield* promises
yield unwrap(rootBlock)
}
return { encrypt, decrypt }
yield* promises;
yield unwrap(rootBlock);
};
return { encrypt, decrypt };
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const chunker = bf(30)
const chunker = bf(30);

function hexStringToUint8Array(hexString: string) {
const length = hexString.length
const uint8Array = new Uint8Array(length / 2)
const length = hexString.length;
const uint8Array = new Uint8Array(length / 2);
for (let i = 0; i < length; i += 2) {
uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16)
uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
}
return uint8Array
return uint8Array;
}

export async function encryptedEncodeCarFile(
crypto: CryptoOpts,
key: string,
rootCid: AnyLink,
t: CarMakeable
): Promise<AnyBlock[]> {
const encryptionKey = hexStringToUint8Array(key)
const encryptedBlocks = new MemoryBlockstore()
const cidsToEncrypt = [] as AnyLink[]
): Promise<AnyBlock> {
const encryptionKey = hexStringToUint8Array(key);
const encryptedBlocks = new MemoryBlockstore();
const cidsToEncrypt = [] as AnyLink[];
for (const { cid } of t.entries()) {
cidsToEncrypt.push(cid)
const g = await t.get(cid)
if (!g) throw new Error('missing cid block')
cidsToEncrypt.push(cid);
const g = await t.get(cid);
if (!g) throw new Error("missing cid block");
}
let last: AnyBlock | null = null
const { encrypt } = makeEncDec(crypto.crypto, crypto.randomBytes)
let last: AnyBlock | null = null;
const { encrypt } = makeEncDec(crypto.crypto, crypto.randomBytes);

for await (const block of encrypt({
cids: cidsToEncrypt,
Expand All @@ -163,34 +192,41 @@ export async function encryptedEncodeCarFile(
chunker,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
cache,
root: rootCid
root: rootCid,
}) as AsyncGenerator<AnyBlock, void, unknown>) {
await encryptedBlocks.put(block.cid, block.bytes)
last = block
await encryptedBlocks.put(block.cid, block.bytes);
last = block;
}
if (!last) throw new Error('no blocks encrypted')
const encryptedCars = await encodeCarFiles([last.cid], encryptedBlocks)
return encryptedCars
if (!last) throw new Error("no blocks encrypted");
const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
return encryptedCar;
}

export async function decodeEncryptedCar(crypto: CryptoOpts, key: string, reader: CarReader) {
const roots = await reader.getRoots()
const root = roots[0]
return await decodeCarBlocks(crypto, root, reader.get.bind(reader), key)
export async function decodeEncryptedCar(
crypto: CryptoOpts,
key: string,
reader: CarReader
) {
const roots = await reader.getRoots();
const root = roots[0];
return await decodeCarBlocks(crypto, root, reader.get.bind(reader), key);
}
async function decodeCarBlocks(
crypto: CryptoOpts,
root: AnyLink,
get: (cid: any) => Promise<AnyBlock | undefined>,
keyMaterial: string
): Promise<{ blocks: MemoryBlockstore; root: AnyLink }> {
const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial)
const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength)
const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
const decryptionKey = decryptionKeyUint8.buffer.slice(
0,
decryptionKeyUint8.byteLength
);

const decryptedBlocks = new MemoryBlockstore()
let last: AnyBlock | null = null
const decryptedBlocks = new MemoryBlockstore();
let last: AnyBlock | null = null;

const { decrypt } = makeEncDec(crypto.crypto, crypto.randomBytes)
const { decrypt } = makeEncDec(crypto.crypto, crypto.randomBytes);

for await (const block of decrypt({
root,
Expand All @@ -200,11 +236,11 @@ async function decodeCarBlocks(
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
chunker,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
cache
cache,
})) {
await decryptedBlocks.put(block.cid, block.bytes)
last = block
await decryptedBlocks.put(block.cid, block.bytes);
last = block;
}
if (!last) throw new Error('no blocks decrypted')
return { blocks: decryptedBlocks, root: last.cid }
if (!last) throw new Error("no blocks decrypted");
return { blocks: decryptedBlocks, root: last.cid };
}
Loading

0 comments on commit 096c7af

Please sign in to comment.