From 06d263da1c1fb52b5affb5e3952f3daf6af41f0e Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 8 Sep 2023 22:49:07 +0200 Subject: [PATCH 1/9] Support for wildcards --- src/lib/encoding.ts | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index ead72d81..c016b151 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -231,6 +231,7 @@ export function parseManifestStream(stream: Readable): EventEmitter { let currentKey: string | undefined; const keyPath: Array = []; let indexPath: string | undefined; + let wildcardpath: string| undefined; let paths: { [k: string]: string } = {}; let hasValidManifestKey = false; // { "manifest": "arweave/paths" } let hasValidManifestVersion = false; // { "version": "0.1.0" } @@ -295,6 +296,18 @@ export function parseManifestStream(stream: Readable): EventEmitter { paths = {}; } } + if ( + keyPath.length === 1 && + keyPath[0] === '*' && + currentKey === 'path' + ) { + wildcardPath = data; + // Resolve if the path id is already known + if (wildcardPath !== undefined && paths[wildcardPath] !== undefined) { + emitter.emit('wildcard', { path: wildcardPath, id: paths[wildcardPath] }); + paths = {}; + } + } // Paths - { "paths": { "some/path/file.html": { "id": "" } } if ( @@ -311,6 +324,9 @@ export function parseManifestStream(stream: Readable): EventEmitter { } else if (p === indexPath) { emitter.emit('index', { path: p, id: data }); paths = {}; + } else if (p === wildcardPath){ + emitter.emit('wildcard', { path: p, id: data }); + paths = {}; } } }); @@ -324,20 +340,23 @@ export function resolveManifestStreamPath( ): Promise { return new Promise((resolve, reject) => { const emitter = parseManifestStream(stream); - + let resolved=false // Remove trailing slashes from path - treat /path and /path/ the same const sanitizedPath = path !== undefined ? path.replace(/\/+$/g, '') : ''; emitter.on('error', (err) => { + resolved=true reject(err); }); emitter.on('end', () => { + resolved=true resolve(undefined); }); emitter.on('index', (data) => { if (sanitizedPath === '') { + resolved=true resolve(data.id); } }); @@ -345,8 +364,16 @@ export function resolveManifestStreamPath( emitter.on('path', (data) => { const trimmedDataPath = data.path.replace(/\/+$/g, ''); if (sanitizedPath !== '' && trimmedDataPath === sanitizedPath) { + resolved=true resolve(data.id); } }); + + emitter.on('wildcard',(data) => { + if(!resolved){ + resolved=true + resolve(data.id) + } + }) }); } From 4a903f1fc21fba87acd27cdd713da91c2e729bb9 Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 8 Sep 2023 22:55:02 +0200 Subject: [PATCH 2/9] Fix for wildcard resolving --- src/lib/encoding.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index c016b151..8a130cdf 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -342,6 +342,7 @@ export function resolveManifestStreamPath( const emitter = parseManifestStream(stream); let resolved=false // Remove trailing slashes from path - treat /path and /path/ the same + let wildcard; const sanitizedPath = path !== undefined ? path.replace(/\/+$/g, '') : ''; emitter.on('error', (err) => { @@ -351,7 +352,7 @@ export function resolveManifestStreamPath( emitter.on('end', () => { resolved=true - resolve(undefined); + resolve(wildcard); }); emitter.on('index', (data) => { @@ -370,10 +371,7 @@ export function resolveManifestStreamPath( }); emitter.on('wildcard',(data) => { - if(!resolved){ - resolved=true - resolve(data.id) - } + wildcard=data }) }); } From beabec426e44d426da6abe276ae2063d5139c468 Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 8 Sep 2023 22:56:36 +0200 Subject: [PATCH 3/9] Check for was it already resolved or not --- src/lib/encoding.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index 8a130cdf..2d3e2aaa 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -351,8 +351,10 @@ export function resolveManifestStreamPath( }); emitter.on('end', () => { + if(!resolved){ resolved=true resolve(wildcard); + } }); emitter.on('index', (data) => { From fe1f39033d776b5a69254ac35a5043b06a9b0afd Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:16:52 +0200 Subject: [PATCH 4/9] Refactor --- src/lib/encoding.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index 2d3e2aaa..ed4c542c 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -231,7 +231,7 @@ export function parseManifestStream(stream: Readable): EventEmitter { let currentKey: string | undefined; const keyPath: Array = []; let indexPath: string | undefined; - let wildcardpath: string| undefined; + let wildcardpath: string | undefined; let paths: { [k: string]: string } = {}; let hasValidManifestKey = false; // { "manifest": "arweave/paths" } let hasValidManifestVersion = false; // { "version": "0.1.0" } @@ -296,6 +296,8 @@ export function parseManifestStream(stream: Readable): EventEmitter { paths = {}; } } + + // Wildcard - { "*": { "path": "404.html" } } if ( keyPath.length === 1 && keyPath[0] === '*' && @@ -375,5 +377,6 @@ export function resolveManifestStreamPath( emitter.on('wildcard',(data) => { wildcard=data }) + }); } From 9f25c9ce9ceccb647541baf3139869c2bde96200 Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:20:42 +0200 Subject: [PATCH 5/9] fix typo of wildcardpath > wildcardPath --- src/lib/encoding.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index ed4c542c..b4fefcda 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -231,7 +231,7 @@ export function parseManifestStream(stream: Readable): EventEmitter { let currentKey: string | undefined; const keyPath: Array = []; let indexPath: string | undefined; - let wildcardpath: string | undefined; + let wildcardPath: string | undefined; let paths: { [k: string]: string } = {}; let hasValidManifestKey = false; // { "manifest": "arweave/paths" } let hasValidManifestVersion = false; // { "version": "0.1.0" } From 77ef5f8f63d85f656d3c6fc630901a09a78a6871 Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Sat, 9 Sep 2023 00:52:20 +0200 Subject: [PATCH 6/9] Resolving of wildcard using only id --- src/lib/encoding.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index b4fefcda..f418b422 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -278,8 +278,8 @@ export function parseManifestStream(stream: Readable): EventEmitter { hasValidManifestKey = true; } - // Manifest version - { "version": "0.1.0" } - if (keyPath.length === 0 && currentKey === 'version' && data === '0.1.0') { + // Manifest version - { "version": "0.1.1" } + if (keyPath.length === 0 && currentKey === 'version' && ["0.1.0", "0.1.1"].includes(data)) { hasValidManifestVersion = true; } @@ -311,6 +311,16 @@ export function parseManifestStream(stream: Readable): EventEmitter { } } + // Wildcard - { "*": { "id": "wildcard-id" } } + if ( + keyPath.length === 1 && + keyPath[0] === '*' && + currentKey === 'id' + ) { + emitter.emit('wildcard', { path: null, id: data }); + paths = {}; + } + // Paths - { "paths": { "some/path/file.html": { "id": "" } } if ( keyPath.length === 2 && From 4f52ee7830b67967ce26066e5a01666969cc2cde Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Fri, 15 Sep 2023 20:32:42 +0200 Subject: [PATCH 7/9] Bump version --- src/lib/encoding.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/encoding.ts b/src/lib/encoding.ts index f418b422..a3e21de1 100644 --- a/src/lib/encoding.ts +++ b/src/lib/encoding.ts @@ -278,8 +278,8 @@ export function parseManifestStream(stream: Readable): EventEmitter { hasValidManifestKey = true; } - // Manifest version - { "version": "0.1.1" } - if (keyPath.length === 0 && currentKey === 'version' && ["0.1.0", "0.1.1"].includes(data)) { + // Manifest version - { "version": "0.2.0" } or { "version": "0.1.0" } + if (keyPath.length === 0 && currentKey === 'version' && ["0.1.0", "0.2.0"].includes(data)) { hasValidManifestVersion = true; } From 76319bd754c11cdbfbbc9f2ef208b297d14b8f60 Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Sat, 16 Sep 2023 00:33:47 +0200 Subject: [PATCH 8/9] Put wildcard into example manifest --- test/mock_files/manifests/example_manifest.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/mock_files/manifests/example_manifest.json b/test/mock_files/manifests/example_manifest.json index 95672c70..e12a273b 100644 --- a/test/mock_files/manifests/example_manifest.json +++ b/test/mock_files/manifests/example_manifest.json @@ -1,8 +1,11 @@ { "manifest": "arweave/paths", - "version": "0.1.0", + "version": "0.2.0", + "*":{ + "id":"vNIcG7HdiMpFqJ8NZN4KopG4_iTQPoH_j7iP7EYgQJ-" + }, "index": { - "path": "index.html" + "path": "index.html", }, "paths": { "index.html": { From 515e1ce2d7dbeafa174602e44530fc4a59550e2e Mon Sep 17 00:00:00 2001 From: Nikita Rykov <40735471+angrymouse@users.noreply.github.com> Date: Sat, 16 Sep 2023 00:37:39 +0200 Subject: [PATCH 9/9] Add test for resolving wildcard --- src/lib/encoding.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/encoding.test.ts b/src/lib/encoding.test.ts index 2f4cd9ad..9d4857ef 100644 --- a/src/lib/encoding.test.ts +++ b/src/lib/encoding.test.ts @@ -247,6 +247,14 @@ describe('Manifest parsing', () => { expect(id).to.equal('cG7Hdi_iTQPoEYgQJFqJ8NMpN4KoZ-vH_j7pG4iP7NI'); }); + it('should return wildcard file for files not present in manifest', async ()=>{ + const id=await resolveManifestStreamPath( + exampleManifestStream(), + 'path/that/does/not/exist.txt' + ); + expect(id).to.equal('vNIcG7HdiMpFqJ8NZN4KopG4_iTQPoH_j7iP7EYgQJ-') + }) + it('should return the ID for non-index paths', async () => { // TODO use an array here const id1 = await resolveManifestStreamPath(