Skip to content

Commit

Permalink
fix: copy more than just metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@maxakuru
maxakuru committed Jul 17, 2024
1 parent 23f327b commit 80b63e8
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 11 deletions.
22 changes: 18 additions & 4 deletions packages/helix-shared-storage/src/storage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,12 @@ class Bucket {
}
}

/** @type {S3Client} */
get client() {
return this._s3;
}

/** @type {string} */
get bucket() {
return this._bucket;
}
Expand Down Expand Up @@ -312,8 +314,14 @@ class Bucket {

try {
if (opts.addMetadata) {
const meta = await this.metadata(key) ?? {};
input.Metadata = { ...meta, ...opts.addMetadata };
const headers = await this.head(key);
if (headers) {
input.ContentType = headers.ContentType;
input.ContentEncoding = headers.ContentEncoding;
input.CacheControl = headers.CacheControl;
input.ContentDisposition = headers.ContentDisposition;
}
input.Metadata = { ...(headers?.Metadata ?? {}), ...opts.addMetadata };
input.MetadataDirective = 'REPLACE';
}
// write to s3 and r2 (mirror) in parallel
Expand Down Expand Up @@ -465,8 +473,14 @@ class Bucket {
};
try {
if (opts.addMetadata) {
const meta = await this.metadata(task.src) ?? {};
input.Metadata = { ...meta, ...opts.addMetadata };
const headers = await this.head(task.src);
if (headers) {
input.ContentType = headers.ContentType;
input.ContentEncoding = headers.ContentEncoding;
input.CacheControl = headers.CacheControl;
input.ContentDisposition = headers.ContentDisposition;
}
input.Metadata = { ...(headers?.Metadata ?? {}), ...opts.addMetadata };
input.MetadataDirective = 'REPLACE';
}
// write to s3 and r2 (mirror) in parallel
Expand Down
50 changes: 43 additions & 7 deletions packages/helix-shared-storage/test/storage.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,9 @@ describe('Storage test', () => {
if (heads.length <= 2) {
return [404];
}
return [200, undefined, {
return [200, [], {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'anonymous',
}];
Expand All @@ -504,6 +506,8 @@ describe('Storage test', () => {
.reply(function f(uri) {
puts.s3.push(uri);
putsHeaders.s3.push({
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -520,6 +524,8 @@ describe('Storage test', () => {
.reply(function f(uri) {
puts.r2.push(uri);
putsHeaders.r2.push({
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -544,11 +550,23 @@ describe('Storage test', () => {
Object.values(putsHeaders).forEach((s3r2) => {
s3r2.forEach((headers, i) => {
// first 2 returned 404, so no meta existed
assert.deepEqual(headers, {
'x-amz-meta-x-dont-overwrite': i <= 1 ? undefined : 'foo',
'x-amz-meta-x-last-modified-by': '[email protected]',
'x-amz-metadata-directive': 'REPLACE',
});
if (i <= 1) {
assert.deepEqual(headers, {
'content-type': undefined,
'content-encoding': undefined,
'x-amz-meta-x-dont-overwrite': undefined,
'x-amz-meta-x-last-modified-by': '[email protected]',
'x-amz-metadata-directive': 'REPLACE',
});
} else {
assert.deepEqual(headers, {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': '[email protected]',
'x-amz-metadata-directive': 'REPLACE',
});
}
});
});

Expand Down Expand Up @@ -614,13 +632,17 @@ describe('Storage test', () => {
const putsHeaders = { s3: undefined, r2: undefined };
nock('https://helix-code-bus.s3.fake.amazonaws.com')
.head('/owner/repo/ref/foo.md')
.reply(200, undefined, {
.reply(200, [], {
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'anonymous',
'content-type': 'text/plain',
'content-encoding': 'gzip',
})
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.s3 = {
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -632,6 +654,8 @@ describe('Storage test', () => {
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.r2 = {
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -652,11 +676,15 @@ describe('Storage test', () => {
assert.deepEqual(puts.r2, expectedPuts);

assert.deepEqual(putsHeaders.s3, {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': '[email protected]',
});
assert.deepEqual(putsHeaders.r2, {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': '[email protected]',
Expand All @@ -672,6 +700,8 @@ describe('Storage test', () => {
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.s3 = {
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -683,6 +713,8 @@ describe('Storage test', () => {
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.r2 = {
'content-type': this.req.headers['content-type'],
'content-encoding': this.req.headers['content-encoding'],
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
Expand All @@ -703,11 +735,15 @@ describe('Storage test', () => {
assert.deepEqual(puts.r2, expectedPuts);

assert.deepEqual(putsHeaders.s3, {
'content-type': undefined,
'content-encoding': undefined,
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': undefined,
'x-amz-meta-x-last-modified-by': '[email protected]',
});
assert.deepEqual(putsHeaders.r2, {
'content-type': undefined,
'content-encoding': undefined,
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': undefined,
'x-amz-meta-x-last-modified-by': '[email protected]',
Expand Down

0 comments on commit 80b63e8

Please sign in to comment.