Skip to content

Commit f7f46e0

Browse files
SylvainSenechalSylvainSenechal
committed
debug
1 parent a8963c5 commit f7f46e0

File tree

2 files changed

+155
-89
lines changed

2 files changed

+155
-89
lines changed

‎tests/zenko_tests/node_tests/backbeat/ReplicationUtility.js‎

Lines changed: 147 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -62,41 +62,32 @@ class ReplicationUtility {
6262
return cb();
6363
}
6464

65-
console.log(`[DEBUG] Attempting to delete ${versionList.length} objects from bucket: ${bucketName}`);
66-
console.log('[DEBUG] Objects to delete:', versionList.map(item => ({ Key: item.Key, VersionId: item.VersionId })));
67-
68-
const params = {
69-
Bucket: bucketName,
70-
Delete: {
71-
Objects: versionList.map(item => ({
72-
Key: item.Key,
73-
VersionId: item.VersionId,
74-
})),
75-
},
76-
};
77-
78-
const command = new DeleteObjectsCommand(params);
65+
// Use individual DeleteObjectCommand calls for all S3 clients to avoid Content-MD5 issues
66+
// This is more reliable than DeleteObjectsCommand and works consistently across all S3-compatible endpoints
67+
const deletePromises = versionList.map(item => {
68+
const params = {
69+
Bucket: bucketName,
70+
Key: item.Key,
71+
};
72+
if (item.VersionId) {
73+
params.VersionId = item.VersionId;
74+
}
75+
return this.s3.send(new DeleteObjectCommand(params));
76+
});
7977

80-
// Disable automatic checksum calculation to avoid Content-MD5 mismatch
81-
return this.s3.send(command, {
82-
requestChecksumCalculation: 'WHEN_REQUIRED',
83-
responseChecksumValidation: 'WHEN_REQUIRED'
84-
})
78+
Promise.all(deletePromises)
8579
.then(() => {
86-
console.log(`[DEBUG] Successfully deleted ${versionList.length} objects from bucket: ${bucketName}`);
8780
cb();
8881
})
8982
.catch(err => {
90-
console.error(`[ERROR] Failed to delete objects from bucket: ${bucketName}`);
91-
console.error('[ERROR] Error details:', {
83+
console.error('[ERROR] _deleteVersionList: Error details:', {
9284
name: err.name,
9385
message: err.message,
9486
code: err.code,
9587
statusCode: err.$metadata?.httpStatusCode,
9688
requestId: err.$metadata?.requestId,
9789
extendedRequestId: err.$metadata?.extendedRequestId
9890
});
99-
console.error('[ERROR] Full error object:', err);
10091
cb(err);
10192
});
10293
}
@@ -111,22 +102,47 @@ class ReplicationUtility {
111102
.then(data => {
112103
let versions = data.Versions || [];
113104
let deleteMarkers = data.DeleteMarkers || [];
105+
114106
// If replicating to a multiple backend bucket, we only want to
115107
// remove versions that we have put with our tests.
116108
if (keyPrefix) {
109+
const originalVersionCount = versions.length;
110+
const originalMarkerCount = deleteMarkers.length;
117111
versions = versions.filter(version => version.Key.startsWith(keyPrefix));
118112
deleteMarkers = deleteMarkers.filter(marker => marker.Key.startsWith(keyPrefix));
119113
}
114+
115+
if (versions.length === 0 && deleteMarkers.length === 0) {
116+
return cb();
117+
}
118+
120119
return async.series([
121-
next => this._deleteVersionList(
122-
deleteMarkers,
123-
bucketName,
124-
next,
125-
),
126-
next => this._deleteVersionList(versions, bucketName, next),
127-
], err => cb(err));
120+
next => {
121+
this._deleteVersionList(deleteMarkers, bucketName, next);
122+
},
123+
next => {
124+
this._deleteVersionList(versions, bucketName, next);
125+
},
126+
], err => {
127+
if (err) {
128+
console.error(`[ERROR] deleteAllVersions: Error during deletion process:`, err);
129+
} else {
130+
console.log(`[DEBUG] deleteAllVersions: All deletions completed successfully for bucket: ${bucketName}`);
131+
}
132+
cb(err);
133+
});
128134
})
129-
.catch(err => cb(err));
135+
.catch(err => {
136+
console.error('[ERROR] deleteAllVersions: ListObjectVersionsCommand error details:', {
137+
name: err.name,
138+
message: err.message,
139+
code: err.code,
140+
statusCode: err.$metadata?.httpStatusCode,
141+
requestId: err.$metadata?.requestId,
142+
extendedRequestId: err.$metadata?.extendedRequestId
143+
});
144+
cb(err);
145+
});
130146
}
131147

132148
deleteAllBlobs(containerName, keyPrefix, cb) {
@@ -156,7 +172,6 @@ class ReplicationUtility {
156172
Bucket: bucketName,
157173
Key: objectName,
158174
Body: content,
159-
ChecksumAlgorithm: 'SHA256',
160175
}))
161176
.then(data => cb(null, data))
162177
.catch(err => cb(err));
@@ -249,7 +264,6 @@ class ReplicationUtility {
249264
Bucket: bucketName,
250265
CopySource: copySource,
251266
Key: objectName,
252-
ChecksumAlgorithm: 'SHA256',
253267
}))
254268
.then(data => cb(null, data))
255269
.catch(err => cb(err));
@@ -272,7 +286,6 @@ class ReplicationUtility {
272286
const initiateMPUParams = {
273287
Bucket: bucketName,
274288
Key: objectName,
275-
ChecksumAlgorithm: 'SHA256',
276289
};
277290
if (hasOptionalFields) {
278291
Object.assign(initiateMPUParams, {
@@ -298,7 +311,6 @@ class ReplicationUtility {
298311
PartNumber: partNumber + 1,
299312
UploadId: uploadId,
300313
Body: Buffer.alloc(partSize).fill(partNumber + 1),
301-
ChecksumAlgorithm: 'SHA256',
302314
};
303315

304316
return this.s3.send(new UploadPartCommand(uploadPartParams))
@@ -322,7 +334,6 @@ class ReplicationUtility {
322334
})),
323335
},
324336
UploadId: uploadId,
325-
ChecksumAlgorithm: 'SHA256',
326337
};
327338
return this.s3.send(new CompleteMultipartUploadCommand(params))
328339
.then(data => next(null, data))
@@ -415,7 +426,6 @@ class ReplicationUtility {
415426
next => this.s3.send(new CreateMultipartUploadCommand({
416427
Bucket: bucketName,
417428
Key: objectName,
418-
ChecksumAlgorithm: 'SHA256',
419429
}))
420430
.then(data => {
421431
uploadId = data.UploadId;
@@ -431,7 +441,6 @@ class ReplicationUtility {
431441
Key: objectName,
432442
PartNumber: partNumber + 1,
433443
UploadId: uploadId,
434-
ChecksumAlgorithm: 'SHA256',
435444
};
436445
return this.s3.send(new UploadPartCopyCommand(uploadPartCopyParams))
437446
.then(data => callback(null, data.ETag))
@@ -453,7 +462,6 @@ class ReplicationUtility {
453462
})),
454463
},
455464
UploadId: uploadId,
456-
ChecksumAlgorithm: 'SHA256',
457465
}))
458466
.then(data => next(null, data))
459467
.catch(err => next(err)),
@@ -474,7 +482,18 @@ class ReplicationUtility {
474482
Bucket: bucketName,
475483
Key: objName,
476484
}))
477-
.then(data => cb(null, data))
485+
.then(async (data) => {
486+
// AWS SDK v3 returns a readable stream in data.Body
487+
// We need to collect the stream data into a buffer
488+
if (data.Body) {
489+
const chunks = [];
490+
for await (const chunk of data.Body) {
491+
chunks.push(chunk);
492+
}
493+
data.Body = Buffer.concat(chunks);
494+
}
495+
cb(null, data);
496+
})
478497
.catch(err => cb(err));
479498
}
480499

@@ -669,13 +688,30 @@ class ReplicationUtility {
669688
}
670689

671690
deleteObject(bucketName, key, versionId, cb) {
672-
this.s3.send(new DeleteObjectCommand({
691+
console.log(`[DEBUG] deleteObject called: bucket=${bucketName}, key=${key}, versionId=${versionId}`);
692+
const params = {
673693
Bucket: bucketName,
674694
Key: key,
675-
VersionId: versionId,
676-
}))
677-
.then(data => cb(null, data))
678-
.catch(err => cb(err));
695+
};
696+
// Only add VersionId if it's not null/undefined
697+
if (versionId) {
698+
params.VersionId = versionId;
699+
}
700+
this.s3.send(new DeleteObjectCommand(params))
701+
.then(data => {
702+
console.log(`[DEBUG] deleteObject succeeded: bucket=${bucketName}, key=${key}`);
703+
cb(null, data);
704+
})
705+
.catch(err => {
706+
console.error(`[ERROR] deleteObject failed: bucket=${bucketName}, key=${key}`, {
707+
name: err.name,
708+
message: err.message,
709+
code: err.code,
710+
statusCode: err.$metadata?.httpStatusCode,
711+
requestId: err.$metadata?.requestId,
712+
});
713+
cb(err);
714+
});
679715
}
680716

681717
expectReplicationStatus(bucketName, key, versionId, expectedStatus, cb) {
@@ -729,8 +765,16 @@ class ReplicationUtility {
729765
return async.doWhilst(
730766
callback => this[method](bucketName, key, err => {
731767
const cbOnce = jsutil.once(callback);
732-
if (err && err.code !== expectedCode) {
733-
return cbOnce(err);
768+
console.error(`[DEBUG] waitUntilDeleted: Checking existence of ${bucketName}`);
769+
console.error(`[DEBUG] waitUntilDeleted: Checking existence of ${key}`);
770+
console.error(`[DEBUG] waitUntilDeleted: Checking existence of ${client}`);
771+
console.error(`[DEBUG] waitUntilDeleted: Checking existence of ${err}`);
772+
if (err) {
773+
// AWS SDK v3 uses err.name instead of err.code
774+
const errorCode = err.name || err.code;
775+
if (errorCode !== expectedCode) {
776+
return cbOnce(err);
777+
}
734778
}
735779
objectExists = err === null;
736780
if (!objectExists) {
@@ -803,52 +847,71 @@ class ReplicationUtility {
803847

804848
compareObjectsAWS(srcBucket, destBucket, key, optionalField, cb) {
805849
return async.series([
806-
next => this.waitUntilReplicated(srcBucket, key, undefined, next),
807-
next => this.getObject(srcBucket, key, next),
808-
next => this._setS3Client(awsS3Client)
809-
.getObject(destBucket, `${srcBucket}/${key}`, next),
850+
next => {
851+
this.waitUntilReplicated(srcBucket, key, undefined, next);
852+
},
853+
next => {
854+
this.getObject(srcBucket, key, next);
855+
},
856+
next => {
857+
this._setS3Client(awsS3Client)
858+
.getObject(destBucket, `${srcBucket}/${key}`, next);
859+
},
810860
], (err, data) => {
811861
this._setS3Client(scalityS3Client);
812862
if (err) {
863+
console.error(`[ERROR] Failed during AWS object comparison for ${srcBucket}/${key}:`, err);
813864
return cb(err);
814865
}
866+
815867
const srcData = data[1];
816868
const destData = data[2];
817-
assert.strictEqual(srcData.ReplicationStatus, 'COMPLETED');
818-
assert.strictEqual(
819-
srcData.ContentLength,
820-
destData.ContentLength,
821-
);
822-
this._compareObjectBody(srcData.Body, destData.Body);
823-
const srcUserMD = srcData.Metadata;
824-
assert.strictEqual(
825-
srcUserMD[`${destAWSLocation}-version-id`],
826-
destData.VersionId,
827-
);
828-
assert.strictEqual(
829-
srcUserMD[`${destAWSLocation}-replication-status`],
830-
'COMPLETED',
831-
);
832-
const destUserMD = destData.Metadata;
833-
assert.strictEqual(
834-
destUserMD['scal-version-id'],
835-
srcData.VersionId,
836-
);
837-
assert.strictEqual(
838-
destUserMD['scal-replication-status'],
839-
'REPLICA',
840-
);
841-
if (optionalField === 'Metadata') {
842-
assert.strictEqual(srcUserMD.customkey, 'customValue');
843-
assert.strictEqual(destUserMD.customkey, 'customValue');
844-
}
845-
if (optionalField && optionalField !== 'Metadata') {
869+
870+
try {
871+
assert.strictEqual(srcData.ReplicationStatus, 'COMPLETED');
872+
assert.strictEqual(
873+
srcData.ContentLength,
874+
destData.ContentLength,
875+
);
876+
this._compareObjectBody(srcData.Body, destData.Body);
877+
878+
const srcUserMD = srcData.Metadata;
879+
const destUserMD = destData.Metadata;
880+
846881
assert.strictEqual(
847-
srcData[optionalField],
848-
destData[optionalField],
882+
srcUserMD[`${destAWSLocation}-version-id`],
883+
destData.VersionId,
849884
);
885+
assert.strictEqual(
886+
srcUserMD[`${destAWSLocation}-replication-status`],
887+
'COMPLETED',
888+
);
889+
assert.strictEqual(
890+
destUserMD['scal-version-id'],
891+
srcData.VersionId,
892+
);
893+
assert.strictEqual(
894+
destUserMD['scal-replication-status'],
895+
'REPLICA',
896+
);
897+
898+
if (optionalField === 'Metadata') {
899+
assert.strictEqual(srcUserMD.customkey, 'customValue');
900+
assert.strictEqual(destUserMD.customkey, 'customValue');
901+
}
902+
if (optionalField && optionalField !== 'Metadata') {
903+
assert.strictEqual(
904+
srcData[optionalField],
905+
destData[optionalField],
906+
);
907+
}
908+
return cb();
909+
} catch (assertionError) {
910+
console.error(`[ERROR] Assertion failed during AWS object comparison for ${srcBucket}/${key}:`, assertionError.message);
911+
console.error(`[ERROR] Source metadata:`, srcUserMD);
912+
console.error(`[ERROR] Dest metadata:`, destUserMD);
913+
return cb(assertionError);
850914
}
851-
return cb();
852915
});
853916
}
854917

@@ -1191,6 +1254,7 @@ class ReplicationUtility {
11911254

11921255
assertNoObject(bucketName, key, cb) {
11931256
this.getObject(bucketName, key, err => {
1257+
console.log(`[DEBUG] assertNoObject: Checking existence of ${bucketName}/${key}`);
11941258
assert.strictEqual(err.name, 'NoSuchKey');
11951259
return cb();
11961260
});

‎tests/zenko_tests/node_tests/backbeat/tests/crr/awsBackend.js‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ const copyKey = `${key}-copy`;
1818
const copySource = `/${srcBucket}/${key}`;
1919
// eslint-disable-next-line
2020
const keyutf8 = `${keyPrefix}/%EA%9D%8崰㈌㒈保轖䳷䀰⺩ቆ楪僷ꈅꓜ퇬枅࿷염곞召㸾⌙ꪊᆐ庍뉆䌗↎幐냂詴 끴鹲萯⇂쫤ᛩ꺶㖭簹릍铰᫫暨鿐魪셑蛃춧㡡竺뫁噛̷ᗰⷑ錜⑔痴䧫㾵᏷ำꎆ꼵껪멷㄀誕㳓腜쒃컹㑻鳃삚舿췈孨੦⮀NJ곓⵪꺼꜈嗼뫘悕錸瑺⁤⑬১㵀⡸Ҏ礄䧛졼⮦ٞ쫁퓡垻ㆩꝿ詀펉ᆙ舑䜾힑藪碙ꀎꂰ췊Ᏻ 㘺幽醛잯ද汧Ꟑꛒⶨ쪸숞헹㭔ꡔᘼ뺓ᡆ᡾ᑟ䅅퀭耓弧⢠⇙폪ް蛧⃪Ἔ돫ꕢ븥ヲ캂䝄쟐颺ᓾ둾Ұ껗礞ᾰ瘹蒯硳풛瞋襎奺熝妒컚쉴⿂㽝㝳駵鈚䄖戭䌸᫲ᇁ䙪鸮ᐴ稫ⶭ뀟ھ⦿䴳稉ꉕ捈袿놾띐✯伤䃫⸧ꠏ瘌틳藔ˋ㫣敀䔩㭘식↴⧵佶痊牌ꪌ搒꾛æᤈべ쉴挜炩⽍舘ꆗ줣徭Z䐨 敗羥誜嘳ֶꫜ걵ࣀ묟ኋ拃秷䨸菥䟆곘縧멀煣⧃⏶혣뎧邕⢄⭖陙䣎灏ꗛ僚䌁䠒䲎둘ꪎ傩쿌ᨌ뀻阥눉넠猌ㆯ㰢船戦跏灳蝒礯鞰諾벥煸珬㟑孫鞹Ƭꄹ孙ꢱ钐삺ᓧ鈠䁞〯蘼᫩헸ῖ"`;
21-
const REPLICATION_TIMEOUT = 300000;
21+
const REPLICATION_TIMEOUT = 500000;
2222

2323
describe('Replication with AWS backend', function () {
2424
// eslint-disable-next-line
@@ -41,11 +41,13 @@ describe('Replication with AWS backend', function () {
4141
next => scalityUtils.deleteVersionedBucket(srcBucket, next),
4242
// Destination location has falsy bucket match property, so we update
4343
// the key prefix.
44-
next => awsUtils.deleteAllVersions(
45-
destBucket,
46-
`${srcBucket}/${keyPrefix}`,
47-
next,
48-
),
44+
next => {
45+
awsUtils.deleteAllVersions(
46+
destBucket,
47+
`${srcBucket}/${keyPrefix}`,
48+
next,
49+
);
50+
},
4951
], done));
5052

5153
it('should replicate an object', done => series([

0 commit comments

Comments
 (0)