diff --git a/manual_tests/test_image.jpeg b/manual_tests/test_image.jpeg new file mode 100644 index 00000000..bd33c459 Binary files /dev/null and b/manual_tests/test_image.jpeg differ diff --git a/manual_tests/upload_new.ts b/manual_tests/upload_new.ts index 9e31b21a..ade0eba7 100644 --- a/manual_tests/upload_new.ts +++ b/manual_tests/upload_new.ts @@ -54,7 +54,8 @@ const createFile = (size = 10 * 1024 * 1024) => Buffer.alloc(size).fill('a'); // } (async () => { - const file = await getFile(createFile()); + const file = await getFile('./test_image.jpeg'); + // const file = await getFile(createFile()); // const file1 = await getFile(createFile()); file.name = 'test.txt'; console.log('Uploadinf file', file.size); diff --git a/package-lock.json b/package-lock.json index ba36c004..5d932f1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1750,25 +1750,10 @@ "webpack-sources": "^1.0.0" } }, - "@sentry/cli": { - "version": "1.49.0", - "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-1.49.0.tgz", - "integrity": "sha512-Augz7c42Cxz/xWQ/NOVjUGePKVA370quvskWbCICMUwxcTvKnCLI+7KDdzEoCexj4MSuxFfBzLnrrn4w2+c9TQ==", - "dev": true, - "requires": { - "fs-copy-file-sync": "^1.1.1", - "https-proxy-agent": "^3.0.0", - "mkdirp": "^0.5.1", - "node-fetch": "^2.1.2", - "progress": "2.0.0", - "proxy-from-env": "^1.0.0" - } - }, "@sentry/hub": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.2.1.tgz", "integrity": "sha512-pG7wCQeRpzeP6t0bT4T0X029R19dbDS3/qswF8BL6bg0AI3afjfjBAZm/fqn1Uwe/uBoMHVVdbxgJDZeQ5d4rQ==", - "dev": true, "requires": { "@sentry/types": "6.2.1", "@sentry/utils": "6.2.1", @@ -1779,7 +1764,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.2.1.tgz", "integrity": "sha512-wuSXB4Ayxv9rBEQ4pm7fnG4UU2ZPtPnnChoEfd4/mw1UthXSvmPFEn6O4pdo2G8fTkl8eqm6wT/Q7uIXMEmw+A==", - "dev": true, "requires": { "@sentry/hub": "6.2.1", "@sentry/types": "6.2.1", @@ -1789,14 +1773,12 @@ "@sentry/types": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.2.1.tgz", - "integrity": "sha512-h0OV1QT+fv5ojfK5/+iEXClu33HirmvbjcQC2jf05IHj9yXIOWy6EB10S8nBjuLiiFqQiAQYj3FN9Ip4eN8NJA==", - "dev": true + "integrity": "sha512-h0OV1QT+fv5ojfK5/+iEXClu33HirmvbjcQC2jf05IHj9yXIOWy6EB10S8nBjuLiiFqQiAQYj3FN9Ip4eN8NJA==" }, "@sentry/utils": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.2.1.tgz", "integrity": "sha512-6kQgM/yBPdXu+3qbJnI6HBcWztN9QfiMkH++ZiKk4ERhg9d2LYWlze478uTU5Fyo/JQYcp+McpjtjpR9QIrr0g==", - "dev": true, "requires": { "@sentry/types": "6.2.1", "tslib": "^1.9.3" @@ -2368,15 +2350,6 @@ "integrity": "sha1-asL8ICtQD4jaj09VMM+hAPTGotA=", "dev": true }, - "agent-base": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", - "dev": true, - "requires": { - "es6-promisify": "^5.0.0" - } - }, "aggregate-error": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", @@ -6261,21 +6234,6 @@ "es6-symbol": "^3.1.1" } }, - "es6-promise": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==", - "dev": true - }, - "es6-promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", - "dev": true, - "requires": { - "es6-promise": "^4.0.3" - } - }, "es6-symbol": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", @@ -7203,12 +7161,6 @@ "null-check": "^1.0.0" } }, - "fs-copy-file-sync": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/fs-copy-file-sync/-/fs-copy-file-sync-1.1.1.tgz", - "integrity": "sha512-2QY5eeqVv4m2PfyMiEuy9adxNP+ajf+8AR05cEi+OAzPcOj90hvFImeZhTmKLBgSd9EvG33jsD7ZRxsx9dThkQ==", - "dev": true - }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -7481,7 +7433,8 @@ }, "ini": { "version": "1.3.5", - "resolved": "", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true, "optional": true }, @@ -8872,27 +8825,6 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, - "https-proxy-agent": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz", - "integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==", - "dev": true, - "requires": { - "agent-base": "^4.3.0", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - } - } - }, "human-signals": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", @@ -15006,12 +14938,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, - "progress": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", - "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=", - "dev": true - }, "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", @@ -15034,12 +14960,6 @@ "ipaddr.js": "1.9.1" } }, - "proxy-from-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", - "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=", - "dev": true - }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -17849,8 +17769,7 @@ "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", - "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==", - "dev": true + "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==" }, "tslint": { "version": "6.0.0", diff --git a/package.json b/package.json index 75b04c8b..16d72935 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "dependencies": { "@babel/runtime": "^7.8.4", "@filestack/loader": "^1.0.4", + "@sentry/minimal": "^6.2.1", "abab": "^2.0.3", "debug": "^4.1.1", "eventemitter3": "^4.0.0", @@ -52,9 +53,7 @@ "jsonschema": "^1.2.5", "lodash.clonedeep": "^4.5.0", "p-queue": "^4.0.0", - "spark-md5": "^3.0.0", - "ts-node": "^8.10.2", - "@sentry/minimal": "^6.2.1" + "spark-md5": "^3.0.0" }, "devDependencies": { "@babel/core": "^7.8.4", diff --git a/src/lib/api/upload/file.ts b/src/lib/api/upload/file.ts index 88a8124f..3b65d7b7 100644 --- a/src/lib/api/upload/file.ts +++ b/src/lib/api/upload/file.ts @@ -16,6 +16,8 @@ */ import { md5, sanitizeName, SanitizeOptions } from './../../utils'; +const MAGIC_BYTES_LEN = 300; + export interface UploadTags { [key: string]: string; } @@ -216,6 +218,13 @@ export class File { }); } + /** + * Returns magic bytes for file + */ + public async getMagicBytes() { + return String.fromCharCode.apply(null, new Uint16Array(await this._file.slice(0, MAGIC_BYTES_LEN))); + } + /** * Returns part chunk * diff --git a/src/lib/api/upload/uploaders/s3.spec.ts b/src/lib/api/upload/uploaders/s3.spec.ts index 494a58ed..142e69dc 100644 --- a/src/lib/api/upload/uploaders/s3.spec.ts +++ b/src/lib/api/upload/uploaders/s3.spec.ts @@ -513,10 +513,13 @@ describe('Api/Upload/Uploaders/S3', () => { expect(res[0].handle).toEqual('test_handle'); const testFile = getSmallTestFile(); + const bytes = await testFile.getMagicBytes(); + expect(mockStart).toHaveBeenCalledWith({ filename: testFile.name, mimetype: testFile.mimetype, size: testFile.size, + bytes, store: { location: DEFAULT_STORE_LOCATION, }, @@ -857,20 +860,23 @@ describe('Api/Upload/Uploaders/S3', () => { it('should upload file', async () => { const partSize = 1024 * 1024 * 7; + const testFile = getTestFile(); + const u = new S3Uploader({}); u.setUrl(testHost); u.setApikey(testApikey); u.setPartSize(partSize); - u.addFile(getTestFile()); + u.addFile(testFile); const res = await u.execute(); - const testFile = getTestFile(); + const bytes = await testFile.getMagicBytes(); expect(mockStart).toHaveBeenCalledWith({ filename: testFile.name, mimetype: testFile.mimetype, size: testFile.size, + bytes, store: { location: DEFAULT_STORE_LOCATION, }, @@ -941,11 +947,13 @@ describe('Api/Upload/Uploaders/S3', () => { const res = await u.execute(); const testFile = getTestFile(); + const bytes = await testFile.getMagicBytes(); expect(mockStart).toHaveBeenCalledWith({ filename: testFile.name, mimetype: testFile.mimetype, size: testFile.size, + bytes, store: { location: DEFAULT_STORE_LOCATION, }, diff --git a/src/lib/api/upload/uploaders/s3.ts b/src/lib/api/upload/uploaders/s3.ts index 7478d2d6..bf4f0777 100644 --- a/src/lib/api/upload/uploaders/s3.ts +++ b/src/lib/api/upload/uploaders/s3.ts @@ -285,7 +285,7 @@ export class S3Uploader extends UploaderAbstract { * @returns {Promise} * @memberof S3Uploader */ - private startRequest(id: string): Promise { + private async startRequest(id: string): Promise { const payload = this.getPayloadById(id); if (payload.file.size === 0) { @@ -294,12 +294,15 @@ export class S3Uploader extends UploaderAbstract { } debug(`[${id}] Make start request`); + const bytes = await payload.file.getMagicBytes(); + return FsRequest.post( `${this.getUrl()}/multipart/start`, { filename: payload.file.name, mimetype: payload.file.type, size: payload.file.size, + bytes: bytes.toString(), ...this.getDefaultFields(id, ['apikey', 'policy', 'signature', 'fii'], true), }, {