From 33dbf7c6ba1653031df61c1d76e1411574999768 Mon Sep 17 00:00:00 2001 From: Zach Wright Date: Fri, 12 Jun 2020 09:50:55 -0400 Subject: [PATCH] Add LFU scripts --- scripts/processLargeFileUpload/.env | 4 + .../application.properties | 33 ++ .../processLargeFileUpload/package-lock.json | 389 ++++++++++++++++++ scripts/processLargeFileUpload/package.json | 7 + .../processLargeFileUploadNoGlobus.sh | 41 ++ scripts/processLargeFileUpload/updateMongo.js | 52 +++ scripts/processLargeFileUpload/updateState.sh | 9 + 7 files changed, 535 insertions(+) create mode 100644 scripts/processLargeFileUpload/.env create mode 100755 scripts/processLargeFileUpload/application.properties create mode 100644 scripts/processLargeFileUpload/package-lock.json create mode 100644 scripts/processLargeFileUpload/package.json create mode 100755 scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh create mode 100644 scripts/processLargeFileUpload/updateMongo.js create mode 100755 scripts/processLargeFileUpload/updateState.sh diff --git a/scripts/processLargeFileUpload/.env b/scripts/processLargeFileUpload/.env new file mode 100644 index 00000000..ec2d8388 --- /dev/null +++ b/scripts/processLargeFileUpload/.env @@ -0,0 +1,4 @@ +export GLOBUS_DROPBOX_ENDPOINT=936381c8-1653-11ea-b94a-0e16720bb42f +export GLOBUS_DROPBOX_PATH=/PROD_INBOX/ +export LOCAL_ENDPOINT=95870406-9548-11ea-8ec8-02c81b96a709 +export DLU_INSTANCE=upload_kpmp_org diff --git a/scripts/processLargeFileUpload/application.properties b/scripts/processLargeFileUpload/application.properties new file mode 100755 index 00000000..dce733b9 --- /dev/null +++ b/scripts/processLargeFileUpload/application.properties @@ -0,0 +1,33 @@ +orion.data.version=${jar.baseName}-${jar.version} + +server.port=3030 + +file.base.path=/data/dataLake + +state.service.host=http://state-spring:3060 +state.service.endpoint=/v1/state + +user.auth.host=http://user-auth:8080 +user.auth.endpoint=/v1/user/info +user.auth.allowed.groups=uw_rit_kpmp_app_data-lake-uploader +user.auth.kpmp.group=uw_rit_kpmp_role_kpmp-user + + +spring.servlet.multipart.max-file-size=102400MB +spring.servlet.multipart.max-request-size=102400MB +spring.http.multipart.enabled=false + +spring.data.mongodb.uri=mongodb://localhost:27017/dataLake + +logging.level.org.springframework.web=DEBUG +logging.level.org.springframework.test.context.TestContextManager=DEBUG +logging.file=orion-data.log + +package.state.metadata.received=METADATA_RECEIVED +package.state.upload.succeeded=UPLOAD_SUCCEEDED +package.state.upload.started=UPLOAD_STARTED +package.state.files.received=FILES_RECEIVED +package.state.upload.failed=UPLOAD_FAILED + +globus.endpoint.ID=936381c8-1653-11ea-b94a-0e16720bb42f +globus.file.manager.url=https://app.globus.org/file-manager diff --git a/scripts/processLargeFileUpload/package-lock.json b/scripts/processLargeFileUpload/package-lock.json new file mode 100644 index 00000000..f9b4a64f --- /dev/null +++ b/scripts/processLargeFileUpload/package-lock.json @@ -0,0 +1,389 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "array-filter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-1.0.0.tgz", + "integrity": "sha1-uveeYubvTCpMC4MSMtr/7CUfnYM=" + }, + "assert": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/assert/-/assert-2.0.0.tgz", + "integrity": "sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==", + "requires": { + "es6-object-assign": "1.1.0", + "is-nan": "1.3.0", + "object-is": "1.1.2", + "util": "0.12.3" + } + }, + "available-typed-arrays": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz", + "integrity": "sha512-XWX3OX8Onv97LMk/ftVyBibpGwY5a8SmuxZPzeOxqmuEqUCOM9ZE+uIaD1VNJ5QnvU2UQusvmKbuM1FR8QWGfQ==", + "requires": { + "array-filter": "1.0.0" + } + }, + "bl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz", + "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==", + "requires": { + "readable-stream": "2.3.7", + "safe-buffer": "5.2.1" + } + }, + "bson": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz", + "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q==" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "1.1.1" + } + }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, + "es-abstract": { + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", + "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", + "requires": { + "es-to-primitive": "1.2.1", + "function-bind": "1.1.1", + "has": "1.0.3", + "has-symbols": "1.0.1", + "is-callable": "1.1.5", + "is-regex": "1.0.5", + "object-inspect": "1.7.0", + "object-keys": "1.1.1", + "object.assign": "4.1.0", + "string.prototype.trimleft": "2.1.2", + "string.prototype.trimright": "2.1.2" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "1.1.5", + "is-date-object": "1.0.2", + "is-symbol": "1.0.3" + } + }, + "es6-object-assign": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz", + "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=" + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "1.1.1" + } + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==" + }, + "is-callable": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", + "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==" + }, + "is-date-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==" + }, + "is-generator-function": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", + "integrity": "sha512-YZc5EwyO4f2kWCax7oegfuSr9mFz1ZvieNYBEjmukLxgXfBUbxAWGVF7GZf0zidYtoBl3WvC07YK0wT76a+Rtw==" + }, + "is-nan": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.0.tgz", + "integrity": "sha512-z7bbREymOqt2CCaZVly8aC4ML3Xhfi0ekuOnjO2L8vKdl+CttdVoGZQhd4adMFAsxQ5VeRVwORs4tU8RH+HFtQ==", + "requires": { + "define-properties": "1.1.3" + } + }, + "is-regex": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", + "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", + "requires": { + "has": "1.0.3" + } + }, + "is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "requires": { + "has-symbols": "1.0.1" + } + }, + "is-typed-array": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.3.tgz", + "integrity": "sha512-BSYUBOK/HJibQ30wWkWold5txYwMUXQct9YHAQJr8fSwvZoiglcqB0pd7vEN23+Tsi9IUEjztdOSzl4qLVYGTQ==", + "requires": { + "available-typed-arrays": "1.0.2", + "es-abstract": "1.17.5", + "foreach": "2.0.5", + "has-symbols": "1.0.1" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, + "mongodb": { + "version": "3.5.7", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.7.tgz", + "integrity": "sha512-lMtleRT+vIgY/JhhTn1nyGwnSMmJkJELp+4ZbrjctrnBxuLbj6rmLuJFz8W2xUzUqWmqoyVxJLYuC58ZKpcTYQ==", + "requires": { + "bl": "2.2.0", + "bson": "1.1.4", + "denque": "1.4.1", + "require_optional": "1.0.1", + "safe-buffer": "5.2.1", + "saslprep": "1.0.3" + } + }, + "object-inspect": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", + "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==" + }, + "object-is": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.2.tgz", + "integrity": "sha512-5lHCz+0uufF6wZ7CRFWJN3hp8Jqblpgve06U5CMQ3f//6iDjPr2PEo9MWCjEssDsa+UZEL4PkFpr+BMop6aKzQ==", + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.17.5" + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "requires": { + "define-properties": "1.1.3", + "function-bind": "1.1.1", + "has-symbols": "1.0.1", + "object-keys": "1.1.1" + } + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.4", + "isarray": "1.0.0", + "process-nextick-args": "2.0.1", + "safe-buffer": "5.1.2", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "require_optional": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", + "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", + "requires": { + "resolve-from": "2.0.0", + "semver": "5.7.1" + } + }, + "resolve-from": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", + "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "3.0.3" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "1.5.0" + } + }, + "string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.17.5" + } + }, + "string.prototype.trimleft": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", + "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.17.5", + "string.prototype.trimstart": "1.0.1" + } + }, + "string.prototype.trimright": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", + "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.17.5", + "string.prototype.trimend": "1.0.1" + } + }, + "string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "requires": { + "define-properties": "1.1.3", + "es-abstract": "1.17.5" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "util": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.3.tgz", + "integrity": "sha512-I8XkoQwE+fPQEhy9v012V+TSdH2kp9ts29i20TaaDUXsg7x/onePbhFJUExBfv/2ay1ZOp/Vsm3nDlmnFGSAog==", + "requires": { + "inherits": "2.0.4", + "is-arguments": "1.0.4", + "is-generator-function": "1.0.7", + "is-typed-array": "1.1.3", + "safe-buffer": "5.2.1", + "which-typed-array": "1.1.2" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==" + }, + "which-typed-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.2.tgz", + "integrity": "sha512-KT6okrd1tE6JdZAy3o2VhMoYPh3+J6EMZLyrxBQsZflI1QCZIxMrIYLkosd8Twf+YfknVIHmYQPgJt238p8dnQ==", + "requires": { + "available-typed-arrays": "1.0.2", + "es-abstract": "1.17.5", + "foreach": "2.0.5", + "function-bind": "1.1.1", + "has-symbols": "1.0.1", + "is-typed-array": "1.1.3" + } + } + } +} diff --git a/scripts/processLargeFileUpload/package.json b/scripts/processLargeFileUpload/package.json new file mode 100644 index 00000000..3322198a --- /dev/null +++ b/scripts/processLargeFileUpload/package.json @@ -0,0 +1,7 @@ +{ + "dependencies": { + "assert": "2.0.0", + "mongodb": "3.5.7", + "uuid": "^8.0.0" + } +} diff --git a/scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh b/scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh new file mode 100755 index 00000000..a89f4980 --- /dev/null +++ b/scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +source "${currentDir}"/.env + +echo "**** It would be wise to start me in Screen ***" + +echo "Package ID:" +read packageId + +packageDir="/data/dataLake/package_$packageId" +rm $packageDir/metadata.json + +cp /globus/PROD_INBOX/"${packageId}"/* "${packageDir}" + +node updateMongo.js "${packageId}" + +result=$? + +echo "Run the rezip process...and when that is done, run the update state script" + +#if [ $result == 0 ]; then +# java -cp /home/pathadmin/apps/orion-data/build/libs/orion-data.jar -Dloader.main=org.kpmp.RegenerateZipFiles org.springframework.boot.loader.PropertiesLauncher +# zipResult=$? +# if [ $zipResult == 0 ]; then +# data='{"packageId":"' +# data+=$packageId +# data+='", "state":"UPLOAD_SUCCEEDED", "largeUploadChecked":true}' +# url="http://localhost:3060/v1/state/host/${DLU_INSTANCE}" +# curl --header "Content-Type: application/json" --request POST --data "${data}" "${url}" +# else +# echo "Zip failed...exiting" +# exit -1 +# fi +#else +# echo "Mongo update failed...exiting." +# exit -1 +#fi +# +# +# +#exit diff --git a/scripts/processLargeFileUpload/updateMongo.js b/scripts/processLargeFileUpload/updateMongo.js new file mode 100644 index 00000000..2cddf777 --- /dev/null +++ b/scripts/processLargeFileUpload/updateMongo.js @@ -0,0 +1,52 @@ +const MongoClient = require("mongodb").MongoClient; +const assert = require("assert"); +const filesystem = require("fs"); +const path = require("path"); +const { v4: uuidGenerator } = require("uuid"); + +const url = "mongodb://localhost:27017"; +const dbName = "dataLake"; + +const updatePackage = function(packageId, files, db, callback) { + + var packages = db.collection("packages"); + packages.findOneAndUpdate({ "_id": packageId }, {$set: { files, regenerateZip: true }}, {returnOriginal: true}, function(err, doc) { + if(err) { + throw err; + } else if (typeof doc === "undefined") { + throw "Unable to find package with id ${packageId}"; + } else { + // eslint-disable-next-line no-console + console.log("Successfully updated"); + } + callback(); + }); +}; + + +MongoClient.connect(url, { useUnifiedTopology: true, "forceServerObjectId" : true }, function(err, client) { + assert.equal(null, err); + + const db = client.db(dbName); + + const args = process.argv.slice(2); + const packageId = args[0]; + + const packageDir = "package_" + packageId; + const directory = path.join("/data/dataLake", packageDir); + var files = filesystem.readdirSync(directory); + let fileInfos = []; + files.forEach(function(file) { + const filePath = path.join(directory, file); + let stats = filesystem.statSync(filePath); + let fileInfo = {}; + fileInfo.fileName= file; + fileInfo.size=stats["size"]; + fileInfo._id = uuidGenerator(); + fileInfos.push(fileInfo); + }); + + updatePackage(packageId, fileInfos , db, function() { + client.close(); + }); +}); diff --git a/scripts/processLargeFileUpload/updateState.sh b/scripts/processLargeFileUpload/updateState.sh new file mode 100755 index 00000000..656015ad --- /dev/null +++ b/scripts/processLargeFileUpload/updateState.sh @@ -0,0 +1,9 @@ +#!/bin/bash +echo "Package ID:" +read packageId + +data='{"packageId":"' +data+=$packageId +data+='", "state":"UPLOAD_SUCCEEDED", "largeUploadChecked":true}' +url="http://localhost:3060/v1/state/host/upload_kpmp_org" +curl --header "Content-Type: application/json" --request POST --data "${data}" "${url}"