diff --git a/build-scripts/gulp/download-translations.js b/build-scripts/gulp/download-translations.js index a8cf9b536bcf..c1e4064f67a8 100644 --- a/build-scripts/gulp/download-translations.js +++ b/build-scripts/gulp/download-translations.js @@ -1,10 +1,14 @@ import fs from "fs/promises"; import gulp from "gulp"; +import path from "path"; import mapStream from "map-stream"; import transform from "gulp-json-transform"; +import { LokaliseApi } from "@lokalise/node-api"; +import JSZip from "jszip"; -const inDirFrontend = "translations/frontend"; -const inDirBackend = "translations/backend"; +const inDir = "translations"; +const inDirFrontend = `${inDir}/frontend`; +const inDirBackend = `${inDir}/backend`; const srcMeta = "src/translations/translationMetadata.json"; const encoding = "utf8"; @@ -68,8 +72,9 @@ gulp.task("convert-backend-translations", function () { }); gulp.task("check-translations-html", function () { - // We exclude backend translations because they are not compliant with the HTML rule for now - return gulp.src([`${inDirFrontend}/*.json`]).pipe(checkHtml()); + return gulp + .src([`${inDirFrontend}/*.json`, `${inDirBackend}/*.json`]) + .pipe(checkHtml()); }); gulp.task("check-all-files-exist", async function () { @@ -89,7 +94,83 @@ gulp.task("check-all-files-exist", async function () { await Promise.allSettled(writings); }); +const lokaliseProjects = { + backend: "130246255a974bd3b5e8a1.51616605", + frontend: "3420425759f6d6d241f598.13594006", +}; + +gulp.task("fetch-lokalise", async function () { + let apiKey; + try { + apiKey = + process.env.LOKALISE_TOKEN || + (await fs.readFile(".lokalise_token", { encoding })); + } catch { + throw new Error( + "An Administrator Lokalise API token is required to download the latest set of translations. Place your token in a new file `.lokalise_token` in the repo root directory." + ); + } + const lokaliseApi = new LokaliseApi({ apiKey }); + + const mkdirPromise = Promise.all([ + fs.mkdir(inDirFrontend, { recursive: true }), + fs.mkdir(inDirBackend, { recursive: true }), + ]); + + await Promise.all( + Object.entries(lokaliseProjects).map(([project, projectId]) => + lokaliseApi + .files() + .download(projectId, { + format: "json", + original_filenames: false, + replace_breaks: false, + json_unescaped_slashes: true, + export_empty_as: "skip", + }) + .then((download) => fetch(download.bundle_url)) + .then((response) => { + if (response.status === 200 || response.status === 0) { + return response.arrayBuffer(); + } + throw new Error(response.statusText); + }) + .then(JSZip.loadAsync) + .then(async (contents) => { + await mkdirPromise; + return Promise.all( + Object.keys(contents.files).map(async (filename) => { + const file = contents.file(filename); + if (!file) { + // no file, probably a directory + return Promise.resolve(); + } + return file + .async("nodebuffer") + .then((content) => + fs.writeFile( + path.join( + inDir, + project, + filename.split("/").splice(-1)[0] + ), + content, + { flag: "w", encoding } + ) + ); + }) + ); + }) + ) + ); +}); + gulp.task( - "check-downloaded-translations", - gulp.series("check-translations-html", "check-all-files-exist") + "download-translations", + gulp.series( + "fetch-lokalise", + "convert-backend-translations", + "check-translations-html", + "check-all-files-exist" + ) ); diff --git a/package.json b/package.json index 2bbaff434c50..2e91294f7689 100644 --- a/package.json +++ b/package.json @@ -159,6 +159,7 @@ "@babel/preset-env": "7.22.14", "@babel/preset-typescript": "7.22.11", "@koa/cors": "4.0.0", + "@lokalise/node-api": "11.0.1", "@octokit/auth-oauth-device": "6.0.0", "@octokit/plugin-retry": "6.0.0", "@octokit/rest": "20.0.1", diff --git a/script/translations_download b/script/translations_download index 25e0a61e0218..6c1770f2dfb2 100755 --- a/script/translations_download +++ b/script/translations_download @@ -8,40 +8,4 @@ set -eu -o pipefail cd "$(dirname "$0")/.." -if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then - echo "Lokalise API token is required to download the latest set of" \ - "translations. Please create an account by using the following link:" \ - "https://lokalise.co/signup/3420425759f6d6d241f598.13594006/all/" \ - "Place your token in a new file \".lokalise_token\" in the repo" \ - "root directory." - exit 1 -fi - -# Load token from file if not already in the environment -[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)" - -declare -A PROJECT_ID=( \ - [frontend]="3420425759f6d6d241f598.13594006" \ - [backend]="130246255a974bd3b5e8a1.51616605" \ - ) - -for project in ${!PROJECT_ID[*]}; do -LOCAL_DIR=`pwd`/translations/${project} -rm -f ${LOCAL_DIR}/* || mkdir -p ${LOCAL_DIR} - docker run \ - -v ${LOCAL_DIR}:/opt/dest/locale \ - --rm \ - lokalise/lokalise-cli-2@sha256:f1860b26be22fa73b8c93bc5f8690f2afc867610a42de6fc27adc790e5d4425d \ - lokalise2 \ - --token ${LOKALISE_TOKEN} \ - --project-id ${PROJECT_ID[${project}]} \ - file download \ - --export-empty-as skip \ - --format json \ - --json-unescaped-slashes=true \ - --replace-breaks=false \ - --original-filenames=false \ - --unzip-to /opt/dest -done - -./node_modules/.bin/gulp check-downloaded-translations \ No newline at end of file +./node_modules/.bin/gulp download-translations \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index fa06ed75d12c..8633fd043e40 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2123,6 +2123,15 @@ __metadata: languageName: node linkType: hard +"@lokalise/node-api@npm:11.0.1": + version: 11.0.1 + resolution: "@lokalise/node-api@npm:11.0.1" + dependencies: + got: ^13.0.0 + checksum: 22435a252810d26e1bd0ca685a650d19267069aad80807d27275b2662d7c2866d927116038baf526bfb690039ea5c1d3d15548e85f7813b5735b2c5cc8435605 + languageName: node + linkType: hard + "@lrnwebcomponents/simple-tooltip@npm:7.0.16": version: 7.0.16 resolution: "@lrnwebcomponents/simple-tooltip@npm:7.0.16" @@ -3949,6 +3958,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/is@npm:^5.2.0": + version: 5.6.0 + resolution: "@sindresorhus/is@npm:5.6.0" + checksum: 2e6e0c3acf188dcd9aea0f324ac1b6ad04c9fc672392a7b5a1218512fcde066965797eba8b9fe2108657a504388bd4a6664e6e6602555168e828a6df08b9f10e + languageName: node + linkType: hard + "@sinonjs/commons@npm:^2.0.0": version: 2.0.0 resolution: "@sinonjs/commons@npm:2.0.0" @@ -4006,6 +4022,15 @@ __metadata: languageName: node linkType: hard +"@szmarczak/http-timer@npm:^5.0.1": + version: 5.0.1 + resolution: "@szmarczak/http-timer@npm:5.0.1" + dependencies: + defer-to-connect: ^2.0.1 + checksum: fc9cb993e808806692e4a3337c90ece0ec00c89f4b67e3652a356b89730da98bc824273a6d67ca84d5f33cd85f317dcd5ce39d8cc0a2f060145a608a7cb8ce92 + languageName: node + linkType: hard + "@thomasloven/round-slider@npm:0.6.0": version: 0.6.0 resolution: "@thomasloven/round-slider@npm:0.6.0" @@ -4264,6 +4289,13 @@ __metadata: languageName: node linkType: hard +"@types/http-cache-semantics@npm:^4.0.1": + version: 4.0.1 + resolution: "@types/http-cache-semantics@npm:4.0.1" + checksum: 1048aacf627829f0d5f00184e16548205cd9f964bf0841c29b36bc504509230c40bc57c39778703a1c965a6f5b416ae2cbf4c1d4589c889d2838dd9dbfccf6e9 + languageName: node + linkType: hard + "@types/http-errors@npm:*": version: 2.0.1 resolution: "@types/http-errors@npm:2.0.1" @@ -6439,6 +6471,28 @@ __metadata: languageName: node linkType: hard +"cacheable-lookup@npm:^7.0.0": + version: 7.0.0 + resolution: "cacheable-lookup@npm:7.0.0" + checksum: 9e2856763fc0a7347ab34d704c010440b819d4bb5e3593b664381b7433e942dd22e67ee5581f12256f908e79b82d30b86ebbacf40a081bfe10ee93fbfbc2d6a9 + languageName: node + linkType: hard + +"cacheable-request@npm:^10.2.8": + version: 10.2.13 + resolution: "cacheable-request@npm:10.2.13" + dependencies: + "@types/http-cache-semantics": ^4.0.1 + get-stream: ^6.0.1 + http-cache-semantics: ^4.1.1 + keyv: ^4.5.3 + mimic-response: ^4.0.0 + normalize-url: ^8.0.0 + responselike: ^3.0.0 + checksum: 1a2e9a20558ff2e23156bf945110f16d08037830a57c7b97ba9a145f6526fff1e1da21b1a1f9f4ee5fda77a482374e1a537b60dc23dab5df506f5a1cea5be9ab + languageName: node + linkType: hard + "call-bind@npm:^1.0.0, call-bind@npm:^1.0.2": version: 1.0.2 resolution: "call-bind@npm:1.0.2" @@ -7244,6 +7298,15 @@ __metadata: languageName: node linkType: hard +"decompress-response@npm:^6.0.0": + version: 6.0.0 + resolution: "decompress-response@npm:6.0.0" + dependencies: + mimic-response: ^3.1.0 + checksum: d377cf47e02d805e283866c3f50d3d21578b779731e8c5072d6ce8c13cc31493db1c2f6784da9d1d5250822120cefa44f1deab112d5981015f2e17444b763812 + languageName: node + linkType: hard + "deep-clone-simple@npm:1.1.1": version: 1.1.1 resolution: "deep-clone-simple@npm:1.1.1" @@ -7344,6 +7407,13 @@ __metadata: languageName: node linkType: hard +"defer-to-connect@npm:^2.0.1": + version: 2.0.1 + resolution: "defer-to-connect@npm:2.0.1" + checksum: 8a9b50d2f25446c0bfefb55a48e90afd58f85b21bcf78e9207cd7b804354f6409032a1705c2491686e202e64fc05f147aa5aa45f9aa82627563f045937f5791b + languageName: node + linkType: hard + "define-lazy-prop@npm:^2.0.0": version: 2.0.0 resolution: "define-lazy-prop@npm:2.0.0" @@ -8822,6 +8892,13 @@ __metadata: languageName: node linkType: hard +"form-data-encoder@npm:^2.1.2": + version: 2.1.4 + resolution: "form-data-encoder@npm:2.1.4" + checksum: e0b3e5950fb69b3f32c273944620f9861f1933df9d3e42066e038e26dfb343d0f4465de9f27e0ead1a09d9df20bc2eed06a63c2ca2f8f00949e7202bae9e29dd + languageName: node + linkType: hard + "forwarded@npm:0.2.0": version: 0.2.0 resolution: "forwarded@npm:0.2.0" @@ -9322,6 +9399,25 @@ __metadata: languageName: node linkType: hard +"got@npm:^13.0.0": + version: 13.0.0 + resolution: "got@npm:13.0.0" + dependencies: + "@sindresorhus/is": ^5.2.0 + "@szmarczak/http-timer": ^5.0.1 + cacheable-lookup: ^7.0.0 + cacheable-request: ^10.2.8 + decompress-response: ^6.0.0 + form-data-encoder: ^2.1.2 + get-stream: ^6.0.1 + http2-wrapper: ^2.1.10 + lowercase-keys: ^3.0.0 + p-cancelable: ^3.0.0 + responselike: ^3.0.0 + checksum: bcae6601efd710bc6c5b454c5e44bcb16fcfe57a1065e2d61ff918c1d69c3cf124984ebf509ca64ed10f0da2d2b5531b77da05aa786e75849d084fb8fbea711b + languageName: node + linkType: hard + "graceful-fs@npm:^4.0.0, graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.10, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" @@ -9615,6 +9711,7 @@ __metadata: "@lit-labs/context": 0.4.0 "@lit-labs/motion": 1.0.4 "@lit-labs/virtualizer": 2.0.7 + "@lokalise/node-api": 11.0.1 "@lrnwebcomponents/simple-tooltip": 7.0.16 "@material/chips": =14.0.0-canary.53b3cad2f.0 "@material/data-table": =14.0.0-canary.53b3cad2f.0 @@ -9990,6 +10087,16 @@ __metadata: languageName: node linkType: hard +"http2-wrapper@npm:^2.1.10": + version: 2.2.0 + resolution: "http2-wrapper@npm:2.2.0" + dependencies: + quick-lru: ^5.1.1 + resolve-alpn: ^1.2.0 + checksum: 6fd20e5cb6a58151715b3581e06a62a47df943187d2d1f69e538a50cccb7175dd334ecfde7900a37d18f3e13a1a199518a2c211f39860e81e9a16210c199cfaa + languageName: node + linkType: hard + "https-proxy-agent@npm:^5.0.0": version: 5.0.1 resolution: "https-proxy-agent@npm:5.0.1" @@ -10954,6 +11061,13 @@ __metadata: languageName: node linkType: hard +"json-buffer@npm:3.0.1": + version: 3.0.1 + resolution: "json-buffer@npm:3.0.1" + checksum: 9026b03edc2847eefa2e37646c579300a1f3a4586cfb62bf857832b60c852042d0d6ae55d1afb8926163fa54c2b01d83ae24705f34990348bdac6273a29d4581 + languageName: node + linkType: hard + "json-parse-even-better-errors@npm:^2.3.1": version: 2.3.1 resolution: "json-parse-even-better-errors@npm:2.3.1" @@ -11064,6 +11178,15 @@ __metadata: languageName: node linkType: hard +"keyv@npm:^4.5.3": + version: 4.5.3 + resolution: "keyv@npm:4.5.3" + dependencies: + json-buffer: 3.0.1 + checksum: 3ffb4d5b72b6b4b4af443bbb75ca2526b23c750fccb5ac4c267c6116888b4b65681015c2833cb20d26cf3e6e32dac6b988c77f7f022e1a571b7d90f1442257da + languageName: node + linkType: hard + "kind-of@npm:^1.1.0": version: 1.1.0 resolution: "kind-of@npm:1.1.0" @@ -11607,6 +11730,13 @@ __metadata: languageName: node linkType: hard +"lowercase-keys@npm:^3.0.0": + version: 3.0.0 + resolution: "lowercase-keys@npm:3.0.0" + checksum: 67a3f81409af969bc0c4ca0e76cd7d16adb1e25aa1c197229587eaf8671275c8c067cd421795dbca4c81be0098e4c426a086a05e30de8a9c587b7a13c0c7ccc5 + languageName: node + linkType: hard + "lru-cache@npm:^5.1.1": version: 5.1.1 resolution: "lru-cache@npm:5.1.1" @@ -11893,6 +12023,20 @@ __metadata: languageName: node linkType: hard +"mimic-response@npm:^3.1.0": + version: 3.1.0 + resolution: "mimic-response@npm:3.1.0" + checksum: 25739fee32c17f433626bf19f016df9036b75b3d84a3046c7d156e72ec963dd29d7fc8a302f55a3d6c5a4ff24259676b15d915aad6480815a969ff2ec0836867 + languageName: node + linkType: hard + +"mimic-response@npm:^4.0.0": + version: 4.0.0 + resolution: "mimic-response@npm:4.0.0" + checksum: 33b804cc961efe206efdb1fca6a22540decdcfce6c14eb5c0c50e5ae9022267ab22ce8f5568b1f7247ba67500fe20d523d81e0e9f009b321ccd9d472e78d1850 + languageName: node + linkType: hard + "min-document@npm:^2.19.0": version: 2.19.0 resolution: "min-document@npm:2.19.0" @@ -12361,6 +12505,13 @@ __metadata: languageName: node linkType: hard +"normalize-url@npm:^8.0.0": + version: 8.0.0 + resolution: "normalize-url@npm:8.0.0" + checksum: 24c20b75ebfd526d8453084692720b49d111c63c0911f1b7447427829597841eef5a8ba3f6bb93d6654007b991c1f5cd85da2c907800e439e2e2ec6c2abd0fc0 + languageName: node + linkType: hard + "now-and-later@npm:^2.0.0": version: 2.0.1 resolution: "now-and-later@npm:2.0.1" @@ -12665,6 +12816,13 @@ __metadata: languageName: node linkType: hard +"p-cancelable@npm:^3.0.0": + version: 3.0.0 + resolution: "p-cancelable@npm:3.0.0" + checksum: 2b5ae34218f9c2cf7a7c18e5d9a726ef9b165ef07e6c959f6738371509e747334b5f78f3bcdeb03d8a12dcb978faf641fd87eb21486ed7d36fb823b8ddef3219 + languageName: node + linkType: hard + "p-limit@npm:^2.2.0": version: 2.3.0 resolution: "p-limit@npm:2.3.0" @@ -13390,6 +13548,13 @@ __metadata: languageName: node linkType: hard +"quick-lru@npm:^5.1.1": + version: 5.1.1 + resolution: "quick-lru@npm:5.1.1" + checksum: a516faa25574be7947969883e6068dbe4aa19e8ef8e8e0fd96cddd6d36485e9106d85c0041a27153286b0770b381328f4072aa40d3b18a19f5f7d2b78b94b5ed + languageName: node + linkType: hard + "randombytes@npm:^2.1.0": version: 2.1.0 resolution: "randombytes@npm:2.1.0" @@ -13720,6 +13885,13 @@ __metadata: languageName: node linkType: hard +"resolve-alpn@npm:^1.2.0": + version: 1.2.1 + resolution: "resolve-alpn@npm:1.2.1" + checksum: f558071fcb2c60b04054c99aebd572a2af97ef64128d59bef7ab73bd50d896a222a056de40ffc545b633d99b304c259ea9d0c06830d5c867c34f0bfa60b8eae0 + languageName: node + linkType: hard + "resolve-cwd@npm:^3.0.0": version: 3.0.0 resolution: "resolve-cwd@npm:3.0.0" @@ -13831,6 +14003,15 @@ __metadata: languageName: node linkType: hard +"responselike@npm:^3.0.0": + version: 3.0.0 + resolution: "responselike@npm:3.0.0" + dependencies: + lowercase-keys: ^3.0.0 + checksum: e0cc9be30df4f415d6d83cdede3c5c887cd4a73e7cc1708bcaab1d50a28d15acb68460ac5b02bcc55a42f3d493729c8856427dcf6e57e6e128ad05cba4cfb95e + languageName: node + linkType: hard + "restore-cursor@npm:^4.0.0": version: 4.0.0 resolution: "restore-cursor@npm:4.0.0"