Move translation fetching to gulp action (#17827)

home-assistant · Sep 5, 2023 · 29aed53 · 29aed53
1 parent 76c878d
commit 29aed53
Show file tree

Hide file tree

Showing 4 changed files with 270 additions and 43 deletions.
diff --git a/build-scripts/gulp/download-translations.js b/build-scripts/gulp/download-translations.js
@@ -1,10 +1,14 @@
 import fs from "fs/promises";
 import gulp from "gulp";
+import path from "path";
 import mapStream from "map-stream";
 import transform from "gulp-json-transform";
+import { LokaliseApi } from "@lokalise/node-api";
+import JSZip from "jszip";
 
-const inDirFrontend = "translations/frontend";
-const inDirBackend = "translations/backend";
+const inDir = "translations";
+const inDirFrontend = `${inDir}/frontend`;
+const inDirBackend = `${inDir}/backend`;
 const srcMeta = "src/translations/translationMetadata.json";
 const encoding = "utf8";
 
@@ -68,8 +72,9 @@ gulp.task("convert-backend-translations", function () {
 });
 
 gulp.task("check-translations-html", function () {
-  // We exclude backend translations because they are not compliant with the HTML rule for now
-  return gulp.src([`${inDirFrontend}/*.json`]).pipe(checkHtml());
+  return gulp
+    .src([`${inDirFrontend}/*.json`, `${inDirBackend}/*.json`])
+    .pipe(checkHtml());
 });
 
 gulp.task("check-all-files-exist", async function () {
@@ -89,7 +94,83 @@ gulp.task("check-all-files-exist", async function () {
   await Promise.allSettled(writings);
 });
 
+const lokaliseProjects = {
+  backend: "130246255a974bd3b5e8a1.51616605",
+  frontend: "3420425759f6d6d241f598.13594006",
+};
+
+gulp.task("fetch-lokalise", async function () {
+  let apiKey;
+  try {
+    apiKey =
+      process.env.LOKALISE_TOKEN ||
+      (await fs.readFile(".lokalise_token", { encoding }));
+  } catch {
+    throw new Error(
+      "An Administrator Lokalise API token is required to download the latest set of translations. Place your token in a new file `.lokalise_token` in the repo root directory."
+    );
+  }
+  const lokaliseApi = new LokaliseApi({ apiKey });
+
+  const mkdirPromise = Promise.all([
+    fs.mkdir(inDirFrontend, { recursive: true }),
+    fs.mkdir(inDirBackend, { recursive: true }),
+  ]);
+
+  await Promise.all(
+    Object.entries(lokaliseProjects).map(([project, projectId]) =>
+      lokaliseApi
+        .files()
+        .download(projectId, {
+          format: "json",
+          original_filenames: false,
+          replace_breaks: false,
+          json_unescaped_slashes: true,
+          export_empty_as: "skip",
+        })
+        .then((download) => fetch(download.bundle_url))
+        .then((response) => {
+          if (response.status === 200 || response.status === 0) {
+            return response.arrayBuffer();
+          }
+          throw new Error(response.statusText);
+        })
+        .then(JSZip.loadAsync)
+        .then(async (contents) => {
+          await mkdirPromise;
+          return Promise.all(
+            Object.keys(contents.files).map(async (filename) => {
+              const file = contents.file(filename);
+              if (!file) {
+                // no file, probably a directory
+                return Promise.resolve();
+              }
+              return file
+                .async("nodebuffer")
+                .then((content) =>
+                  fs.writeFile(
+                    path.join(
+                      inDir,
+                      project,
+                      filename.split("/").splice(-1)[0]
+                    ),
+                    content,
+                    { flag: "w", encoding }
+                  )
+                );
+            })
+          );
+        })
+    )
+  );
+});
+
 gulp.task(
-  "check-downloaded-translations",
-  gulp.series("check-translations-html", "check-all-files-exist")
+  "download-translations",
+  gulp.series(
+    "fetch-lokalise",
+    "convert-backend-translations",
+    "check-translations-html",
+    "check-all-files-exist"
+  )
 );
diff --git a/package.json b/package.json
@@ -159,6 +159,7 @@
     "@babel/preset-env": "7.22.14",
     "@babel/preset-typescript": "7.22.11",
     "@koa/cors": "4.0.0",
+    "@lokalise/node-api": "11.0.1",
     "@octokit/auth-oauth-device": "6.0.0",
     "@octokit/plugin-retry": "6.0.0",
     "@octokit/rest": "20.0.1",

diff --git a/script/translations_download b/script/translations_download
@@ -8,40 +8,4 @@ set -eu -o pipefail
 
 cd "$(dirname "$0")/.."
 
-if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then
-    echo "Lokalise API token is required to download the latest set of" \
-        "translations. Please create an account by using the following link:" \
-        "https://lokalise.co/signup/3420425759f6d6d241f598.13594006/all/" \
-        "Place your token in a new file \".lokalise_token\" in the repo" \
-        "root directory."
-    exit 1
-fi
-
-# Load token from file if not already in the environment
-[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)"
-
-declare -A PROJECT_ID=( \
-    [frontend]="3420425759f6d6d241f598.13594006" \
-    [backend]="130246255a974bd3b5e8a1.51616605" \
-    )
-
-for project in ${!PROJECT_ID[*]}; do
-LOCAL_DIR=`pwd`/translations/${project}
-rm -f ${LOCAL_DIR}/* || mkdir -p ${LOCAL_DIR}
-    docker run \
-        -v ${LOCAL_DIR}:/opt/dest/locale \
-        --rm \
-        lokalise/lokalise-cli-2@sha256:f1860b26be22fa73b8c93bc5f8690f2afc867610a42de6fc27adc790e5d4425d \
-        lokalise2 \
-            --token ${LOKALISE_TOKEN} \
-            --project-id ${PROJECT_ID[${project}]} \
-            file download \
-            --export-empty-as skip \
-            --format json \
-            --json-unescaped-slashes=true \
-            --replace-breaks=false \
-            --original-filenames=false \
-            --unzip-to /opt/dest
-done
-
-./node_modules/.bin/gulp check-downloaded-translations
+./node_modules/.bin/gulp download-translations