From 9f8b4654ffe4704629588d5ed41d67274e07d826 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Sun, 14 Apr 2024 22:40:34 +1000 Subject: [PATCH 1/9] #3516 Fixed released @magda NPM package esbuild commonjs bundle issue --- CHANGES.md | 1 + packages/arbitraries/esbuild.js | 7 +++++++ packages/auth-api-client/esbuild.js | 1 + packages/authentication-plugin-sdk/esbuild.js | 7 +++++++ packages/cjs-shim.js | 7 +++++++ packages/connector-sdk/esbuild.js | 1 + packages/connector-test-utils/esbuild.js | 7 +++++++ packages/create-secrets/esbuild.js | 1 + packages/docker-utils/esbuild.js | 7 ++++++- packages/minion-sdk/esbuild.js | 1 + packages/org-tree/esbuild.js | 1 + packages/registry-client/esbuild.js | 1 + packages/utils/esbuild.js | 8 ++++++-- 13 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 packages/cjs-shim.js diff --git a/CHANGES.md b/CHANGES.md index 167822197c..84b18ac176 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ## v3.0.1 - #3513 Fixed Indexer reindex trigger job doesn't start +- #3516 Fixed released @magda NPM package esbuild commonjs bundle issue ## v3.0.1 diff --git a/packages/arbitraries/esbuild.js b/packages/arbitraries/esbuild.js index be4ea80420..b744e0427b 100644 --- a/packages/arbitraries/esbuild.js +++ b/packages/arbitraries/esbuild.js @@ -1,4 +1,6 @@ import * as esbuild from "esbuild"; +import { require } from "@magda/esm-utils"; +const pkg = require("./package.json"); await esbuild.build({ entryPoints: ["./src/index.ts"], @@ -6,5 +8,10 @@ await esbuild.build({ platform: "node", target: ["es2022"], outdir: "dist", + inject: ["../cjs-shim.js"], + external: [ + ...Object.keys(pkg.dependencies || {}), + ...Object.keys(pkg.peerDependencies || {}) + ], format: "esm" }); diff --git a/packages/auth-api-client/esbuild.js b/packages/auth-api-client/esbuild.js index be4ea80420..e452056d1b 100644 --- a/packages/auth-api-client/esbuild.js +++ b/packages/auth-api-client/esbuild.js @@ -6,5 +6,6 @@ await esbuild.build({ platform: "node", target: ["es2022"], outdir: "dist", + inject: ["../cjs-shim.js"], format: "esm" }); diff --git a/packages/authentication-plugin-sdk/esbuild.js b/packages/authentication-plugin-sdk/esbuild.js index 19b0a55754..d4345d11dd 100644 --- a/packages/authentication-plugin-sdk/esbuild.js +++ b/packages/authentication-plugin-sdk/esbuild.js @@ -1,4 +1,6 @@ import * as esbuild from "esbuild"; +import { require } from "@magda/esm-utils"; +const pkg = require("./package.json"); await esbuild.build({ entryPoints: ["./src/index.ts"], @@ -7,5 +9,10 @@ await esbuild.build({ target: ["es2022"], outdir: "dist", format: "esm", + inject: ["../cjs-shim.js"], + external: [ + ...Object.keys(pkg.dependencies || {}), + ...Object.keys(pkg.peerDependencies || {}) + ], external: ["pg"] }); diff --git a/packages/cjs-shim.js b/packages/cjs-shim.js new file mode 100644 index 0000000000..7d7bbfd79f --- /dev/null +++ b/packages/cjs-shim.js @@ -0,0 +1,7 @@ +import { createRequire } from "node:module"; +import path from "node:path"; +import url from "node:url"; + +globalThis.require = createRequire(import.meta.url); +globalThis.__filename = url.fileURLToPath(import.meta.url); +globalThis.__dirname = path.dirname(__filename); diff --git a/packages/connector-sdk/esbuild.js b/packages/connector-sdk/esbuild.js index d2365921a5..ce87f7b257 100644 --- a/packages/connector-sdk/esbuild.js +++ b/packages/connector-sdk/esbuild.js @@ -13,6 +13,7 @@ async function build(web = false) { platform: platform, target: [target], outdir: "dist", + inject: ["../cjs-shim.js"], // should always be esm format: "esm" }); diff --git a/packages/connector-test-utils/esbuild.js b/packages/connector-test-utils/esbuild.js index be4ea80420..b744e0427b 100644 --- a/packages/connector-test-utils/esbuild.js +++ b/packages/connector-test-utils/esbuild.js @@ -1,4 +1,6 @@ import * as esbuild from "esbuild"; +import { require } from "@magda/esm-utils"; +const pkg = require("./package.json"); await esbuild.build({ entryPoints: ["./src/index.ts"], @@ -6,5 +8,10 @@ await esbuild.build({ platform: "node", target: ["es2022"], outdir: "dist", + inject: ["../cjs-shim.js"], + external: [ + ...Object.keys(pkg.dependencies || {}), + ...Object.keys(pkg.peerDependencies || {}) + ], format: "esm" }); diff --git a/packages/create-secrets/esbuild.js b/packages/create-secrets/esbuild.js index 96b7ee59ff..5b15ce6b8e 100644 --- a/packages/create-secrets/esbuild.js +++ b/packages/create-secrets/esbuild.js @@ -10,6 +10,7 @@ await esbuild.build({ bundle: true, platform: "node", target: ["es2022"], + inject: ["../cjs-shim.js"], outdir: "bin", format: "esm" }); diff --git a/packages/docker-utils/esbuild.js b/packages/docker-utils/esbuild.js index fa7d539ad5..0dad509a4d 100644 --- a/packages/docker-utils/esbuild.js +++ b/packages/docker-utils/esbuild.js @@ -1,5 +1,6 @@ import * as esbuild from "esbuild"; -import { requireResolve } from "@magda/esm-utils"; +import { requireResolve, require } from "@magda/esm-utils"; +const pkg = require("./package.json"); const entries = { "create-docker-context-for-node-component": requireResolve( @@ -13,6 +14,10 @@ await esbuild.build({ bundle: true, platform: "node", target: ["es2022"], + external: [ + ...Object.keys(pkg.dependencies || {}), + ...Object.keys(pkg.peerDependencies || {}) + ], outdir: "dist", format: "esm" }); diff --git a/packages/minion-sdk/esbuild.js b/packages/minion-sdk/esbuild.js index be4ea80420..e452056d1b 100644 --- a/packages/minion-sdk/esbuild.js +++ b/packages/minion-sdk/esbuild.js @@ -6,5 +6,6 @@ await esbuild.build({ platform: "node", target: ["es2022"], outdir: "dist", + inject: ["../cjs-shim.js"], format: "esm" }); diff --git a/packages/org-tree/esbuild.js b/packages/org-tree/esbuild.js index db017b6c09..641bd31e12 100644 --- a/packages/org-tree/esbuild.js +++ b/packages/org-tree/esbuild.js @@ -32,6 +32,7 @@ await esbuild.build({ target: ["es2022"], outdir: "dist", format: "esm", + inject: ["../cjs-shim.js"], external: [ ...Object.keys(pkg.dependencies || {}), ...Object.keys(pkg.peerDependencies || {}) diff --git a/packages/registry-client/esbuild.js b/packages/registry-client/esbuild.js index be4ea80420..e452056d1b 100644 --- a/packages/registry-client/esbuild.js +++ b/packages/registry-client/esbuild.js @@ -6,5 +6,6 @@ await esbuild.build({ platform: "node", target: ["es2022"], outdir: "dist", + inject: ["../cjs-shim.js"], format: "esm" }); diff --git a/packages/utils/esbuild.js b/packages/utils/esbuild.js index a3bd8c1fa3..1e1250de01 100644 --- a/packages/utils/esbuild.js +++ b/packages/utils/esbuild.js @@ -7,7 +7,7 @@ async function build(web = false) { console.log( `Building for ${entryPoints} ${platform} with target ${target}` ); - await esbuild.build({ + const options = { entryPoints: entryPoints, bundle: true, platform: platform, @@ -15,7 +15,11 @@ async function build(web = false) { outdir: "dist", // should always be esm format: "esm" - }); + }; + if (!web) { + options.inject = ["../cjs-shim.js"]; + } + await esbuild.build(options); } await build(); From 1d75ead5793ef825ea8c9a699d78405e13e498ff Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Mon, 15 Apr 2024 14:55:51 +1000 Subject: [PATCH 2/9] Added the API document for the sitemap endpoint --- CHANGES.md | 1 + magda-web-server/src/buildSitemapRouter.ts | 38 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 84b18ac176..af465da01e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ - #3513 Fixed Indexer reindex trigger job doesn't start - #3516 Fixed released @magda NPM package esbuild commonjs bundle issue +- Added the API document for the sitemap endpoint ## v3.0.1 diff --git a/magda-web-server/src/buildSitemapRouter.ts b/magda-web-server/src/buildSitemapRouter.ts index 1101b0c1bb..aa668d2d93 100644 --- a/magda-web-server/src/buildSitemapRouter.ts +++ b/magda-web-server/src/buildSitemapRouter.ts @@ -49,6 +49,27 @@ export default function buildSitemapRouter({ return pageTokens; } + /** + * @apiGroup Sitemaps + * @api {get} /sitemap.xml Sitemaps entrypoint + * @apiDescription A [sitemaps protocol interface](https://www.sitemaps.org/protocol.html) that is prepared for external search engines to harvest datasets from Magda. + * The sitemap index is produced based on the live data in the metadata store database. By default, the sitemap index will be cached for 86400 seconds (24 hours). + * This setting can be adjusted via `sitemapCacheSeconds` of [web-server](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/web-server) module helm chart. + * Please note: due to the cache and the search engine indexing delay, the total number of datasets in the sitemap index may be different from the dataset total count from the search API. + * This sitemaps endpoint is recorded on the default /robots.txt endpoint that follows the [Robots Exclusion Standard](https://en.wikipedia.org/wiki/Robots_exclusion_standard#About_the_standard). + * + * @apiSuccessExample {json} 200 + * + * + * + * https://example.com/sitemap/main.xml + * + * + * https://example.com/sitemap/dataset/afterToken/0.xml + * + * + * + */ app.get("/sitemap.xml", async (req, res) => { try { const smis = new SitemapIndexStream({ level: ErrorLevel.WARN }); @@ -91,6 +112,23 @@ export default function buildSitemapRouter({ } }); + /** + * @apiGroup Sitemaps + * @api {get} /sitemap/main.xml Sitemaps main index + * @apiDescription List + * + * @apiSuccessExample {json} 200 + * + * + * + * https://example.com/sitemap/main.xml + * + * + * https://example.com/sitemap/dataset/afterToken/0.xml + * + * + * + */ app.get("/sitemap/main.xml", async (req, res) => { try { // For now we just put the homepage in here, seeing as everything except the datasets should be reachable From 804aa51d0cb82213d60c8690b548b84bd5d7031f Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Mon, 15 Apr 2024 15:25:43 +1000 Subject: [PATCH 3/9] update sitemaps docs --- magda-web-server/src/buildSitemapRouter.ts | 46 +++++++++++++++------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/magda-web-server/src/buildSitemapRouter.ts b/magda-web-server/src/buildSitemapRouter.ts index aa668d2d93..399e2a6957 100644 --- a/magda-web-server/src/buildSitemapRouter.ts +++ b/magda-web-server/src/buildSitemapRouter.ts @@ -55,10 +55,12 @@ export default function buildSitemapRouter({ * @apiDescription A [sitemaps protocol interface](https://www.sitemaps.org/protocol.html) that is prepared for external search engines to harvest datasets from Magda. * The sitemap index is produced based on the live data in the metadata store database. By default, the sitemap index will be cached for 86400 seconds (24 hours). * This setting can be adjusted via `sitemapCacheSeconds` of [web-server](https://github.com/magda-io/magda/tree/main/deploy/helm/internal-charts/web-server) module helm chart. - * Please note: due to the cache and the search engine indexing delay, the total number of datasets in the sitemap index may be different from the dataset total count from the search API. - * This sitemaps endpoint is recorded on the default /robots.txt endpoint that follows the [Robots Exclusion Standard](https://en.wikipedia.org/wiki/Robots_exclusion_standard#About_the_standard). + * Please note: + * 1> due to the cache and the search engine indexing delay, the total number of datasets in the sitemap index may be different from the dataset total count from the search API. + * 2> This sitemaps endpoint is recorded on the default /robots.txt endpoint that follows the [Robots Exclusion Standard](https://en.wikipedia.org/wiki/Robots_exclusion_standard#About_the_standard). + * 3> Only public datasets are included in the sitemaps. In fact, the sitemaps index is generated with anonymous user access. * - * @apiSuccessExample {json} 200 + * @apiSuccessExample {xml} 200 * * * @@ -114,19 +116,17 @@ export default function buildSitemapRouter({ /** * @apiGroup Sitemaps - * @api {get} /sitemap/main.xml Sitemaps main index - * @apiDescription List + * @api {get} /sitemap/main.xml Sitemaps index entry for the home page + * @apiDescription Sitemaps index entry for the home page * - * @apiSuccessExample {json} 200 + * @apiSuccessExample {xml} 200 * - * - * - * https://example.com/sitemap/main.xml - * - * - * https://example.com/sitemap/dataset/afterToken/0.xml - * - * + * + * + * https://example.com/ + * daily + * + * * */ app.get("/sitemap/main.xml", async (req, res) => { @@ -152,6 +152,24 @@ export default function buildSitemapRouter({ } }); + /** + * @apiGroup Sitemaps + * @api {get} /sitemap/dataset/afterToken/{pageToken}.xml Sitemaps index entry for datasets + * @apiDescription The sitemaps entrypoint `/sitemap.xml` contains links to all index entries for datasets (paginated by the `pageToken` parameter). + * Each index entry contains a list of dataset page links. + * You can either visit the dataset page directly (A text based SEO friendly view will be shown if the `User-Agent` request header is [recognised as search engine bot](https://github.com/magda-io/magda/blob/main/magda-web-server/src/shouldRenderCrawlerView.ts#L20)). + * Or extract the dataset ID from the dataset page link and access the detailed metadata in JSON format via the [registry get a record by ID API](#api-Registry_Record_Service-GetV0RegistryRecordsId). + * + * @apiSuccessExample {xml} 200 + * + * + * + * https://example.com/dataset/ds-a3d075b9-70d7-40b3-a693-70db2a415765 + * weekly + * + * + * + */ app.get("/sitemap/dataset/afterToken/:afterToken.xml", async (req, res) => { try { const sms = new SitemapStream({ From c4af5d6e85b944b95746a6ba9760bdbe7f2bbe33 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Mon, 15 Apr 2024 15:35:52 +1000 Subject: [PATCH 4/9] upgrade to npm@10.5.2 --- magda-builder-nodejs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magda-builder-nodejs/Dockerfile b/magda-builder-nodejs/Dockerfile index bf357c9d02..e31eb4e879 100644 --- a/magda-builder-nodejs/Dockerfile +++ b/magda-builder-nodejs/Dockerfile @@ -1,6 +1,6 @@ FROM node:18-alpine3.18 -RUN npm install -g lerna@3.22.1 +RUN npm install -g npm@10.5.2 && npm install -g lerna@3.22.1 # Install haveged and run it so that we don't get lockups due to a lack of entropy RUN apk --no-cache add haveged From cc16a5e3ae9da6b74e28b1701efd7e33df9c1b87 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 16 Apr 2024 17:18:13 +1000 Subject: [PATCH 5/9] - move getStorageApiResourceAccessUrl to typescript-common - release it from @magda/utils --- .../src/getStorageApiResourceAccessUrl.ts | 40 +++++++++++++++++++ .../helpers/getStorageApiResourceAccessUrl.ts | 29 +++----------- packages/utils/src/index.ts | 1 + 3 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 magda-typescript-common/src/getStorageApiResourceAccessUrl.ts diff --git a/magda-typescript-common/src/getStorageApiResourceAccessUrl.ts b/magda-typescript-common/src/getStorageApiResourceAccessUrl.ts new file mode 100644 index 0000000000..65d208e5f2 --- /dev/null +++ b/magda-typescript-common/src/getStorageApiResourceAccessUrl.ts @@ -0,0 +1,40 @@ +import urijs from "urijs"; + +/** + * Get the access url of a storage api resource from [pseudo storage api resource URL](https://github.com/magda-io/magda/issues/3000) + * If the input url is not a pseudo storage api resource URL, return the input url directly + * + * @export + * @param {string} resourceUrl pseudo storage api resource URL or ordinary HTTP access url + * @param {string} storageApiBaseUrl storage api base url + * @param {string} datasetsBucket datasets storage bucket name + * @return {*} + */ +export default function getStorageApiResourceAccessUrl( + resourceUrl: string, + storageApiBaseUrl: string, + datasetsBucket: string +) { + if (!resourceUrl) { + return resourceUrl; + } + const uri = urijs(resourceUrl); + if (uri.protocol() === "magda" && uri.hostname() === "storage-api") { + // --- convert [pseudo storage api resource URL](https://github.com/magda-io/magda/issues/3000) to http access url + const [datasetId, distributionId, fileName] = uri.segmentCoded(); + const accessUri = urijs(storageApiBaseUrl); + const segments = [ + ...accessUri.segmentCoded(), + datasetsBucket, + datasetId, + distributionId, + fileName + ]; + return accessUri.segmentCoded(segments).toString(); + } else { + // --- return legacy url directly + // --- legacy url is a HTTP access url generated when create the resource + // --- can be used to access storage api directly + return resourceUrl; + } +} diff --git a/magda-web-client/src/helpers/getStorageApiResourceAccessUrl.ts b/magda-web-client/src/helpers/getStorageApiResourceAccessUrl.ts index d1f0c02a9d..b0929bdc89 100644 --- a/magda-web-client/src/helpers/getStorageApiResourceAccessUrl.ts +++ b/magda-web-client/src/helpers/getStorageApiResourceAccessUrl.ts @@ -1,27 +1,10 @@ import { config, DATASETS_BUCKET } from "../config"; -import urijs from "urijs"; +import { default as getStorageApiResourceAccessUrlOriginal } from "@magda/typescript-common/dist/getStorageApiResourceAccessUrl.js"; export default function getStorageApiResourceAccessUrl(resourceUrl: string) { - if (!resourceUrl) { - return resourceUrl; - } - const uri = urijs(resourceUrl); - if (uri.protocol() === "magda" && uri.hostname() === "storage-api") { - // --- convert [pseudo storage api resource URL](https://github.com/magda-io/magda/issues/3000) to http access url - const [datasetId, distributionId, fileName] = uri.segmentCoded(); - const accessUri = urijs(config.storageApiBaseUrl); - const segments = [ - ...accessUri.segmentCoded(), - DATASETS_BUCKET, - datasetId, - distributionId, - fileName - ]; - return accessUri.segmentCoded(segments).toString(); - } else { - // --- return legacy url directly - // --- legacy url is a HTTP access url generated when create the resource - // --- can be used to access storage api directly - return resourceUrl; - } + return getStorageApiResourceAccessUrlOriginal( + resourceUrl, + config.storageApiBaseUrl, + DATASETS_BUCKET + ); } diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 07d9386783..ba270f1d91 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -28,3 +28,4 @@ export { ServiceError, BadRequestError } from "@magda/typescript-common/dist/createServiceError.js"; +export { default as getStorageApiResourceAccessUrl } from "@magda/typescript-common/dist/getStorageApiResourceAccessUrl.js"; From 6e4c8817afab8fef833d06a110e1ff363c03015b Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 16 Apr 2024 17:23:57 +1000 Subject: [PATCH 6/9] - update node engines requirements to all SDKs to 18.19.0 - registry aspects migrator should use node 18 docker image --- magda-content-schemas/package.json | 2 +- magda-migrator-registry-aspects/Dockerfile | 2 +- magda-registry-aspects/package.json | 2 +- packages/acs-cmd/package.json | 2 +- packages/arbitraries/package.json | 2 +- packages/auth-api-client/package.json | 2 +- packages/authentication-plugin-sdk/package.json | 2 +- packages/connector-sdk/package.json | 2 +- packages/create-secrets/package.json | 2 +- packages/docker-utils/package.json | 2 +- packages/minion-sdk/package.json | 2 +- packages/org-tree/package.json | 2 +- packages/registry-client/package.json | 2 +- packages/utils/package.json | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/magda-content-schemas/package.json b/magda-content-schemas/package.json index 536e6218f6..cd8e8705aa 100644 --- a/magda-content-schemas/package.json +++ b/magda-content-schemas/package.json @@ -11,7 +11,7 @@ "./*.json": "./*.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "description": "Common schemas for use with the content api.", "scripts": { diff --git a/magda-migrator-registry-aspects/Dockerfile b/magda-migrator-registry-aspects/Dockerfile index 832169ee60..1c4d1c60f7 100644 --- a/magda-migrator-registry-aspects/Dockerfile +++ b/magda-migrator-registry-aspects/Dockerfile @@ -1,4 +1,4 @@ -FROM node:14-alpine +FROM node:18-alpine RUN mkdir -p /usr/src/app COPY . /usr/src/app diff --git a/magda-registry-aspects/package.json b/magda-registry-aspects/package.json index 56a467ff63..d52916f5e7 100644 --- a/magda-registry-aspects/package.json +++ b/magda-registry-aspects/package.json @@ -9,7 +9,7 @@ "./*.json": "./*.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "build": "echo \"ok\"", diff --git a/packages/acs-cmd/package.json b/packages/acs-cmd/package.json index 468047fde3..6b071a8594 100644 --- a/packages/acs-cmd/package.json +++ b/packages/acs-cmd/package.json @@ -11,7 +11,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "acs-cmd": "./bin/acs-cmd.js", diff --git a/packages/arbitraries/package.json b/packages/arbitraries/package.json index 3bf3973b10..cb8ed41234 100644 --- a/packages/arbitraries/package.json +++ b/packages/arbitraries/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/auth-api-client/package.json b/packages/auth-api-client/package.json index 5734495335..be263a24ec 100644 --- a/packages/auth-api-client/package.json +++ b/packages/auth-api-client/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/authentication-plugin-sdk/package.json b/packages/authentication-plugin-sdk/package.json index 4b9b4b2c41..34cf17c71c 100644 --- a/packages/authentication-plugin-sdk/package.json +++ b/packages/authentication-plugin-sdk/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/connector-sdk/package.json b/packages/connector-sdk/package.json index 6fedb26f42..d3688bcbcd 100644 --- a/packages/connector-sdk/package.json +++ b/packages/connector-sdk/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/create-secrets/package.json b/packages/create-secrets/package.json index acde11800c..52148f614a 100644 --- a/packages/create-secrets/package.json +++ b/packages/create-secrets/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "bin": { "create-secrets": "./bin/index.js" diff --git a/packages/docker-utils/package.json b/packages/docker-utils/package.json index d91915b6ea..1976f0d373 100644 --- a/packages/docker-utils/package.json +++ b/packages/docker-utils/package.json @@ -9,7 +9,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "bin": { "create-docker-context-for-node-component": "./dist/create-docker-context-for-node-component.js", diff --git a/packages/minion-sdk/package.json b/packages/minion-sdk/package.json index 212665e5dc..2ebfd00465 100644 --- a/packages/minion-sdk/package.json +++ b/packages/minion-sdk/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/org-tree/package.json b/packages/org-tree/package.json index f88304f5bd..55edc4ce67 100644 --- a/packages/org-tree/package.json +++ b/packages/org-tree/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "bin": { "org-tree": "./dist/index.js" diff --git a/packages/registry-client/package.json b/packages/registry-client/package.json index 8fa9c1ac41..0a37cb0e32 100644 --- a/packages/registry-client/package.json +++ b/packages/registry-client/package.json @@ -8,7 +8,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", diff --git a/packages/utils/package.json b/packages/utils/package.json index 8b55d83f6d..c245b69417 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "./package.json": "./package.json" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.19.0" }, "scripts": { "prebuild": "rimraf dist tsconfig.tsbuildinfo", From e8cb22a1ab0255b18d051185d575ea8fe9e0d6ec Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Tue, 16 Apr 2024 23:50:28 +1000 Subject: [PATCH 7/9] update docs & CHANGES.md --- CHANGES.md | 2 + packages/utils/README.md | 115 ++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index af465da01e..fd5646a81a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,8 @@ - #3513 Fixed Indexer reindex trigger job doesn't start - #3516 Fixed released @magda NPM package esbuild commonjs bundle issue - Added the API document for the sitemap endpoint +- Make getStorageApiResourceAccessUrl available via @magda/typescript-common & @magda/utils +- Fixed: update node engines requirements to all SDKs to 18.19.0 ## v3.0.1 diff --git a/packages/utils/README.md b/packages/utils/README.md index 427743782e..4f4f7a320f 100644 --- a/packages/utils/README.md +++ b/packages/utils/README.md @@ -3,6 +3,14 @@ This package includes the following common utilities that may help magda miniors / connectors development: ```typescript +/** + * Checks to see whether the passed argv has a jwtSecret object. If not, + * tries to add one by looking at the JWT_SECRET env var and failing that, + * the jwtSecret value in package.json config. + * + * If it can't find one and required is true (or unprovided), this will + * throw an Error. + */ export declare function addJwtSecretFromEnvVar( argv: { [key in keyof Arguments]: Arguments[key]; @@ -38,38 +46,53 @@ export declare function buildJwt( jwtSecret: string, userId: string, session?: any -): any; - -export declare const coerceJson: (param: string) => (json?: string) => any; - -export declare function createServiceError(e: any): Error; - -export declare function encodeURIComponentWithApost(string: string): string; +): string; -export declare function forEachAsync( - page: AsyncPage, - maxConcurrency: number, - callbackFn: (data: T) => Promise -): Promise; +export declare const coerceJson: ( + param: string +) => (json?: string | object | any[]) => any; -export declare function formatServiceError( - baseMessage: string, - e: any, - retriesLeft: number -): string; +declare interface CreateAsyncPage { + (): Promise>; +} -export declare function getMinikubeIP(): string; +export declare function createNoCacheFetchOptions( + fetchOptions?: RequestInit +): { + body?: BodyInit; + cache?: RequestCache; + credentials?: RequestCredentials; + headers?: HeadersInit; + integrity?: string; + keepalive?: boolean; + method?: string; + mode?: RequestMode; + redirect?: RequestRedirect; + referrer?: string; + referrerPolicy?: ReferrerPolicy; + signal?: AbortSignal; + window?: null; +}; -export declare const isUuid: (id: any) => boolean; +/** + * Creates a {@link ServiceError} from the result of a failed call to an API generated + * by swagger-codegen. The result typically includes `response` (with a status code) and + * a `body` (the JSON the server returned with the error), but may be other things if, + * e.g., an exception occurred while attempting to invoke the service. + * + * @export + * @param {*} e The result of the failed call. + * @returns {Error} An Error created from the failed result. + */ +export declare function createServiceError(e: any): Error; -// deprecated. Please use fetchRequest instead -export declare const request: any; +export declare function encodeURIComponentWithApost(string: string): string; export declare function fetchRequest( method: string, url: string, body?: any, - contentType?: CT | RequestContentType | undefined, + contentType?: CT | RequestContentType | undefined | null, returnHeaders?: false, extraRequestOptions?: RequestInit ): Promise; @@ -78,28 +101,44 @@ export declare function fetchRequest( method: string, url: string, body?: any, - contentType?: CT | RequestContentType | undefined, + contentType?: CT | RequestContentType | undefined | null, returnHeaders?: true, extraRequestOptions?: RequestInit ): Promise<[T, Headers]>; -export declare function getDefaultRequestInitOptions(): RequestInit; -export declare function setDefaultRequestInitOptions( - options: RequestInit -): void; +export declare function forEachAsync( + page: AsyncPage, + maxConcurrency: number, + callbackFn: (data: T) => Promise +): Promise; -export declare function getRequest( - url: string, - noCache?: boolean, - extraFetchOptions?: RequestInit -): Promise; +export declare function formatServiceError( + baseMessage: string, + e: any, + retriesLeft: number +): string; -export declare function getRequestNoCache( - url: string, - extraFetchOptions?: RequestInit -): Promise; +export declare function getMinikubeIP(): string; + +/** + * Get the access url of a storage api resource from [pseudo storage api resource URL](https://github.com/magda-io/magda/issues/3000) + * If the input url is not a pseudo storage api resource URL, return the input url directly + * + * @export + * @param {string} resourceUrl pseudo storage api resource URL or ordinary HTTP access url + * @param {string} storageApiBaseUrl storage api base url + * @param {string} datasetsBucket datasets storage bucket name + * @return {*} + */ +export declare function getStorageApiResourceAccessUrl( + resourceUrl: string, + storageApiBaseUrl: string, + datasetsBucket: string +): string; + +export declare const isUuid: (id: any) => boolean; -export declare function retry( +export declare function retry( op: () => Promise, delaySeconds: number, retries: number, @@ -125,5 +164,5 @@ export declare class ServiceError extends Error { constructor(message: string, e: any); } -export declare function unionToThrowable(input: T | Error): T; +export declare function unionToThrowable(input: T | Error | ServerError): T; ``` From 86d3a98ef3571b71b5f06ff5a4bdb735da953921 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 18 Apr 2024 00:03:13 +1000 Subject: [PATCH 8/9] upgrade broken link minion to v3.0.0 --- deploy/helm/magda/Chart.lock | 6 +++--- deploy/helm/magda/Chart.yaml | 2 +- deploy/helm/magda/README.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/helm/magda/Chart.lock b/deploy/helm/magda/Chart.lock index 4a2ebd7413..4cdd661377 100644 --- a/deploy/helm/magda/Chart.lock +++ b/deploy/helm/magda/Chart.lock @@ -10,7 +10,7 @@ dependencies: version: 2.0.0 - name: magda-minion-broken-link repository: oci://ghcr.io/magda-io/charts - version: 2.0.0 + version: 3.0.0 - name: magda-minion-format repository: oci://ghcr.io/magda-io/charts version: 2.0.1 @@ -29,5 +29,5 @@ dependencies: - name: magda-function-esri-url-processor repository: oci://ghcr.io/magda-io/charts version: 2.0.0 -digest: sha256:637bc2e418fd3e29d9162b263e9afa26e0355e3547ab7aaecd9e0a865b952d1b -generated: "2024-03-26T03:55:48.726443758Z" +digest: sha256:68193166fa130ec0771de1897538b8f7e255f450716f2060616c7166d8bd16fb +generated: "2024-04-18T00:00:16.359432+10:00" diff --git a/deploy/helm/magda/Chart.yaml b/deploy/helm/magda/Chart.yaml index fee79f4efe..6c66244b8d 100644 --- a/deploy/helm/magda/Chart.yaml +++ b/deploy/helm/magda/Chart.yaml @@ -32,7 +32,7 @@ dependencies: - name: magda-minion-broken-link alias: minion-broken-link - version: "2.0.0" + version: "3.0.0" repository: "oci://ghcr.io/magda-io/charts" tags: - minions diff --git a/deploy/helm/magda/README.md b/deploy/helm/magda/README.md index 4a956098e7..faeb365378 100644 --- a/deploy/helm/magda/README.md +++ b/deploy/helm/magda/README.md @@ -18,7 +18,7 @@ A complete solution for managing, publishing and discovering government data, pr | oci://ghcr.io/magda-io/charts | ckan-connector-functions(magda-ckan-connector) | 2.0.0 | | oci://ghcr.io/magda-io/charts | magda-function-esri-url-processor | 2.0.0 | | oci://ghcr.io/magda-io/charts | magda-function-history-report | 2.0.0 | -| oci://ghcr.io/magda-io/charts | minion-broken-link(magda-minion-broken-link) | 2.0.0 | +| oci://ghcr.io/magda-io/charts | minion-broken-link(magda-minion-broken-link) | 3.0.0 | | oci://ghcr.io/magda-io/charts | magda-minion-ckan-exporter | 2.0.0 | | oci://ghcr.io/magda-io/charts | minion-format(magda-minion-format) | 2.0.1 | | oci://ghcr.io/magda-io/charts | minion-linked-data-rating(magda-minion-linked-data-rating) | 2.0.0 | From c8ba61f890e3442a11f1370981466dc236416505 Mon Sep 17 00:00:00 2001 From: Jacky Jiang Date: Thu, 18 Apr 2024 14:47:17 +1000 Subject: [PATCH 9/9] fixed version number on CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fd5646a81a..6c57c352de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # CHANGELOG -## v3.0.1 +## v3.0.2 - #3513 Fixed Indexer reindex trigger job doesn't start - #3516 Fixed released @magda NPM package esbuild commonjs bundle issue