From e2afb25c6c4981b3c18b575709f7e38bb8b4dc2b Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 19 Dec 2024 13:31:08 -0800 Subject: [PATCH 01/10] fix(worker): Add expected dotfiles to default bidsignore rules --- services/datalad/hooks/pre-receive | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/datalad/hooks/pre-receive b/services/datalad/hooks/pre-receive index fddc74c05..15926e6b7 100755 --- a/services/datalad/hooks/pre-receive +++ b/services/datalad/hooks/pre-receive @@ -130,6 +130,9 @@ function main() { python -m bidsschematools pre-receive-hook < Date: Thu, 19 Dec 2024 14:11:30 -0800 Subject: [PATCH 02/10] fix(worker): Workaround bids-validator structure for affects field --- services/datalad/datalad_service/tasks/validator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/datalad/datalad_service/tasks/validator.py b/services/datalad/datalad_service/tasks/validator.py index 4401f4727..2e85a46d2 100644 --- a/services/datalad/datalad_service/tasks/validator.py +++ b/services/datalad/datalad_service/tasks/validator.py @@ -76,6 +76,10 @@ def issues_mutation(dataset_id, ref, issues, validator_metadata): """ Return the OpenNeuro mutation to update any validation issues. """ + # Workaround for bad "affects" values - drop them + for issue in issues["issues"]: + if "affects" in issue and not isinstance(issue["affects"], str): + del issue["affects"] validatorInput = { 'datasetId': dataset_id, 'id': ref, From dcd9625fdee465b5593f6f73b2bf34cacfb9db75 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 19 Dec 2024 14:14:04 -0800 Subject: [PATCH 03/10] deps: Update @bids/validator to 2.0.1 --- services/datalad/datalad_service/tasks/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/datalad/datalad_service/tasks/validator.py b/services/datalad/datalad_service/tasks/validator.py index 2e85a46d2..8fa705d4b 100644 --- a/services/datalad/datalad_service/tasks/validator.py +++ b/services/datalad/datalad_service/tasks/validator.py @@ -10,7 +10,7 @@ logger = logging.getLogger('datalad_service.' + __name__) -DENO_VALIDATOR_VERSION = '2.0.0' +DENO_VALIDATOR_VERSION = '2.0.1' DENO_METADATA = { 'validator': 'schema', From 5a6184d0c1aed7a2082592505ce3ab82c839a78f Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 19 Dec 2024 14:24:09 -0800 Subject: [PATCH 04/10] chore: Bump storage for worker 1 --- helm/openneuro/values-production.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/openneuro/values-production.yaml b/helm/openneuro/values-production.yaml index bb74507d3..6e59d2ea6 100644 --- a/helm/openneuro/values-production.yaml +++ b/helm/openneuro/values-production.yaml @@ -54,7 +54,7 @@ workerDiskSize: - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-datasets-0 size: 47000Gi - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-datasets-1 - size: 38000Gi + size: 40000Gi - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-datasets-2 size: 31000Gi - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-datasets-3 From 992fc9e3044c67f6d85757ecf252845cfda6e809 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 19 Dec 2024 14:27:07 -0800 Subject: [PATCH 05/10] v4.29.9 --- cli/deno.json | 2 +- lerna.json | 2 +- packages/openneuro-app/package.json | 4 ++-- packages/openneuro-cli/package.json | 4 ++-- packages/openneuro-client/package.json | 4 ++-- packages/openneuro-components/package.json | 2 +- packages/openneuro-indexer/package.json | 6 +++--- packages/openneuro-search/package.json | 2 +- packages/openneuro-server/package.json | 4 ++-- yarn.lock | 20 ++++++++++---------- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cli/deno.json b/cli/deno.json index 050858e49..8dd9db50b 100644 --- a/cli/deno.json +++ b/cli/deno.json @@ -1,7 +1,7 @@ { "name": "@openneuro/cli", "license": "MIT", - "version": "4.29.8", + "version": "4.29.9", "exports": "./mod.ts", "publish": { "include": [ diff --git a/lerna.json b/lerna.json index 76db45a69..fd67ab73f 100644 --- a/lerna.json +++ b/lerna.json @@ -5,5 +5,5 @@ ], "npmClient": "yarn", "useWorkspaces": true, - "version": "4.29.8" + "version": "4.29.9" } diff --git a/packages/openneuro-app/package.json b/packages/openneuro-app/package.json index 6d1d347dd..f3328f7eb 100644 --- a/packages/openneuro-app/package.json +++ b/packages/openneuro-app/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/app", - "version": "4.29.8", + "version": "4.29.9", "description": "React JS web frontend for the OpenNeuro platform.", "license": "MIT", "main": "public/client.js", @@ -20,7 +20,7 @@ "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", "@niivue/niivue": "0.45.1", - "@openneuro/components": "^4.29.8", + "@openneuro/components": "^4.29.9", "@sentry/react": "^8.25.0", "@tanstack/react-table": "^8.9.3", "buffer": "^6.0.3", diff --git a/packages/openneuro-cli/package.json b/packages/openneuro-cli/package.json index bae9155c0..bfda067a2 100644 --- a/packages/openneuro-cli/package.json +++ b/packages/openneuro-cli/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/cli", - "version": "4.29.8", + "version": "4.29.9", "description": "OpenNeuro command line uploader / editor.", "main": "index.js", "repository": "git@github.com:OpenNeuroOrg/openneuro.git", @@ -16,7 +16,7 @@ }, "dependencies": { "@apollo/client": "3.11.8", - "@openneuro/client": "^4.29.8", + "@openneuro/client": "^4.29.9", "bids-validator": "1.14.8", "cli-progress": "^3.8.2", "commander": "7.2.0", diff --git a/packages/openneuro-client/package.json b/packages/openneuro-client/package.json index 4bfc89adb..34bcfc9ad 100644 --- a/packages/openneuro-client/package.json +++ b/packages/openneuro-client/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/client", - "version": "4.29.8", + "version": "4.29.9", "description": "OpenNeuro shared client library.", "main": "dist/index.js", "browser": "src/index.js", @@ -27,7 +27,7 @@ }, "devDependencies": { "@graphql-tools/schema": "^10.0.0", - "@openneuro/server": "^4.29.8", + "@openneuro/server": "^4.29.9", "core-js": "^3.10.1", "ioredis-mock": "^8.8.1", "vitest": "2.1.2" diff --git a/packages/openneuro-components/package.json b/packages/openneuro-components/package.json index 1cce76a7f..4239291e3 100644 --- a/packages/openneuro-components/package.json +++ b/packages/openneuro-components/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/components", - "version": "4.29.8", + "version": "4.29.9", "description": "OpenNeuro component library and Storybook configuration", "main": "dist/index.js", "browser": "src/index.js", diff --git a/packages/openneuro-indexer/package.json b/packages/openneuro-indexer/package.json index 0dc4274db..241240d2f 100644 --- a/packages/openneuro-indexer/package.json +++ b/packages/openneuro-indexer/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/indexer", - "version": "4.29.8", + "version": "4.29.9", "description": "OpenNeuro elastic search indexing tool", "main": "./dist/index.js", "types": "./src/index.d.ts", @@ -11,8 +11,8 @@ "dependencies": { "@apollo/client": "3.11.8", "@elastic/elasticsearch": "8.13.1", - "@openneuro/client": "^4.29.8", - "@openneuro/search": "^4.29.8", + "@openneuro/client": "^4.29.9", + "@openneuro/search": "^4.29.9", "graphql": "16.8.1", "ts-node": "10.9.2", "typescript": "5.6.3" diff --git a/packages/openneuro-search/package.json b/packages/openneuro-search/package.json index 0985abb04..432c3e014 100644 --- a/packages/openneuro-search/package.json +++ b/packages/openneuro-search/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/search", - "version": "4.29.8", + "version": "4.29.9", "description": "OpenNeuro search client functions.", "main": "dist/index.js", "browser": "src/index.ts", diff --git a/packages/openneuro-server/package.json b/packages/openneuro-server/package.json index f5adfdd04..8469e8fbc 100644 --- a/packages/openneuro-server/package.json +++ b/packages/openneuro-server/package.json @@ -1,6 +1,6 @@ { "name": "@openneuro/server", - "version": "4.29.8", + "version": "4.29.9", "description": "Core service for the OpenNeuro platform.", "license": "MIT", "main": "src/server.js", @@ -21,7 +21,7 @@ "@elastic/elasticsearch": "8.13.1", "@graphql-tools/schema": "^10.0.0", "@keyv/redis": "^2.7.0", - "@openneuro/search": "^4.29.8", + "@openneuro/search": "^4.29.9", "@sentry/node": "^8.25.0", "@sentry/profiling-node": "^8.25.0", "base64url": "^3.0.0", diff --git a/yarn.lock b/yarn.lock index 4731cbef5..367b0df35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5915,7 +5915,7 @@ __metadata: "@emotion/react": "npm:11.11.1" "@emotion/styled": "npm:11.11.0" "@niivue/niivue": "npm:0.45.1" - "@openneuro/components": "npm:^4.29.8" + "@openneuro/components": "npm:^4.29.9" "@sentry/react": "npm:^8.25.0" "@tanstack/react-table": "npm:^8.9.3" "@testing-library/jest-dom": "npm:6.4.2" @@ -5971,7 +5971,7 @@ __metadata: resolution: "@openneuro/cli@workspace:packages/openneuro-cli" dependencies: "@apollo/client": "npm:3.11.8" - "@openneuro/client": "npm:^4.29.8" + "@openneuro/client": "npm:^4.29.9" "@types/mkdirp": "npm:1.0.2" "@types/node": "npm:20.12.7" bids-validator: "npm:1.14.8" @@ -5994,14 +5994,14 @@ __metadata: languageName: unknown linkType: soft -"@openneuro/client@npm:^4.29.8, @openneuro/client@workspace:packages/openneuro-client": +"@openneuro/client@npm:^4.29.9, @openneuro/client@workspace:packages/openneuro-client": version: 0.0.0-use.local resolution: "@openneuro/client@workspace:packages/openneuro-client" dependencies: "@apollo/client": "npm:3.11.8" "@graphql-tools/mock": "npm:^9.0.0" "@graphql-tools/schema": "npm:^10.0.0" - "@openneuro/server": "npm:^4.29.8" + "@openneuro/server": "npm:^4.29.9" core-js: "npm:^3.10.1" crypto-hash: "npm:^1.3.0" graphql: "npm:16.8.1" @@ -6013,7 +6013,7 @@ __metadata: languageName: unknown linkType: soft -"@openneuro/components@npm:^4.29.8, @openneuro/components@workspace:packages/openneuro-components": +"@openneuro/components@npm:^4.29.9, @openneuro/components@workspace:packages/openneuro-components": version: 0.0.0-use.local resolution: "@openneuro/components@workspace:packages/openneuro-components" dependencies: @@ -6054,8 +6054,8 @@ __metadata: dependencies: "@apollo/client": "npm:3.11.8" "@elastic/elasticsearch": "npm:8.13.1" - "@openneuro/client": "npm:^4.29.8" - "@openneuro/search": "npm:^4.29.8" + "@openneuro/client": "npm:^4.29.9" + "@openneuro/search": "npm:^4.29.9" "@types/jsonwebtoken": "npm:^8" "@types/node": "npm:20.12.7" "@types/tsc-watch": "npm:^4" @@ -6069,7 +6069,7 @@ __metadata: languageName: unknown linkType: soft -"@openneuro/search@npm:^4.29.8, @openneuro/search@workspace:packages/openneuro-search": +"@openneuro/search@npm:^4.29.9, @openneuro/search@workspace:packages/openneuro-search": version: 0.0.0-use.local resolution: "@openneuro/search@workspace:packages/openneuro-search" dependencies: @@ -6084,7 +6084,7 @@ __metadata: languageName: unknown linkType: soft -"@openneuro/server@npm:^4.29.8, @openneuro/server@workspace:packages/openneuro-server": +"@openneuro/server@npm:^4.29.9, @openneuro/server@workspace:packages/openneuro-server": version: 0.0.0-use.local resolution: "@openneuro/server@workspace:packages/openneuro-server" dependencies: @@ -6094,7 +6094,7 @@ __metadata: "@elastic/elasticsearch": "npm:8.13.1" "@graphql-tools/schema": "npm:^10.0.0" "@keyv/redis": "npm:^2.7.0" - "@openneuro/search": "npm:^4.29.8" + "@openneuro/search": "npm:^4.29.9" "@sentry/node": "npm:^8.25.0" "@sentry/profiling-node": "npm:^8.25.0" "@types/cors": "npm:^2" From f59fe01cde22c0b942f5c59cac49c9347964ebc8 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 19 Dec 2024 14:35:48 -0800 Subject: [PATCH 06/10] chore: Bump released version to 4.29.9 --- helm/openneuro/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/openneuro/Chart.yaml b/helm/openneuro/Chart.yaml index a4792c082..e806e123a 100644 --- a/helm/openneuro/Chart.yaml +++ b/helm/openneuro/Chart.yaml @@ -5,7 +5,7 @@ description: OpenNeuro production deployment chart home: https://openneuro.org sources: - https://github.com/openNeuroOrg/openneuro -appVersion: 4.29.8 +appVersion: 4.29.9 dependencies: - name: redis version: 17.1.4 From ad5a7340a9883354a94b18398cc73530114d80f6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 6 Jan 2025 10:49:10 -0500 Subject: [PATCH 07/10] fix(pre-receive): Skip validation of git-annex branches --- services/datalad/hooks/pre-receive | 37 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/services/datalad/hooks/pre-receive b/services/datalad/hooks/pre-receive index 15926e6b7..3648bb1d2 100755 --- a/services/datalad/hooks/pre-receive +++ b/services/datalad/hooks/pre-receive @@ -121,6 +121,27 @@ function main() { continue fi + IFS=$'\n' + for file in $large_files; do + if [[ "$status" == 0 ]]; then + echo "" + echo "-------------------------------------------------------------------------" + echo "Your push was rejected because it contains files larger than 10MB." + echo "Please use git-annex to store larger files." + echo "-------------------------------------------------------------------------" + echo "" + echo "Offending files:" + status="$EXIT_FAILURE" + fi + echo " - ${file} (ref: ${refname})" + done + unset IFS + + if [[ "$branch" == "git-annex" || "$branch" == "synced/git-annex" ]]; then + continue + fi + # Additional validation and filtering of main/master branches + filterDotFiles # Check if .bidsignore is in the new tree @@ -147,22 +168,6 @@ END exit $EXIT_FAILURE fi - IFS=$'\n' - for file in $large_files; do - if [[ "$status" == 0 ]]; then - echo "" - echo "-------------------------------------------------------------------------" - echo "Your push was rejected because it contains files larger than 10MB." - echo "Please use git-annex to store larger files." - echo "-------------------------------------------------------------------------" - echo "" - echo "Offending files:" - status="$EXIT_FAILURE" - fi - echo " - ${file} (ref: ${refname})" - done - unset IFS - validateGitAttributes done From d0f681da5c381cf02db66737ed290d1ac6ec271a Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Mon, 6 Jan 2025 08:32:27 -0800 Subject: [PATCH 08/10] chore: Format readme with deno fmt --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index a8e0ad654..47da40e04 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ GOOGLE_CLIENT_SECRET= [podman-compose](https://github.com/containers/podman-compose) is used to run a local copy of all required services together. - - macOS Development Setup - If you're working on macOS and encounter issues with file watching in Vite, use the following environment variables (in config.env): ``` From 16d6d2e2c9f043fe60595878a797e9c89a9fa0e5 Mon Sep 17 00:00:00 2001 From: Gregory Noack Date: Mon, 6 Jan 2025 10:51:21 -0800 Subject: [PATCH 09/10] adding workaround for elastic search container on mac --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 11c260124..75da162c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -134,6 +134,8 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 environment: + ES_JAVA_OPTS: -XX:UseSVE=0 + CLI_JAVA_OPTS: -XX:UseSVE=0 discovery.type: single-node cluster.routing.allocation.disk.threshold_enabled: "true" cluster.routing.allocation.disk.watermark.flood_stage: 1gb @@ -141,7 +143,6 @@ services: cluster.routing.allocation.disk.watermark.high: 5gb xpack.security.enabled: false xpack.security.enrollment.enabled: false - ES_JAVA_OPTS: "-Xms750m -Xmx750m" healthcheck: test: "curl -s -f http://localhost:9200 || exit 1" interval: 10s From 8abbeeafbe6a0072a05dfd2ab569e79f168528d0 Mon Sep 17 00:00:00 2001 From: Gregory Noack Date: Mon, 6 Jan 2025 13:25:52 -0800 Subject: [PATCH 10/10] adding java opts to config.env --- README.md | 10 +++++++++- config.env.example | 3 +++ docker-compose.yml | 2 -- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 47da40e04..b11e6b8d3 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,20 @@ GOOGLE_CLIENT_SECRET= [podman-compose](https://github.com/containers/podman-compose) is used to run a local copy of all required services together. -macOS Development Setup - If you're working on macOS and encounter issues with file watching in Vite, use the following environment variables (in config.env): +#### macOS Development Setup +If you're working on macOS and encounter issues with file watching in Vite, use the following environment variables (in config.env): ``` CHOKIDAR_USEPOLLING=true ``` +If you're working on macOS and encounter issues with elastic search, use the following environment variables (in config.env): + +``` +ES_JAVA_OPTS: -XX:UseSVE=0 +CLI_JAVA_OPTS: -XX:UseSVE=0 +``` + ```shell # This will run podman-compose in the background (-d flag is --detach) podman-compose up -d diff --git a/config.env.example b/config.env.example index da15fce36..dc80925d2 100644 --- a/config.env.example +++ b/config.env.example @@ -21,6 +21,9 @@ GLOBUS_CLIENT_SECRET= # macOS Development Setup - Fixes issues with Vite file watching - set to CHOKIDAR_USEPOLLING=true CHOKIDAR_USEPOLLING=false CHOKIDAR_INTERVAL=1000 +# macOS Development Setup - Fixes issues with elastic search container - set to ES_JAVA_OPTS: -XX:UseSVE=0 and CLI_JAVA_OPTS: -XX:UseSVE=0 +# ES_JAVA_OPTS: -XX:UseSVE=0 +# CLI_JAVA_OPTS: -XX:UseSVE=0 # Relay for enabling mail - leave blank to disable email CRN_SERVER_MAIL_API_KEY= diff --git a/docker-compose.yml b/docker-compose.yml index 75da162c3..cc59e8c05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -134,8 +134,6 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 environment: - ES_JAVA_OPTS: -XX:UseSVE=0 - CLI_JAVA_OPTS: -XX:UseSVE=0 discovery.type: single-node cluster.routing.allocation.disk.threshold_enabled: "true" cluster.routing.allocation.disk.watermark.flood_stage: 1gb