From 8166db7555d39e40ed05b8b49988221aa3e8b29b Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 3 Oct 2024 13:01:10 -0700 Subject: [PATCH] Add support for configurable containers engines (#963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add initial Exacloud page to documentation * Update Exacloud support * Clean up console debug output [no ci] * Add default Worker engine ('docker') [no ci] * Update configurable Container Engine commands * Revert changes to docker implementation and add engine interface [no ci] * Update Exacloud/Container Engine implementation [no ci] * Update ContainerEngine interface [no ci] * Rename ContainerEngine to ContainerType * Add initial Exadocker unit test [no ci] * Update TaskService to pass config to Funnel worker [no ci] * Add small fix in HPC Template config [no ci] * Update worker config path in HPC Template [no ci] * Add debugging from Exacloud testing * Update input/output working example (md5sum) - Test command: ```sh funnel task create examples/md5sum.json ``` [no ci] * Update docker.go to implement ContainerEngine interface - Can be used in HPC environment with unique docker setups (e.g. exadocker) as well as future command templating efforts. [no ci] * Remove exadocker implementation in favor of docker * Add initial HPC documentation and example * Update docker and kin-openapi to compatible versions - github.com/docker/docker versions above v24.0.9 resulted in `panic: Can't connect docker client`: ``` ➜ go test tests/auth/auth_test.go --- FAIL: TestBasicAuthFail (0.01s) panic: Can't connect docker client [recovered] panic: Can't connect docker client FAIL command-line-arguments 0.673s FAIL ``` - This error appears to be originating from [util/dockerutil/docker.go](https://github.com/ohsu-comp-bio/funnel/blob/f5acbabe85d28f45f2f86db959c4d7bc929334a4/util/dockerutil/docker.go#L28). - github.com/getkin/kin-openapi versions above v0.112.0 resulted in type errors: v0.113.0: ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType FAIL command-line-arguments [build failed] FAIL ``` v0.123.0 (latest): ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:43:7: cannot convert "integer" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:45:7: cannot convert "boolean" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:47:7: cannot convert "number" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:49:7: cannot convert "object" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType util/openapi2proto/main.go:60:7: cannot convert "array" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:72:17: cannot use p.Value.Type (variable of type *openapi3.Types) as string value in return statement util/openapi2proto/main.go:84:32: invalid operation: param.Schema.Value.Type != "" (mismatched types *openapi3.Types and untyped string) util/openapi2proto/main.go:175:17: resp.Get undefined (type *openapi3.Responses has no field or method Get) util/openapi2proto/main.go:175:17: too many errors FAIL command-line-arguments [build failed] FAIL ``` * Update util/openapi2proto/main.go with latest kin-openapi version * Update github.com/docker/docker to latest version * Update Docker client creation in tests * Add `WithAPIVersionNegotiation` to docker client creation * Rename 'Driver' to 'DriverCommand' in ContainerConfig * Fix minor linting errors * Update GitHub Actions workflow (linting) * fix: updated config check in hpc_backend.go * Revert tabs/space change in storage/local.go * Update output copying in scratch directory * Update compliance-test.yaml * Debug: add tmate step to Nextflow test * Update Nextflow tests * Update file copying behavior for Nextflow tests * Update local file copying behavior to support directory dests * Update local dest for files with glob patterns * Update Base64Encode test to match Github Actions test - TODO: investigate why this is necessary... * Fix Base64Encoding test value * Revert glob output change based on TesOutput spec - Wildcards in outputs will be uploaded to a directory, with no special handling for the filename. ```yaml tesOutput: properties: url: description: |- URL at which the TES server makes the output accessible after the task is complete. When tesOutput.path contains wildcards, it must be a directory; see `tesOutput.path_prefix` for details on how output URLs are constructed in this case. ``` * Debug: Add tmate step to Nextflow tests * Update wildcard file output destination * Add initial command templating for container engines * Add initial Docker/ContainerEngine tests * Run `go mod tidy` * Update Github Actions * Update container engine IO and command execution * Re-enable all tests for container engines * Update test suite * Nextflow test debug * Nextflow test debug * Nextflow test debug * Nextflow test debug * Update compliance test workflow * Update S3 tests * Update S3 tests * Add comments and clean up debug statements * Clean up code for 0.11.1-rc.1 release * Add container command event to match existing pattern * Update .goreleaser.yaml to replace deprecated fields * Update Nextflow instructions * Update K8s support and documentation - Update docker images to use quay.io/ohsu-comp-bio/funnel * Update README.md * fix: add support for wildcards in AWS S3 paths * Add support for wildcards in AWS S3 output paths * Update actions/upload-artifact to v4 in Github Actions - https://github.blog/changelog/2024-02-13-deprecation-notice-v1-and-v2-of-the-artifact-actions/ * Update version to 0.11.1-rc.4 in Makefile * Fix bug where FileMapper output was being overwritten in every loop * Add initial Exacloud page to documentation * Update Exacloud support * Clean up console debug output [no ci] * Add default Worker engine ('docker') [no ci] * Update configurable Container Engine commands * Revert changes to docker implementation and add engine interface [no ci] * Update Exacloud/Container Engine implementation [no ci] * Update ContainerEngine interface [no ci] * Rename ContainerEngine to ContainerType * Add initial Exadocker unit test [no ci] * Update TaskService to pass config to Funnel worker [no ci] * Add small fix in HPC Template config [no ci] * Update worker config path in HPC Template [no ci] * Add debugging from Exacloud testing * Update input/output working example (md5sum) - Test command: ```sh funnel task create examples/md5sum.json ``` [no ci] * Update docker.go to implement ContainerEngine interface - Can be used in HPC environment with unique docker setups (e.g. exadocker) as well as future command templating efforts. [no ci] * Remove exadocker implementation in favor of docker * Add initial HPC documentation and example * Update docker and kin-openapi to compatible versions - github.com/docker/docker versions above v24.0.9 resulted in `panic: Can't connect docker client`: ``` ➜ go test tests/auth/auth_test.go --- FAIL: TestBasicAuthFail (0.01s) panic: Can't connect docker client [recovered] panic: Can't connect docker client FAIL command-line-arguments 0.673s FAIL ``` - This error appears to be originating from [util/dockerutil/docker.go](https://github.com/ohsu-comp-bio/funnel/blob/f5acbabe85d28f45f2f86db959c4d7bc929334a4/util/dockerutil/docker.go#L28). - github.com/getkin/kin-openapi versions above v0.112.0 resulted in type errors: v0.113.0: ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType FAIL command-line-arguments [build failed] FAIL ``` v0.123.0 (latest): ``` ➜ go test util/openapi2proto/main.go util/openapi2proto/main.go:43:7: cannot convert "integer" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:45:7: cannot convert "boolean" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:47:7: cannot convert "number" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:49:7: cannot convert "object" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:54:38: invalid operation: p.Value.AdditionalProperties != nil (mismatched types openapi3.AdditionalProperties and untyped nil) util/openapi2proto/main.go:55:24: cannot use p.Value.AdditionalProperties (variable of type openapi3.AdditionalProperties) as *openapi3.SchemaRef value in argument to getType util/openapi2proto/main.go:60:7: cannot convert "array" (untyped string constant) to type *openapi3.Types util/openapi2proto/main.go:72:17: cannot use p.Value.Type (variable of type *openapi3.Types) as string value in return statement util/openapi2proto/main.go:84:32: invalid operation: param.Schema.Value.Type != "" (mismatched types *openapi3.Types and untyped string) util/openapi2proto/main.go:175:17: resp.Get undefined (type *openapi3.Responses has no field or method Get) util/openapi2proto/main.go:175:17: too many errors FAIL command-line-arguments [build failed] FAIL ``` * Update util/openapi2proto/main.go with latest kin-openapi version * Update github.com/docker/docker to latest version * Update Docker client creation in tests * Add `WithAPIVersionNegotiation` to docker client creation * Rename 'Driver' to 'DriverCommand' in ContainerConfig * Fix minor linting errors * Update GitHub Actions workflow (linting) * fix: updated config check in hpc_backend.go * Revert tabs/space change in storage/local.go * Update output copying in scratch directory * Update compliance-test.yaml * Debug: add tmate step to Nextflow test * Update Nextflow tests * Update file copying behavior for Nextflow tests * Update local file copying behavior to support directory dests * Update local dest for files with glob patterns * Update Base64Encode test to match Github Actions test - TODO: investigate why this is necessary... * Fix Base64Encoding test value * Revert glob output change based on TesOutput spec - Wildcards in outputs will be uploaded to a directory, with no special handling for the filename. ```yaml tesOutput: properties: url: description: |- URL at which the TES server makes the output accessible after the task is complete. When tesOutput.path contains wildcards, it must be a directory; see `tesOutput.path_prefix` for details on how output URLs are constructed in this case. ``` * Debug: Add tmate step to Nextflow tests * Update wildcard file output destination * Add initial command templating for container engines * Add initial Docker/ContainerEngine tests * Run `go mod tidy` * Update Github Actions * Update container engine IO and command execution * Re-enable all tests for container engines * Update test suite * Nextflow test debug * Nextflow test debug * Nextflow test debug * Nextflow test debug * Update compliance test workflow * Update S3 tests * Update S3 tests * Add comments and clean up debug statements * Clean up code for 0.11.1-rc.1 release * Add container command event to match existing pattern * Update .goreleaser.yaml to replace deprecated fields * Update Nextflow instructions * Update K8s support and documentation - Update docker images to use quay.io/ohsu-comp-bio/funnel * Update README.md * fix: add support for wildcards in AWS S3 paths * Add support for wildcards in AWS S3 output paths * Update actions/upload-artifact to v4 in Github Actions - https://github.blog/changelog/2024-02-13-deprecation-notice-v1-and-v2-of-the-artifact-actions/ * Update version to 0.11.1-rc.4 in Makefile * Fix bug where FileMapper output was being overwritten in every loop * Add initial search functionality to website - Uses Pagefind: https://pagefind.app/docs/ * Update GitHub Actions workflows * Add Pagefind step to hugo.yaml * Run `go mod tidy` * Fix lint errors * debug: Slurm Github Action workflow * debug: Slurm GitHub Actions Workflow * debug: Slurm GitHub Actions Workflow * debug: Slurm GitHub Actions Workflow * debug: Slurm GitHub Actions Workflow * debug: Slurm GitHub Actions Workflow --- .github/workflows/build.yml | 43 ++ .github/workflows/compliance-test.yaml | 135 ------ .github/workflows/compliance.yaml | 85 ++++ .github/workflows/hugo.yml | 9 +- .github/workflows/nextflow.yaml | 38 +- .github/workflows/s3-test.yaml | 48 -- .github/workflows/s3.yaml | 52 +++ .github/workflows/tests.yaml | 73 +-- .goreleaser.yaml | 4 +- Dockerfile | 2 +- Dockerfile.dind | 4 +- Dockerfile.dind-rootless | 4 +- Makefile | 2 +- cmd/node/run.go | 4 +- cmd/server/run.go | 7 +- cmd/util/flags.go | 2 + cmd/worker/run.go | 12 +- compute/hpc_backend.go | 21 +- compute/hpc_backend_test.go | 3 +- compute/kubernetes/backend.go | 33 +- compute/kubernetes/backend_test.go | 4 +- compute/scheduler/node.go | 4 +- compute/scheduler/testutils_test.go | 18 +- compute/scheduler/util.go | 4 +- config/config.go | 25 + config/default.go | 12 + database/boltdb/events.go | 3 + database/boltdb/scheduler.go | 2 +- database/elastic/scheduler.go | 2 +- database/mongodb/scheduler.go | 8 +- deployments/kubernetes/README.md | 52 ++- deployments/kubernetes/funnel-deployment.yml | 27 +- .../kubernetes/funnel-server-config.yml | 60 +++ events/data.json | 14 + examples/s3.json | 8 +- go.mod | 172 ++++++- go.sum | 439 +++++++++++++++++- server/marshal.go | 54 +-- server/server.go | 45 +- server/tes.go | 9 +- storage/amazon_s3.go | 22 +- storage/anonymous_get_test.go | 10 +- storage/generic_s3.go | 6 + storage/local.go | 169 +++---- storage/local_test.go | 25 + tes/utils_test.go | 4 +- tests/funnel_utils.go | 5 +- util/dockerutil/docker.go | 9 +- util/openapi2proto/main.go | 43 +- webdash/src/Pages.js | 2 - website/config.yaml | 5 + .../{interop => integrations}/nextflow.md | 40 +- .../docs/{interop => integrations}/py-tes.md | 2 +- website/layouts/_default/single.html | 2 +- website/layouts/partials/head.html | 11 + .../default-config.yaml | 25 +- worker/command.go | 20 +- worker/container_engine.go | 63 +++ worker/docker.go | 179 +++---- worker/docker_test.go | 139 ++++++ worker/file_mapper.go | 167 ++++++- worker/interfaces.go | 2 +- worker/kubernetes.go | 24 +- worker/step.go | 17 +- worker/storage.go | 40 ++ worker/taskreader_test.go | 6 + worker/worker.go | 116 +++-- 67 files changed, 2015 insertions(+), 681 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/compliance-test.yaml create mode 100644 .github/workflows/compliance.yaml delete mode 100644 .github/workflows/s3-test.yaml create mode 100644 .github/workflows/s3.yaml create mode 100644 deployments/kubernetes/funnel-server-config.yml create mode 100644 events/data.json rename website/content/docs/{interop => integrations}/nextflow.md (74%) rename website/content/docs/{interop => integrations}/py-tes.md (97%) create mode 100644 worker/container_engine.go create mode 100644 worker/docker_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..b853e0630 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,43 @@ +name: Funnel Build and Cache + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Cache Funnel binary + uses: actions/cache@v3 + with: + path: ./funnel + key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }} + restore-keys: | + ${{ runner.os }}-funnel-bin-${{ github.ref }} + + - name: Build Funnel (if cache doesn't exist) + run: | + if [ ! -f ./funnel ]; then + make build + fi + + - name: Cache Funnel binary (after build) + uses: actions/cache@v3 + with: + path: ./funnel + key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }} + + - name: Upload Funnel binary as artifact + uses: actions/upload-artifact@v4 + with: + name: funnel + path: funnel diff --git a/.github/workflows/compliance-test.yaml b/.github/workflows/compliance-test.yaml deleted file mode 100644 index 45990039c..000000000 --- a/.github/workflows/compliance-test.yaml +++ /dev/null @@ -1,135 +0,0 @@ -# Workflow for running the TES compliance suite against Funnel - -# This includes the following steps: -# 1. Build Funnel and store the resulting binary artifact -# 2. Install tes-compliance-suite and run against every version of TES simultaneously -# 3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish -# the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/ - -# Optionally debug via SSH -# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability -# -# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and -# output a ssh address associated with the Github action worker. Helpful for debugging build steps and -# and intermediary files/artifacts. -# -# - name: "Debug: Package dependancies for tmate (CentOS)" -# run: | -# yum install -y xz -# ln -s /bin/true /bin/apt-get -# -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 - -name: Compliance Test - -on: - push: - -jobs: - build: - runs-on: ubuntu-latest - container: quay.io/ohsu-comp-bio/slurm - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: 1.21 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Build - run: make build - - - name: Store funnel - uses: actions/upload-artifact@v2 - with: - name: funnelBin - path: funnel - - compliance: - strategy: - fail-fast: false - matrix: - version: [1.0.0, 1.1.0] - db: ["boltdb", "mongodb"] - compute: ["local", "slurm"] - needs: build - runs-on: ubuntu-latest - container: - image: quay.io/ohsu-comp-bio/slurm - options: --hostname slurmctl --cap-add sys_admin - steps: - # Required to access the 'tests/mongo.config.yml' file - # Perhaps uploading it as an artifact would be more efficient? - - name: Check out code - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v3 - with: - name: funnelBin - - - name: Start Funnel server - run: | - touch config.yml - if [ ${{ matrix.db }} = "mongodb" ]; then - make start-mongodb - cat `pwd`/tests/mongo.config.yml >> config.yml - # Required for Funnel to connect MongoDB - echo "172.17.0.1 localhost" >> /etc/hosts - elif [ ${{ matrix.compute }} = "slurm" ]; then - cat `pwd`/tests/slurm.config.yml >> config.yml - cp config.yml /opt/funnel_config.yml - # Start Slurm - /usr/local/bin/docker-entrypoint.sh - fi - chmod +x funnel - FLAGS="--config `pwd`/config.yml" - ./funnel server run $FLAGS &> funnel.logs & - - - name: Run OpenAPI Test Runner - run: | - # Clone the 'upstream' OpenAPI Test Runner when PR #65 is merged - # https://github.com/elixir-cloud-aai/openapi-test-runner/pull/65 - # git clone https://github.com/elixir-cloud-aai/openapi-test-runner - git clone https://github.com/ohsu-comp-bio/openapi-test-runner -b fix/create-and-filter-task - - cd openapi-test-runner - python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt - python setup.py install - openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" - - - name: Install TES compliance suite - run: | - git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1 - cd tes-compliance-suite - /root/.pyenv/shims/python3 -m venv venv - source venv/bin/activate - pip install -r requirements.txt - python setup.py install - mkdir reports - - - name: Test compliance (Report Publishing) - run: | - cd tes-compliance-suite - source venv/bin/activate - tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/" - - start-report-deployment: - needs: compliance - runs-on: ubuntu-latest - steps: - # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event - - name: Start report generation - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_AUTHOR: "Funnel" - PAYLOAD_REVISION: "3" - with: - repository: ohsu-comp-bio/funnel-compliance - event: start-report - github_pat: ${{ secrets.ACTIONS_TOKEN }} - diff --git a/.github/workflows/compliance.yaml b/.github/workflows/compliance.yaml new file mode 100644 index 000000000..0c7676e0a --- /dev/null +++ b/.github/workflows/compliance.yaml @@ -0,0 +1,85 @@ +# Workflow for running the TES compliance suite against Funnel +# +# This includes the following steps: +# 1. Build Funnel and store the resulting binary artifact +# 2. Install tes-compliance-suite and run against every version of TES simultaneously +# 3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish +# the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/ +# +# Optionally debug via SSH +# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability +# +# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and +# output a ssh address associated with the Github action worker. Helpful for debugging build steps and +# and intermediary files/artifacts. +# +# - name: "Debug: Package dependancies for tmate (CentOS)" +# run: | +# yum install -y xz +# ln -s /bin/true /bin/apt-get +# +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 + +name: Compliance Test + +on: + push: + pull_request: + +jobs: + build: + uses: ./.github/workflows/build.yml + + compliance: + strategy: + fail-fast: false + matrix: + version: [1.0.0, 1.1.0] + db: ["boltdb", "mongodb"] + compute: ["local", "kubernetes"] + storage: ["local", "s3"] + needs: build + runs-on: ubuntu-latest + steps: + # Required to access the 'tests/mongo.config.yml' file + # Perhaps uploading it as an artifact would be more efficient? + - name: Check out code + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v4 + with: + name: funnel + + - name: Start Funnel server + run: | + touch config.yml + + if [ ${{ matrix.db }} = "mongodb" ]; then + make start-mongodb + cat `pwd`/tests/mongo.config.yml >> config.yml + fi + + if [ ${{ matrix.storage }} = "s3" ]; then + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ROOT_USER=minioadmin" \ + -e "MINIO_ROOT_PASSWORD=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + cat `pwd`/tests/s3.config.yml >> config.yml + fi + + chmod +x funnel + ./funnel server run --config `pwd`/config.yml &> funnel.logs & + + - name: Run OpenAPI Test Runner + run: | + git clone https://github.com/elixir-cloud-aai/openapi-test-runner + cd openapi-test-runner + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + python setup.py install + openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" + diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 0e3d40fc5..a26f69bdb 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -4,7 +4,9 @@ name: Deploy Hugo site to Pages on: # Runs on pushes targeting the default branch push: - branches: ["master"] + branches: + - main + - master # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -59,6 +61,11 @@ jobs: --baseURL "${{ steps.pages.outputs.base_url }}/" \ --source website \ --destination public + + # Run pagefind to add search functionality + # Reference: https://pagefind.app/docs/ + npx -y pagefind --site public + - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml index 57685e720..1b04b0180 100644 --- a/.github/workflows/nextflow.yaml +++ b/.github/workflows/nextflow.yaml @@ -2,55 +2,34 @@ name: Nextflow Test on: push: + pull_request: jobs: build: - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: 1.21 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Build Funnel (if cache does not exist) - run: make build + uses: ./.github/workflows/build.yml - - name: Store Funnel - uses: actions/upload-artifact@v2 - with: - name: funnelBin - path: funnel - nextflow: - runs-on: ubuntu-latest needs: build + runs-on: ubuntu-latest steps: - name: Download Funnel - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin - path: funnel + name: funnel - name: Start Funnel - working-directory: run: | - cd funnel/ chmod +x ./funnel ./funnel server --LocalStorage.AllowedDirs $HOME run & - name: Install Nextflow run: | - cd .. - git clone https://github.com/nextflow-io/nextflow/ -b tes-update-1.1 + git clone https://github.com/nextflow-io/nextflow cd nextflow make compile - name: Install nf-canary and GA4GH-TES plugin run: | - cd .. git clone https://github.com/seqeralabs/nf-canary cd nf-canary cat <> nextflow.config @@ -60,8 +39,7 @@ jobs: process.executor = 'tes' tes.endpoint = 'http://localhost:8000' EOF - + - name: Run nf-canary tests run: | - cd ../nf-canary - ../nextflow/launch.sh run main.nf + ./nextflow/nextflow run nf-canary/main.nf diff --git a/.github/workflows/s3-test.yaml b/.github/workflows/s3-test.yaml deleted file mode 100644 index a6b271814..000000000 --- a/.github/workflows/s3-test.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Credit: rhnvrm -# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/ - -name: S3 Integration Test - -on: - push: - branches: - - main - -jobs: - s3Test-integration: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Setup minio - run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ROOT_USER=minioadmin" \ - -e "MINIO_ROOT_PASSWORD=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - - cat < funnel.config.yml - LocalStorage: - Disabled: true - AmazonS3: - Disabled: true - GoogleStorage: - Disabled: true - HTTPStorage: - Disabled: true - FTPStorage: - Disabled: true - GenericS3: - - Disabled: false - Endpoint: "localhost:9000" - Key: "minioadmin" - Secret: "minioadmin" - EOF - - wget https://github.com/ohsu-comp-bio/funnel/releases/download/untagged-217841a99d14ccfe289c/funnel-darwin-arm64-0.11.0.tar.gz - tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz - chmod +x funnel - ./funnel server run --config funnel.config.yml & - ./funnel task run examples/s3-test.yml diff --git a/.github/workflows/s3.yaml b/.github/workflows/s3.yaml new file mode 100644 index 000000000..1855fd519 --- /dev/null +++ b/.github/workflows/s3.yaml @@ -0,0 +1,52 @@ +# Credit: rhnvrm +# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/ + +name: S3 Integration Test + +on: + push: + pull_request: + +jobs: + build: + uses: ./.github/workflows/build.yml + + s3Test: + needs: build + runs-on: ubuntu-latest + steps: + - name: Setup minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ROOT_USER=minioadmin" \ + -e "MINIO_ROOT_PASSWORD=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + + - uses: actions/download-artifact@v4 + with: + name: funnel + + - name: Start Funnel server + run: | + cat < config.yml + LocalStorage: + Disabled: true + AmazonS3: + Disabled: true + GoogleStorage: + Disabled: true + HTTPStorage: + Disabled: true + FTPStorage: + Disabled: true + GenericS3: + - Disabled: false + Endpoint: "localhost:9000" + Key: "minioadmin" + Secret: "minioadmin" + EOF + chmod +x funnel + ./funnel server run --config `pwd`/config.yml &> funnel.logs & + ./funnel task run examples/s3-test.yml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a156af450..911b61aab 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,6 +2,7 @@ name: Go Tests on: push: + pull_request: jobs: lint: @@ -11,38 +12,37 @@ jobs: - uses: actions/setup-go@v3 with: go-version: 1.21 + - uses: actions/checkout@v3 + - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: version: latest - args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign - - build: - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: 1.21 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Build - run: make build - - - name: Store funnel - uses: actions/upload-artifact@v2 - with: - name: funnelBin - path: funnel + # Matches the "primary" golangci-lint command in the Makefile + args: | + --timeout 3m --disable-all --enable=govet --enable=gofmt --enable=goimports --enable=misspell \ + --skip-dirs "vendor" \ + --skip-dirs "webdash" \ + --skip-dirs "cmd/webdash" \ + --skip-dirs "funnel-work-dir" \ + -e '.*bundle.go' -e ".*pb.go" -e ".*pb.gw.go" \ + ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: latest + # Matches the "termdash" golangci-lint command in the Makefile + args: | + --timeout 3m --disable-all --enable=vet --enable=gofmt --enable=goimports --enable=misspell \ + ./cmd/termdash/... unitTest: runs-on: ubuntu-latest steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code @@ -51,12 +51,15 @@ jobs: - name: Unit Tests run: make test-verbose + build: + uses: ./.github/workflows/build.yml + mongoTest: runs-on: ubuntu-latest needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 @@ -64,9 +67,9 @@ jobs: uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: MongoTest run: | @@ -80,16 +83,16 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: Badger Test run: | chmod +x funnel @@ -100,16 +103,16 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: Slurm Test run: | @@ -121,16 +124,16 @@ jobs: needs: build steps: - name: Set up Go 1.x - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - name: Check out code uses: actions/checkout@v2 - name: Download funnel bin - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: funnelBin + name: funnel - name: S3 Test run: | @@ -138,4 +141,4 @@ jobs: make start-generic-s3 sleep 10 make test-generic-s3 - \ No newline at end of file + diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 54dbde04d..47c496f38 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -28,10 +28,10 @@ archives: name_template: "{{.ProjectName}}-{{.Os}}-{{.Arch}}-{{.Version}}" brews: - - tap: + - repository: owner: ohsu-comp-bio name: homebrew-formula - folder: Formula + directory: Formula skip_upload: true description: "distributed task execution toolkit" homepage: "https://ohsu-comp-bio.github.io/funnel/" diff --git a/Dockerfile b/Dockerfile index da946f810..71520ed07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build stage -FROM golang:1.21-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" diff --git a/Dockerfile.dind b/Dockerfile.dind index 488fb0fd2..d25631e60 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -11,7 +11,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/go-build make build # final stage -FROM docker:stable-dind +FROM docker:dind WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/Dockerfile.dind-rootless b/Dockerfile.dind-rootless index ced21148c..79866c7d7 100644 --- a/Dockerfile.dind-rootless +++ b/Dockerfile.dind-rootless @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -11,7 +11,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/go-build make build # final stage -FROM docker:stable-dind-rootless +FROM docker:dind-rootless WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/Makefile b/Makefile index 1923e8dca..49bd35bb0 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git export GIT_BRANCH = $(git_branch) export GIT_UPSTREAM = $(git_upstream) -export FUNNEL_VERSION=0.11.0 +export FUNNEL_VERSION=0.11.1-rc.5 # LAST_PR_NUMBER is used by the release notes builder to generate notes # based on pull requests (PR) up until the last release. diff --git a/cmd/node/run.go b/cmd/node/run.go index c0492273a..c2dc791bd 100644 --- a/cmd/node/run.go +++ b/cmd/node/run.go @@ -34,8 +34,8 @@ func Run(ctx context.Context, conf config.Config, log *logger.Logger) error { return err } - runctx, cancel := context.WithCancel(context.Background()) - runctx = util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM) + _, cancel := context.WithCancel(context.Background()) + runctx := util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM) defer cancel() hupsig := make(chan os.Signal, 1) diff --git a/cmd/server/run.go b/cmd/server/run.go index 2213cb04f..79e808c2e 100644 --- a/cmd/server/run.go +++ b/cmd/server/run.go @@ -147,7 +147,11 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se case strings.ToLower(conf.Database): continue case "log": - continue + if conf.Compute == "slurm" { + writer = &events.Logger{Log: log} + } else { + continue + } case "boltdb": writer, err = boltdb.NewBoltDB(conf.BoltDB) case "badger": @@ -271,6 +275,7 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se Compute: compute, Read: reader, Log: log, + Config: conf, }, Events: &events.Service{Writer: writer}, Nodes: nodes, diff --git a/cmd/util/flags.go b/cmd/util/flags.go index b97febbaa..4a9830744 100644 --- a/cmd/util/flags.go +++ b/cmd/util/flags.go @@ -100,7 +100,9 @@ func workerFlags(flagConf *config.Config) *pflag.FlagSet { f.Var(&flagConf.Worker.LogUpdateRate, "Worker.LogUpdateRate", "How often to send stdout/stderr log updates") f.Var(&flagConf.Worker.PollingRate, "Worker.PollingRate", "How often to poll for cancel signals") f.StringVar(&flagConf.Worker.WorkDir, "Worker.WorkDir", flagConf.Worker.WorkDir, "Working directory") + f.StringVar(&flagConf.Worker.ScratchPath, "Worker.ScratchPath", flagConf.Worker.ScratchPath, "Scratch directory") f.BoolVar(&flagConf.Worker.LeaveWorkDir, "Worker.LeaveWorkDir", flagConf.Worker.LeaveWorkDir, "Leave working directory after execution") + f.StringVar(&flagConf.Worker.DriverCommand, "Worker.DriverCommand", flagConf.Worker.DriverCommand, "Overrides the default command used to run containers.") return f } diff --git a/cmd/worker/run.go b/cmd/worker/run.go index 960066bb3..72d73463a 100644 --- a/cmd/worker/run.go +++ b/cmd/worker/run.go @@ -83,7 +83,7 @@ func NewWorker(ctx context.Context, conf config.Config, log *logger.Logger, opts var executor = worker.Executor{ Backend: "docker", } - + if conf.Kubernetes.Executor == "kubernetes" { executor.Backend = "kubernetes" executor.Template = conf.Kubernetes.ExecutorTemplate @@ -91,11 +91,11 @@ func NewWorker(ctx context.Context, conf config.Config, log *logger.Logger, opts } return &worker.DefaultWorker{ - Executor: executor, - Conf: conf.Worker, - Store: store, - TaskReader: reader, - EventWriter: writer, + Executor: executor, + Conf: conf.Worker, + Store: store, + TaskReader: reader, + EventWriter: writer, }, nil } diff --git a/compute/hpc_backend.go b/compute/hpc_backend.go index 1f09cee9e..022e7e395 100644 --- a/compute/hpc_backend.go +++ b/compute/hpc_backend.go @@ -48,7 +48,7 @@ type HPCBackend struct { func (b *HPCBackend) WriteEvent(ctx context.Context, ev *events.Event) error { switch ev.Type { case events.Type_TASK_CREATED: - return b.Submit(ev.GetTask()) + return b.Submit(ctx, ev.GetTask()) case events.Type_TASK_STATE: if ev.GetState() == tes.State_CANCELED { @@ -61,10 +61,8 @@ func (b *HPCBackend) WriteEvent(ctx context.Context, ev *events.Event) error { func (b *HPCBackend) Close() {} // Submit submits a task via "qsub", "condor_submit", "sbatch", etc. -func (b *HPCBackend) Submit(task *tes.Task) error { - ctx := context.Background() - - submitPath, err := b.setupTemplatedHPCSubmit(task) +func (b *HPCBackend) Submit(ctx context.Context, task *tes.Task) error { + submitPath, err := b.setupTemplatedHPCSubmit(ctx, task) if err != nil { return err } @@ -229,7 +227,7 @@ ReconcileLoop: // setupTemplatedHPCSubmit sets up a task submission in a HPC environment with // a shared file system. It generates a submission file based on a template for // schedulers such as SLURM, HTCondor, SGE, PBS/Torque, etc. -func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { +func (b *HPCBackend) setupTemplatedHPCSubmit(ctx context.Context, task *tes.Task) (string, error) { var err error // TODO document that these working dirs need manual cleanup @@ -264,6 +262,16 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { zone = zones[0] } + var args string + if ctx.Value("Config") != nil { + conf := ctx.Value("Config").(config.Config) + configFile := filepath.Join(workdir, "config.yaml") + err = config.ToYamlFile(conf, configFile) + if err != nil { + return "", err + } + args = fmt.Sprintf("--config %v", configFile) + } err = submitTpl.Execute(f, map[string]interface{}{ "TaskId": task.Id, "WorkDir": workdir, @@ -271,6 +279,7 @@ func (b *HPCBackend) setupTemplatedHPCSubmit(task *tes.Task) (string, error) { "RamGb": res.GetRamGb(), "DiskGb": res.GetDiskGb(), "Zone": zone, + "Args": args, }) if err != nil { return "", err diff --git a/compute/hpc_backend_test.go b/compute/hpc_backend_test.go index 5b91abb3e..2c2add627 100644 --- a/compute/hpc_backend_test.go +++ b/compute/hpc_backend_test.go @@ -1,6 +1,7 @@ package compute import ( + "context" "fmt" "io/ioutil" "testing" @@ -58,7 +59,7 @@ funnel worker run --taskID {{.TaskId}} Conf: conf, } - sf, err := b.setupTemplatedHPCSubmit(task) + sf, err := b.setupTemplatedHPCSubmit(context.Background(), task) if err != nil { t.Fatal(err) } diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go index 9a29c263e..d2cd3159d 100644 --- a/compute/kubernetes/backend.go +++ b/compute/kubernetes/backend.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io/ioutil" "text/template" @@ -82,15 +83,32 @@ func NewBackend(ctx context.Context, conf config.Kubernetes, reader tes.ReadOnly // Backend represents the local backend. type Backend struct { - client batchv1.JobInterface - namespace string - template string - event events.Writer - database tes.ReadOnlyServer - log *logger.Logger + client batchv1.JobInterface + namespace string + template string + event events.Writer + database tes.ReadOnlyServer + log *logger.Logger + backendParameters map[string]string events.Computer } +func (b Backend) CheckBackendParameterSupport(task *tes.Task) error { + if !task.Resources.GetBackendParametersStrict() { + return nil + } + + taskBackendParameters := task.Resources.GetBackendParameters() + for k := range taskBackendParameters { + _, ok := b.backendParameters[k] + if !ok { + return errors.New("backend parameters not supported") + } + } + + return nil +} + // WriteEvent writes an event to the compute backend. // Currently, only TASK_CREATED is handled, which calls Submit. func (b *Backend) WriteEvent(ctx context.Context, ev *events.Event) error { @@ -153,7 +171,8 @@ func (b *Backend) Submit(ctx context.Context, task *tes.Task) error { if err != nil { return fmt.Errorf("creating job spec: %v", err) } - _, err = b.client.Create(ctx, job, metav1.CreateOptions{ + ctx = context.Background() + job, err = b.client.Create(ctx, job, metav1.CreateOptions{ FieldManager: task.Id, }) if err != nil { diff --git a/compute/kubernetes/backend_test.go b/compute/kubernetes/backend_test.go index 313247ea6..96fed7721 100644 --- a/compute/kubernetes/backend_test.go +++ b/compute/kubernetes/backend_test.go @@ -2,7 +2,7 @@ package kubernetes import ( "fmt" - "io/ioutil" + "os" "testing" "github.com/ohsu-comp-bio/funnel/config" @@ -12,7 +12,7 @@ import ( func TestCreateJobc(t *testing.T) { conf := config.DefaultConfig().Kubernetes - content, err := ioutil.ReadFile("../../config/kubernetes-template.yaml") + content, err := os.ReadFile("../../config/kubernetes-template.yaml") if err != nil { t.Fatal(fmt.Errorf("reading template: %v", err)) } diff --git a/compute/scheduler/node.go b/compute/scheduler/node.go index 6f4df3bcf..4da0bece0 100644 --- a/compute/scheduler/node.go +++ b/compute/scheduler/node.go @@ -41,7 +41,7 @@ func NewNodeProcess(ctx context.Context, conf config.Config, factory Worker, log conf: conf, client: cli, log: log, - resources: &res, + resources: res, workerRun: factory, workers: newRunSet(), timeout: timeout, @@ -154,7 +154,7 @@ func (n *NodeProcess) sync(ctx context.Context) { // Node data has been updated. Send back to server for database update. var derr error - *n.resources, derr = detectResources(n.conf.Node, n.conf.Worker.WorkDir) + n.resources, derr = detectResources(n.conf.Node, n.conf.Worker.WorkDir) if derr != nil { n.log.Error("error detecting resources", "error", derr) } diff --git a/compute/scheduler/testutils_test.go b/compute/scheduler/testutils_test.go index 6be88cd4c..ab816a909 100644 --- a/compute/scheduler/testutils_test.go +++ b/compute/scheduler/testutils_test.go @@ -2,6 +2,7 @@ package scheduler import ( "context" + "fmt" "io/ioutil" "testing" "time" @@ -32,7 +33,7 @@ func newTestNode(conf config.Config, t *testing.T) testNode { conf: conf, client: s, log: log, - resources: &res, + resources: res, workerRun: NoopWorker, workers: newRunSet(), timeout: util.NewIdleTimeout(time.Duration(conf.Node.Timeout)), @@ -81,14 +82,27 @@ func (t *testNode) AddTasks(ids ...string) { func timeLimit(t *testing.T, d time.Duration) func() { stop := make(chan struct{}) + errCh := make(chan error, 1) // Channel to report errors + go func() { select { case <-time.NewTimer(d).C: - t.Fatal("time limit expired") + errCh <- fmt.Errorf("time limit expired") // Send error case <-stop: + return } }() + + // This is the cancel function that will be returned return func() { close(stop) + select { + case err := <-errCh: + if err != nil { + t.Fatal(err) // Report error from the main goroutine + } + default: + // No error, do nothing + } } } diff --git a/compute/scheduler/util.go b/compute/scheduler/util.go index 5e99b5e20..3e11a7966 100644 --- a/compute/scheduler/util.go +++ b/compute/scheduler/util.go @@ -23,8 +23,8 @@ func GenNodeID() string { // // Upon error, detectResources will return the resources given by the config // with the error. -func detectResources(conf config.Node, workdir string) (Resources, error) { - res := Resources{ +func detectResources(conf config.Node, workdir string) (*Resources, error) { + res := &Resources{ Cpus: conf.Resources.Cpus, RamGb: conf.Resources.RamGb, DiskGb: conf.Resources.DiskGb, diff --git a/config/config.go b/config/config.go index 440aee887..e8bf613a7 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( + "io" "os" "github.com/ohsu-comp-bio/funnel/logger" @@ -137,6 +138,8 @@ type Node struct { type Worker struct { // Directory to write task files to WorkDir string + // Additional directory to symlink to the working directory. + ScratchPath string // How often the worker should poll for cancel signals PollingRate Duration // How often to update stdout/stderr log fields. @@ -151,6 +154,28 @@ type Worker struct { LeaveWorkDir bool // Limit the number of concurrent downloads/uploads MaxParallelTransfers int + // Container engine to use for executing tasks. + Container ContainerConfig + // Command to use for the container engine. + // This can be used to override the default command used to run containers. + DriverCommand string +} + +type ContainerConfig struct { + Id string + Image string + Name string + Command []string + Workdir string + RemoveContainer bool + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + DriverCommand string + RunCommand string // template string + PullCommand string // template string + StopCommand string // template string } // HPCBackend describes the configuration for a HPC scheduler backend such as diff --git a/config/default.go b/config/default.go index 643f3f92f..1db89782a 100644 --- a/config/default.go +++ b/config/default.go @@ -61,6 +61,18 @@ func DefaultConfig() Config { LogUpdateRate: Duration(time.Second * 5), LogTailSize: 10000, MaxParallelTransfers: 10, + Container: ContainerConfig{ + DriverCommand: "docker", + RunCommand: "run -i --read-only " + + "{{if .RemoveContainer}}--rm{{end}} " + + "{{range $k, $v := .Env}}-e {{$k}}={{$v}} {{end}} " + + "{{if .Name}}--name {{.Name}}{{end}} " + + "{{if .Workdir}}-w {{.Workdir}}{{end}} " + + "{{range .Volumes}}-v {{.HostPath}}:{{.ContainerPath}}:{{if .Readonly}}ro{{else}}rw{{end}} {{end}} " + + "{{.Image}} {{.Command}}", + PullCommand: "pull {{.Image}}", + StopCommand: "rm -f {{.Name}}", + }, }, Logger: logger.DefaultConfig(), // databases / event handlers diff --git a/database/boltdb/events.go b/database/boltdb/events.go index d06cccd77..e5f649a71 100644 --- a/database/boltdb/events.go +++ b/database/boltdb/events.go @@ -146,6 +146,9 @@ func (taskBolt *BoltDB) WriteEvent(ctx context.Context, req *events.Event) error tx.Bucket(SysLogs).Put(idBytes, logbytes) return nil }) + if err != nil { + return err + } } return err diff --git a/database/boltdb/scheduler.go b/database/boltdb/scheduler.go index bfa032c46..dfdac5d0f 100644 --- a/database/boltdb/scheduler.go +++ b/database/boltdb/scheduler.go @@ -89,7 +89,7 @@ func (taskBolt *BoltDB) GetNode(ctx context.Context, req *scheduler.GetNodeReque }) if err == errNotFound { - return nil, status.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", err.Error(), req.Id)) + return nil, status.Errorf(codes.NotFound, "%v: nodeID: %s", err.Error(), req.Id) } if err != nil { diff --git a/database/elastic/scheduler.go b/database/elastic/scheduler.go index 5a5791464..06d22c453 100644 --- a/database/elastic/scheduler.go +++ b/database/elastic/scheduler.go @@ -54,7 +54,7 @@ func (es *Elastic) GetNode(ctx context.Context, req *scheduler.GetNodeRequest) ( Do(ctx) if elastic.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", err.Error(), req.Id)) + return nil, status.Errorf(codes.NotFound, "%v: nodeID: %s", err.Error(), req.Id) } if err != nil { return nil, err diff --git a/database/mongodb/scheduler.go b/database/mongodb/scheduler.go index 33de34765..9b69a1223 100644 --- a/database/mongodb/scheduler.go +++ b/database/mongodb/scheduler.go @@ -19,6 +19,10 @@ func (db *MongoDB) ReadQueue(n int) []*tes.Task { var tasks []*tes.Task opts := options.Find().SetSort(bson.M{"creationtime": 1}).SetLimit(int64(n)) cursor, err := db.tasks(db.client).Find(context.TODO(), bson.M{"state": tes.State_QUEUED}, opts) + if err != nil { + fmt.Println(err) + return nil + } err = cursor.All(context.TODO(), &tasks) if err != nil { @@ -66,7 +70,7 @@ func (db *MongoDB) GetNode(ctx context.Context, req *scheduler.GetNodeRequest) ( var node scheduler.Node err := db.nodes(db.client).FindOne(context.TODO(), bson.M{"id": req.Id}).Decode(&node) if err == mongo.ErrNoDocuments { - return nil, status.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", err, req.Id)) + return nil, status.Errorf(codes.NotFound, "%v: nodeID: %s", err, req.Id) } return &node, nil @@ -78,7 +82,7 @@ func (db *MongoDB) DeleteNode(ctx context.Context, req *scheduler.Node) (*schedu _, err := db.nodes(db.client).DeleteOne(context.TODO(), bson.M{"id": req.Id}) fmt.Println("DeleteNode", req.Id, err) if err == mongo.ErrNoDocuments { - return nil, status.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", err, req.Id)) + return nil, status.Errorf(codes.NotFound, "%v: nodeID: %s", err, req.Id) } return nil, err } diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md index 2f6a54a24..7a7eb9adf 100644 --- a/deployments/kubernetes/README.md +++ b/deployments/kubernetes/README.md @@ -1,3 +1,6 @@ +> [!NOTE] +> Funnel's Kubernetes support is in active development and may involve frequent updates + # Using Funnel with Kubernetes Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/deployments/kubernetes @@ -6,7 +9,7 @@ Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/ *funnel-service.yml* -``` +```yaml apiVersion: v1 kind: Service metadata: @@ -23,18 +26,17 @@ spec: protocol: TCP port: 9090 targetPort: 9090 - ``` Deploy it: -``` +```sh kubectl apply -f funnel-service.yml ``` Get the clusterIP: -``` +```sh kubectl get services funnel --output=yaml | grep clusterIP ``` @@ -42,13 +44,17 @@ Use this value to configure the server hostname of the worker config. #### Create Funnel config files -*Note*: The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `ohsucompbio/funnel-dind-rootless:latest`. +> [!TIP] +> The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `quay.io/ohsu-comp-bio/funnel-dind-rootless:latest` *funnel-server-config.yml* -``` +```yaml Database: boltdb +BoltDB: + Path: /opt/funnel/funnel-work-dir/funnel.bolt.db + Compute: kubernetes Logger: @@ -74,7 +80,7 @@ Kubernetes: restartPolicy: Never containers: - name: {{printf "funnel-worker-%s" .TaskId}} - image: ohsucompbio/funnel-kube-dind:latest + image: quay.io/ohsu-comp-bio/funnel-dind:latest imagePullPolicy: IfNotPresent args: - "funnel" @@ -87,8 +93,8 @@ Kubernetes: resources: requests: cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} - memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}} - ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} volumeMounts: - name: {{printf "funnel-storage-%s" .TaskId}} mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} @@ -112,7 +118,7 @@ I recommend setting `DisableJobCleanup` to `true` for debugging - otherwise fail ***Remember to modify the template below to have the actual server hostname.*** -``` +```yaml Database: boltdb BoltDB: @@ -138,7 +144,7 @@ Server: #### Create a ConfigMap -``` +```sh kubectl create configmap funnel-config --from-file=funnel-server-config.yml --from-file=funnel-worker-config.yml ``` @@ -148,7 +154,7 @@ Define a Role and RoleBinding: *role.yml* -``` +```yaml kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -168,7 +174,7 @@ rules: *role_binding.yml* -``` +```yaml kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -195,7 +201,7 @@ kubectl create -f role_binding.yml *funnel-deployment.yml* -``` +```yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -215,7 +221,7 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest + image: quay.io/ohsu-comp-bio/funnel:latest imagePullPolicy: IfNotPresent command: - 'funnel' @@ -225,9 +231,13 @@ spec: - '/etc/config/funnel-server-config.yml' resources: requests: - cpu: 2 + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: + cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -247,25 +257,25 @@ spec: Deploy it: -``` +```sh kubectl apply -f funnel-deployment.yml ``` #### Proxy the Service for local testing -``` +```sh kubectl port-forward service/funnel 8000:8000 ``` Now you can access the funnel server locally. Verify by running: -``` +```sh funnel task list ``` Now try running a task: -``` +```sh funnel examples hello-world > hello.json funnel task create hello.json ``` diff --git a/deployments/kubernetes/funnel-deployment.yml b/deployments/kubernetes/funnel-deployment.yml index 45ebcb548..8856f2dfe 100644 --- a/deployments/kubernetes/funnel-deployment.yml +++ b/deployments/kubernetes/funnel-deployment.yml @@ -17,18 +17,23 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest - imagePullPolicy: Never - args: - - "server" - - "run" - - "--config" - - "/etc/config/funnel-server-config.yml" - resources: - requests: + image: quay.io/ohsu-comp-bio/funnel:latest + imagePullPolicy: IfNotPresent + command: + - 'funnel' + - 'server' + - 'run' + - '--config' + - '/etc/config/funnel-server-config.yml' + resources: + requests: + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -44,4 +49,4 @@ spec: claimName: storage-pvc - name: config-volume configMap: - name: funnel-config + name: funnel-config \ No newline at end of file diff --git a/deployments/kubernetes/funnel-server-config.yml b/deployments/kubernetes/funnel-server-config.yml new file mode 100644 index 000000000..c9c10f6c8 --- /dev/null +++ b/deployments/kubernetes/funnel-server-config.yml @@ -0,0 +1,60 @@ +Database: boltdb + +BoltDB: + Path: /opt/funnel/funnel-work-dir/funnel.bolt.db + +Compute: kubernetes + +Logger: + Level: debug + +Kubernetes: + DisableJobCleanup: false + DisableReconciler: false + ReconcileRate: 5m + Namespace: default + Template: | + apiVersion: batch/v1 + kind: Job + metadata: + ## DO NOT CHANGE NAME + name: {{.TaskId}} + namespace: {{.Namespace}} + spec: + backoffLimit: 0 + completions: 1 + template: + spec: + restartPolicy: Never + containers: + - name: {{printf "funnel-worker-%s" .TaskId}} + image: quay.io/ohsu-comp-bio/funnel-dind:latest + imagePullPolicy: IfNotPresent + args: + - "funnel" + - "worker" + - "run" + - "--config" + - "/etc/config/funnel-worker-config.yml" + - "--taskID" + - {{.TaskId}} + resources: + requests: + cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} + volumeMounts: + - name: {{printf "funnel-storage-%s" .TaskId}} + mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} + - name: config-volume + mountPath: /etc/config + + securityContext: + privileged: true + + volumes: + - name: {{printf "funnel-storage-%s" .TaskId}} + emptyDir: {} + - name: config-volume + configMap: + name: funnel-config diff --git a/events/data.json b/events/data.json new file mode 100644 index 000000000..ac0e930f9 --- /dev/null +++ b/events/data.json @@ -0,0 +1,14 @@ +{ + "id": "event123", + "timestamp": "2024-05-20T12:34:56Z", + "type": "TASK_CREATED", + "task": { + "id": "task123", + "executors": [ + { + "image": "alpine", + "command": ["echo", "hello world"] + } + ] + } +} diff --git a/examples/s3.json b/examples/s3.json index c89c885dc..9f0e3064f 100644 --- a/examples/s3.json +++ b/examples/s3.json @@ -4,15 +4,15 @@ "executors": [ { "image": "ubuntu", - "command": ["md5sum", "/tmp/file.xml"] + "command": ["md5sum", "/tmp/release.json"] } ], "inputs": [ { "name": "input", - "description": "Download a public file from S3 Storage (IRS form 990 data)", - "url": "s3://irs-form-990/200931393493000150_public.xml", - "path": "/tmp/file.xml" + "description": "Download a public file from S3 Storage (CZ CELLxGENE Discover Census Data: https://registry.opendata.aws/tag/bioinformatics/)", + "url": "s3://cellxgene-census-public-us-west-2/cell-census/release.json", + "path": "/tmp/release.json" } ] } diff --git a/go.mod b/go.mod index 7064f9fd4..262893005 100644 --- a/go.mod +++ b/go.mod @@ -80,42 +80,158 @@ require ( cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/iam v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/alessio/shellescape v1.4.1 // indirect + github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22 // indirect + github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect + github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect + github.com/anchore/quill v0.4.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/atc0005/go-teams-notify/v2 v2.10.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.28.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.23.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect + github.com/aws/smithy-go v1.20.2 // indirect + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240514230400-03fa26f5508f // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blacktop/go-dwarf v1.0.9 // indirect + github.com/blacktop/go-macho v1.1.162 // indirect + github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect + github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d // indirect + github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/caarlos0/ctrlc v1.2.0 // indirect + github.com/caarlos0/env/v11 v11.0.1 // indirect + github.com/caarlos0/go-reddit/v3 v3.0.1 // indirect + github.com/caarlos0/go-shellwords v1.0.12 // indirect + github.com/caarlos0/go-version v0.1.1 // indirect + github.com/caarlos0/log v0.4.4 // indirect + github.com/carlmjohnson/versioninfo v0.22.5 // indirect + github.com/cavaliergopher/cpio v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/bubbletea v0.22.1 // indirect + github.com/charmbracelet/lipgloss v0.10.0 // indirect + github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d // indirect + github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect + github.com/cloudflare/circl v1.3.8 // indirect + github.com/containerd/console v1.0.4 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb // indirect + github.com/dghubble/oauth1 v0.7.3 // indirect + github.com/dghubble/sling v1.4.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v25.0.4+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gammazero/deque v0.2.1 // indirect + github.com/github/smimesign v0.2.0 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-containerregistry v0.19.1 // indirect + github.com/google/go-github v17.0.0+incompatible // indirect + github.com/google/go-github/v62 v62.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.12.0 // indirect github.com/invopop/yaml v0.3.1 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-block-format v0.2.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect + github.com/ipfs/go-ipfs-util v0.0.3 // indirect + github.com/ipfs/go-ipld-cbor v0.1.0 // indirect + github.com/ipfs/go-ipld-format v0.6.0 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -126,39 +242,83 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/maruel/panicparse v1.6.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-mastodon v0.0.8 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/mango v0.1.0 // indirect + github.com/muesli/mango-cobra v1.2.0 // indirect + github.com/muesli/mango-pflag v0.1.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/roff v0.1.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nsf/termbox-go v1.1.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/vbatts/tar-split v0.11.5 // indirect + github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect + github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c // indirect + github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xanzy/go-gitlab v0.105.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect @@ -172,13 +332,19 @@ require ( golang.org/x/text v0.18.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/mail.v2 v2.3.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + lukechampine.com/blake3 v1.2.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kind v0.23.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect + software.sslmate.com/src/go-pkcs12 v0.4.0 // indirect ) diff --git a/go.sum b/go.sum index 846e710a4..ac5792cc9 100644 --- a/go.sum +++ b/go.sum @@ -20,57 +20,168 @@ cloud.google.com/go/pubsub v1.43.0 h1:s3Qx+F96J7Kwey/uVHdK3QxFLIlOvvw4SfMYw2jFjb cloud.google.com/go/pubsub v1.43.0/go.mod h1:LNLfqItblovg7mHWgU5g84Vhza4J8kTxx0YqIeTzcXY= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22 h1:5NFK6VGgqBUOAX2SYyzFYvNdOiYDxzim8jga386FlZY= +github.com/anchore/bubbly v0.0.0-20230518153401-87b6af8ccf22/go.mod h1:Kv+Mm9CdtnV8iem48iEPIwy7/N4Wmk0hpxYNH5gTwKQ= +github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw= +github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a/go.mod h1:ubLFmlsv8/DFUQrZwY5syT5/8Er3ugSr4rDFwHsE3hg= +github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb h1:iDMnx6LIjtjZ46C0akqveX83WFzhpTD3eqOthawb5vU= +github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb/go.mod h1:DmTY2Mfcv38hsHbG78xMiTDdxFtkHpgYNVDPsF2TgHk= +github.com/anchore/quill v0.4.1 h1:mffDnvnER3ZgPjN5hexc3nr/4Y1dtKdDB6td5K8uInk= +github.com/anchore/quill v0.4.1/go.mod h1:t6hOPYDohN8wn2SRWQdNkJBkhmK8s3gzuHzzgcEvzQU= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/atc0005/go-teams-notify/v2 v2.10.0 h1:eQvRIkyESQgBvlUdQ/iPol/lj3QcRyrdEQM3+c/nXhM= +github.com/atc0005/go-teams-notify/v2 v2.10.0/go.mod h1:SIeE1UfCcVRYMqP5b+r1ZteHyA/2UAjzWF5COnZ8q0w= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blacktop/go-dwarf v1.0.9 h1:eT/L7gt0gllvvgnRXY0MFKjNB6+jtOY5DTm2ynVX2dY= +github.com/blacktop/go-dwarf v1.0.9/go.mod h1:4W2FKgSFYcZLDwnR7k+apv5i3nrau4NGl9N6VQ9DSTo= +github.com/blacktop/go-macho v1.1.162 h1:FjM3XAsJTAOGZ1eppRSX9ZBX3Bk11JMTC1amsZAOA5I= +github.com/blacktop/go-macho v1.1.162/go.mod h1:f2X4noFBob4G5bWUrzvPBKDVcFWZgDCM7rIn7ygTID0= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d h1:xxPhzCOpmOntzVe8S6tqsMdFgaB8B4NXSV54lG4B1qk= +github.com/bluesky-social/indigo v0.0.0-20240411170459-440932307e0d/go.mod h1:ysMQ0a4RYWjgyvKrl5ME352oHA6QgK900g5sB9XXgPE= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= +github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb h1:1JIKG3zt7fIMG3Hr1sZ3LcQxEROJYJ2qwpjdvD3FCw4= +github.com/buchanae/github-release-notes v0.0.0-20180827045457-200e1dacadbb/go.mod h1:YlY7IAd5TVq6+Bv/iDwCjc122k6aynvrHRJ2E+E82pg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/caarlos0/ctrlc v1.2.0 h1:AtbThhmbeYx1WW3WXdWrd94EHKi+0NPRGS4/4pzrjwk= +github.com/caarlos0/ctrlc v1.2.0/go.mod h1:n3gDlSjsXZ7rbD9/RprIR040b7oaLfNStikPd4gFago= +github.com/caarlos0/env/v11 v11.0.1 h1:A8dDt9Ub9ybqRSUF3fQc/TA/gTam2bKT4Pit+cwrsPs= +github.com/caarlos0/env/v11 v11.0.1/go.mod h1:2RC3HQu8BQqtEK3V4iHPxj0jOdWdbPpWJ6pOueeU1xM= +github.com/caarlos0/go-reddit/v3 v3.0.1 h1:w8ugvsrHhaE/m4ez0BO/sTBOBWI9WZTjG7VTecHnql4= +github.com/caarlos0/go-reddit/v3 v3.0.1/go.mod h1:QlwgmG5SAqxMeQvg/A2dD1x9cIZCO56BMnMdjXLoisI= +github.com/caarlos0/go-shellwords v1.0.12 h1:HWrUnu6lGbWfrDcFiHcZiwOLzHWjjrPVehULaTFgPp8= +github.com/caarlos0/go-shellwords v1.0.12/go.mod h1:bYeeX1GrTLPl5cAMYEzdm272qdsQAZiaHgeF0KTk1Gw= +github.com/caarlos0/go-version v0.1.1 h1:1bikKHkGGVIIxqCmufhSSs3hpBScgHGacrvsi8FuIfc= +github.com/caarlos0/go-version v0.1.1/go.mod h1:Ze5Qx4TsBBi5FyrSKVg1Ibc44KGV/llAaKGp86oTwZ0= +github.com/caarlos0/log v0.4.4 h1:LnvgBz/ofsJ00AupP/cEfksJSZglb1L69g4Obk/sdAc= +github.com/caarlos0/log v0.4.4/go.mod h1:+AmCI9Liv5LKXmzFmFI1htuHdTTj/0R3KuoP9DMY7Mo= +github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk= +github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= +github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbletea v0.22.1 h1:z66q0LWdJNOWEH9zadiAIXp2GN1AWrwNXU8obVY9X24= +github.com/charmbracelet/bubbletea v0.22.1/go.mod h1:8/7hVvbPN6ZZPkczLiB8YpLkLJ0n7DMho5Wvfd2X1C0= +github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= +github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= +github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d h1:+o+e/8hf7cG0SbAzEAm/usJ8qoZPgFXhudLjop+TM0g= +github.com/charmbracelet/x/exp/ordered v0.0.0-20231010190216-1cb11efc897d/go.mod h1:aoG4bThKYIOnyB55r202eHqo6TkN7ZXV+cu4Do3eoBQ= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= +github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb h1:7ENzkH+O3juL+yj2undESLTaAeRllHwCs/b8z6aWSfc= +github.com/dghubble/go-twitter v0.0.0-20211115160449-93a8679adecb/go.mod h1:qhZBgV9e4WyB1JNjHpcXVkUe3knWUwYuAPB1hITdm50= +github.com/dghubble/oauth1 v0.7.3 h1:EkEM/zMDMp3zOsX2DC/ZQ2vnEX3ELK0/l9kb+vs4ptE= +github.com/dghubble/oauth1 v0.7.3/go.mod h1:oxTe+az9NSMIucDPDCCtzJGsPhciJV33xocHfcR2sVY= +github.com/dghubble/sling v1.4.0 h1:/n8MRosVTthvMbwlNZgLx579OGVjUOy3GNEv5BIqAWY= +github.com/dghubble/sling v1.4.0/go.mod h1:0r40aNsU9EdDUVBNhfCstAtFgutjgJGYbO1oNzkMoM8= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -79,6 +190,8 @@ github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkz github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= @@ -98,20 +211,27 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= -github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -123,8 +243,18 @@ github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYI github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a/go.mod h1:AEugkNu3BjBxyz958nJ5holD9PRjta6iprcoUauDbU4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/github/smimesign v0.2.0 h1:Hho4YcX5N1I9XNqhq0fNx0Sts8MhLonHd+HRXVGNjvk= +github.com/github/smimesign v0.2.0/go.mod h1:iZiiwNT4HbtGRVqCQu7uJPEZCuEE5sfSSttcnePkDl4= github.com/gizak/termui v2.3.0+incompatible h1:S8wJoNumYfc/rR5UezUM4HsPEo3RJh0LKdiuDWQpjqw= github.com/gizak/termui v2.3.0+incompatible/go.mod h1:PkJoWUt/zacQKysNfQtcw1RW+eK2SxkieVBtl+4ovLA= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -137,21 +267,49 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSrdV7Nv3/gkvc= +github.com/go-restruct/restruct v1.2.0-alpha/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc= +github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -180,8 +338,8 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -189,14 +347,26 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= +github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= +github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/ko v0.15.4 h1:0blRbIdPmSy6v4LvedGxbI/8krdJYQgbSih3v6Y8V1c= +github.com/google/ko v0.15.4/go.mod h1:ZkcmfV91Xt6ZzOBHc/cXXGYnqWdNWDVy/gHoUU9sjag= github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= @@ -211,6 +381,9 @@ github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDP github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= @@ -219,21 +392,68 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= +github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= +github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= +github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= +github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= +github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= +github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= +github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= +github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= +github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= +github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -246,6 +466,7 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -257,8 +478,11 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -277,30 +501,55 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/panicparse v1.6.2 h1:tZuGQTlbOY5jCprrWMJTikREqKPn+UAKdR4CHSpj834= github.com/maruel/panicparse v1.6.2/go.mod h1:uoxI4w9gJL6XahaYPMq/z9uadrdr1SyHuQwV2q80Mm0= github.com/maruel/panicparse/v2 v2.1.1/go.mod h1:AeTWdCE4lcq8OKsLb6cHSj1RWHVSnV9HBCk7sKLF4Jg= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-mastodon v0.0.8 h1:UgKs4SmQ5JeawxMIPP7NQ9xncmOXA+5q6jYk4erR7vk= +github.com/mattn/go-mastodon v0.0.8/go.mod h1:8YkqetHoAVEktRkK15qeiv/aaIMfJ/Gc89etisPZtHU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -316,6 +565,36 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI= +github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4= +github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg= +github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA= +github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg= +github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= +github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= @@ -324,6 +603,8 @@ github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY= github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olivere/elastic/v7 v7.0.12/go.mod h1:14rWX28Pnh3qCKYRVnSGXWLf9MbLonYS/4FDCY3LAPo= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= @@ -334,12 +615,23 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/getopt v0.0.0-20180811024354-2b5b3bfb099b/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -358,34 +650,60 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigstore/cosign/v2 v2.2.4 h1:iY4vtEacmu2hkNj1Fh+8EBqBwKs2DHM27/lbNWDFJro= +github.com/sigstore/cosign/v2 v2.2.4/go.mod h1:JZlRD2uaEjVAvZ1XJ3QkkZJhTqSDVtLaet+C/TMR81Y= +github.com/sigstore/rekor v1.3.6 h1:QvpMMJVWAp69a3CHzdrLelqEqpTM3ByQRt5B5Kspbi8= +github.com/sigstore/rekor v1.3.6/go.mod h1:JDTSNNMdQ/PxdsS49DJkJ+pRJCO/83nbR5p3aZQteXc= +github.com/sigstore/sigstore v1.8.3 h1:G7LVXqL+ekgYtYdksBks9B38dPoIsbscjQJX/MGWkA4= +github.com/sigstore/sigstore v1.8.3/go.mod h1:mqbTEariiGA94cn6G3xnDiV6BD8eSLdL/eA7bvJ0fVs= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/slack-go/slack v0.13.0 h1:7my/pR2ubZJ9912p9FtvALYpbt0cQPAqkRy2jaSI1PQ= +github.com/slack-go/slack v0.13.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -397,6 +715,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -406,21 +726,48 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= +github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= +github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 h1:jIVmlAFIqV3d+DOxazTR9v+zgj8+VYuQBzPgBZvWBHA= +github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651/go.mod h1:b26F2tHLqaoRQf8DywqzVaV1MQ9yvjb0OMcNl7Nxu20= +github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c h1:gFwUKtkv6QzQsFdIjvPqd0Qdw42DHUEbbUdiUTI1uco= +github.com/wagoodman/go-progress v0.0.0-20220614130704-4b1c25a33c7c/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302 h1:MhInbXe4SzcImAKktUvWBCWZgcw6MYf5NfumTj1BhAw= +github.com/whyrusleeping/cbor-gen v0.1.1-0.20240311221002-68b9f235c302/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xanzy/go-gitlab v0.105.0 h1:3nyLq0ESez0crcaM19o5S//SvezOQguuIHZ3wgX64hM= +github.com/xanzy/go-gitlab v0.105.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -432,6 +779,7 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zU github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -460,28 +808,61 @@ go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt3 go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +gocloud.dev v0.37.0 h1:XF1rN6R0qZI/9DYjN16Uy0durAmSlf58DHOcb28GPro= +gocloud.dev v0.37.0/go.mod h1:7/O4kqdInCNsc6LqgmuFnS0GRew4XNNYWpA44yQnwco= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= +golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -495,7 +876,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -511,6 +895,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -522,33 +907,46 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -559,19 +957,29 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.199.0 h1:aWUXClp+VFJmqE0JPvpZOK3LDQMyFKYIow4etYd9qxs= google.golang.org/api v0.199.0/go.mod h1:ohG4qSztDJmZdjK/Ar6MhbAmb/Rpi4JHOqagsh90K28= @@ -611,18 +1019,29 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= +gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -645,9 +1064,15 @@ k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGc k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.23.0 h1:8fyDGWbWTeCcCTwA04v4Nfr45KKxbSPH1WO9K+jVrBg= +sigs.k8s.io/kind v0.23.0/go.mod h1:ZQ1iZuJLh3T+O8fzhdi3VWcFTzsdXtNv2ppsHc8JQ7s= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/server/marshal.go b/server/marshal.go index d52925fc1..0da374224 100644 --- a/server/marshal.go +++ b/server/marshal.go @@ -22,12 +22,12 @@ type CustomMarshal struct { func NewMarshaler() runtime.Marshaler { return &CustomMarshal{ m: &runtime.JSONPb{ - protojson.MarshalOptions{ + MarshalOptions: protojson.MarshalOptions{ Indent: " ", EmitUnpopulated: true, UseProtoNames: true, }, - protojson.UnmarshalOptions{}, + UnmarshalOptions: protojson.UnmarshalOptions{}, }, } } @@ -44,10 +44,10 @@ func (marshal *CustomMarshal) ContentType(i interface{}) string { func (mclean *CustomMarshal) Marshal(v interface{}) ([]byte, error) { list, ok := v.(*tes.ListTasksResponse) - if ok { - // v is of type *tes.ListTasksResponse + if ok { + // v is of type *tes.ListTasksResponse return mclean.MarshalList(list) - } + } task, ok := v.(*tes.Task) if ok { @@ -98,10 +98,10 @@ func (mclean *CustomMarshal) DetectView(task *tes.Task) (tes.View, error) { // return a MINIMAL view return tes.View_MINIMAL, nil } - + if len(task.Logs[0].SystemLogs) == 0 { return tes.View_BASIC, nil - } + } // view = "FULL" return tes.View_FULL, nil @@ -116,18 +116,18 @@ func (mclean *CustomMarshal) TranslateTask(task *tes.Task, view tes.View) interf } return min } - + // view = "BASIC" if view == tes.View_BASIC { executors := []*tes.ExecutorBasic{} for _, executor := range task.Executors { executors = append(executors, &tes.ExecutorBasic{ - Command: executor.Command, - Env: executor.Env, + Command: executor.Command, + Env: executor.Env, IgnoreError: executor.IgnoreError, - Image: executor.Image, - Stdin: executor.Stdin, - Workdir: executor.Workdir, + Image: executor.Image, + Stdin: executor.Stdin, + Workdir: executor.Workdir, }) } @@ -135,11 +135,11 @@ func (mclean *CustomMarshal) TranslateTask(task *tes.Task, view tes.View) interf for _, input := range task.Inputs { inputs = append(inputs, &tes.InputBasic{ Description: input.Description, - Name: input.Name, - Path: input.Path, + Name: input.Name, + Path: input.Path, Streamable: input.Streamable, - Type: input.Type, - Url: input.Url, + Type: input.Type, + Url: input.Url, }) } @@ -154,19 +154,19 @@ func (mclean *CustomMarshal) TranslateTask(task *tes.Task, view tes.View) interf }) } - basic := &tes.TaskBasic { + basic := &tes.TaskBasic{ CreationTime: task.CreationTime, Description: task.Description, Executors: executors, - Id: task.Id, - Inputs: inputs, - Logs: logs, - Name: task.Name, - Outputs: task.Outputs, - Resources: task.Resources, - State: task.State, - Tags: task.Tags, - Volumes: task.Volumes, + Id: task.Id, + Inputs: inputs, + Logs: logs, + Name: task.Name, + Outputs: task.Outputs, + Resources: task.Resources, + State: task.State, + Tags: task.Tags, + Volumes: task.Volumes, } return basic diff --git a/server/server.go b/server/server.go index 871b382e3..4c3a7ecb9 100644 --- a/server/server.go +++ b/server/server.go @@ -53,6 +53,7 @@ func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor { "resp", resp, "err", err, ) + return resp, err } } @@ -61,16 +62,16 @@ func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor { // Returns '400' for invalid backend parameters and '500' for all other errors // Required for TES Compliance Tests func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, r *http.Request, err error) { - const fallback = `{"error": "failed to process the request"}` + const fallback = `{"error": "failed to process the request"}` - st, ok := status.FromError(err) - if !ok { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fallback)) - return - } + st, ok := status.FromError(err) + if !ok { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fallback)) + return + } - // Map specific gRPC error codes to HTTP status codes + // Map specific gRPC error codes to HTTP status codes switch st.Code() { case codes.Unauthenticated: w.WriteHeader(http.StatusUnauthorized) // 401 @@ -78,27 +79,31 @@ func customErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler ru w.WriteHeader(http.StatusForbidden) // 403 case codes.NotFound: // Special case for missing tasks (TES Compliance Suite) - if (strings.Contains(st.Message(), "task not found")) { + if strings.Contains(st.Message(), "task not found") { w.WriteHeader(http.StatusInternalServerError) // 500 } else { w.WriteHeader(http.StatusNotFound) // 404 } default: - w.WriteHeader(http.StatusInternalServerError) // 500 + if strings.Contains(st.Message(), "backend parameters not supported") { + w.WriteHeader(http.StatusBadRequest) // 400 + } else { + w.WriteHeader(http.StatusInternalServerError) // 500 + } } - // Write the error message - jErr := JSONError{Error: st.Message()} - jErrBytes, mErr := marshaler.Marshal(jErr) - if mErr != nil { - w.Write([]byte(fallback)) - return - } - w.Write(jErrBytes) + // Write the error message + jErr := JSONError{Error: st.Message()} + jErrBytes, mErr := marshaler.Marshal(jErr) + if mErr != nil { + w.Write([]byte(fallback)) + return + } + w.Write(jErrBytes) } type JSONError struct { - Error string `json:"error"` + Error string `json:"error"` } // Serve starts the server and does not block. This will open TCP ports @@ -133,7 +138,7 @@ func (s *Server) Serve(pctx context.Context) error { marsh := NewMarshaler() grpcMux := runtime.NewServeMux( runtime.WithMarshalerOption(runtime.MIMEWildcard, marsh), runtime.WithErrorHandler(customErrorHandler)) - + // m := protojson.MarshalOptions{ // Indent: " ", // EmitUnpopulated: true, diff --git a/server/tes.go b/server/tes.go index 953dcefcb..20b705d27 100644 --- a/server/tes.go +++ b/server/tes.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/ohsu-comp-bio/funnel/config" "github.com/ohsu-comp-bio/funnel/events" "github.com/ohsu-comp-bio/funnel/logger" "github.com/ohsu-comp-bio/funnel/tes" @@ -29,6 +30,7 @@ type TaskService struct { Compute events.Computer Read tes.ReadOnlyServer Log *logger.Logger + Config config.Config } // CreateTask provides an HTTP/gRPC endpoint for creating a task. @@ -36,7 +38,7 @@ type TaskService struct { func (ts *TaskService) CreateTask(ctx context.Context, task *tes.Task) (*tes.CreateTaskResponse, error) { if err := tes.InitTask(task, true); err != nil { - return nil, status.Errorf(codes.InvalidArgument, err.Error()) + return nil, status.Errorf(codes.InvalidArgument, "%v", err.Error()) } err := ts.Compute.CheckBackendParameterSupport(task) @@ -44,6 +46,7 @@ func (ts *TaskService) CreateTask(ctx context.Context, task *tes.Task) (*tes.Cre return nil, fmt.Errorf("error from backend: %s", err) } + ctx = context.WithValue(ctx, "Config", ts.Config) if err := ts.Event.WriteEvent(ctx, events.NewTaskCreated(task)); err != nil { return nil, fmt.Errorf("error creating task: %s", err) } @@ -64,7 +67,7 @@ func (ts *TaskService) CreateTask(ctx context.Context, task *tes.Task) (*tes.Cre func (ts *TaskService) GetTask(ctx context.Context, req *tes.GetTaskRequest) (*tes.Task, error) { task, err := ts.Read.GetTask(ctx, req) if err == tes.ErrNotFound { - err = status.Errorf(codes.NotFound, fmt.Sprintf("%v: taskID: %s", err.Error(), req.Id)) + err = status.Errorf(codes.NotFound, "%v: taskID: %s", err.Error(), req.Id) } return task, err } @@ -85,7 +88,7 @@ func (ts *TaskService) CancelTask(ctx context.Context, req *tes.CancelTaskReques // updated database and other event streams err = ts.Event.WriteEvent(ctx, events.NewState(req.Id, tes.Canceled)) if err == tes.ErrNotFound { - err = status.Errorf(codes.NotFound, fmt.Sprintf("%v: taskID: %s", err.Error(), req.Id)) + err = status.Errorf(codes.NotFound, "%v: taskID: %s", err.Error(), req.Id) } return &tes.CancelTaskResponse{}, err } diff --git a/storage/amazon_s3.go b/storage/amazon_s3.go index 4d6aa2753..e67d68359 100644 --- a/storage/amazon_s3.go +++ b/storage/amazon_s3.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "regexp" "strings" @@ -232,9 +233,24 @@ func (s3b *AmazonS3) Put(ctx context.Context, url, path string) (*Object, error) sess := s3b.sess.Copy(&aws.Config{Region: aws.String(region)}) manager := s3manager.NewUploader(sess) - hf, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("amazonS3: opening file: %v", err) + var hf *os.File + // If path contains a wildcard, then handle globbing + if strings.Contains(path, "*") { + globs, err := filepath.Glob(path) + if err != nil { + return nil, fmt.Errorf("amazonS3: failed to resolve path %v: %v", path, err) + } + for _, glob := range globs { + hf, err = os.Open(glob) + if err != nil { + return nil, fmt.Errorf("amazonS3: opening file %v: %v", glob, err) + } + } + } else { + hf, err = os.Open(path) + if err != nil { + return nil, fmt.Errorf("amazonS3: opening fil %v: %v", path, err) + } } defer hf.Close() diff --git a/storage/anonymous_get_test.go b/storage/anonymous_get_test.go index d6ee19869..3e20251be 100644 --- a/storage/anonymous_get_test.go +++ b/storage/anonymous_get_test.go @@ -41,6 +41,10 @@ func TestAmazonS3AnonymousGet(t *testing.T) { endpoint: "", } + // AWS S3 Public Datasets: + // - https://registry.opendata.aws/ + // 1000 Genomes Public Dataset: + // - https://registry.opendata.aws/1000-genomes/ _, err = store.Get(context.Background(), "s3://1000genomes/README.analysis_history", "_test_download/README.analysis_history") if err != nil { t.Error("Error downloading file:", err) @@ -55,7 +59,11 @@ func TestGoogleStorageAnonymousGet(t *testing.T) { store := &GoogleCloud{svc} - _, err = store.Get(context.Background(), "gs://uspto-pair/applications/07820856.zip", "_test_download/07820856.zip") + // Google Cloud Public Datasets: + // - https://cloud.google.com/datasets?hl=en + // Broad Institute Public Dataset: + // - https://console.cloud.google.com/storage/browser/gcp-public-data--broad-references;tab=objects?prefix=&forceOnObjectsSortingFiltering=false + _, err = store.Get(context.Background(), "gs://gcp-public-data--broad-references/C.elegans/WBcel235/README.txt", "_test_download/README.txt") if err != nil { t.Error("Error downloading file:", err) } diff --git a/storage/generic_s3.go b/storage/generic_s3.go index bd37c5a27..ea4ea66d1 100644 --- a/storage/generic_s3.go +++ b/storage/generic_s3.go @@ -73,6 +73,9 @@ func (s3 *GenericS3) Stat(ctx context.Context, url string) (*Object, error) { } isDir, err := isDir(s3.client, u.bucket, u.path) + if err != nil { + return nil, fmt.Errorf("genericS3: stat object: %s", err) + } if isDir { return &Object{ URL: url, @@ -135,6 +138,9 @@ func (s3 *GenericS3) Get(ctx context.Context, url, path string) (*Object, error) } isDir, err := isDir(s3.client, u.bucket, u.path) + if err != nil { + return nil, fmt.Errorf("genericS3: getting object %s: %v", url, err) + } if isDir { objects, err := s3.List(ctx, url) if err != nil { diff --git a/storage/local.go b/storage/local.go index 75f6401bc..dae397277 100644 --- a/storage/local.go +++ b/storage/local.go @@ -168,100 +168,105 @@ func copyFile(ctx context.Context, source string, dest string) (err error) { // Hard links file source to destination dest. func linkFile(ctx context.Context, source string, dest string) error { - // If source has a glob or wildcard, get the filepath using the filepath.Glob function - if strings.Contains(source, "*") { - globs, err := filepath.Glob(source) - if err != nil { - return fmt.Errorf("failed to get filepath using Glob: %v", err) - } - for _, glob := range globs { - // Correctly calculate the destination for each file - destFile := filepath.Join(dest, filepath.Base(glob)) - err := processItem(ctx, glob, destFile) - if err != nil { - return err - } - } - return nil - } else { - return processItem(ctx, source, dest) - } -} + // If source has a glob or wildcard, get the filepath using the filepath.Glob function + if strings.Contains(source, "*") { + globs, err := filepath.Glob(source) + if err != nil { + return fmt.Errorf("failed to get filepath using Glob: %v", err) + } + for _, glob := range globs { + // Since tesOutput.path contains a wildcard, then the dest path must be a directory + // Ref: https://github.com/ga4gh/task-execution-schemas/blob/v1.1/openapi/task_execution_service.openapi.yaml#L528 + // TODO: Verify that path_prefix is being removed from the dest filepath + dest = filepath.Join(dest, filepath.Base(glob)) + err := processItem(ctx, glob, dest) + if err != nil { + return err + } + } + return nil + } else { + return processItem(ctx, source, dest) + } +} // Process a single item (file or directory) -func processItem(ctx context.Context, source, dest string) error { - fileInfo, err := os.Stat(source) - if err != nil { - return err - } - - if fileInfo.IsDir() { - return processDirectory(ctx, source, dest) - } else { - return processFile(ctx, source, dest) - } +func processItem(ctx context.Context, source string, dest string) error { + fileInfo, err := os.Stat(source) + if err != nil { + return err + } + + if fileInfo.IsDir() { + return processDirectory(ctx, source, dest) + } else { + return processFile(ctx, source, dest) + } } // Process a directory -func processDirectory(ctx context.Context, source, dest string) error { - // Create destination directory - err := os.MkdirAll(dest, 0755) // Adjust permissions as needed - if err != nil { - return err - } - - entries, err := os.ReadDir(source) - if err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(source, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - - if entry.IsDir() { - err = processDirectory(ctx, srcPath, destPath) - } else { - err = processFile(ctx, srcPath, destPath) - } - - if err != nil { - return err - } - } - return nil +func processDirectory(ctx context.Context, source string, dest string) error { + // Create destination directory + err := os.MkdirAll(dest, 0755) // Adjust permissions as needed + if err != nil { + return err + } + + entries, err := os.ReadDir(source) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(source, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + if entry.IsDir() { + err = processDirectory(ctx, srcPath, destPath) + } else { + err = processFile(ctx, srcPath, destPath) + } + + if err != nil { + return err + } + } + return nil } // Process a single file -func processFile(ctx context.Context, source, dest string) error { - // without this resulting link could be a symlink - parent, err := filepath.EvalSymlinks(source) - - same, err := sameFile(parent, dest) - if err != nil { - return err - } - if same { - return nil - } - - err = os.Link(parent, dest) - if err != nil { - return copyFile(ctx, parent, dest) - } - return nil +func processFile(ctx context.Context, source string, dest string) error { + // without this resulting link could be a symlink + parent, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + + same, err := sameFile(parent, dest) + if err != nil { + return err + } + if same { + return nil + } + + err = os.Link(parent, dest) + if err != nil { + return copyFile(ctx, parent, dest) + } + return nil } func FilePathWalkDir(root string) ([]string, error) { - var files []string - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - files = append(files, path) - } - return nil - }) - return files, err + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + return files, err } func sameFile(source string, dest string) (bool, error) { diff --git a/storage/local_test.go b/storage/local_test.go index 587b6443e..9631cd83c 100644 --- a/storage/local_test.go +++ b/storage/local_test.go @@ -234,6 +234,31 @@ func TestLocalPutPath(t *testing.T) { } } +func TestProcessFile(t *testing.T) { + ctx := context.Background() + tmp, err := os.MkdirTemp("", "funnel-test-local-storage") + if err != nil { + t.Fatal(err) + } + + cp := path.Join(tmp, "input.txt") + op := path.Join(tmp, "output.txt") + os.WriteFile(cp, []byte("foo"), os.ModePerm) + + err = processFile(ctx, cp, op) + if err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile(op) + if err != nil { + t.Fatal(err) + } + if string(b) != "foo" { + t.Fatal("Unexpected content") + } +} + // Tests Put when source and dest reference the same file (inode) // Since Local storage hard-links files when possible we need to protect // against the case where the same path is 'Put' twice diff --git a/tes/utils_test.go b/tes/utils_test.go index b89a6c1b2..b903286e6 100644 --- a/tes/utils_test.go +++ b/tes/utils_test.go @@ -16,7 +16,9 @@ func TestBase64Encode(t *testing.T) { }, } - expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" + // TODO: Investigate strange whitespace behavior in Github Actions + // expected := "ewogICJleGVjdXRvcnMiOiBbCiAgICB7CiAgICAgICJjb21tYW5kIjogWwogICAgICAgICJlY2hvIiwKICAgICAgICAiaGVsbG8gd29ybGQiCiAgICAgIF0sCiAgICAgICJpbWFnZSI6ICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAidGFzazEiCn0=" + expected := "ewogICJleGVjdXRvcnMiOiAgWwogICAgewogICAgICAiY29tbWFuZCI6ICBbCiAgICAgICAgImVjaG8iLAogICAgICAgICJoZWxsbyB3b3JsZCIKICAgICAgXSwKICAgICAgImltYWdlIjogICJhbHBpbmUiCiAgICB9CiAgXSwKICAiaWQiOiAgInRhc2sxIgp9" encoded, err := Base64Encode(task) if err != nil { diff --git a/tests/funnel_utils.go b/tests/funnel_utils.go index a418bfb1f..32f027932 100644 --- a/tests/funnel_utils.go +++ b/tests/funnel_utils.go @@ -332,6 +332,9 @@ func (f *Funnel) StartServerInDocker(containerName, imageName string, extraArgs if gopath == "" { funnelBinary, err = filepath.Abs("../../funnel") + if err != nil { + log.Error("Error finding funnel binary", err) + } } else { if runtime.GOOS == "linux" { funnelBinary, err = filepath.Abs(filepath.Join( @@ -411,7 +414,7 @@ func (f *Funnel) StartServerInDocker(containerName, imageName string, extraArgs func (f *Funnel) findTestServerContainers() []string { res := []string{} - containers, err := f.Docker.ContainerList(context.Background(), dockerTypes.ContainerListOptions{}) + containers, err := f.Docker.ContainerList(context.Background(), container.ListOptions{}) if err != nil { panic(err) } diff --git a/util/dockerutil/docker.go b/util/dockerutil/docker.go index e4ecd247b..8dd0f0358 100644 --- a/util/dockerutil/docker.go +++ b/util/dockerutil/docker.go @@ -13,7 +13,10 @@ import ( // NewDockerClient returns a new docker client. This util handles // working around some client/server API version mismatch issues. func NewDockerClient() (*client.Client, error) { - dclient, err := client.NewEnvClient() + dclient, err := client.NewClientWithOpts( + client.FromEnv, + client.WithAPIVersionNegotiation(), + ) if err != nil { return nil, err } @@ -34,9 +37,7 @@ func NewDockerClient() (*client.Client, error) { } // Error message example: // Error getting metadata for container: Error response from daemon: client is newer than server (client API version: 1.26, server API version: 1.24) - // Hardcoding Docker version until version[1] returns valid version - // os.Setenv("DOCKER_API_VERSION", version[1]) - os.Setenv("DOCKER_API_VERSION", "1.24") + os.Setenv("DOCKER_API_VERSION", version[1]) return NewDockerClient() } } diff --git a/util/openapi2proto/main.go b/util/openapi2proto/main.go index 5177a2a10..4d4010991 100644 --- a/util/openapi2proto/main.go +++ b/util/openapi2proto/main.go @@ -39,38 +39,39 @@ type ServicePath struct { } func getType(p *openapi3.SchemaRef) (bool, string) { - switch p.Value.Type { - case "integer": + if p.Value.Type.Includes("integer") { return false, "int32" - case "boolean": + } + if p.Value.Type.Includes("boolean") { return false, "bool" - case "number": + } + if p.Value.Type.Includes("number") { return false, "double" - case "object": + } + if p.Value.Type.Includes("object") { if p.Ref != "" { t := strings.Split(p.Ref, "/") return false, t[len(t)-1] } - if p.Value.AdditionalProperties != nil { - _, aType := getType(p.Value.AdditionalProperties) + if p.Value.AdditionalProperties.Has != nil && *p.Value.AdditionalProperties.Has != false { + _, aType := getType(p.Value.AdditionalProperties.Schema) return false, fmt.Sprintf("map", aType) } return false, "map" - //return fmt.Sprintf("%#v", p.Value) - case "array": + } + if p.Value.Type.Includes("array") { if p.Value.Items.Ref != "" { t := strings.Split(p.Value.Items.Ref, "/") return true, t[len(t)-1] } _, aType := getType(p.Value.Items) return true, aType - default: - if p.Ref != "" { - t := strings.Split(p.Ref, "/") - return false, t[len(t)-1] - } - return false, p.Value.Type } + if p.Ref != "" { + t := strings.Split(p.Ref, "/") + return false, t[len(t)-1] + } + return false, p.Value.Type.Slice()[0] } func getParamType(param *openapi3.Parameter) (bool, string) { @@ -81,7 +82,7 @@ func getParamType(param *openapi3.Parameter) (bool, string) { return false, t[len(t)-1] } - if param.Schema.Value.Type != "" { + if param.Schema.Value.Type != nil && param.Schema.Value.Type.Includes("") == false { return getType(param.Schema) } return false, "" @@ -172,7 +173,7 @@ func cleanSchema(messages []Message, enums []Enum, services []ServicePath) { } func getResponseMessage(resp *openapi3.Responses) string { - schema := resp.Get(200).Value.Content.Get("application/json").Schema + schema := resp.Status(200).Value.Content.Get("application/json").Schema s := strings.Split(schema.Ref, "/") return s[len(s)-1] } @@ -218,7 +219,7 @@ func main() { } } - for path, req := range doc.Paths { + for path, req := range doc.Paths.Map() { if req.Get != nil { //log.Printf("Get: %s %s\n", path, req.Get.OperationID) reqFields := []Field{} @@ -252,14 +253,14 @@ func main() { } } - for path, req := range doc.Paths { + for path, req := range doc.Paths.Map() { p := ServicePath{} if req.Get != nil { p.Name = req.Get.OperationID p.Path = path p.Mode = "get" p.InputType = req.Get.OperationID + "Request" - p.OutputType = getResponseMessage(&req.Get.Responses) + p.OutputType = getResponseMessage(req.Get.Responses) service = append(service, p) } if req.Post != nil { @@ -274,7 +275,7 @@ func main() { } else { p.InputType = req.Post.OperationID + "Request" } - p.OutputType = getResponseMessage(&req.Post.Responses) + p.OutputType = getResponseMessage(req.Post.Responses) service = append(service, p) } } diff --git a/webdash/src/Pages.js b/webdash/src/Pages.js index b0841e0fb..cf7686b9e 100644 --- a/webdash/src/Pages.js +++ b/webdash/src/Pages.js @@ -315,10 +315,8 @@ function ServiceInfo() { React.useEffect(() => { var url = new URL("/v1/service-info", window.location.origin); - console.log("DEBUG: ServiceInfo url:", url); get(url).then( (info) => { - console.log("DEBUG: ServiceInfo info:", info); setInfo(info); }); }, []); diff --git a/website/config.yaml b/website/config.yaml index 6dd8c8858..3393f0df9 100644 --- a/website/config.yaml +++ b/website/config.yaml @@ -13,3 +13,8 @@ security: funcs: getenv: - ^FUNNEL_ + +markup: + goldmark: + renderer: + unsafe: true diff --git a/website/content/docs/interop/nextflow.md b/website/content/docs/integrations/nextflow.md similarity index 74% rename from website/content/docs/interop/nextflow.md rename to website/content/docs/integrations/nextflow.md index cf9a6687c..3090a9415 100644 --- a/website/content/docs/interop/nextflow.md +++ b/website/content/docs/integrations/nextflow.md @@ -2,7 +2,7 @@ title: Nextflow menu: main: - parent: Interop + parent: Integrations --- > ⚠️ Nextflow support is currently in development and requires a few additional steps to run which are included below. @@ -23,38 +23,62 @@ To set up Nextflow to use Funnel as the TES executor, run the following steps: ### 1. Install Nextflow +*Adapted from the [Nextflow Documentation](https://nextflow.io/docs/latest/install.html)* + +#### a. Install Nextflow: + +```sh +curl -s https://get.nextflow.io | bash +``` + +This will create the nextflow executable in the current directory. + +#### b. Make Nextflow executable: + ```sh -git clone https://github.com/nextflow-io/nextflow -b tes-update-1.1 +chmod +x nextflow +``` -cd nextflow +#### c. Move Nextflow into an executable path: -make compile +```sh +sudo mv nextflow /usr/local/bin ``` -This will create a new `launch.sh` file that can be used to run the Nextflow workflow below. +#### d. Confirm that Nextflow is installed correctly: + +```sh +nextflow info +``` ### 2. Update Nextflow Config Add the following to your `nextflow.config` in order to use the GA4GH TES plugin: ```yaml +cat <> nextflow.config plugins { id 'nf-ga4gh' } process.executor = 'tes' tes.endpoint = 'http://localhost:8000' # <--- Funnel's default address +EOF ``` -### 3. Start Funnel and Run Workflow +### 3. Start the Funnel Server -Finally, start the Funnel server and launch Nextflow: +Start the Funnel server: ```sh funnel server run +``` + +### 4. Run Nextflow -alias nextflow=~/nextflow/launch.sh # <--- Change this line to match your local nextflow directory +In another window, run the workflow: +```sh nextflow run main.nf -c nextflow.config ``` diff --git a/website/content/docs/interop/py-tes.md b/website/content/docs/integrations/py-tes.md similarity index 97% rename from website/content/docs/interop/py-tes.md rename to website/content/docs/integrations/py-tes.md index e07d2bb9d..385255729 100644 --- a/website/content/docs/interop/py-tes.md +++ b/website/content/docs/integrations/py-tes.md @@ -2,7 +2,7 @@ title: py-tes menu: main: - parent: Interop + parent: Integrations --- > ⚠️ py-tes support is in active development and may be subject to change. diff --git a/website/layouts/_default/single.html b/website/layouts/_default/single.html index 9bcb97c5e..11c37ed28 100644 --- a/website/layouts/_default/single.html +++ b/website/layouts/_default/single.html @@ -31,7 +31,7 @@
- {{ .Content }} + {{ .Content | safeHTML }}
diff --git a/website/layouts/partials/head.html b/website/layouts/partials/head.html index 683833b07..e7189503b 100644 --- a/website/layouts/partials/head.html +++ b/website/layouts/partials/head.html @@ -28,6 +28,15 @@ + + + + + @@ -48,3 +57,5 @@ + + diff --git a/website/static/funnel-config-examples/default-config.yaml b/website/static/funnel-config-examples/default-config.yaml index f929e3b39..1c87e79f2 100644 --- a/website/static/funnel-config-examples/default-config.yaml +++ b/website/static/funnel-config-examples/default-config.yaml @@ -8,7 +8,7 @@ Compute: local # The name of the active event writer backend(s). # Available backends: log, boltdb, badger, datastore, dynamodb, elastic, mongodb, kafka -EventWriters: +EventWriters: - boltdb - log @@ -42,7 +42,7 @@ Server: DisableHTTPCache: true RPCClient: - # RPC server address + # RPC server address ServerAddress: localhost:9090 # Credentials for Basic authentication for the server APIs using a password. @@ -59,7 +59,7 @@ RPCClient: # up to 1 minute MaxRetries: 10 -# The scheduler is used for the Manual compute backend. +# The scheduler is used for the Manual compute backend. Scheduler: # How often to run a scheduler iteration. ScheduleRate: 1s @@ -78,7 +78,7 @@ Node: # -1 means there is no timeout. 0 means timeout immediately after the first task. Timeout: -1s - # A Node will automatically try to detect what resources are available to it. + # A Node will automatically try to detect what resources are available to it. # Defining Resources in the Node configuration overrides this behavior. Resources: # CPUs available. @@ -149,7 +149,7 @@ Datastore: # Optional. If possible, credentials will be automatically discovered # from the environment. CredentialsFile: "" - + MongoDB: # Addrs holds the addresses for the seed servers. Addrs: @@ -277,7 +277,7 @@ Slurm: {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} {{- end}} - {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} --Server.hostname exahead1 + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} # AWSBatch describes the configuration for the AWS Batch compute backend. AWSBatch: @@ -298,6 +298,8 @@ AWSBatch: # Kubernetes describes the configuration for the Kubernetes compute backend. Kubernetes: + # The executor used to execute tasks. Available executors: docker, kubernetes + Executor: "docker" # Turn off task state reconciler. When enabled, Funnel communicates with Kubernetes # to find tasks that are stuck in a queued state or errored and # updates the task state accordingly. @@ -307,11 +309,18 @@ Kubernetes: ReconcileRate: 10m # Kubernetes Namespace to spawn jobs within Namespace: "" - # Batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch + # Master batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch Template: "" - # TemplateFile is the path to the job template. + # TemplateFile is the path to the master job template. TemplateFile: "" + # Configuration of the Kubernetes executor. + + # Job template used for executing the tasks. + ExecutorTemplate: "" + # ExecutorTemplateFile is the path to the executor template. + ExecutorTemplateFile: "" + #------------------------------------------------------------------------------- # Storage #------------------------------------------------------------------------------- diff --git a/worker/command.go b/worker/command.go index 8e32dc888..60b6057ea 100644 --- a/worker/command.go +++ b/worker/command.go @@ -7,15 +7,15 @@ import ( ) type Command struct { - Image string - ShellCommand []string - Volumes []Volume - Workdir string - Env map[string]string - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - Event *events.ExecutorWriter + Image string + ShellCommand []string + Volumes []Volume + Workdir string + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Event *events.ExecutorWriter TaskCommand } @@ -37,4 +37,4 @@ func (c *Command) SetStderr(w io.Writer) { func (c *Command) SetStdin(r io.Reader) { c.Stdin = r -} \ No newline at end of file +} diff --git a/worker/container_engine.go b/worker/container_engine.go new file mode 100644 index 000000000..b0a8915e8 --- /dev/null +++ b/worker/container_engine.go @@ -0,0 +1,63 @@ +package worker + +import ( + "context" + "io" + + "github.com/ohsu-comp-bio/funnel/events" +) + +type ContainerEngine interface { + // Run runs the container. + Run(ctx context.Context) error + + // Stop stops the container. + Stop() error + + // Inspect returns the container configuration. + InspectContainer(ctx context.Context) ContainerConfig + + // GetImage returns the image name for the container. + GetImage() string + + // GetIO returns the stdin, stdout, and stderr for the container. + GetIO() (io.Reader, io.Writer, io.Writer) + + // SetIO sets the stdin, stdout, and stderr for the container. + SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) + + // SyncAPIVersions ensures that the client uses the same API version as the server. + SyncAPIVersion() error +} + +type ContainerConfig struct { + Id string + Image string + Name string + Command []string + Volumes []Volume + Workdir string + RemoveContainer bool + Env map[string]string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Event *events.ExecutorWriter + DriverCommand string + RunCommand string // template string + PullCommand string // template string + StopCommand string // template string +} + +type ContainerVersion struct { + Client string + Server string +} + +type ContainerEngineFactory struct{} + +func (f *ContainerEngineFactory) NewContainerEngine(containerConfig ContainerConfig) (ContainerEngine, error) { + return &Docker{ + ContainerConfig: containerConfig, + }, nil +} diff --git a/worker/docker.go b/worker/docker.go index afb5125ff..56072461d 100644 --- a/worker/docker.go +++ b/worker/docker.go @@ -1,94 +1,103 @@ package worker import ( + "bytes" "context" "encoding/json" "fmt" + "io" "os" "os/exec" "strings" + "text/template" "time" ) -// DockerCommand is responsible for configuring and running a docker container. -type DockerCommand struct { - ContainerName string - RemoveContainer bool - Command +// Docker is responsible for configuring and running a docker container. +type Docker struct { + ContainerConfig } // Run runs the Docker command and blocks until done. -func (dcmd DockerCommand) Run(ctx context.Context) error { +func (docker Docker) Run(ctx context.Context) error { // Sync docker API version info. - err := SyncDockerAPIVersion() + err := docker.SyncAPIVersion() if err != nil { - dcmd.Event.Error("failed to sync docker client API version", err) + docker.Event.Error("failed to sync docker client API version", err) } - pullcmd := exec.Command("docker", "pull", dcmd.Image) - err = pullcmd.Run() + err = docker.executeCommand(ctx, docker.PullCommand, false) if err != nil { - dcmd.Event.Error("failed to pull docker image", err) + docker.Event.Error("failed to pull docker image", err) } - args := []string{"run", "-i", "--read-only"} - - if dcmd.RemoveContainer { - args = append(args, "--rm") + err = docker.executeCommand(ctx, docker.RunCommand, true) + if err != nil { + docker.Event.Error("failed to run docker container", err) } - if dcmd.Env != nil { - for k, v := range dcmd.Env { - args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) - } - } + return err +} - if dcmd.ContainerName != "" { - args = append(args, "--name", dcmd.ContainerName) +// Stop stops the container. +func (docker Docker) Stop() error { + docker.Event.Info("Stopping container", "container", docker.Name) + err := docker.executeCommand(context.Background(), docker.StopCommand, false) + if err != nil { + docker.Event.Error("failed to stop docker container", err) + return err } + return nil +} - if dcmd.Workdir != "" { - args = append(args, "-w", dcmd.Workdir) +func (docker Docker) executeCommand(ctx context.Context, commandTemplate string, enableIO bool) error { + var usingCommand bool = false + if strings.Contains(commandTemplate, "{{.Command}}") { + usingCommand = true + commandTemplate = strings.ReplaceAll(commandTemplate, "{{.Command}}", "") } - for _, vol := range dcmd.Volumes { - arg := formatVolumeArg(vol) - args = append(args, "-v", arg) + tmpl, err := template.New("command").Parse(commandTemplate) + if err != nil { + return fmt.Errorf("failed to parse template for command: %w", err) } - args = append(args, dcmd.Image) - // if dcmd.Command contains the ["/bin/bash", "-ue"] string, replace it with "/bin/bash -ue" - c := strings.Join(dcmd.Command, " ") - if strings.Contains(c, "[/bin/bash, -ue]") { - dcmd.Command = transformCommandSlice(dcmd.Command) + var cmdBuffer bytes.Buffer + err = tmpl.Execute(&cmdBuffer, docker.ContainerConfig) + if err != nil { + return fmt.Errorf("failed to execute template for command: %w", err) } - args = append(args, dcmd.Command...) - - // Roughly: `docker run --rm -i --read-only -w [workdir] -v [bindings] [imageName] [cmd]` - err = dcmd.Event.Info(fmt.Sprintf("args: %s", args)) - dcmd.Event.Info("Running command", "cmd", "docker "+strings.Join(args, " ")) - cmd := exec.Command("docker", args...) - if dcmd.Stdin != nil { - cmd.Stdin = dcmd.Stdin + cmdParts := strings.Fields(cmdBuffer.String()) + if usingCommand { + go docker.InspectContainer(ctx) + cmdParts = append(cmdParts, docker.Command...) } - if dcmd.Stdout != nil { - cmd.Stdout = dcmd.Stdout + + driverCmd := strings.Fields(docker.DriverCommand) + var cmd *exec.Cmd + if len(driverCmd) > 1 { + cmdArgs := append(driverCmd[1:], cmdParts...) + cmd = exec.CommandContext(ctx, driverCmd[0], cmdArgs...) + } else { + cmd = exec.CommandContext(ctx, driverCmd[0], cmdParts...) } - if dcmd.Stderr != nil { - cmd.Stderr = dcmd.Stderr + + if enableIO { + if docker.Stdin != nil { + cmd.Stdin = docker.Stdin + } + if docker.Stdout != nil { + cmd.Stdout = docker.Stdout + } + if docker.Stderr != nil { + cmd.Stderr = docker.Stderr + } } - go dcmd.inspectContainer(ctx) - out := cmd.Run() - dcmd.Event.Info("Command %s Complete exit=%s", strings.Join(args, " "), out) - return out -} -// Stop stops the container. -func (dcmd DockerCommand) Stop() error { - dcmd.Event.Info("Stopping container", "container", dcmd.ContainerName) - // cmd := exec.Command("docker", "stop", dcmd.ContainerName) - cmd := exec.Command("docker", "rm", "-f", dcmd.ContainerName) //switching to this to be a bit more forceful + if usingCommand { + docker.Event.Info("Running command", "cmd", cmd.String()) + } return cmd.Run() } @@ -101,23 +110,28 @@ func formatVolumeArg(v Volume) string { return fmt.Sprintf("%s:%s:%s", v.HostPath, v.ContainerPath, mode) } -func transformCommandSlice(cmd []string) []string { - if len(cmd) == 0 { - return cmd // Handle empty slice - } +func (docker Docker) GetImage() string { + return docker.Image +} - cmd[2] = "/bin/bash -ue .command.run &> .command.log" - return cmd +func (docker Docker) GetIO() (io.Reader, io.Writer, io.Writer) { + return docker.Stdin, docker.Stdout, docker.Stderr } -type metadata struct { - ID string - Name string - Image string +func (docker *Docker) SetIO(stdin io.Reader, stdout io.Writer, stderr io.Writer) { + if stdin != nil && stdin != (*os.File)(nil) { + docker.Stdin = stdin + } + if stdout != nil && stdout != (*os.File)(nil) { + docker.Stdout = stdout + } + if stderr != nil && stderr != (*os.File)(nil) { + docker.Stderr = stderr + } } // inspectContainer inspects the docker container for metadata. -func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { +func (docker *Docker) InspectContainer(ctx context.Context) ContainerConfig { // Give the container time to start. time.Sleep(2 * time.Second) @@ -128,40 +142,47 @@ func (dcmd *DockerCommand) inspectContainer(ctx context.Context) { for i := 0; i < 5; i++ { select { case <-ctx.Done(): - return + return ContainerConfig{} case <-ticker.C: - cmd := exec.CommandContext(ctx, "docker", "inspect", dcmd.ContainerName) + cmd := exec.CommandContext(ctx, "docker", "inspect", docker.Name) out, err := cmd.Output() if err == nil { - meta := []metadata{} - err := json.Unmarshal(out, &meta) + + meta := []ContainerConfig{} + err = json.Unmarshal(out, &meta) if err == nil && len(meta) == 1 { - dcmd.Event.Info("container metadata", - "containerID", meta[0].ID, + docker.Event.Info("container metadata", + "containerID", meta[0].Id, "containerName", meta[0].Name, "containerImageHash", meta[0].Image) - return + return meta[0] } } } } -} -type dockerVersion struct { - Client string - Server string + return ContainerConfig{} } // SyncDockerAPIVersion ensures that the client uses the same API version as // the server. -func SyncDockerAPIVersion() error { +func (docker *Docker) SyncAPIVersion() error { if os.Getenv("DOCKER_API_VERSION") == "" { - cmd := exec.Command("docker", "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + var args []string + driverCmd := strings.Fields(docker.ContainerConfig.DriverCommand) + + if len(docker.ContainerConfig.DriverCommand) > 1 { + // Merge driver parts and command parts + args = append(args, driverCmd[1:]...) + } + + args = append(args, "version", "--format", `{"Server": "{{.Server.APIVersion}}", "Client": "{{.Client.APIVersion}}"}`) + cmd := exec.Command(driverCmd[0], args...) out, err := cmd.Output() if err != nil { return fmt.Errorf("docker version command failed: %v", err) } - version := &dockerVersion{} + version := &ContainerVersion{} err = json.Unmarshal(out, version) if err != nil { return fmt.Errorf("failed to unmarshal docker version: %v", err) diff --git a/worker/docker_test.go b/worker/docker_test.go new file mode 100644 index 000000000..4bb87ea86 --- /dev/null +++ b/worker/docker_test.go @@ -0,0 +1,139 @@ +package worker + +import ( + "bytes" + "context" + "testing" + "time" + + "math/rand" + + "github.com/ohsu-comp-bio/funnel/events" + "github.com/ohsu-comp-bio/funnel/logger" +) + +var defaultConfig = ContainerConfig{ + Id: "123", + Image: "alpine", + Name: "funnel-test-" + RandomString(6), + DriverCommand: "docker", + Command: []string{"sh", "-c", "echo Hello, World!"}, + RunCommand: "run --name {{.Name}} {{.Image}} {{.Command}}", + PullCommand: "pull {{.Image}}", + RemoveContainer: true, + Event: events.NewExecutorWriter("123", 1, 1, &events.Logger{ + Log: logger.NewLogger("test", logger.DefaultConfig()), + }), +} + +func TestDockerRun(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.Run(context.Background()) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} + +func TestDockerExecuteCommand(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.executeCommand(context.Background(), "run --rm alpine echo Hello, World!", true) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} + +func TestDockerStop(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + ctx, cancel := context.WithCancel(context.Background()) + + // Run the container command in a separate goroutine + go func() { + err := docker.executeCommand(ctx, "run --rm alpine sleep 30", true) + if err != nil && ctx.Err() == nil { + t.Errorf("Expected no error, but got: %v", err) + } + }() + + // Give the container some time to start + time.Sleep(2 * time.Second) + + // Stop the container + err := docker.Stop() + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + // Cancel the context to stop the goroutine if it is still running + cancel() +} + +func TestFormatVolumeArg(t *testing.T) { + volume := Volume{ + HostPath: "/path/to/source", + ContainerPath: "/path/to/destination", + Readonly: true, + } + expected := "/path/to/source:/path/to/destination:ro" + result := formatVolumeArg(volume) + if result != expected { + t.Errorf("Expected %s, but got %s", expected, result) + } +} + +func TestDockerGetImage(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + expected := "alpine" + result := docker.GetImage() + if result != expected { + t.Errorf("Expected %s, but got %s", expected, result) + } +} + +func TestDockerSetIO(t *testing.T) { + docker := Docker{} + stdin := &bytes.Buffer{} + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + docker.SetIO(stdin, stdout, stderr) + if docker.Stdin != stdin || docker.Stdout != stdout || docker.Stderr != stderr { + t.Errorf("Expected stdin, stdout, and stderr to be set correctly") + } +} + +func TestDockerInspectContainer(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + config := docker.InspectContainer(context.Background()) + if config.Id == "" { + t.Errorf("Expected non-nil container config") + } +} + +func TestDockerSyncAPIVersion(t *testing.T) { + docker := Docker{ + ContainerConfig: defaultConfig, + } + err := docker.SyncAPIVersion() + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } +} + +// RandomString generates a random string of length n +func RandomString(n int) string { + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz0123456789") + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} diff --git a/worker/file_mapper.go b/worker/file_mapper.go index 4219c724d..6aa5407f9 100644 --- a/worker/file_mapper.go +++ b/worker/file_mapper.go @@ -2,6 +2,7 @@ package worker import ( "fmt" + "io" "io/ioutil" "os" "path" @@ -21,10 +22,11 @@ import ( // outputs, uploads, stdin/out/err, etc. FileMapper helps the worker engine // manage all these paths. type FileMapper struct { - Volumes []Volume - Inputs []*tes.Input - Outputs []*tes.Output - WorkDir string + Volumes []Volume + Inputs []*tes.Input + Outputs []*tes.Output + WorkDir string + ScratchDir string } // Volume represents a volume mounted into a docker container. @@ -96,6 +98,146 @@ func (mapper *FileMapper) MapTask(task *tes.Task) error { return nil } +func (mapper *FileMapper) CopyInputsToScratch(scratchDir string) error { + scratchAbsDir, err := filepath.Abs(scratchDir) + if err != nil { + return err + } + + // Copy the input file or directory to the scratch target + for _, input := range mapper.Inputs { + scratchTarget := filepath.Join(scratchAbsDir, input.Path) + + info, err := os.Stat(input.Path) + if err != nil { + fmt.Println(err) + } + if info.IsDir() { + // Ensure the scratch target directory exists + if err := os.MkdirAll(scratchTarget, 0755); err != nil { + return fmt.Errorf("failed to create scratch directory: %w", err) + } + copyDir(input.Path, scratchTarget) + } else { + // Ensure the scratch target directory exists + if err := os.MkdirAll(path.Dir(scratchTarget), 0755); err != nil { + return fmt.Errorf("failed to create scratch directory: %w", err) + } + err = copyFile(input.Path, scratchTarget) + if err != nil { + return fmt.Errorf("failed to copy input file to scratch directory: %w", err) + } + _, _ = os.ReadFile(scratchTarget) + } + } + + return nil +} + +func (mapper *FileMapper) CopyOutputsToWorkDir(scratchDir string) error { + scratchAbsDir, err := filepath.Abs(scratchDir) + if err != nil { + return err + } + err = filepath.Walk(scratchAbsDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + return nil + }) + if err != nil { + fmt.Println(err) + } + + // Copy the input file or directory to the scratch target + for _, output := range mapper.Outputs { + scratchTarget := filepath.Join(scratchAbsDir, output.Path) + + matches, err := filepath.Glob(scratchTarget) + if err != nil { + return fmt.Errorf("invalid pattern %s: %w", scratchTarget, err) + } + if len(matches) == 0 { + return fmt.Errorf("no files matched the pattern: %s", scratchTarget) + } + + parentDir := filepath.Dir(output.Path) + for _, src := range matches { + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("failed to stat output path: %w", err) + } + // If output is a directory + if info.IsDir() { + // Ensure the scratch target directory exists + if err = os.MkdirAll(output.Path, 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + copyDir(src, output.Path) + // If output is a file + } else { + // Ensure the scratch target directory exists + if err = os.MkdirAll(path.Dir(parentDir), 0755); err != nil { + return fmt.Errorf("failed to create output path: %w", err) + } + + workDirPath := filepath.Join(parentDir, info.Name()) + + err = copyFile(src, workDirPath) + if err != nil { + return fmt.Errorf("failed to copy output file to output directory: %w", err) + } + } + } + } + + return nil +} + +func copyDir(src string, dst string) error { + entries, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + if err := os.MkdirAll(dstPath, 0755); err != nil { + return err + } + if err := copyDir(srcPath, dstPath); err != nil { + return err + } + } else { + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + } + return nil +} + +func copyFile(src string, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(dst) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, sourceFile) + return err +} + // AddVolume adds a mapped volume to the mapper. A corresponding Volume record // is added to mapper.Volumes. // @@ -148,6 +290,15 @@ func (mapper *FileMapper) HostPath(src string) (string, error) { return p, nil } +func (mapper *FileMapper) HostScratchPath(src string) (string, error) { + p := path.Join(mapper.ScratchDir, src) + p = path.Clean(p) + if !mapper.IsSubpath(p, mapper.ScratchDir) { + return "", fmt.Errorf("Invalid path: %s is not a valid subpath of %s", p, mapper.ScratchDir) + } + return p, nil +} + // OpenHostFile opens a file on the host file system at a mapped path. // "src" is an unmapped path. This function will handle mapping the path. // @@ -173,7 +324,13 @@ func (mapper *FileMapper) OpenHostFile(src string) (*os.File, error) { // // If the path can't be mapped or the file can't be created, an error is returned. func (mapper *FileMapper) CreateHostFile(src string) (*os.File, error) { - p, perr := mapper.HostPath(src) + var p string + var perr error + if mapper.ScratchDir != "" { + p, perr = mapper.HostScratchPath(src) + } else { + p, perr = mapper.HostPath(src) + } if perr != nil { return nil, perr } diff --git a/worker/interfaces.go b/worker/interfaces.go index 5886fa72b..69b416b28 100644 --- a/worker/interfaces.go +++ b/worker/interfaces.go @@ -23,4 +23,4 @@ type TaskCommand interface { SetStdout(io.Writer) SetStderr(io.Writer) SetStdin(io.Reader) -} \ No newline at end of file +} diff --git a/worker/kubernetes.go b/worker/kubernetes.go index c1e7e8d91..e3ca4dd30 100644 --- a/worker/kubernetes.go +++ b/worker/kubernetes.go @@ -19,18 +19,18 @@ import ( // KubernetesCommand is responsible for configuring and running a task in a Kubernetes cluster. type KubernetesCommand struct { - TaskId string - JobId int - StdinFile string - TaskTemplate string - Namespace string - Resources *tes.Resources + TaskId string + JobId int + StdinFile string + TaskTemplate string + Namespace string + Resources *tes.Resources Command } // Creates a new Kuberntes Job which will run the task. func (kcmd KubernetesCommand) Run(ctx context.Context) error { - var taskId = kcmd.TaskId + var taskId = kcmd.TaskId tpl, err := template.New(taskId).Parse(kcmd.TaskTemplate) if err != nil { @@ -78,7 +78,7 @@ func (kcmd KubernetesCommand) Run(ctx context.Context) error { var client = clientset.BatchV1().Jobs(kcmd.Namespace) - _ , err = client.Create(ctx, job, metav1.CreateOptions{}) + _, err = client.Create(ctx, job, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("creating job: %v", err) @@ -96,7 +96,7 @@ func (kcmd KubernetesCommand) Run(ctx context.Context) error { for _, v := range pods.Items { req := clientset.CoreV1().Pods(kcmd.Namespace).GetLogs(v.Name, &corev1.PodLogOptions{}) - podLogs, err := req.Stream(ctx) + podLogs, err := req.Stream(ctx) if err != nil { return err @@ -106,12 +106,12 @@ func (kcmd KubernetesCommand) Run(ctx context.Context) error { buf := new(bytes.Buffer) _, err = io.Copy(buf, podLogs) if err != nil { - return err + return err } var bytes = buf.Bytes() kcmd.Stdout.Write(bytes) - } + } return nil } @@ -172,4 +172,4 @@ func getKubernetesClientset() (*kubernetes.Clientset, error) { clientset, err := kubernetes.NewForConfig(kubeconfig) return clientset, err -} \ No newline at end of file +} diff --git a/worker/step.go b/worker/step.go index ab5bd0c2c..bc6ef113c 100644 --- a/worker/step.go +++ b/worker/step.go @@ -3,6 +3,7 @@ package worker import ( "context" "io" + "os" "sync" "time" @@ -12,7 +13,7 @@ import ( type stepWorker struct { Conf config.Worker - Command TaskCommand + Command ContainerEngine Event *events.ExecutorWriter IP string } @@ -42,14 +43,14 @@ func (s *stepWorker) Run(ctx context.Context) error { } // Capture stdout/err to file. - if s.Command.GetStdout() != nil { - stdout = io.MultiWriter(s.Command.GetStdout(), stdout) } - if s.Command.GetStderr() != nil { - stderr = io.MultiWriter(s.Command.GetStderr(), stderr) + _, out, err := s.Command.GetIO() + if out != nil && out != (*os.File)(nil) { + stdout = io.MultiWriter(out, stdout) } - - s.Command.SetStdout(stdout) - s.Command.SetStderr(stderr) + if err != nil && err != (*os.File)(nil) { + stderr = io.MultiWriter(err, stderr) + } + s.Command.SetIO(nil, stdout, stderr) go func() { done <- s.Command.Run(subctx) diff --git a/worker/storage.go b/worker/storage.go index 0edba8d34..a82bcc845 100644 --- a/worker/storage.go +++ b/worker/storage.go @@ -3,10 +3,12 @@ package worker import ( "context" "fmt" + "net/url" "os" "path/filepath" "strings" + proto "github.com/golang/protobuf/proto" "github.com/ohsu-comp-bio/funnel/events" "github.com/ohsu-comp-bio/funnel/storage" "github.com/ohsu-comp-bio/funnel/tes" @@ -246,3 +248,41 @@ func fixLinks(mapper *FileMapper, basepath string) { return nil }) } + +// resolveWildcards resolves any wildcards/globs in the output paths. +func resolveWildcards(mapper *FileMapper) error { + var outputs []*tes.Output + + for _, output := range mapper.Outputs { + // If path contains a wildcard, handle globbing and upload each file individually + if strings.Contains(output.Path, "*") { + globs, err := filepath.Glob(output.Path) + if err != nil { + return fmt.Errorf("failed to resolve path %v: %v", output.Path, err) + } + + for _, glob := range globs { + out := proto.Clone(output).(*tes.Output) + out.Path = glob + + // Construct URL by: + // - removing the mapper.WorkDir from the output.Path + // - then removing the PathPrefix + // - then joining the output.URL + globPath := strings.TrimPrefix(glob, mapper.WorkDir) + fixPath := strings.TrimPrefix(globPath, output.PathPrefix) + out.Url, err = url.JoinPath(output.Url, fixPath) + if err != nil { + return fmt.Errorf("failed to join URL: %v", err) + } + + outputs = append(outputs, out) + } + } else { + outputs = append(outputs, output) + } + } + + mapper.Outputs = outputs + return nil +} diff --git a/worker/taskreader_test.go b/worker/taskreader_test.go index 994d13c28..534b6bb1d 100644 --- a/worker/taskreader_test.go +++ b/worker/taskreader_test.go @@ -13,6 +13,9 @@ func TestFileTaskReader(t *testing.T) { ctx := context.Background() task, err := r.Task(ctx) + if err != nil { + t.Fatal(err) + } if task.Name != "Hello world" { t.Error("unexpected task content") } @@ -30,6 +33,9 @@ func TestBase64TaskReader(t *testing.T) { ctx := context.Background() task, err := r.Task(ctx) + if err != nil { + t.Fatal(err) + } if task.Name != "Hello world" { t.Error("unexpected task content") } diff --git a/worker/worker.go b/worker/worker.go index e4cbb043c..a0f67bdd6 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -19,19 +19,19 @@ import ( // sequential process of task initialization, execution, finalization, // and logging. type DefaultWorker struct { - Executor Executor - Conf config.Worker - Store storage.Storage - TaskReader TaskReader - EventWriter events.Writer + Executor Executor + Conf config.Worker + Store storage.Storage + TaskReader TaskReader + EventWriter events.Writer } // Configuration of the task executor. type Executor struct { // "docker" or "kubernetes" - Backend string + Backend string // Kubernetes executor template - Template string + Template string // Kubernetes namespace Namespace string } @@ -70,6 +70,13 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { // set up task specific utilities event = events.NewTaskWriter(task.GetId(), 0, r.EventWriter) mapper = NewFileMapper(filepath.Join(r.Conf.WorkDir, task.GetId())) + if r.Conf.ScratchPath != "" { + scratchAbsDir, err := filepath.Abs(r.Conf.ScratchPath) + if err != nil { + return err + } + mapper.ScratchDir = filepath.Join(scratchAbsDir, filepath.Join(r.Conf.WorkDir, task.GetId())) + } event.Info("Version", version.LogFields()...) event.State(tes.State_INITIALIZING) @@ -140,48 +147,54 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { event.State(tes.State_RUNNING) } - var resources = task.GetResources() - if resources == nil { - resources = &tes.Resources{} + // Create symlinks between the working directory and the Scratch Directory + if run.ok() && r.Conf.ScratchPath != "" { + err := mapper.CopyInputsToScratch(r.Conf.ScratchPath) + if err != nil { + return err + } + // Get the absolute path of the scratch directory + scratchAbsDir, err := filepath.Abs(r.Conf.ScratchPath) + if err != nil { + return err + } + // For every volume in mapper, add the Scratch Path prefix to the host path + for idx, v := range mapper.Volumes { + v.HostPath = filepath.Join(scratchAbsDir, v.HostPath) + mapper.Volumes[idx] = v + } + } // Run steps if run.ok() { ignoreError := false + f := ContainerEngineFactory{} for i, d := range task.GetExecutors() { - var command = Command{ - Image: d.Image, - ShellCommand: d.Command, - Volumes: mapper.Volumes, - Workdir: d.Workdir, - Env: d.Env, - Event: event.NewExecutorWriter(uint32(i)), + containerConfig := ContainerConfig{ + Image: d.Image, + Command: d.Command, + Env: d.Env, + Volumes: mapper.Volumes, + Workdir: d.Workdir, + Name: fmt.Sprintf("%s-%d", task.Id, i), + // TODO make RemoveContainer configurable + RemoveContainer: true, + Event: event.NewExecutorWriter(uint32(i)), } - - var taskCommand TaskCommand - if r.Executor.Backend == "kubernetes" { - taskCommand = &KubernetesCommand{ - TaskId: task.Id, - JobId: i, - StdinFile: d.Stdin, - TaskTemplate: r.Executor.Template, - Namespace: r.Executor.Namespace, - Resources: resources, - Command: command, - } - } else { - taskCommand = &DockerCommand{ - ContainerName: fmt.Sprintf("%s-%d", task.Id, i), - //TODO Make RemoveContainer configurable - RemoveContainer: true, - Command: command, - } + containerConfig.DriverCommand = r.Conf.Container.DriverCommand + containerConfig.RunCommand = r.Conf.Container.RunCommand + containerConfig.PullCommand = r.Conf.Container.PullCommand + containerConfig.StopCommand = r.Conf.Container.StopCommand + containerEngine, err := f.NewContainerEngine(containerConfig) + if err != nil { + run.syserr = err } s := &stepWorker{ - Conf: r.Conf, - Event: event.NewExecutorWriter(uint32(i)), - Command: taskCommand, + Conf: r.Conf, + Event: event.NewExecutorWriter(uint32(i)), + Command: containerEngine, } // Opens stdin/out/err files and updates those fields on "cmd". @@ -204,6 +217,15 @@ func (r *DefaultWorker) Run(pctx context.Context) (runerr error) { } } + if run.ok() { + // Resolve wildcards in the output paths + resolveWildcards(mapper) + } + + if run.ok() && r.Conf.ScratchPath != "" { + mapper.CopyOutputsToWorkDir(r.Conf.ScratchPath) + } + // Upload outputs var outputLog []*tes.OutputFileLog if run.ok() { @@ -230,10 +252,14 @@ func (r *DefaultWorker) Close() { // openLogs opens/creates the logs files for a step and updates those fields. func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.Executor) error { + var stdin *os.File + var stdout *os.File + var stderr *os.File + var err error + // Find the path for task stdin if d.Stdin != "" { - stdin, err := mapper.OpenHostFile(d.Stdin) - s.Command.SetStdin(stdin) + stdin, err = mapper.CreateHostFile(d.Stdin) if err != nil { s.Event.Error("Couldn't prepare log files", err) return err @@ -242,8 +268,7 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E // Create file for task stdout if d.Stdout != "" { - stdout, err := mapper.CreateHostFile(d.Stdout) - s.Command.SetStdout(stdout) + stdout, err = mapper.CreateHostFile(d.Stdout) if err != nil { s.Event.Error("Couldn't prepare log files", err) return err @@ -252,14 +277,15 @@ func (r *DefaultWorker) openStepLogs(mapper *FileMapper, s *stepWorker, d *tes.E // Create file for task stderr if d.Stderr != "" { - stderr, err := mapper.CreateHostFile(d.Stderr) - s.Command.SetStderr(stderr) + stderr, err = mapper.CreateHostFile(d.Stderr) if err != nil { _ = s.Event.Error("Couldn't prepare log files", err) return err } } + s.Command.SetIO(stdin, stdout, stderr) + return nil }