From 383524346fbe8456d2dec9dc71a66dbd0e0dbae5 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 30 Aug 2024 14:55:49 -0400 Subject: [PATCH 1/5] chore(ci): Update large file test to use platform --- .github/workflows/large-tests.yaml | 137 ++--------------------------- 1 file changed, 8 insertions(+), 129 deletions(-) diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index bf3eb52f..5b14e2fd 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -1,98 +1,10 @@ -name: Build, Test, and Deliver Client +name: Large file test on: schedule: - cron: '0 4 * * 2,4' jobs: - lib: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./lib - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: './lib/package-lock.json' - - run: npm ci - - run: npm test - - uses: actions/upload-artifact@v4 - with: - name: coverage - path: lib/coverage/ - - run: npm audit --omit dev --audit-level moderate - - run: npm run license-check - - run: npm run lint - - run: npm pack - - uses: actions/upload-artifact@v4 - with: - name: opentdf-sdk-lib - path: ./lib/opentdf-sdk-*.tgz - - cli: - needs: - - lib - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./cli - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: './cli/package-lock.json' - - uses: actions/download-artifact@v4 - with: - name: opentdf-sdk-lib - path: lib/ - - run: npm uninstall @opentdf/sdk && npm ci && npm i ../lib/opentdf-sdk-*.tgz - - run: npm test - - run: npm audit --omit dev --audit-level moderate - - run: npm run license-check - - run: npm run lint - - run: npm pack - - uses: actions/upload-artifact@v4 - with: - name: opentdf-ctl - path: ./cli/opentdf-ctl-*.tgz - - web-app: - needs: - - lib - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./web-app - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: './web-app/package-lock.json' - - uses: actions/download-artifact@v4 - with: - name: opentdf-sdk-lib - path: lib/ - - run: npm uninstall @opentdf/sdk && npm ci && npm i ../lib/opentdf-sdk-*.tgz - - run: npm install - - run: npm test - - run: npm audit --omit dev --audit-level moderate - - run: npm run license-check - - run: npm run lint - - run: npm pack - - backend-roundtrip: - needs: - - cli - - lib + large-file-test: runs-on: ubuntu-latest defaults: run: @@ -105,44 +17,11 @@ jobs: node-version: '20' cache: 'npm' cache-dependency-path: './web-app/package-lock.json' - - uses: actions/download-artifact@v4 - with: - name: opentdf-sdk-lib - path: lib/ - - uses: actions/download-artifact@v4 - with: - name: opentdf-ctl - path: cli - - name: Git clone backend - run: | - git clone https://github.com/opentdf/opentdf.git - - uses: yokawasa/action-setup-kube-tools@v0.11.1 - with: - setup-tools: | - kubectl - helm - tilt - # This should be in sync with the minikube-deployed kube version below - kubectl: '1.23.1' - helm: '3.8.0' - tilt: '0.26.2' - - run: | - kubectl version --client - kustomize version - tilt version - - name: start minikube - id: minikube - uses: medyagh/setup-minikube@master - with: - minikube-version: 1.25.1 - # This should be in sync with the setup-tools version above - kubernetes-version: 1.23.1 - - name: Run tilt ci - env: - #path relative to the quickstart Tiltfile - TEST_SCRIPT: ../../wait-and-test.sh - OPENTDF_INGRESS_HOST_PORT: '5432' - OPENTDF_LOAD_FRONTEND: 'false' + - run: make all && make test + - name: run platform + uses: opentdf/platform/test/start-up-with-containers@main + - run: docker compose up -d --wait --wait-timeout 240 + - env: PLAYWRIGHT_TESTS_TO_RUN: huge roundtrip run: |- - tilt ci --file opentdf/quickstart/Tiltfile + ./wait-and-test.sh platform From 614e086a24bb11d8432eca6c623386d4eefa3ffd Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 30 Aug 2024 15:12:10 -0400 Subject: [PATCH 2/5] Update large-tests.yaml --- .github/workflows/large-tests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index 5b14e2fd..84bc7164 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -1,6 +1,11 @@ name: Large file test on: + pull_request: + push: + branches: + - main + - feature/large-okish schedule: - cron: '0 4 * * 2,4' jobs: From 9bbaa80af211237b3a7fd10dfdeaea1d30d8e4c8 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 30 Aug 2024 15:16:35 -0400 Subject: [PATCH 3/5] Update large-tests.yaml Update Makefile Update build.yaml Update wait-and-test.sh --- .github/workflows/build.yaml | 10 +++++----- .github/workflows/large-tests.yaml | 5 ++--- .github/workflows/roundtrip/wait-and-test.sh | 3 ++- Makefile | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index eb6a6474..d027d69c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,14 +6,14 @@ env: github.event.pull_request.head.repo.full_name == github.repository }} on: - pull_request: + # pull_request: push: branches: - main - - release/[0-9]+.[0-9]+.[0-9]+ - release: - types: - - created + # - release/[0-9]+.[0-9]+.[0-9]+ + # release: + # types: + # - created jobs: lib: runs-on: ubuntu-latest diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index 84bc7164..1cc994ba 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -11,9 +11,6 @@ on: jobs: large-file-test: runs-on: ubuntu-latest - defaults: - run: - working-directory: .github/workflows/roundtrip timeout-minutes: 120 steps: - uses: actions/checkout@v4 @@ -26,7 +23,9 @@ jobs: - name: run platform uses: opentdf/platform/test/start-up-with-containers@main - run: docker compose up -d --wait --wait-timeout 240 + working-directory: .github/workflows/roundtrip - env: PLAYWRIGHT_TESTS_TO_RUN: huge roundtrip run: |- ./wait-and-test.sh platform + working-directory: .github/workflows/roundtrip diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 3ee41c0d..627f47df 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -17,7 +17,8 @@ _configure_app() { echo "[ERROR] Couldn't ci roundtrip command line app" return 1 fi - if ! npm i "../../../cli/opentdf-ctl-${app_version}.tgz"; then + if ! npm i "${ROOT_DIR}/cli/opentdf-ctl-${app_version}.tgz"; then + echo "[ERROR] Couldn't install current command line app" return 1 fi return 0 diff --git a/Makefile b/Makefile index 548ad71c..a7031780 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ i: (cd lib && npm i && npm pack) for x in $(extras); do (cd $$x && npm uninstall @opentdf/sdk && npm i && npm i ../lib/opentdf-sdk-$(version).tgz) || exit 1; done -all: ci lib/opentdf-sdk-$(version).tgz web-app/opentdf-web-app-$(version).tgz +all: ci lib/opentdf-sdk-$(version).tgz cli/opentdf-ctl-$(version).tgz web-app/opentdf-web-app-$(version).tgz cli/opentdf-ctl-$(version).tgz: lib/opentdf-sdk-$(version).tgz $(shell find cli -not -path '*/dist*' -and -not -path '*/coverage*' -and -not -path '*/node_modules*') (cd cli && npm uninstall @opentdf/sdk && npm ci && npm i ../lib/opentdf-sdk-$(version).tgz && npm pack) From aa43fa673c238ac6f392d117fb755feb77ab18d9 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 7 Nov 2024 07:45:33 -0500 Subject: [PATCH 4/5] Update large-tests.yaml --- .github/workflows/large-tests.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index 1cc994ba..0873af21 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -22,8 +22,6 @@ jobs: - run: make all && make test - name: run platform uses: opentdf/platform/test/start-up-with-containers@main - - run: docker compose up -d --wait --wait-timeout 240 - working-directory: .github/workflows/roundtrip - env: PLAYWRIGHT_TESTS_TO_RUN: huge roundtrip run: |- From f642b99c568311bf7705fb1d1a27a7bf76e9c759 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 7 Nov 2024 08:17:53 -0500 Subject: [PATCH 5/5] fixes --- .github/workflows/large-tests.yaml | 1 - .github/workflows/roundtrip/wait-and-test.sh | 44 ++------------------ 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index 0873af21..255b72ed 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -1,7 +1,6 @@ name: Large file test on: - pull_request: push: branches: - main diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 627f47df..b1cbd1e7 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -24,15 +24,15 @@ _configure_app() { return 0 } -if [ $1 = backend ]; then +if [ "$1" = backend ]; then VITE_PROXY='{"/api":{"target":"http://localhost:5432","xfwd":true},"/auth":{"target":"http://localhost:5432","xfwd":true}}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' else # if [ $1 = platform ]; then VITE_PROXY='{"/kas":{"target":"http://localhost:8080","xfwd":true},"/auth":{"target":"http://localhost:8888","xfwd":true}}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/opentdf","clientId":"browsertest"},"kas":"http://localhost:65432/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' fi -export VITE_PROXY -export VITE_TDF_CFG +export "VITE_PROXY" +export "VITE_TDF_CFG" # VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' npm run dev @@ -103,46 +103,10 @@ _init_platform() { echo "[ERROR] unable to cd ${APP_DIR}" exit 2 fi - svc=github.com/opentdf/platform/service@latest - if [ -f go.work ]; then - svc=github.com/opentdf/platform/service - fi - if ! go run "${svc}" provision keycloak -f "${APP_DIR}/keycloak_data.yaml"; then - echo "[ERROR] unable to provision keycloak" - return 1 - fi if ! ./config-demo-idp.sh; then echo "[ERROR] unable to provision keycloak" return 1 fi - if ! ./init-temp-keys.sh; then - echo "[ERROR] unable to initialize keys" - return 1 - fi - go run "${svc}" start &>"$output" & - server_pid=$! - echo "Platform pid: $server_pid" - echo "Output: $output" - echo "Wait:" - limit=5 - for i in $(seq 1 $limit); do - if grep -q -i 'starting http server' "$output"; then - return 0 - fi - if ! ps $server_pid >/dev/null; then - echo "The server died" >&2 - cat "${output}" - exit 1 - fi - if [[ $i == "$limit" ]]; then - echo "[WARN] Breaking _init_platform loop after ${limit} iterations" - cat "${output}" - break - fi - sleep_for=$((5 + i * i * 2)) - echo "[INFO] retrying in ${sleep_for} seconds... ( ${i} / $limit ) ..." - sleep ${sleep_for} - done } if ! _configure_app; then @@ -155,7 +119,7 @@ if ! _init_webapp; then exit 2 fi -if [ $1 = platform ]; then +if [ "$1" = platform ]; then if ! _init_platform; then echo "[ERROR] Couldn't run platform" exit 2