From 4eb42abccd25b97f689542e678aedb01a3e2446a Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Fri, 23 Feb 2024 09:53:04 -0800 Subject: [PATCH 01/26] re-enable setting annotations on images --- .github/workflows/docker-publish.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 90bee9e2..c7482662 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -84,7 +84,7 @@ jobs: id: meta uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 env: - #DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + DOCKER_METADATA_ANNOTATIONS_LEVELS: index CURRENT_DATETIME: ${{ env.CURRENT_DATETIME }} with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -119,10 +119,9 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - cache-to: type=gha,mode=max - #platforms: linux/amd64 + cache-to: type=gha,mode=max file: Dockerfile - #annotations: ${{ steps.meta.outputs.annotations }} + annotations: ${{ steps.meta.outputs.annotations }} build-args: | OWNER=nam20485 GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} From 2f94afb7d597540869fddf03d5f5c767c33b68e2 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Fri, 23 Feb 2024 09:57:17 -0800 Subject: [PATCH 02/26] minor cleanup --- .github/workflows/docker-scout-scan.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-scout-scan.yml b/.github/workflows/docker-scout-scan.yml index 206ece88..46cdbb61 100644 --- a/.github/workflows/docker-scout-scan.yml +++ b/.github/workflows/docker-scout-scan.yml @@ -66,16 +66,6 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - # # Login against GHCR Docker registry except on PR - # # https://github.com/docker/login-action - # - name: Log into GHCR registry ${{ env.REGISTRY }} - # if: github.event_name != 'pull_request' - # uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - # with: - # registry: ${{ env.REGISTRY }} - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # Login against Docker Hub to allow running Docker Scout # https://github.com/docker/login-action - name: Log into Docker Hub registry @@ -114,8 +104,8 @@ jobs: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} VCPKG_BINARY_SOURCES=clear;nuget,GitHub,readwrite - - name: List Local Images - run: docker image ls + # - name: List Local Images + # run: docker image ls # # Sign the resulting Docker image digest except on PRs. # # This will only write to the public Rekor transparency log when the Docker @@ -159,3 +149,4 @@ jobs: only-severities: critical,high write-comment: true github-token: ${{ secrets.GITHUB_TOKEN }} # to be able to write the comment + \ No newline at end of file From 9b1be41a50e19f31fc7f70f051933d182fb256bf Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Sat, 24 Feb 2024 16:21:37 -0800 Subject: [PATCH 03/26] add odbdesign-server-request-secret files --- .../OdbDesignServer/odbdesign-server-request-secret.ps1 | 1 + .../OdbDesignServer/odbdesign-server-request-secret.yaml | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 create mode 100644 deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml diff --git a/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 b/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 new file mode 100644 index 00000000..7ab764ad --- /dev/null +++ b/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 @@ -0,0 +1 @@ +kubectl create secret generic 'odbdesign-server-request-secret' --from-literal=ODBDESIGN_SERVER_REQUEST_USERNAME=$env:ODBDESIGN_SERVER_REQUEST_USERNAME --from-literal=ODBDESIGN_SERVER_REQUEST_PASSWORD=$env:ODBDESIGN_SERVER_REQUEST_PASSWORD \ No newline at end of file diff --git a/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml b/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml new file mode 100644 index 00000000..0588f059 --- /dev/null +++ b/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: odbdesign-server-request-secret +type: Opaque +data: + ODBDESIGN_SERVER_REQUEST_USERNAME: + ODBDESIGN_SERVER_REQUEST_PASSWORD: \ No newline at end of file From f206bee9a0ab00ab3088332614a9e1253363f8be Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Sat, 24 Feb 2024 16:22:01 -0800 Subject: [PATCH 04/26] remove ods.local from ingress --- deploy/kube/local-ingress.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/deploy/kube/local-ingress.yaml b/deploy/kube/local-ingress.yaml index fa1b4d14..c0e59d4e 100644 --- a/deploy/kube/local-ingress.yaml +++ b/deploy/kube/local-ingress.yaml @@ -3,17 +3,7 @@ kind: Ingress metadata: name: odbdesign-server-ingress spec: - rules: - - host: ods.local - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: odbdesign-server-service - port: - name: ods-svc-port + rules: - host: precision5820 http: paths: From 97837c73a1b8df75052f949d0fd08b48e1e123dd Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Sat, 24 Feb 2024 16:22:17 -0800 Subject: [PATCH 05/26] add healthcheck endpoints and another host --- swagger/odbdesign-server-0.9-swagger.yaml | 38 ++++++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/swagger/odbdesign-server-0.9-swagger.yaml b/swagger/odbdesign-server-0.9-swagger.yaml index 30e40eb7..e403b8d9 100644 --- a/swagger/odbdesign-server-0.9-swagger.yaml +++ b/swagger/odbdesign-server-0.9-swagger.yaml @@ -11,11 +11,12 @@ info: servers: - url: http://default-ingress-1165108808.us-west-2.elb.amazonaws.com - url: http://localhost:8888 + - url: http://precision5820:8081 tags: - name: "file upload" - name: "filemodel" #- name: "hello world" - #- name: "health check" + - name: "health check" - name: "steps" - name: "layers" - name: "symbols" @@ -564,15 +565,36 @@ paths: description: "" security: - BasicAuth: [] + /healthz/live: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" + /healthz/started: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" + /healthz/ready: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" components: securitySchemes: BasicAuth: type: http scheme: basic - schemas: - odbdesign-server-schema: - type: object - properties: - id: - type: integer - format: int64 + # schemas: + # odbdesign-server-schema: + # type: object + # properties: + # id: + # type: integer + # format: int64 From 8b2aa8ce0f5b56e6ffcd7c3129a60d38c59526dc Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Sat, 24 Feb 2024 16:44:20 -0800 Subject: [PATCH 06/26] add context type = "text/plain" to health check endpoint responses --- OdbDesignServer/Controllers/HealthCheckController.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OdbDesignServer/Controllers/HealthCheckController.cpp b/OdbDesignServer/Controllers/HealthCheckController.cpp index 4ba60b26..4881344b 100644 --- a/OdbDesignServer/Controllers/HealthCheckController.cpp +++ b/OdbDesignServer/Controllers/HealthCheckController.cpp @@ -26,16 +26,16 @@ namespace Odb::App::Server crow::response HealthCheckController::health_check_live(const crow::request& req) { - return crow::response(crow::status::OK, "healthy: live"); + return crow::response(crow::status::OK, "text/plain", "healthy: live"); } crow::response HealthCheckController::health_check_ready(const crow::request& req) { - return crow::response(crow::status::OK, "healthy: ready"); + return crow::response(crow::status::OK, "text/plain", "healthy: ready"); } crow::response HealthCheckController::health_check_started(const crow::request& req) { - return crow::response(crow::status::OK, "healthy: started"); + return crow::response(crow::status::OK, "text/plain", "healthy: started"); } } From 41698bb92e741f6cea9d4024ea69773aa4fa48f4 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Sun, 25 Feb 2024 10:40:07 -0800 Subject: [PATCH 07/26] fix exception setting content-type for healthcheck endpoint responses --- OdbDesignServer/Controllers/HealthCheckController.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/OdbDesignServer/Controllers/HealthCheckController.cpp b/OdbDesignServer/Controllers/HealthCheckController.cpp index 4881344b..875ca269 100644 --- a/OdbDesignServer/Controllers/HealthCheckController.cpp +++ b/OdbDesignServer/Controllers/HealthCheckController.cpp @@ -26,16 +26,16 @@ namespace Odb::App::Server crow::response HealthCheckController::health_check_live(const crow::request& req) { - return crow::response(crow::status::OK, "text/plain", "healthy: live"); + return crow::response(crow::status::OK, "txt", "healthy: live"); } crow::response HealthCheckController::health_check_ready(const crow::request& req) { - return crow::response(crow::status::OK, "text/plain", "healthy: ready"); + return crow::response(crow::status::OK, "txt", "healthy: ready"); } crow::response HealthCheckController::health_check_started(const crow::request& req) - { - return crow::response(crow::status::OK, "text/plain", "healthy: started"); + { + return crow::response(crow::status::OK, "txt", "healthy: started"); } } From ec0e1b83d976a5f5a1382896ac6b0aafa2e08ca9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 06:50:46 +0000 Subject: [PATCH 08/26] Bump docker/setup-buildx-action Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 to 0d103c3126aa41d772a8362f6aa67afac040f80c. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/edfb0fe6204400c56fbfd3feba3fe9ad1adfa345...0d103c3126aa41d772a8362f6aa67afac040f80c) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-publish.yml | 2 +- .github/workflows/docker-scout-scan.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 90bee9e2..8da385d6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -56,7 +56,7 @@ jobs: # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # Login against a Docker registry except on PR # https://github.com/docker/login-action diff --git a/.github/workflows/docker-scout-scan.yml b/.github/workflows/docker-scout-scan.yml index 206ece88..c774671b 100644 --- a/.github/workflows/docker-scout-scan.yml +++ b/.github/workflows/docker-scout-scan.yml @@ -64,7 +64,7 @@ jobs: # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # # Login against GHCR Docker registry except on PR # # https://github.com/docker/login-action From 291c2da36bd17e6679ad4f69b61059c4a7a5df35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 06:50:51 +0000 Subject: [PATCH 09/26] Bump github/codeql-action from 3.24.3 to 3.24.5 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.3 to 3.24.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/379614612a29c9e28f31f39a59013eb8012a51f0...47b3d888fe66b639e431abf22ebca059152f1eea) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/docker-scout-scan.yml | 2 +- .github/workflows/scorecard.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b4c0d1ef..c7c26d68 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -52,7 +52,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: languages: ${{ matrix.language }} config-file: ${{ github.workspace }}/.github/codeql-config.yml @@ -83,6 +83,6 @@ jobs: run: cmake --build --preset linux-release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docker-scout-scan.yml b/.github/workflows/docker-scout-scan.yml index 206ece88..dcc7e5d8 100644 --- a/.github/workflows/docker-scout-scan.yml +++ b/.github/workflows/docker-scout-scan.yml @@ -143,7 +143,7 @@ jobs: - name: Upload SARIF result id: upload-sarif - uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: sarif_file: sarif.output.json diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e99065c0..f2af71e4 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -75,6 +75,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: sarif_file: results.sarif From 026ddb326e789c94d34f155d0c8a64fef5b0bb42 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:24:07 -0800 Subject: [PATCH 10/26] add volume when creating cluster --- scripts/create-k3d-cluster.ps1 | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/create-k3d-cluster.ps1 b/scripts/create-k3d-cluster.ps1 index e0fea396..e988133c 100644 --- a/scripts/create-k3d-cluster.ps1 +++ b/scripts/create-k3d-cluster.ps1 @@ -11,7 +11,11 @@ param( # When set to true, the cluster will be deleted first [switch]$DeleteClusterFirst = $false, # When set to true, the cluster will be deleted without asking for confirmation - [switch]$ForceDelete = $false + [switch]$ForceDelete = $false, + # Host Volume Path for PersistentVolume + [Parameter(Mandatory=$true)] + [string]$HostVolumePath = "D:/k3dvolume" + ) # $clusterName="k3dcluster" @@ -46,6 +50,9 @@ k3d cluster create $ClusterName ` --agents $NumAgents ` --k3s-arg="--tls-san=${hostIp}@server:0" ` --k3s-arg="--tls-san=precision5820@server:0" ` - -p "${IngressHostPort}:80@loadbalancer" + --port "${IngressHostPort}:80@loadbalancer" ` + --port "8443:443@loadbalancer" ` + --volume ${HostVolumePath}:/k3dvolume@all + #--volume ${HostVolumePath}:/tmp/k3dvolume@all Write-Host "Cluster '$ClusterName' created." From 77385bfaed570736fabd349569cfe545fa2ae84b Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:24:33 -0800 Subject: [PATCH 11/26] move to scripts dir --- .../odbdesign-server-request-secret.ps1 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {deploy/kube/OdbDesignServer => scripts}/odbdesign-server-request-secret.ps1 (100%) diff --git a/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 b/scripts/odbdesign-server-request-secret.ps1 similarity index 100% rename from deploy/kube/OdbDesignServer/odbdesign-server-request-secret.ps1 rename to scripts/odbdesign-server-request-secret.ps1 From fc35cbf08bf2770ed1df9d3c883726dce02e456b Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:24:53 -0800 Subject: [PATCH 12/26] remove unused file --- .../OdbDesignServer/odbdesign-server-request-secret.yaml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml diff --git a/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml b/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml deleted file mode 100644 index 0588f059..00000000 --- a/deploy/kube/OdbDesignServer/odbdesign-server-request-secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: odbdesign-server-request-secret -type: Opaque -data: - ODBDESIGN_SERVER_REQUEST_USERNAME: - ODBDESIGN_SERVER_REQUEST_PASSWORD: \ No newline at end of file From ce5d75618862f2ccd0dd8b3ff469775aaee9ced0 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:26:10 -0800 Subject: [PATCH 13/26] add persistent volume manifests --- deploy/kube/k3d-volume-pv.yaml | 15 +++++++++++++++ deploy/kube/k3d-volume-pvc.yaml | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 deploy/kube/k3d-volume-pv.yaml create mode 100644 deploy/kube/k3d-volume-pvc.yaml diff --git a/deploy/kube/k3d-volume-pv.yaml b/deploy/kube/k3d-volume-pv.yaml new file mode 100644 index 00000000..00b8cb2d --- /dev/null +++ b/deploy/kube/k3d-volume-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: k3d-volume + #name: task-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 1Ti + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/d/k3dvolume \ No newline at end of file diff --git a/deploy/kube/k3d-volume-pvc.yaml b/deploy/kube/k3d-volume-pvc.yaml new file mode 100644 index 00000000..853fa124 --- /dev/null +++ b/deploy/kube/k3d-volume-pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + #name: task-pv-claim + name: k3d-volume-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi From 941ef5ad96210a5e37de0a317aaf36a98a7a6a68 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:26:42 -0800 Subject: [PATCH 14/26] apply persistent volume manifests in deploy script --- scripts/deploy.ps1 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index 28d3e592..8a2bb299 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -14,10 +14,21 @@ if ($LASTEXITCODE -ne 0) { } # -# odbdesign-server +# Common (pre) # -# apply manifests +# persistent volume +kubectl apply -f deploy/kube/k3d-volume-pv.yaml +kubectl apply -f deploy/kube/k3d-volume-pvc.yaml + +# +# OdbDesignServer +# + +# secrets +& "$PSScriptRoot\odbdesign-server-request-secret.ps1" + +# apply deployment/service manifests kubectl apply -f deploy/kube/OdbDesignServer/deployment.yaml kubectl apply -f deploy/kube/OdbDesignServer/service.yaml @@ -30,7 +41,7 @@ kubectl rollout status deployment/$DeploymentName # Swagger UI # -# apply manifests +# apply deployment/service manifests kubectl apply -f deploy/kube/OdbDesignServer-SwaggerUI/deployment.yaml kubectl apply -f deploy/kube/OdbDesignServer-SwaggerUI/service.yaml @@ -39,7 +50,7 @@ kubectl rollout restart deployment/odbdesign-server-swaggerui-v1 kubectl rollout status deployment/odbdesign-server-swaggerui-v1 # -# common +# Common (post) # # apply ingress manifest From fe3bf75911e9f8f0c1cc4ec2184eeeb767ff5e49 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 13:28:30 -0800 Subject: [PATCH 15/26] add persistent volume mount to OdbDesignServer deployment --- deploy/kube/OdbDesignServer/deployment.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/deploy/kube/OdbDesignServer/deployment.yaml b/deploy/kube/OdbDesignServer/deployment.yaml index e6cb55cf..5615e269 100644 --- a/deploy/kube/OdbDesignServer/deployment.yaml +++ b/deploy/kube/OdbDesignServer/deployment.yaml @@ -19,7 +19,20 @@ spec: labels: app: odbdesign-server version: v1 - spec: + spec: + volumes: + - name: odbdesign-server-storage + persistentVolumeClaim: + claimName: k3d-volume-claim + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: NotIn + values: + - "true" containers: - name: odbdesign-server image: ghcr.io/nam20485/odbdesign:nam20485-latest @@ -27,6 +40,9 @@ spec: ports: - containerPort: 8888 name: ods-dep-port + volumeMounts: + - mountPath: /OdbDesign/data + name: odbdesign-server-storage env: - name: ODBDESIGN_SERVER_REQUEST_USERNAME valueFrom: @@ -64,4 +80,3 @@ spec: failureThreshold: 10 periodSeconds: 1 #initialDelaySeconds: 1 - From f1c1d5ffbd8593c9b00098877fa271ea4f8fe8d3 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 15:28:47 -0800 Subject: [PATCH 16/26] mount volume for designs --- compose.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compose.yml b/compose.yml index 0d1165e5..8fc03db3 100644 --- a/compose.yml +++ b/compose.yml @@ -1,8 +1,8 @@ -name: odbdesignserver-swaggerui +name: odbdesign services: - odbdesign-server: + server: ## enable for passing in branch name as an environment variable #environment: # - BRANCH=nam20485 @@ -12,7 +12,9 @@ services: # build: # context: . # dockerfile: Dockerfile - container_name: odbdesign-server + #container_name: odbdesign-server + volumes: + - ./compose-designs:/OdbDesign/designs ports: - 8888:8888 environment: @@ -26,9 +28,9 @@ services: # - BRANCH=nam20485 # image: ghcr.io/nam20485/odbdesignserver-swaggerui:${BRANCH}-latest image: ghcr.io/nam20485/odbdesignserver-swaggerui:nam20485-latest - container_name: swagger-ui + #container_name: swagger-ui depends_on: - - odbdesign-server + - server ports: - 8080:8080 \ No newline at end of file From ebfc802485b50e9232a97a88c679fca136a3f2f6 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 15:29:51 -0800 Subject: [PATCH 17/26] don't create designs dir; add comment about requiring mounting the volume --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70093daa..b2073a4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,11 +90,12 @@ COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so . COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests . # copy templates directory -RUN mkdir ./templates +RUN mkdir -p ./templates COPY --from=build /src/OdbDesign/OdbDesignServer/templates/* ./templates # create designs directory -RUN mkdir ./designs +# required to be voluume mounted! +#RUN mkdir ./designs # run ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/OdbDesign/bin From 90710eae8c939eddc7e7d3b56b2758ce06706baa Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Mon, 26 Feb 2024 15:30:15 -0800 Subject: [PATCH 18/26] fix logic error in multipart/form-data multiple file upload --- OdbDesignServer/Controllers/FileUploadController.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/OdbDesignServer/Controllers/FileUploadController.cpp b/OdbDesignServer/Controllers/FileUploadController.cpp index 1d97cbe7..4f263cde 100644 --- a/OdbDesignServer/Controllers/FileUploadController.cpp +++ b/OdbDesignServer/Controllers/FileUploadController.cpp @@ -124,11 +124,8 @@ namespace Odb::App::Server { // log to debug and skip rest of the loop CROW_LOG_DEBUG << " Value: " << part_value.body << '\n'; - continue; - } - else - { CROW_LOG_ERROR << "multipart/form-data POST failed! Part name was: [" << part_name << "], which is not supported. Part name should be [" << MULTIPART_FORMDATA_PART_NAME << "]."; + continue; } // Extract the file name From 873b590c0887c9c58376fc7110adb10761016622 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Wed, 28 Feb 2024 08:10:32 -0800 Subject: [PATCH 19/26] increase resources --- deploy/kube/OdbDesignServer/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/kube/OdbDesignServer/deployment.yaml b/deploy/kube/OdbDesignServer/deployment.yaml index 5615e269..268b4988 100644 --- a/deploy/kube/OdbDesignServer/deployment.yaml +++ b/deploy/kube/OdbDesignServer/deployment.yaml @@ -56,11 +56,11 @@ spec: key: ODBDESIGN_SERVER_REQUEST_PASSWORD resources: limits: + cpu: "2" + memory: 2Gi + requests: cpu: "1" memory: 1Gi - requests: - cpu: 500m - memory: 500Mi livenessProbe: httpGet: path: /healthz/live From ee73a1522fcfd8156dfb718c213871f7080587d3 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Wed, 28 Feb 2024 17:50:48 -0800 Subject: [PATCH 20/26] adjust paths in Dockerfile --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2073a4e..6fae52b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,26 +80,26 @@ LABEL org.opencontainers.image.source=https://github.com/nam20485/OdbDesign \ EXPOSE 8888 RUN mkdir --parents /OdbDesign/bin -WORKDIR /OdbDesign/bin +WORKDIR /OdbDesign # copy binaries -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests . +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests ./bin/ # copy templates directory RUN mkdir -p ./templates COPY --from=build /src/OdbDesign/OdbDesignServer/templates/* ./templates # create designs directory -# required to be voluume mounted! +# required to be volume mounted! #RUN mkdir ./designs # run ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/OdbDesign/bin # ENV ODBDESIGN_SERVER_REQUEST_USERNAME=${ODBDESIGN_SERVER_REQUEST_USERNAME} # ENV ODBDESIGN_SERVER_REQUEST_PASSWORD=${ODBDESIGN_SERVER_REQUEST_PASSWORD} -RUN chmod +x ./OdbDesignServer -ENTRYPOINT [ "./OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ] +RUN chmod +x ./bin/OdbDesignServer +ENTRYPOINT [ "./bin/OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ] From c1e18dbf98cb31bdf47cd365658b73533aae95ab Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 08:41:29 -0800 Subject: [PATCH 21/26] ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8326ea80..b003758e 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ OdbDesignServer/PyOdbDesignLib/_PyOdbDesignLib.pyd deploy/kubeconfig Dockerfile.commentedOut scripts/create-release-invoker.js +compose-deisgns/ From 04f45db3d30671d4d6483e92ab25e14247d857ed Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 08:45:50 -0800 Subject: [PATCH 22/26] fix volume mount path in deployment --- deploy/kube/OdbDesignServer/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kube/OdbDesignServer/deployment.yaml b/deploy/kube/OdbDesignServer/deployment.yaml index 268b4988..08a2c691 100644 --- a/deploy/kube/OdbDesignServer/deployment.yaml +++ b/deploy/kube/OdbDesignServer/deployment.yaml @@ -41,7 +41,7 @@ spec: - containerPort: 8888 name: ods-dep-port volumeMounts: - - mountPath: /OdbDesign/data + - mountPath: /OdbDesign/designs name: odbdesign-server-storage env: - name: ODBDESIGN_SERVER_REQUEST_USERNAME From 4d24082a3cf4b44604339d263e29592a64aa6ff2 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 09:53:15 -0800 Subject: [PATCH 23/26] add fastcopy functions for trying to rename before a regular copy --- Utils/CMakeLists.txt | 2 +- Utils/fastcopy.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++ Utils/fastcopy.h | 14 +++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 Utils/fastcopy.cpp create mode 100644 Utils/fastcopy.h diff --git a/Utils/CMakeLists.txt b/Utils/CMakeLists.txt index 64987dbe..04432a2d 100644 --- a/Utils/CMakeLists.txt +++ b/Utils/CMakeLists.txt @@ -1,7 +1,7 @@ # CMakeList.txt : CMake project for OdbDesignServer # -add_library(Utils SHARED "utils_export.h" "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h" "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h") +add_library(Utils SHARED "utils_export.h" "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h" "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h" "fastcopy.h" "fastcopy.cpp") # state that anybody linking to us needs to include the current source dir, # while we don't. diff --git a/Utils/fastcopy.cpp b/Utils/fastcopy.cpp new file mode 100644 index 00000000..974639fd --- /dev/null +++ b/Utils/fastcopy.cpp @@ -0,0 +1,59 @@ +#include "fastcopy.h" + +using namespace std; +using namespace std::filesystem; + +namespace Utils +{ + error_code copy(const path& source, const path& dest, bool overwriteExisting) + { + error_code ec; + + auto options = copy_options::none; + if (overwriteExisting) + { + options = copy_options::overwrite_existing; + } + + if (copy_file(source, dest, options, ec)) + { + remove(source, ec); + } + + return ec; + } + + error_code copy(const string& source, const string& dest, bool overwriteExisting) + { + return copy(path(source), path(dest), overwriteExisting); + } + + error_code fastcopy(const string& source, const string& dest, bool overwriteExisting) + { + return fastcopy(path(source), path(dest), overwriteExisting); + } + + error_code fastcopy(const path& source, const path& dest, bool overwriteExisting) + { + error_code ec; + + try + { + rename(source, dest, ec); + } + catch (filesystem_error& fe) + { + // can't rename across devices- try standard copy and remove + if (fe.code() == std::errc::cross_device_link) + { + ec = copy(source, dest, overwriteExisting); + } + else + { + throw fe; + } + } + + return ec; + } +} \ No newline at end of file diff --git a/Utils/fastcopy.h b/Utils/fastcopy.h new file mode 100644 index 00000000..05d2a373 --- /dev/null +++ b/Utils/fastcopy.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include "utils_export.h" + +namespace Utils +{ + UTILS_EXPORT std::error_code copy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting); + UTILS_EXPORT std::error_code copy(const std::string& source, const std::string& dest, bool overwriteExisting); + + UTILS_EXPORT std::error_code fastcopy(const std::string& source, const std::string& dest, bool overwriteExisting); + UTILS_EXPORT std::error_code fastcopy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting); +} From 6e1989c93ac0d241909b8cd163f8640867e5a3ca Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 09:53:26 -0800 Subject: [PATCH 24/26] ignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b003758e..1d795bf2 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,4 @@ OdbDesignServer/PyOdbDesignLib/_PyOdbDesignLib.pyd deploy/kubeconfig Dockerfile.commentedOut scripts/create-release-invoker.js -compose-deisgns/ +/compose-designs From f9ce8f648f8737673da39b7034729d59e76f767e Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 09:53:45 -0800 Subject: [PATCH 25/26] use new fastcopy function for file upload endpoints --- OdbDesignServer/Controllers/FileUploadController.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/OdbDesignServer/Controllers/FileUploadController.cpp b/OdbDesignServer/Controllers/FileUploadController.cpp index 4f263cde..d2b133b7 100644 --- a/OdbDesignServer/Controllers/FileUploadController.cpp +++ b/OdbDesignServer/Controllers/FileUploadController.cpp @@ -1,7 +1,9 @@ #include "FileUploadController.h" +#include "fastcopy.h" using namespace std::filesystem; using namespace Odb::Lib::App; +using namespace Utils; namespace Odb::App::Server { @@ -104,7 +106,8 @@ namespace Odb::App::Server path finalPath(m_serverApp.args().designsDir()); finalPath /= safeName; - rename(tempPath, finalPath); + //rename(tempPath, finalPath); + auto ec = fastcopy(tempPath, finalPath, false); std::string responseBody = "{ \"filename\": \"" + safeName + "\" }"; @@ -170,7 +173,8 @@ namespace Odb::App::Server auto safeName = sanitizeFilename(outfile_name); path finalPath(m_serverApp.args().designsDir()); finalPath /= safeName; - rename(tempPath, finalPath); + //rename(tempPath, finalPath); + auto ec = fastcopy(tempPath, finalPath, false); CROW_LOG_INFO << " Contents written to " << outfile_name << '\n'; } From 75ce7f3c3c5493579ba8660f38ca82e5757bf770 Mon Sep 17 00:00:00 2001 From: Nathan Miller Date: Thu, 29 Feb 2024 14:22:37 -0800 Subject: [PATCH 26/26] use exception-throwing version of std::rename to fix not resorting to regular copy & delete on failure --- Utils/fastcopy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utils/fastcopy.cpp b/Utils/fastcopy.cpp index 974639fd..f6fab078 100644 --- a/Utils/fastcopy.cpp +++ b/Utils/fastcopy.cpp @@ -39,7 +39,7 @@ namespace Utils try { - rename(source, dest, ec); + rename(source, dest); } catch (filesystem_error& fe) {