diff --git a/.DS_Store b/.DS_Store index 0f4e923f..2a2dc9a3 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/linters/.golangci.yml b/.github/linters/.golangci.yml new file mode 100644 index 00000000..341e3cdb --- /dev/null +++ b/.github/linters/.golangci.yml @@ -0,0 +1,43 @@ +--- +######################### +######################### +## Golang Linter rules ## +######################### +######################### + +# configure golangci-lint +# see https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml +issues: + exclude-dirs: + - ci + exclude-rules: + - path: _test\.go + linters: + - dupl + - gosec + - goconst +linters: + enable: + - gosec + - unconvert + - gocyclo + - goconst + - goimports + - gocritic + - govet + - revive +linters-settings: + errcheck: + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: true + govet: + enable: + # report about shadowed variables + - shadowing + gocyclo: + # minimal code complexity to report, 30 by default + min-complexity: 15 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 89e8bb46..058fda16 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -5,8 +5,10 @@ on: tags: - "*" jobs: - virtual-kubelet: + core-containers: runs-on: ubuntu-latest + env: + GH_REPO_OWNER: ${{ github.repository_owner }} #env: # DOCKER_TARGET_PLATFORM: linux/arm64 steps: @@ -23,21 +25,33 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GH_CR_TOKEN }} - name: Get Repo Owner id: get_repo_owner run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') - - name: Build container base image + - name: Build container base image vk uses: docker/build-push-action@v5 with: context: ./ outputs: "type=registry,push=true" tags: | - ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw:${{ env.RELEASE_VERSION }} - ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw:latest + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw:${{ env.RELEASE_VERSION }} + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw:latest file: ./docker/Dockerfile.vk platforms: linux/amd64, linux/arm64 - + build-args: + - VERSION=${{ env.RELEASE_VERSION }} + - name: Build container base image interlink + uses: docker/build-push-action@v5 + with: + context: ./ + outputs: "type=registry,push=true" + tags: | + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/interlink:${{ env.RELEASE_VERSION }} + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/interlink:latest + file: ./docker/Dockerfile.interlink + platforms: linux/amd64, linux/arm64 + virtual-kubelet-refrest-token: runs-on: ubuntu-latest #env: @@ -56,7 +70,7 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GH_CR_TOKEN }} - name: Get Repo Owner id: get_repo_owner run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') @@ -66,7 +80,7 @@ jobs: context: ./ outputs: "type=registry,push=true" tags: | - ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw-refresh:${{ env.RELEASE_VERSION }} - ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/virtual-kubelet-inttw-refresh:latest + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw-refresh:${{ env.RELEASE_VERSION }} + ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw-refresh:latest file: ./docker/Dockerfile.refresh-token platforms: linux/amd64, linux/arm64 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..a0e42364 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,32 @@ +name: integration-tests + +on: + push: + #branches: [main,next,next2next] + pull_request: + +jobs: + build: + name: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + #password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GH_CR_TOKEN }} + - name: Get Repo Owner + id: get_repo_owner + run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') + - name: Integration Test + uses: dagger/dagger-for-github@v5 + with: + workdir: ci + verb: call + args: -s build-images --source-folder ../ --virtual-kubelet-ref ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw:$GITHUB_SHA --interlink-ref ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/interlink:$GITHUB_SHA new-interlink --manifests $PWD/manifests load-plugin test stdout + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + #dagger-flags: -d diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 55d427c5..d2441dc4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,6 +24,7 @@ jobs: - name: Run Super-Linter on new changes uses: docker://ghcr.io/github/super-linter:slim-v4 env: + FILTER_REGEX_EXCLUDE: ci/.* DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MARKDOWN_CONFIG_FILE: .markdownlint.json diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7667d0a1..f25ebe14 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -7,6 +7,8 @@ before: builds: - id: "virtual-kubelet" binary: virtual-kubelet + hooks: + pre: bash -c "KUBELET_VERSION={{.Version}} ./cmd/virtual-kubelet/set-version.sh" env: - CGO_ENABLED=0 goos: diff --git a/ci/.DS_Store b/ci/.DS_Store new file mode 100644 index 00000000..554d649c Binary files /dev/null and b/ci/.DS_Store differ diff --git a/ci/.gitattributes b/ci/.gitattributes new file mode 100644 index 00000000..3a454933 --- /dev/null +++ b/ci/.gitattributes @@ -0,0 +1,4 @@ +/dagger.gen.go linguist-generated +/internal/dagger/** linguist-generated +/internal/querybuilder/** linguist-generated +/internal/telemetry/** linguist-generated diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 00000000..1d703bb5 --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1,5 @@ +/dagger.gen.go +/internal/dagger +/internal/querybuilder +/internal/telemetry +kubeconfig.yaml diff --git a/ci/LICENSE b/ci/LICENSE new file mode 100644 index 00000000..78ae443a --- /dev/null +++ b/ci/LICENSE @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [2024] [INFN] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/ci/README.md b/ci/README.md index 8faba732..25ff428a 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,12 +1,135 @@ +# INTERLINK E2E INTEGRATION TESTS + +Here you can find how to test a virtual kubelet implementation against the main pod use cases we mean to support. + +## Requirements + +- [Docker engine](https://docs.docker.com/engine/install/) +- [Dagger CLI](https://docs.dagger.io/install/) + +## What's in the Dagger module + +- E2e integration tests: a reproducible test environment (selfcontained in Dagger runtime). Run the very same tests executed by github actions to validate any PR +- A development setup tool: optionally you can use your k8s cluster of choice to run and install different interlink components via this module. + +:warning: by default the docker plugin is the one tested and to be referred to for any change as first thing. + +## Usage + +The whole test suite is based on the application of k8s manifests inside a folder that must be passed at runtime. In `./ci/manifests` of this repo you can find the one executed by default by the github actions. + +That means you can test your code **before** any commit, discovering in advance if anything is breaking. + +### Run e2e tests + +#### Edit manifests with your images + +- `service-account.yaml` is the default set of permission needed by the virtualkubelet. Do not touch unless you know what you are doing. +- `virtual-kubelet-config.yaml` is the configuration mounted into the __virtual kubelet__ component to determine its behaviour. +- `virtual-kubelet.yaml` is the one that you should touch if you are pointing to different interlink endpoints or if you want to change the __virtual kubelet__ image to be tested. +- `interlink-config.yaml` is the configuration mounted into the __interlink API__ component to determine its behaviour. +- `interlink.yaml` is the one that you should touch if you are pointing to different plugin endpoints or if you want to change the __interlink API__ image to be tested. +- `plugin-config.yaml` is the configuration mounted into the __interLink plugin__ component to determine its behaviour. +- `plugin.yaml` is the one that you should touch if you are pointing to different plugin endpoints or if you want to change the __interlink plugin__ image to be tested. + + +#### Run the tests + +First of all, in `ci/manifests/vktest_config.yaml` you will find the pytest configuration file. Please see the [test documentation](https://github.com/interTwin-eu/vk-test-set/tree/main) for understanding how to tweak it. + +The following instructions are thought for building docker images of the virtual-kubelet and interlink api server components at runtime and published on `virtual-kubelet-ref` and `interlink-ref` repositories (in this example it will be dockerHUB repository of the dciangot user). +It basically consists on a chain of Dagger tasks for building core images (`build-images`), creating the kubernetes environment configured with core components (`new-interlink`), installing the plugin of choice indicated in the `manifest` folder (`load-plugin`), and eventually the execution of the tests (`test`) + +To run the default tests you can move to `ci` folder and execute the Dagger pipeline with: ```bash -docker run --rm -it \ - -v $(pwd)/engine.toml:/etc/dagger/engine.toml \ - --privileged \ - -p 1234:1234 \ - registry.dagger.io/engine:v0.8.1 +export YOUR_DOCKERHUB_USER=dciangot + +dagger call build-images --source-folder ../ --virtual-kubelet-ref ${YOUR_DOCKERHUB_USER}/vk --interlink-ref ${YOUR_DOCKERHUB_USER}/interlink \ + new-interlink --manifests $PWD/manifests \ + load-plugin \ + test stdout +``` + +:warning: by default the docker plugin is the one tested and to be referred to for any change as first thing. + +In case of success the output should print something like the following: + +```text +cachedir: .pytest_cache +rootdir: /opt/vk-test-set +configfile: pyproject.toml +collecting ... collected 12 items / 1 deselected / 11 selected + +vktestset/basic_test.py::test_namespace_exists[default] PASSED [ 9%] +vktestset/basic_test.py::test_namespace_exists[kube-system] PASSED [ 18%] +vktestset/basic_test.py::test_namespace_exists[interlink] PASSED [ 27%] +vktestset/basic_test.py::test_node_exists[virtual-kubelet] PASSED [ 36%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-000-hello-world.yaml] PASSED [ 45%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-010-simple-python.yaml] PASSED [ 54%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-020-python-env.yaml] PASSED [ 63%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-030-simple-shared-volume.yaml] PASSED [ 72%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-040-config-volumes.yaml] PASSED [ 81%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-050-limits.yaml] PASSED [ 90%] +vktestset/basic_test.py::test_manifest[virtual-kubelet-060-init-container.yaml] PASSED [100%] + +====================== 11 passed, 1 deselected in 41.71s ======================= ``` +#### Debug with interactive session + +In case something went wrong, you have the possibility to spawn a session inside the final step of the pipeline to debug things: + +```bash +dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ + new-interlink --manifests $PWD/manifests \ + load-plugin \ + run terminal +``` + +with this command (after some minutes) then you should be able to access a bash session doing the following commands: + +```bash +bash +source .venv/bin/activate +export KUBECONFIG=/.kube/config + +## check connectivity with k8s cluster +kubectl get pod -A + +## re-run the tests +pytest -vk 'not rclone' +``` + + +### Deploy on existing K8s cluster + +You might want to hijack the test machinery in order to have it instantiating the test environemnt on your own kubernetes cluster (e.g. to debug and develop plugins in a efficient way). We are introducing options for this purpose and it is expected to be extended even more in the future. + +If you have a kubernetes cluster **publically accessible**, you can pass your kubeconfig to the Dagger pipeline and use that instead of the internal one that is "one-shot" for the tests only. + +```bash +dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ + new-interlink --manifests $PWD/manifests --kubeconfig $PWD/kubeconfig.yaml \ + load-plugin \ + test stdout +``` + +If you have a *local* cluster (e.g. via MiniKube), you need to forward the local port of the Kubernetes API server (look inside the kubeconfig file) inside the Dagger runtime with the following: + ```bash -_EXPERIMENTAL_DAGGER_RUNNER_HOST=docker-container://$container_name +dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ + new-interlink --manifests $PWD/manifests --kubeconfig $PWD/kubeconfig.yaml --local-cluster tcp://localhost:59127 \ + load-plugin \ + test stdout ``` + +### Develop Virtual Kubelet code + +:warning: Coming soon + +### Develop Interlink API code + +:warning: Coming soon + + diff --git a/ci/dagger.json b/ci/dagger.json new file mode 100644 index 00000000..44df161b --- /dev/null +++ b/ci/dagger.json @@ -0,0 +1,6 @@ +{ + "name": "interlink", + "sdk": "go", + "source": ".", + "engineVersion": "v0.11.4" +} diff --git a/ci/engine.toml b/ci/engine.toml deleted file mode 100644 index 214701a8..00000000 --- a/ci/engine.toml +++ /dev/null @@ -1,5 +0,0 @@ -# debug = true -insecure-entitlements = ["security.insecure"] - -[registry."registry:5432"] - http = true diff --git a/ci/go.mod b/ci/go.mod index ae0f949a..8149d7ae 100644 --- a/ci/go.mod +++ b/ci/go.mod @@ -1,60 +1,36 @@ -module ci +module dagger/interlink -go 1.22.1 +go 1.22.2 require ( - dagger.io/dagger v0.11.0 - k8s.io/api v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 + github.com/99designs/gqlgen v0.17.44 + github.com/Khan/genqlient v0.7.0 + github.com/vektah/gqlparser/v2 v2.5.11 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 + go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/sync v0.7.0 + google.golang.org/grpc v1.63.2 ) require ( - github.com/99designs/gqlgen v0.17.31 // indirect - github.com/Khan/genqlient v0.6.0 // indirect - github.com/adrg/xdg v0.4.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/vektah/gqlparser/v2 v2.5.6 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect + github.com/sosodev/duration v1.2.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.16.0 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/ci/go.sum b/ci/go.sum index fcfcffab..13039de9 100644 --- a/ci/go.sum +++ b/ci/go.sum @@ -1,191 +1,71 @@ -dagger.io/dagger v0.11.0 h1:7McBEIWVt1L2cuhuTDIA4m6ZpEXbZlnNwhVBO/VnVyY= -dagger.io/dagger v0.11.0/go.mod h1:eJZr2HyDP+2bNrV2Z0RSKjH6X/gMIA52gk48NINAoEo= -github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= -github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= -github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= -github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= -github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= -github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= -github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/99designs/gqlgen v0.17.44 h1:OS2wLk/67Y+vXM75XHbwRnNYJcbuJd4OBL76RX3NQQA= +github.com/99designs/gqlgen v0.17.44/go.mod h1:UTCu3xpK2mLI5qcMNw+HKDiEL77it/1XtAjisC4sLwM= +github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= +github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/sosodev/duration v1.2.0 h1:pqK/FLSjsAADWY74SyWDCjOcd5l7H8GSnnOGEB9A1Us= +github.com/sosodev/duration v1.2.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU= -github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8= +github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa h1:RBgMaUMP+6soRkik4VoN8ojR2nex2TqZwjSSogic+eo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/ci/k8s.go b/ci/k8s.go index 73349dcf..a683d909 100644 --- a/ci/k8s.go +++ b/ci/k8s.go @@ -5,16 +5,8 @@ import ( "fmt" "strings" "time" - - "dagger.io/dagger" ) -const registries = `mirrors: - "registry:5432": - endpoint: - - "http://registry:5432" -` - // entrypoint to setup cgroup nesting since k3s only does it // when running as PID 1. This doesn't happen in Dagger given that we're using // our custom shim @@ -43,89 +35,151 @@ fi exec "$@" ` -func NewK8sInstance(ctx context.Context, client *dagger.Client) *K8sInstance { +func NewK8sInstance(ctx context.Context) *K8sInstance { return &K8sInstance{ - ctx: ctx, - client: client, - container: nil, - registry: nil, - configCache: client.CacheVolume("k3s_config"), + KContainer: nil, + Registry: nil, + ConfigCache: dag.CacheVolume("k3s_config"), + ContainersCache: dag.CacheVolume("k3s_containers"), } } type K8sInstance struct { - ctx context.Context - client *dagger.Client - container *dagger.Container - registry *dagger.Service - configCache *dagger.CacheVolume + KContainer *Container + K3s *Container + Registry *Service + ConfigCache *CacheVolume + ContainersCache *CacheVolume } -func (k *K8sInstance) start() error { - - registry := k.client.Host().Service( - []dagger.PortForward{ - { - Backend: 5432, - Frontend: 5432, - }, - }) - - // create k3s service container - k3s := k.client.Pipeline("k3s init").Container(). - From("rancher/k3s"). - WithNewFile("/usr/bin/entrypoint.sh", dagger.ContainerWithNewFileOpts{ - Contents: entrypoint, - Permissions: 0o755, - }). - WithNewFile("/etc/rancher/k3s/registries.yaml", dagger.ContainerWithNewFileOpts{ - Contents: registries, - }). - WithEntrypoint([]string{"entrypoint.sh"}). - WithServiceBinding("registry", registry). - WithMountedCache("/etc/rancher/k3s", k.configCache). - WithMountedTemp("/etc/lib/cni"). - WithMountedTemp("/var/lib/kubelet"). - WithMountedTemp("/var/lib/rancher/k3s"). - WithMountedTemp("/var/log"). - WithExec([]string{"sh", "-c", "k3s server --bind-address $(ip route | grep src | awk '{print $NF}') --disable traefik --disable metrics-server --kube-apiserver-arg \"--disable-admission-plugins=ServiceAccount\" --egress-selector-mode=disabled"}, dagger.ContainerWithExecOpts{InsecureRootCapabilities: true}). - WithExposedPort(6443) - - k.container = k.client.Container(). - From("bitnami/kubectl"). - WithMountedCache("/cache/k3s", k.configCache). - WithMountedDirectory("/tests", k.client.Host().Directory("./tests")). - WithServiceBinding("k3s", k3s.AsService()). - WithEnvVariable("CACHE", time.Now().String()). - WithUser("root"). - WithExec([]string{"cp", "/cache/k3s/k3s.yaml", "/.kube/config"}, dagger.ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"chown", "1001:0", "/.kube/config"}, dagger.ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithEntrypoint([]string{"sh", "-c"}) - - if err := k.waitForNodes(); err != nil { - return fmt.Errorf("failed to start k8s: %v", err) +func (k *K8sInstance) start( + ctx context.Context, + manifests *Directory, + // +optional + bufferVK string, + // +optional + bufferIL string, + // +optional + kubeconfig *File, + // +optional + localCluster *Service) error { + + if kubeconfig == nil { + // create k3s service container + k.K3s = dag.Pipeline("k3s init").Container(). + From("rancher/k3s"). + WithNewFile("/usr/bin/entrypoint.sh", ContainerWithNewFileOpts{ + Contents: entrypoint, + Permissions: 0o755, + }). + WithEntrypoint([]string{"entrypoint.sh"}). + WithMountedCache("/etc/rancher/k3s", k.ConfigCache). + WithMountedTemp("/etc/lib/cni"). + WithMountedCache("/etc/lib/containers", k.ContainersCache). + WithMountedTemp("/var/lib/kubelet"). + WithMountedTemp("/var/lib/rancher/k3s"). + WithMountedTemp("/var/log"). + WithExec([]string{"sh", "-c", "k3s server --bind-address $(ip route | grep src | awk '{print $NF}') --disable traefik --disable metrics-server --egress-selector-mode=disabled"}, ContainerWithExecOpts{InsecureRootCapabilities: true}). + WithExposedPort(6443) + + k.KContainer = dag.Container(). + From("bitnami/kubectl"). + WithUser("root"). + WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithMountedCache("/cache/k3s", k.ConfigCache). + WithMountedDirectory("/manifests", manifests). + WithServiceBinding("k3s", k.K3s.AsService()). + WithEnvVariable("CACHE", time.Now().String()). + WithUser("root"). + WithExec([]string{"cp", "/cache/k3s/k3s.yaml", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithEntrypoint([]string{"sh", "-c"}) + + } else if localCluster != nil { + + // k.KContainer, err = dag.Container().From("ubuntu"). + // WithServiceBinding("localhost", localCluster). + // WithMountedDirectory("/manifests", manifests). + // WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). + // WithExec([]string{"apt", "install", "-y", "curl"}, ContainerWithExecOpts{SkipEntrypoint: true}). + // WithExec([]string{"curl", "-vvv", "localhost:59127"}).Sync(k.Ctx) + // if err != nil { + // return err + // } + + fileName, _ := kubeconfig.Name(ctx) + + k.KContainer = dag.Container(). + From("bitnami/kubectl"). + WithUser("root"). + WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithDirectory("/manifests", manifests). + WithServiceBinding("minikube", localCluster). + WithEnvVariable("CACHE", time.Now().String()). + WithUser("root"). + WithFile(fmt.Sprintf("/src/%s", fileName), kubeconfig). + WithExec([]string{"cp", fmt.Sprintf("/src/%s", fileName), "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithEntrypoint([]string{"sh", "-c"}) + } else if localCluster == nil { + + fileName, _ := kubeconfig.Name(ctx) + k.KContainer = dag.Container(). + From("bitnami/kubectl"). + WithUser("root"). + WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithDirectory("/manifests", manifests). + WithEnvVariable("CACHE", time.Now().String()). + WithUser("root"). + WithFile(fmt.Sprintf("/src/%s", fileName), kubeconfig). + WithExec([]string{"cp", fmt.Sprintf("/src/%s", fileName), "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithUser("1001"). + WithEntrypoint([]string{"sh", "-c"}) + + } + + if bufferIL != "" { + k.KContainer = k.KContainer. + WithNewFile("/manifests/virtual-kubelet-merge.yaml", ContainerWithNewFileOpts{ + Contents: bufferVK, + Permissions: 0o755, + }) + } + if bufferIL != "" { + k.KContainer = k.KContainer. + WithNewFile("/manifests/interlink-merge.yaml", ContainerWithNewFileOpts{ + Contents: bufferIL, + Permissions: 0o755, + }) } return nil } -func (k *K8sInstance) kubectl(command string) (string, error) { - return k.exec("kubectl", fmt.Sprintf("kubectl %v", command)) +func (k *K8sInstance) kubectl(ctx context.Context, command string) (string, error) { + return k.exec(ctx, "kubectl", fmt.Sprintf("kubectl %v", command)) } -func (k *K8sInstance) exec(name, command string) (string, error) { - return k.container.Pipeline(name).Pipeline(command). +func (k *K8sInstance) exec(ctx context.Context, name, command string) (string, error) { + return k.KContainer.Pipeline(name).Pipeline(command). WithEnvVariable("CACHE", time.Now().String()). WithExec([]string{command}). - Stdout(k.ctx) + Stdout(ctx) } -func (k *K8sInstance) waitForNodes() (err error) { - maxRetries := 5 - retryBackoff := 15 * time.Second +func (k *K8sInstance) waitForNodes(ctx context.Context) (err error) { + maxRetries := 10 + retryBackoff := 30 * time.Second for i := 0; i < maxRetries; i++ { - time.Sleep(retryBackoff) - kubectlGetNodes, err := k.kubectl("get nodes -o wide") + kubectlGetNodes, err := k.kubectl(ctx, "get nodes -o wide") if err != nil { fmt.Println(fmt.Errorf("could not fetch nodes: %v", err)) continue @@ -134,6 +188,82 @@ func (k *K8sInstance) waitForNodes() (err error) { return nil } fmt.Println("waiting for k8s to start:", kubectlGetNodes) + time.Sleep(retryBackoff) + } + return fmt.Errorf("k8s took too long to start") +} + +func (k *K8sInstance) waitForVirtualKubelet(ctx context.Context) (err error) { + maxRetries := 10 + retryBackoff := 30 * time.Second + for i := 0; i < maxRetries; i++ { + time.Sleep(retryBackoff) + kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l nodeName=virtual-kubelet") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) + continue + } + if strings.Contains(kubectlGetPod, "1/1") || strings.Contains(kubectlGetPod, "2/2") { + return nil + } + fmt.Println("waiting for k8s to start:", kubectlGetPod) + describePod, err := k.kubectl(ctx, "logs -n interlink -l nodeName=virtual-kubelet -c inttw-vk") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) + continue + } + fmt.Println(describePod) + } return fmt.Errorf("k8s took too long to start") } + +func (k *K8sInstance) waitForInterlink(ctx context.Context) (err error) { + maxRetries := 10 + retryBackoff := 30 * time.Second + for i := 0; i < maxRetries; i++ { + time.Sleep(retryBackoff) + kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l app=interlink") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) + continue + } + if strings.Contains(kubectlGetPod, "1/1") { + return nil + } + fmt.Println("waiting for k8s to start:", kubectlGetPod) + describePod, err := k.kubectl(ctx, "logs -n interlink -l app=interlink") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) + continue + } + fmt.Println(describePod) + + } + return fmt.Errorf("interlink took too long to start") +} + +func (k *K8sInstance) waitForPlugin(ctx context.Context) (err error) { + maxRetries := 10 + retryBackoff := 30 * time.Second + for i := 0; i < maxRetries; i++ { + time.Sleep(retryBackoff) + kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l app=plugin") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) + continue + } + if strings.Contains(kubectlGetPod, "1/1") { + return nil + } + fmt.Println("waiting for k8s to start:", kubectlGetPod) + describePod, err := k.kubectl(ctx, "logs -n interlink -l app=plugin") + if err != nil { + fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) + continue + } + fmt.Println(describePod) + + } + return fmt.Errorf("plugin took too long to start") +} diff --git a/ci/main.go b/ci/main.go index 20c3a5fc..3040e2ef 100644 --- a/ci/main.go +++ b/ci/main.go @@ -1,83 +1,293 @@ package main import ( + "bytes" "context" "fmt" - "os" + "html/template" "strings" - "time" - "dagger.io/dagger" + "dagger/interlink/internal/dagger" ) -func main() { - ctx := context.Background() +var ( + interLinkPatch = ` +kind: Deployment +metadata: + name: interlink + namespace: interlink +spec: + template: + spec: + containers: + - name: interlink + image: "{{.InterLinkRef}}" - // create Dagger client - client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr)) +` + virtualKubeletPatch = ` +kind: Deployment +metadata: + name: virtual-kubelet + namespace: interlink +spec: + template: + spec: + containers: + - name: inttw-vk + image: "{{.VirtualKubeletRef}}" +` +) + +type patchSchema struct { + InterLinkRef string + VirtualKubeletRef string +} + +type Interlink struct { + K8s *K8sInstance + VirtualKubeletRef string + InterlinkRef string + Manifests *Directory + // TODO: services on NodePort? + //virtualkubelet bool + //interlink bool + //plugin bool + CleanupCluster bool +} + +func (i *Interlink) BuildImages( + ctx context.Context, + // +optional + // +default="ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw" + virtualKubeletRef string, + // +optional + // +default="ghcr.io/intertwin-eu/interlink/interlink" + interlinkRef string, + // +optional + // +default="ghcr.io/intertwin-eu/interlink/plugin-test" + pluginRef string, + sourceFolder *Directory, +) (*Interlink, error) { + + // TODO: get tag + + i.VirtualKubeletRef = virtualKubeletRef + i.InterlinkRef = interlinkRef + + workspace := dag.Container(). + WithDirectory("/src", sourceFolder). + WithWorkdir("/src"). + Directory("/src") + + vkVersionSplits := strings.Split(virtualKubeletRef, ":") + + vkVersion := vkVersionSplits[len(vkVersionSplits)-1] + if vkVersion == "" { + return nil, fmt.Errorf("no tag specified on the image for VK") + } + + modulesCache := dag.CacheVolume("go-mod-122") + + _, err := dag.Container(). + WithMountedCache("/go/pkg/mod", modulesCache). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). + Build(workspace, dagger.ContainerBuildOpts{ + Dockerfile: "docker/Dockerfile.vk", + BuildArgs: []dagger.BuildArg{ + {"VERSION", vkVersion}, + }, + }). + Publish(ctx, virtualKubeletRef) + if err != nil { + return nil, err + } + + _, err = dag.Container(). + WithMountedCache("/go/pkg/mod", modulesCache). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). + Build(workspace, dagger.ContainerBuildOpts{ + Dockerfile: "docker/Dockerfile.interlink", + }). + Publish(ctx, interlinkRef) + if err != nil { + return nil, err + } + + return i, nil +} + +func (i *Interlink) NewInterlink( + ctx context.Context, + manifests *Directory, + // +optional + kubeconfig *File, + // +optional + localCluster *Service, +) (*Interlink, error) { + + // create Kustomize patch for images to be used + patch := patchSchema{} + if i.InterlinkRef != "" && i.VirtualKubeletRef != "" { + patch = patchSchema{ + InterLinkRef: i.InterlinkRef, + VirtualKubeletRef: i.VirtualKubeletRef, + } + } else { + patch = patchSchema{ + InterLinkRef: "ghcr.io/intertwin-eu/interlink/interlink", + VirtualKubeletRef: "ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw", + } + } + + interLinkCompiler, err := template.New("interlink").Parse(interLinkPatch) + if err != nil { + return nil, err + } + + bufferIL := new(bytes.Buffer) + + err = interLinkCompiler.Execute(bufferIL, patch) + if err != nil { + return nil, err + } + + virtualKubeletCompiler, err := template.New("vk").Parse(virtualKubeletPatch) + if err != nil { + return nil, err + } + + bufferVK := new(bytes.Buffer) + + err = virtualKubeletCompiler.Execute(bufferVK, patch) if err != nil { - panic(err) + return nil, err } - defer client.Close() - k8s := NewK8sInstance(ctx, client) - if err = k8s.start(); err != nil { - panic(err) + // use the manifest folder defined in the chain and install components + + if manifests != nil { + i.Manifests = manifests } - ns, err := k8s.kubectl("create ns interlink") + fmt.Println(bufferVK.String()) + + i.K8s = NewK8sInstance(ctx) + if err := i.K8s.start(ctx, i.Manifests, bufferVK.String(), bufferIL.String(), kubeconfig, localCluster); err != nil { + return nil, err + } + + err = i.K8s.waitForNodes(ctx) if err != nil { - panic(err) + return nil, err } + + ns, _ := i.K8s.kubectl(ctx, "create ns interlink") fmt.Println(ns) - sa, err := k8s.kubectl("apply -f /tests/manifests/service-account.yaml") + sa, err := i.K8s.kubectl(ctx, "apply -f /manifests/service-account.yaml") if err != nil { - panic(err) + return nil, err } fmt.Println(sa) - vkConfig, err := k8s.kubectl("apply -f /tests/manifests/virtual-kubelet-config.yaml") + vkConfig, err := i.K8s.kubectl(ctx, "apply -k /manifests/") if err != nil { - panic(err) + return nil, err } fmt.Println(vkConfig) - vk, err := k8s.kubectl("apply -f /tests/manifests/virtual-kubelet.yaml") + return i, nil +} + +func (i *Interlink) LoadPlugin(ctx context.Context) (*Interlink, error) { + pluginConfig, err := i.K8s.kubectl(ctx, "apply -f /manifests/plugin-config.yaml") if err != nil { - panic(err) + return nil, err } - fmt.Println(vk) + fmt.Println(pluginConfig) - if err := k8s.waitForVirtualKubelet(); err != nil { - panic(err) + plugin, err := i.K8s.kubectl(ctx, "apply -f /manifests/plugin.yaml") + if err != nil { + return nil, err } + fmt.Println(plugin) - // build interlink and push - // build mock and push + return i, nil } -func (k *K8sInstance) waitForVirtualKubelet() (err error) { - maxRetries := 5 - retryBackoff := 15 * time.Second - for i := 0; i < maxRetries; i++ { - time.Sleep(retryBackoff) - kubectlGetPod, err := k.kubectl("get pod -n interlink -l nodeName=virtual-kubelet") +func (i *Interlink) Cleanup(ctx context.Context) error { + + cleanup, err := i.K8s.kubectl(ctx, "delete -f /manifests/") + if err != nil { + return err + } + fmt.Println(cleanup) + + return nil +} + +func (i *Interlink) Test( + ctx context.Context, + // +optional + manifests *Directory, + // +optional + kubeconfig *File, + // +optional + localCluster *Service, + // +optional + // +default false + cleanup bool, +) (*Container, error) { + + if manifests != nil { + i.Manifests = manifests + } + + if err := i.K8s.waitForVirtualKubelet(ctx); err != nil { + return nil, err + } + if err := i.K8s.waitForInterlink(ctx); err != nil { + return nil, err + } + if err := i.K8s.waitForPlugin(ctx); err != nil { + return nil, err + } + + result := i.K8s.KContainer. + WithWorkdir("/opt"). + WithExec([]string{"bash", "-c", "git clone https://github.com/interTwin-eu/vk-test-set.git"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/vk-test-set/vktest_config.yaml"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithWorkdir("/opt/vk-test-set"). + WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./ "}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config && pytest -vk 'not rclone' || echo OPS "}, ContainerWithExecOpts{SkipEntrypoint: true}) + + if i.CleanupCluster { + err := i.Cleanup(ctx) if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) - continue + return nil, err } - if strings.Contains(kubectlGetPod, "2/2") { - return nil - } - fmt.Println("waiting for k8s to start:", kubectlGetPod) - describePod, err := k.kubectl("describe pod -n interlink -l nodeName=virtual-kubelet") + } + + return result, nil + +} + +func (i *Interlink) Run( + ctx context.Context, +) (*Container, error) { + + if i.CleanupCluster { + err := i.Cleanup(ctx) if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) - continue + return nil, err } - fmt.Println(describePod) - } - return fmt.Errorf("k8s took too long to start") + + return i.K8s.KContainer. + WithWorkdir("/opt"). + WithExec([]string{"bash", "-c", "git clone https://github.com/interTwin-eu/vk-test-set.git"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/vk-test-set/vktest_config.yaml"}, ContainerWithExecOpts{SkipEntrypoint: true}). + WithWorkdir("/opt/vk-test-set"). + WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./ "}, ContainerWithExecOpts{SkipEntrypoint: true}), nil + } diff --git a/ci/manifests/interlink-config.yaml b/ci/manifests/interlink-config.yaml new file mode 100644 index 00000000..7a0cb817 --- /dev/null +++ b/ci/manifests/interlink-config.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "interlink-config" + namespace: interlink +data: + InterLinkConfig.yaml: | + InterlinkURL: "http://localhost" + InterlinkPort: "3000" + SidecarURL: "http://plugin.interlink.svc.cluster.local" + SidecarPort: "4000" + VerboseLogging: true + ErrorsOnlyLogging: false + ExportPodData: true + DataRootFolder: "~/.interlink" diff --git a/ci/manifests/interlink.yaml b/ci/manifests/interlink.yaml new file mode 100644 index 00000000..a11f6e7e --- /dev/null +++ b/ci/manifests/interlink.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Service +metadata: + name: interlink + namespace: interlink +spec: + selector: + app: interlink + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: interlink + namespace: interlink + labels: + app: interlink +spec: + replicas: 1 + selector: + matchLabels: + app: interlink + template: + metadata: + labels: + app: interlink + spec: + containers: + - name: interlink + image: "ghcr.io/intertwin-eu/interlink/interlink" + imagePullPolicy: Always + env: + - name: CONFIGPATH + value: "/etc/interlink/InterLinkConfig.yaml" + volumeMounts: + - name: config + mountPath: /etc/interlink/InterLinkConfig.yaml + subPath: InterLinkConfig.yaml + volumes: + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: interlink-config diff --git a/ci/manifests/kustomization.yaml b/ci/manifests/kustomization.yaml new file mode 100644 index 00000000..694260d7 --- /dev/null +++ b/ci/manifests/kustomization.yaml @@ -0,0 +1,15 @@ +resources: +- virtual-kubelet-config.yaml +- virtual-kubelet.yaml +- interlink-config.yaml +- interlink.yaml +patches: +- path: virtual-kubelet-merge.yaml + target: + kind: Deployment + labelSelector: nodeName=virtual-kubelet +- path: interlink-merge.yaml + target: + kind: Deployment + labelSelector: app=interlink + diff --git a/ci/manifests/plugin-config.yaml b/ci/manifests/plugin-config.yaml new file mode 100644 index 00000000..e4aae3ba --- /dev/null +++ b/ci/manifests/plugin-config.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "plugin-config" + namespace: interlink +data: + InterLinkConfig.yaml: | + InterlinkURL: "http://localhost" + InterlinkPort: "3000" + SidecarURL: "http://0.0.0.0" + SidecarPort: "4000" + VerboseLogging: true + ErrorsOnlyLogging: false + ExportPodData: true + DataRootFolder: "~/.interlink" + SbatchPath: "/usr/bin/sbatch" + ScancelPath: "/usr/bin/scancel" + SqueuePath: "/usr/bin/squeue" + CommandPrefix: "" + SingularityPrefix: "" + Namespace: "vk" + Tsocks: false + TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" + TsocksLoginNode: "login01" + BashPath: /bin/bash diff --git a/ci/manifests/plugin.yaml b/ci/manifests/plugin.yaml new file mode 100644 index 00000000..046f963d --- /dev/null +++ b/ci/manifests/plugin.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: Service +metadata: + name: plugin + namespace: interlink +spec: + selector: + app: plugin + ports: + - protocol: TCP + port: 4000 + targetPort: 4000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: plugin + namespace: interlink + labels: + app: plugin +spec: + replicas: 1 + selector: + matchLabels: + app: plugin + template: + metadata: + labels: + app: plugin + spec: + containers: + - name: plugin + image: "ghcr.io/intertwin-eu/interlink-docker-plugin/docker-plugin:0.0.16-no-gpu" + #image: "ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.2.3" + imagePullPolicy: Always + command: + - bash + - -c + args: + - dockerd --mtu 1450 & /sidecar/docker-sidecar + securityContext: + privileged: true + env: + - name: INTERLINKCONFIGPATH + value: "/etc/interlink/InterLinkConfig.yaml" + volumeMounts: + - name: config + mountPath: /etc/interlink/InterLinkConfig.yaml + subPath: InterLinkConfig.yaml + volumes: + - name: config + configMap: + # Provide the name of the ConfigMap you want to mount. + name: plugin-config diff --git a/ci/tests/manifests/service-account.yaml b/ci/manifests/service-account.yaml similarity index 88% rename from ci/tests/manifests/service-account.yaml rename to ci/manifests/service-account.yaml index 09d92fe1..8a0b3542 100644 --- a/ci/tests/manifests/service-account.yaml +++ b/ci/manifests/service-account.yaml @@ -10,6 +10,17 @@ metadata: name: virtual-kubelet namespace: interlink rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - update + - create + - get + - list + - watch + - patch - apiGroups: - "" resources: diff --git a/ci/tests/manifests/virtual-kubelet-config.yaml b/ci/manifests/virtual-kubelet-config.yaml similarity index 68% rename from ci/tests/manifests/virtual-kubelet-config.yaml rename to ci/manifests/virtual-kubelet-config.yaml index 0b7ac174..59fd6791 100644 --- a/ci/tests/manifests/virtual-kubelet-config.yaml +++ b/ci/manifests/virtual-kubelet-config.yaml @@ -5,12 +5,14 @@ metadata: namespace: interlink data: InterLinkConfig.yaml: | - InterlinkURL: http://interlink.interlink.svc.cluster + InterlinkURL: http://interlink.interlink.svc.cluster.local InterlinkPort: 3000 ExportPodData: true + VerboseLogging: true + ErrorsOnlyLogging: false ServiceAccount: "virtual-kubelet" Namespace: interlink VKTokenFile: /dev/null CPU: "100" - Memory: "128GiB", + Memory: "128GiB" Pods: "100" diff --git a/ci/tests/manifests/virtual-kubelet.yaml b/ci/manifests/virtual-kubelet.yaml similarity index 83% rename from ci/tests/manifests/virtual-kubelet.yaml rename to ci/manifests/virtual-kubelet.yaml index e996431d..51b34a88 100644 --- a/ci/tests/manifests/virtual-kubelet.yaml +++ b/ci/manifests/virtual-kubelet.yaml @@ -15,30 +15,27 @@ spec: labels: nodeName: virtual-kubelet spec: + automountServiceAccountToken: true serviceAccountName: virtual-kubelet containers: - name: inttw-vk - image: "registry:5432/virtual-kubelet:latest" + image: "ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw" imagePullPolicy: Always env: - name: NODENAME value: virtual-kubelet - - name: CONFIGPATH - value: /etc/interlink/interlink-cfg.json - name: KUBELET_PORT value: "10250" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: INTERLINKCONFIGPATH + - name: CONFIGPATH value: "/etc/interlink/InterLinkConfig.yaml" volumeMounts: - name: config mountPath: /etc/interlink/InterLinkConfig.yaml subPath: InterLinkConfig.yaml - - name: jaeger - image: jaegertracing/all-in-one:1.51 volumes: - name: config configMap: diff --git a/ci/manifests/vktest_config.yaml b/ci/manifests/vktest_config.yaml new file mode 100644 index 00000000..ab393912 --- /dev/null +++ b/ci/manifests/vktest_config.yaml @@ -0,0 +1,20 @@ +target_nodes: + - virtual-kubelet + +required_namespaces: + - default + - kube-system + - interlink + +timeout_multiplier: 2. +values: + namespace: interlink + + annotations: + slurm-job.vk.io/flags: "--job-name=test-pod-cfg -t 2800 --ntasks=8 --nodes=1 --mem-per-cpu=2000" + + tolerations: + - key: virtual-node.interlink/no-schedule + operator: Exists + effect: NoSchedule + diff --git a/ci/tests/create_test.go b/ci/tests/create_test.go deleted file mode 100644 index a03e644a..00000000 --- a/ci/tests/create_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package tests - -import ( - "context" - "flag" - "path/filepath" - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -func TestCreatePod(t *testing.T) { - var kubeconfig *string - if home := homedir.HomeDir(); home != "" { - kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") - } else { - kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file") - } - flag.Parse() - - // use the current context in kubeconfig - config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - panic(err.Error()) - } - - // create the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - panic(err.Error()) - } - - checks := []struct { - name string - selector corev1.NodeSelector - tolerations corev1.Toleration - volumes []corev1.Volume - attachment []corev1.VolumeMount - fail bool - }{ - { - name: "valid-selector-no-volumes", - selector: corev1.NodeSelector{}, - tolerations: corev1.Toleration{}, - volumes: []corev1.Volume{}, - attachment: []corev1.VolumeMount{}, - fail: false, - }, - } - - for _, tt := range checks { - t.Run(tt.name, func(t *testing.T) { - - podManifest := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.name, - }, - } - pod, err := clientset.CoreV1().Pods("").Create(context.TODO(), &podManifest, metav1.CreateOptions{}) - if err != nil && !tt.fail { - t.Fatal("did not expect creation to fail:", err) - } else if err == nil { - _, err = clientset.CoreV1().Pods("").Get(context.TODO(), pod.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil && !tt.fail { - t.Fatal("did not expect creation to fail:", err) - } - } - }) - } - -} diff --git a/ci/tests/manifests/bkp/docker-compose.yaml b/ci/tests/manifests/bkp/docker-compose.yaml deleted file mode 100644 index 6efed940..00000000 --- a/ci/tests/manifests/bkp/docker-compose.yaml +++ /dev/null @@ -1,49 +0,0 @@ -version: '3.7' -services: - interlink: - build: - context: ../../../../ - dockerfile: docker/Dockerfile.interlink - restart: always - #network_mode: "host" - ports: - - 3000:3000 - volumes: - - type: bind - source: ./interlink_cfg.yaml - target: /etc/interlink - environment: - - INTERLINKCONFIGPATH=/etc/interlink/interlink_cfg.yaml - # healthcheck: - # test: ["CMD", "/check.sh"] - # interval: 10s - # timeout: 10s - # retries: 3 - # start_period: 5s - docker-sidecar: - image: ghcr.io - #build: - # context: ../../../ - # dockerfile: docker/Dockerfile.sidecar-docker - restart: always - privileged: true - cap_add: - - SYS_ADMIN - #network_mode: "host" - ports: - - 4000:4000 - environment: - - INTERLINKCONFIGPATH=/etc/interlink/InterLinkConfig.yaml - volumes: - - type: bind - source: ../vk - target: /etc/interlink - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - # healthcheck: - # test: ["CMD", "/check.sh"] - # interval: 10s - # timeout: 10s - # retries: 3 - # start_period: 5s diff --git a/ci/tests/manifests/bkp/interlink_cfg.yaml b/ci/tests/manifests/bkp/interlink_cfg.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/ci/tests/manifests/bkp/virtual-kubelet.yaml b/ci/tests/manifests/bkp/virtual-kubelet.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/cmd/.DS_Store b/cmd/.DS_Store new file mode 100644 index 00000000..01c579cd Binary files /dev/null and b/cmd/.DS_Store differ diff --git a/cmd/installer/templates/deployment.yaml b/cmd/installer/templates/deployment.yaml index 0522c6ab..b186dd9a 100644 --- a/cmd/installer/templates/deployment.yaml +++ b/cmd/installer/templates/deployment.yaml @@ -20,7 +20,7 @@ spec: - name: jaeger image: jaegertracing/all-in-one:1.51 - name: inttw-vk - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw:latest + image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:latest imagePullPolicy: Always env: - name: NODENAME diff --git a/cmd/installer/templates/interlink-install.sh b/cmd/installer/templates/interlink-install.sh index dbb69dde..88c883fc 100644 --- a/cmd/installer/templates/interlink-install.sh +++ b/cmd/installer/templates/interlink-install.sh @@ -106,6 +106,7 @@ start() { --https-address 0.0.0.0:{{.InterLinkPort}} \ --tls-cert-file ${HOME}/.interlink/config/tls.crt \ --tls-key-file ${HOME}/.interlink/config/tls.key \ + --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \ --skip-jwt-bearer-tokens true > $HOME/.interlink/logs/oauth2-proxy.log 2>&1 & echo $! > $HOME/.interlink/oauth2-proxy.pid @@ -127,6 +128,7 @@ start() { --https-address 0.0.0.0:{{.InterLinkPort}} \ --tls-cert-file ${HOME}/.interlink/config/tls.crt \ --tls-key-file ${HOME}/.interlink/config/tls.key \ + --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \ --skip-jwt-bearer-tokens true > $HOME/.interlink/logs/oauth2-proxy.log 2>&1 & echo $! > $HOME/.interlink/oauth2-proxy.pid diff --git a/cmd/virtual-kubelet/main.go b/cmd/virtual-kubelet/main.go index 1437667a..fe5919ab 100644 --- a/cmd/virtual-kubelet/main.go +++ b/cmd/virtual-kubelet/main.go @@ -32,6 +32,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes/scheme" + lease "k8s.io/client-go/kubernetes/typed/coordination/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" @@ -74,6 +75,7 @@ func PodInformerFilter(node string) informers.SharedInformerOption { type Config struct { ConfigPath string NodeName string + NodeVersion string OperatingSystem string InternalIP string DaemonPort int32 @@ -145,6 +147,11 @@ func initProvider() (func(context.Context) error, error) { return tracerProvider.Shutdown, nil } +func tlsConfig(tls *tls.Config) error { + tls.InsecureSkipVerify = true + return nil +} + func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -185,20 +192,22 @@ func main() { } log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) - shutdown, err := initProvider() - if err != nil { - log.G(ctx).Fatal(err) - } - defer func() { - if err = shutdown(ctx); err != nil { - log.G(ctx).Fatal("failed to shutdown TracerProvider: %w", err) + if os.Getenv("ENABLE_TRACING") == "1" { + shutdown, err := initProvider() + if err != nil { + log.G(ctx).Fatal(err) } - }() + defer func() { + if err = shutdown(ctx); err != nil { + log.G(ctx).Fatal("failed to shutdown TracerProvider: %w", err) + } + }() - log.G(ctx).Info("Tracer setup succeeded") + log.G(ctx).Info("Tracer setup succeeded") - // TODO: disable this through options - trace.T = opentelemetry.Adapter{} + // TODO: disable this through options + trace.T = opentelemetry.Adapter{} + } // TODO: if token specified http.DefaultClient = ... // and remove reading from file @@ -213,6 +222,7 @@ func main() { cfg := Config{ ConfigPath: configpath, NodeName: nodename, + NodeVersion: commonIL.KubeletVersion, OperatingSystem: "Linux", // https://github.com/liqotech/liqo/blob/d8798732002abb7452c2ff1c99b3e5098f848c93/deployments/liqo/templates/liqo-gateway-deployment.yaml#L69 InternalIP: os.Getenv("POD_IP"), @@ -245,34 +255,14 @@ func main() { localClient := kubernetes.NewForConfigOrDie(kubecfg) - nodeProvider, err := commonIL.NewProvider(cfg.ConfigPath, cfg.NodeName, cfg.OperatingSystem, cfg.InternalIP, cfg.DaemonPort, ctx) - go func() { - - ILbind := false - retValue := -1 - counter := 0 - - for { - ILbind, retValue, err = commonIL.PingInterLink(ctx, interLinkConfig) - - if err != nil { - log.G(ctx).Error(err) - } - - if !ILbind && retValue == 1 { - counter++ - } else if ILbind && retValue == 0 { - counter = 0 - } - - if counter > 10 { - log.G(ctx).Fatal("Unable to communicate with the InterLink API, exiting...") - } - - time.Sleep(time.Second * 10) - - } - }() + nodeProvider, err := commonIL.NewProvider( + cfg.ConfigPath, + cfg.NodeName, + cfg.NodeVersion, + cfg.OperatingSystem, + cfg.InternalIP, + cfg.DaemonPort, + ctx) if err != nil { log.G(ctx).Fatal(err) @@ -280,6 +270,10 @@ func main() { nc, _ := node.NewNodeController( nodeProvider, nodeProvider.GetNode(), localClient.CoreV1().Nodes(), + node.WithNodeEnableLeaseV1( + lease.NewForConfigOrDie(kubecfg).Leases(v1.NamespaceNodeLease), + 300, + ), ) go func() error { @@ -289,8 +283,6 @@ func main() { } return nil }() - // <-nc.Ready() - // close(nc) eb := record.NewBroadcaster() diff --git a/cmd/virtual-kubelet/set-version.sh b/cmd/virtual-kubelet/set-version.sh new file mode 100755 index 00000000..9af21950 --- /dev/null +++ b/cmd/virtual-kubelet/set-version.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z ${KUBELET_VERSION} ]; then +cat << EOF > pkg/virtualkubelet/version.go +package main + +var ( + kubeletVersion = "$KUBELET_VERSION" +) +EOF +fi diff --git a/docker/Dockerfile.interlink b/docker/Dockerfile.interlink index 55f9c663..a81767ee 100644 --- a/docker/Dockerfile.interlink +++ b/docker/Dockerfile.interlink @@ -1,9 +1,17 @@ -FROM golang:1.21 as build-stage +FROM golang:1.22 as build-stage WORKDIR /app COPY .. . +ENV GOMODCACHE="/go/pkg/mod" + +ENV GOCACHE="/go/build-cache" + +RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE + + +RUN go mod tidy RUN CGO_ENABLED=0 GOOS=linux go build -o bin/interlink # Deploy the application binary into a lean image diff --git a/docker/Dockerfile.vk b/docker/Dockerfile.vk index 55633acf..9a455a24 100644 --- a/docker/Dockerfile.vk +++ b/docker/Dockerfile.vk @@ -1,11 +1,22 @@ -FROM bitnami/kubectl as kubectl +FROM bitnami/kubectl:1.27.14 as kubectl -FROM golang:1.21 as build-stage +FROM golang:1.22 as build-stage WORKDIR /app COPY .. . +ARG VERSION +ENV KUBELET_VERSION=${VERSION} + +ENV GOMODCACHE="/go/pkg/mod" +ENV GOCACHE="/go/build-cache" + +RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE + + +RUN bash -c "./cmd/virtual-kubelet/set-version.sh" +RUN go mod tidy RUN CGO_ENABLED=0 GOOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go # Deploy the application binary into a lean image diff --git a/docs/docs/tutorial-users/01-quick-start.md b/docs/docs/tutorial-users/01-quick-start.md index c9ed0787..ab0725c9 100644 --- a/docs/docs/tutorial-users/01-quick-start.md +++ b/docs/docs/tutorial-users/01-quick-start.md @@ -113,161 +113,3 @@ Also you can see with `docker ps` the container appearing on the `interlink-dock ```bash docker exec interlink-docker-sidecar-1 docker ps ``` - -## Connect a SLURM batch system - -Let's connect a cluster to a SLURM batch. Move to example location: - -```bash -cd interLink/examples/interlink-slurm -``` - -### Setup Kubernetes cluster - -:::danger - -__N.B.__ in the demo the oauth2 proxy authN/Z is disabled. DO NOT USE THIS IN PRODUCTION unless you know what you are doing. - -::: - -### Bootstrap a minikube cluster - -```bash -minikube start --kubernetes-version=1.26.10 -``` - -Once finished you should check that everything went well with a simple `kubectl get node`. - -:::note - -If you don't have `kubectl` installed on your machine, you can install it as describe in the [official documentation](https://kubernetes.io/docs/tasks/tools/) - -::: - -### Configure interLink - -You need to provide the interLink IP address that should be reachable from the kubernetes pods. - -:::note - -In case of this demo setup, that address __is the address of your machine__ - -::: - -```bash -export INTERLINK_IP_ADDRESS=XXX.XX.X.XXX - -sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' vk/InterLinkConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' vk/InterLinkConfig.yaml -``` - -### Deploy the interLink components - -#### Deploy the interLink virtual node - -Create a `vk` namespace: - -```bash -kubectl create ns vk -``` - -Deploy the vk resources on the cluster with: - -```bash -kubectl apply -n vk -k vk/ -``` - -Check that both the pods and the node are in ready status - -```bash -kubectl get pod -n vk - -kubectl get node -``` - -#### Deploy interLink remote components - -With the following commands you are going to deploy a docker compose that emulates a remote center managing resources via a SLURM batch system. - -The following containers are going to be deployed: - -- **interLink API server**: the API layer responsible of receiving requests from the kubernetes virtual node and forward a digested vertion to the interLink plugin -- **interLink SLURM plugin**: translates the information from the API server into a SLURM job -- **a SLURM local daemon**: a local instance of a SLURM dummy queue with [singularity/apptainer](https://apptainer.org/) available as container runtime. - -```bash -cd interlink - -docker compose up -d -``` - -Check logs for both interLink APIs and SLURM sidecar: - -```bash -docker logs interlink-interlink-1 - -docker logs interlink-docker-sidecar-1 -``` - -### Deploy a sample application - -Congratulation! Now it's all set up for the execution of your first pod on a virtual node! - -What you have to do, is just explicitly allow a pod of yours in the following way: - -```yaml title="./examples/interlink-slurm/test_pod.yaml" -apiVersion: v1 -kind: Pod -metadata: - name: test-pod-cowsay - namespace: vk - annotations: - slurm-job.knoc.io/flags: "--job-name=test-pod-cfg -t 2800 --ntasks=8 --nodes=1 --mem-per-cpu=2000" -spec: - restartPolicy: Never - containers: - - image: docker://ghcr.io/grycap/cowsay - command: ["/bin/sh"] - args: ["-c", "\"touch /tmp/test.txt && sleep 60 && echo \\\"hello muu\\\" | /usr/games/cowsay \" " ] - imagePullPolicy: Always - name: cowsayo - dnsPolicy: ClusterFirst - // highlight-start - nodeSelector: - kubernetes.io/hostname: test-vk - tolerations: - - key: virtual-node.interlink/no-schedule - operator: Exists - // highlight-end -``` - -Then, you are good to go: - -```bash -kubectl apply -f ../test_pod.yaml -``` - -Now observe the application running and eventually succeeding via: - -```bash -kubectl get pod -n vk --watch -``` - -When finished, interrupt the watch with `Ctrl+C` and retrieve the logs with: - -```bash -kubectl logs -n vk test-pod-cfg-cowsay-dciangot -``` - -Also you can see with `squeue --me` the jobs appearing on the `interlink-docker-sidecar-1` container with: - -```bash -docker exec interlink-docker-sidecar-1 squeue --me -``` - -Or, if you need more debug, you can log into the sidecar and look for your POD_UID folder in `.local/interlink/jobs`: - -```bash -docker exec -ti interlink-docker-sidecar-1 bash - -ls -altrh .local/interlink/jobs -``` diff --git a/example/.DS_Store b/example/.DS_Store new file mode 100644 index 00000000..eb4bb013 Binary files /dev/null and b/example/.DS_Store differ diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 00000000..1d17dae1 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1 @@ +.venv diff --git a/example/interlink-docker/test_pod.yaml b/example/interlink-docker/test_pod.yaml index 8b8e309e..7dcd1ecf 100644 --- a/example/interlink-docker/test_pod.yaml +++ b/example/interlink-docker/test_pod.yaml @@ -4,7 +4,7 @@ metadata: name: test-pod-cfg-cowsay-dciangot namespace: vk annotations: - slurm-job.knoc.io/flags: "--job-name=test-pod-cfg -t 2800 --ntasks=8 --nodes=1 --mem-per-cpu=2000" + slurm-job.vk.io/flags: "--job-name=test-pod-cfg -t 2800 --ntasks=8 --nodes=1 --mem-per-cpu=2000" spec: restartPolicy: Never containers: diff --git a/example/interlink-docker/vk/deployment.yaml b/example/interlink-docker/vk/deployment.yaml index 4f700a6e..109d1a4a 100644 --- a/example/interlink-docker/vk/deployment.yaml +++ b/example/interlink-docker/vk/deployment.yaml @@ -28,7 +28,7 @@ spec: - name: jaeger image: jaegertracing/all-in-one:1.51 - name: inttw-vk - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw:latest + image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:latest #image: dciangot/vk:latest imagePullPolicy: Always #command: ["sleep", "infinity"] diff --git a/example/provider_mock.py b/example/provider_mock.py deleted file mode 100644 index 4f84673b..00000000 --- a/example/provider_mock.py +++ /dev/null @@ -1,220 +0,0 @@ -import interlink - -from fastapi.responses import PlainTextResponse -from fastapi import FastAPI, HTTPException -from typing import List -import re -import os - - -app = FastAPI() - - -class MyProvider(interlink.provider.Provider): - def __init__( - self, - ): - - # Recover already running containers refs - self.CONTAINER_POD_MAP = {} - - def DumpVolumes( - self, pods: List[interlink.PodVolume], volumes: List[interlink.Volume] - ) -> List[str]: - - dataList = [] - - # Match data source information (actual bytes) to the mount ref in pod description - for v in volumes: - if v.configMaps: - for dataSource in v.configMaps: - for ref in pods: - podMount = ref.volumeSource.configMap - if podMount: - if ref.name == dataSource.metadata.name: - for filename, content in dataSource.data.items(): - # write content to file - path = f"{dataSource.metadata.namespace}-{dataSource.metadata.name}/{filename}" - try: - os.makedirs( - os.path.dirname(path), exist_ok=True - ) - with open(path, "w") as f: - f.write(content) - except Exception as ex: - raise HTTPException(status_code=500, detail=ex) - - # dump list of written files - dataList.append(path) - - if v.secrets: - pass - - if v.emptyDirs: - pass - return dataList - - def Create(self, pod: interlink.Pod) -> None: - container = pod.pod.spec.containers[0] - - if pod.pod.spec.volumes: - _ = self.DumpVolumes(pod.pod.spec.volumes, pod.container) - - volumes = [] - if container.volumeMounts: - for mount in container.volumeMounts: - if mount.subPath: - volumes.append( - f"{pod.pod.metadata.namespace}-{mount.name}/{mount.subPath}:{mount.mountPath}" - ) - else: - volumes.append( - f"{pod.pod.metadata.namespace}-{mount.name}:{mount.mountPath}" - ) - - self.CONTAINER_POD_MAP.update({pod.pod.metadata.uid: [docker_run_id]}) - print(self.CONTAINER_POD_MAP) - - print(pod) - - def Delete(self, pod: interlink.PodRequest) -> None: - try: - print(f"docker rm -f {self.CONTAINER_POD_MAP[pod.metadata.uid][0]}") - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[pod.metadata.uid][0] - ) - container.remove(force=True) - self.CONTAINER_POD_MAP.pop(pod.metadata.uid) - except: - raise HTTPException(status_code=404, detail="No containers found for UUID") - print(pod) - return - - def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: - print(self.CONTAINER_POD_MAP) - print(pod.metadata.uid) - try: - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[pod.metadata.uid][0] - ) - status = container.status - except: - raise HTTPException(status_code=404, detail="No containers found for UUID") - - print(status) - - if status == "running": - try: - statuses = self.DOCKER.api.containers( - filters={"status": "running", "id": container.id} - ) - print(statuses) - startedAt = statuses[0]["Created"] - except Exception as ex: - raise HTTPException(status_code=500, detail=ex) - - return interlink.PodStatus( - name=pod.metadata.name, - UID=pod.metadata.uid, - namespace=pod.metadata.namespace, - containers=[ - interlink.ContainerStatus( - name=pod.spec.containers[0].name, - state=interlink.ContainerStates( - running=interlink.StateRunning(startedAt=startedAt), - waiting=None, - terminated=None, - ), - ) - ], - ) - elif status == "exited": - - try: - statuses = self.DOCKER.api.containers( - filters={"status": "exited", "id": container.id} - ) - print(statuses) - reason = statuses[0]["Status"] - pattern = re.compile(r"Exited \((.*?)\)") - - exitCode = -1 - for match in re.findall(pattern, reason): - exitCode = int(match) - except Exception as ex: - raise HTTPException(status_code=500, detail=ex) - - return interlink.PodStatus( - name=pod.metadata.name, - UID=pod.metadata.uid, - namespace=pod.metadata.namespace, - containers=[ - interlink.ContainerStatus( - name=pod.spec.containers[0].name, - state=interlink.ContainerStates( - running=None, - waiting=None, - terminated=interlink.StateTerminated( - reason=reason, exitCode=exitCode - ), - ), - ) - ], - ) - - return interlink.PodStatus( - name=pod.metadata.name, - UID=pod.metadata.uid, - namespace=pod.metadata.namespace, - containers=[ - interlink.ContainerStatus( - name=pod.spec.containers[0].name, - state=interlink.ContainerStates( - running=None, - waiting=None, - terminated=interlink.StateTerminated( - reason="Completed", exitCode=0 - ), - ), - ) - ], - ) - - def Logs(self, req: interlink.LogRequest) -> bytes: - # TODO: manage more complicated multi container pod - # THIS IS ONLY FOR DEMONSTRATION - print(req.PodUID) - print(self.CONTAINER_POD_MAP[req.PodUID]) - try: - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[req.PodUID][0] - ) - # log = container.logs(timestamps=req.Opts.Timestamps, tail=req.Opts.Tail) - log = container.logs() - print(log) - except: - raise HTTPException(status_code=404, detail="No containers found for UUID") - return log - - -ProviderNew = MyProvider(dockerCLI) - - -@app.post("/create") -async def create_pod(pods: List[interlink.Pod]) -> str: - return ProviderNew.create_pod(pods) - - -@app.post("/delete") -async def delete_pod(pod: interlink.PodRequest) -> str: - return ProviderNew.delete_pod(pod) - - -@app.get("/status") -async def status_pod(pods: List[interlink.PodRequest]) -> List[interlink.PodStatus]: - return ProviderNew.get_status(pods) - - -@app.get("/getLogs", response_class=PlainTextResponse) -async def get_logs(req: interlink.LogRequest) -> bytes: - return ProviderNew.get_logs(req) diff --git a/go.mod b/go.mod index 7b384ade..3453b20f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/intertwin-eu/interlink -go 1.21 - -toolchain go1.21.3 +go 1.22 require ( github.com/containerd/containerd v1.7.6 diff --git a/helm/interlink/Chart.yaml b/helm/interlink/Chart.yaml index 9cca282c..cc8bd96d 100644 --- a/helm/interlink/Chart.yaml +++ b/helm/interlink/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: interlink -description: A Helm chart for Kubernetes +description: A Helm chart for interLink virtual kubelet provider # A chart can be either an 'application' or a 'library' chart. # @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.16.0" +appVersion: "0.2.3" diff --git a/helm/interlink/templates/InterLinkConfig.yaml b/helm/interlink/templates/InterLinkConfig.yaml deleted file mode 100644 index 1f0c66b3..00000000 --- a/helm/interlink/templates/InterLinkConfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -VKTokenFile: "$HOME/interLink/token" -InterlinkURL: "http://10.25.127.213" -SidecarURL: "http://10.25.127.213" -InterlinkPort: "3000" -SidecarPort: "4000" -SbatchPath: "/usr/bin/sbatch" -ScancelPath: "/usr/bin/scancel" -SqueuePath: "/usr/bin/squeue" -CommandPrefix: "" -ExportPodData: true -DataRootFolder: ".local/interlink/jobs/" -ServiceAccount: "interlink" -Namespace: "vk" -Tsocks: false -TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" -TsocksLoginNode: "login01" -BashPath: /bin/bash -VerboseLogging: true -ErrorsOnlyLogging: false diff --git a/helm/interlink/templates/NOTES.txt b/helm/interlink/templates/NOTES.txt index 46b130cf..6eb8cbc5 100644 --- a/helm/interlink/templates/NOTES.txt +++ b/helm/interlink/templates/NOTES.txt @@ -1,22 +1,5 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "interlink.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "interlink.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "interlink.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "interlink.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +Check node status with: + +``` +kubectl get node {{ .Values.nodeName }} +``` diff --git a/helm/interlink/templates/deployment copy.yaml b/helm/interlink/templates/deployment copy.yaml deleted file mode 100644 index c16e19e8..00000000 --- a/helm/interlink/templates/deployment copy.yaml +++ /dev/null @@ -1,106 +0,0 @@ ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: test-vk - labels: - nodeName: test-vk -spec: - replicas: 1 - selector: - matchLabels: - nodeName: test-vk - template: - metadata: - labels: - nodeName: test-vk - spec: - initContainers: - - name: settoken - image: "docker.io/alpine:3" - command: ["sh", "-c"] - args: ["touch /opt/interlink/token"] - volumeMounts: - - name: token - mountPath: /opt/interlink - containers: - - name: jaeger - image: jaegertracing/all-in-one:1.51 - - name: inttw-vk - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw:latest - #image: dciangot/vk:latest - imagePullPolicy: Always - #command: ["sleep", "infinity"] - env: - - name: NODENAME - value: test-vk - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - #- name: KUBECONFIG - # value: /etc/interlink/kubeconfig - - name: KUBELET_PORT - value: "10255" - - name: INTERLINKCONFIGPATH - value: "/etc/interlink/InterLinkConfig.yaml" - - name: VKTOKENFILE - value: "/opt/interlink/token" - - name: CONFIGPATH - value: "/etc/interlink/vk-cfg.json" - volumeMounts: - - name: config - mountPath: /etc/interlink/InterLinkConfig.yaml - subPath: InterLinkConfig.yaml - - name: config-json - mountPath: /etc/interlink/vk-cfg.json - subPath: vk-cfg.json - - name: token - mountPath: /opt/interlink - resources: - limits: - cpu: 500m - memory: 600Mi - requests: - cpu: 50m - memory: 100Mi - - name: refresh-token - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw-refresh:latest - imagePullPolicy: Always - env: - - name: IAM_SERVER - value: "https://dodas-iam.cloud.cnaf.infn.it/" - # TODO load env IAM client from secret - - name: IAM_CLIENT_ID - value: "DUMMY" - - name: IAM_CLIENT_SECRET - value: "DUMMY" - - name: IAM_REFRESH_TOKEN - value: "DUMMY" - - name: IAM_VK_AUD - value: intertw-vk - - name: TOKEN_PATH - value: /opt/interlink/token - resources: - limits: - cpu: 500m - memory: 600Mi - requests: - cpu: 50m - memory: 100Mi - volumeMounts: - - name: token - mountPath: /opt/interlink - serviceAccountName: interlink - volumes: - - name: config - configMap: - # Provide the name of the ConfigMap you want to mount. - name: vk-config - - name: config-json - configMap: - # Provide the name of the ConfigMap you want to mount. - name: test-vk-config - - name: token - emptyDir: {} diff --git a/helm/interlink/templates/deployment.yaml b/helm/interlink/templates/deployment.yaml deleted file mode 100644 index 9847777b..00000000 --- a/helm/interlink/templates/deployment.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "interlink.fullname" . }} - labels: - {{- include "interlink.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "interlink.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "interlink.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "interlink.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 80 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: http - readinessProbe: - httpGet: - path: / - port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/interlink/templates/hpa.yaml b/helm/interlink/templates/hpa.yaml deleted file mode 100644 index faf760e1..00000000 --- a/helm/interlink/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "interlink.fullname" . }} - labels: - {{- include "interlink.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "interlink.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/interlink/templates/ingress.yaml b/helm/interlink/templates/ingress.yaml deleted file mode 100644 index 4b129fb4..00000000 --- a/helm/interlink/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "interlink.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "interlink.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/interlink/templates/kustomization.yaml b/helm/interlink/templates/kustomization.yaml deleted file mode 100644 index a30024c2..00000000 --- a/helm/interlink/templates/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ./deployment.yaml - - ./service-account.yaml - -configMapGenerator: - - name: test-vk-config - files: - - vk-cfg.json=vk-cfg.json - - name: vk-config - files: - - InterLinkConfig.yaml=InterLinkConfig.yaml diff --git a/helm/interlink/templates/otecol_config.yaml b/helm/interlink/templates/otecol_config.yaml deleted file mode 100644 index 2f6bb992..00000000 --- a/helm/interlink/templates/otecol_config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -extensions: - health_check: - pprof: - endpoint: 0.0.0.0:1777 - zpages: - endpoint: 0.0.0.0:55679 - -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4319 - http: - endpoint: 0.0.0.0:4318 - -processors: - batch: - -exporters: - otlp: - endpoint: "localhost:4317" - tls: - insecure: true - logging: - verbosity: detailed - -service: - - pipelines: - - traces: - receivers: [otlp, opencensus, jaeger, zipkin] - processors: [batch] - exporters: [logging] - - metrics: - receivers: [otlp, opencensus, prometheus] - processors: [batch] - exporters: [logging] - - extensions: [health_check, pprof, zpages] \ No newline at end of file diff --git a/helm/interlink/templates/service-account.yaml b/helm/interlink/templates/service-account.yaml index b9cfb2f0..54111570 100644 --- a/helm/interlink/templates/service-account.yaml +++ b/helm/interlink/templates/service-account.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: interlink - namespace: vk + name: {{ .Values.nodeName }} + namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: interlink-role - namespace: vk + name: "{{ .Values.nodeName }}-role" + namespace: {{ .Release.Namespace }} rules: - apiGroups: - "" @@ -63,23 +63,24 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: interlink-rolebinding + name: "{{ .Values.nodeName }}-rolebinding" + namespace: {{ .Release.Namespace }} subjects: - kind: ServiceAccount - name: interlink - namespace: vk + name: {{ .Values.nodeName }} + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: interlink-role + name: "{{ .Values.nodeName }}-role" --- apiVersion: v1 kind: Secret metadata: - name: interlink-secret - namespace: vk + name: "{{ .Values.nodeName }}-secret" + namespace: "{{ .Release.Namespace }}" annotations: - kubernetes.io/service-account.name: interlink + kubernetes.io/service-account.name: {{ .Values.nodeName }} labels: - kubernetes.io/service-account.name: interlink + kubernetes.io/service-account.name: {{ .Values.nodeName }} type: kubernetes.io/service-account-token diff --git a/helm/interlink/templates/service.yaml b/helm/interlink/templates/service.yaml deleted file mode 100644 index 532da7f4..00000000 --- a/helm/interlink/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "interlink.fullname" . }} - labels: - {{- include "interlink.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "interlink.selectorLabels" . | nindent 4 }} diff --git a/helm/interlink/templates/tests/test-connection.yaml b/helm/interlink/templates/tests/test-connection.yaml deleted file mode 100644 index 45634c13..00000000 --- a/helm/interlink/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "interlink.fullname" . }}-test-connection" - labels: - {{- include "interlink.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "interlink.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/helm/interlink/templates/virtual-kubelet-config.yaml b/helm/interlink/templates/virtual-kubelet-config.yaml new file mode 100644 index 00000000..4c32e3e9 --- /dev/null +++ b/helm/interlink/templates/virtual-kubelet-config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "virtual-kubelet-config" + namespace: {{ .Release.Namespace }} +data: + InterLinkConfig.yaml: | + InterlinkURL: {{ .Values.interlink.URL }} + InterlinkPort: {{ .Values.interlink.port }} + ExportPodData: true + VerboseLogging: true + ErrorsOnlyLogging: false + ServiceAccount: "{{ .Values.nodeName }}" + Namespace: "" + VKTokenFile: /opt/interlink/token + CPU: "{{ .Values.virtualNode.CPUs }}" + Memory: "{{ .Values.virtualNode.MemGiB }}GiB" + Pods: "{{ .Values.virtualNode.Pods }}" diff --git a/helm/interlink/templates/virtual-kubelet.yaml b/helm/interlink/templates/virtual-kubelet.yaml new file mode 100644 index 00000000..d4db5fb6 --- /dev/null +++ b/helm/interlink/templates/virtual-kubelet.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.deployment.name}} + namespace: {{ .Release.Namespace}} + labels: + nodeName: {{ .Values.nodeName }} +spec: + replicas: 1 + selector: + matchLabels: + nodeName: {{ .Values.nodeName }} + template: + metadata: + labels: + nodeName: {{ .Values.nodeName }} + spec: + automountServiceAccountToken: true + serviceAccountName: {{ .Values.deployment.name }} + containers: + - name: inttw-vk + image: {{ .Values.deployment.image }} + imagePullPolicy: Always + env: + - name: NODENAME + value: {{ .Values.nodeName }} + - name: KUBELET_PORT + value: "10250" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: CONFIGPATH + value: "/etc/interlink/InterLinkConfig.yaml" + - name: VKTOKENFILE + value: "/opt/interlink/token" + volumeMounts: + - name: config + mountPath: /etc/interlink/InterLinkConfig.yaml + subPath: InterLinkConfig.yaml + - name: token + mountPath: /opt/interlink + - name: jaeger + image: jaegertracing/all-in-one:1.51 + - name: refresh-token + image: ghcr.io/intertwin-eu/virtual-kubelet-inttw-refresh:latest + imagePullPolicy: Always + env: + - name: IAM_TOKEN_ENDPOINT + value: {{.Values.OAUTH.TokenURL}} + # TODO load env IAM client from secret + - name: IAM_CLIENT_ID + value: {{.Values.OAUTH.ClientID}} + - name: IAM_CLIENT_SECRET + value: {{.Values.OAUTH.ClientSecret}} + - name: IAM_REFRESH_TOKEN + value: {{.Values.OAUTH.RefreshToken}} + - name: IAM_VK_AUD + value: {{.Values.OAUTH.Audience}} + - name: TOKEN_PATH + value: /opt/interlink/token + command: + - python3 + - /opt/refresh.py + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 300Mi + volumeMounts: + - name: token + mountPath: /opt/interlink + volumes: + - name: config + configMap: + name: virtual-kubelet-config + - name: token + hostPath: + path: /tmp + type: Directory diff --git a/helm/interlink/values.yaml b/helm/interlink/values.yaml index 557b5482..6d53fd6c 100644 --- a/helm/interlink/values.yaml +++ b/helm/interlink/values.yaml @@ -2,81 +2,25 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 +nodeName: default-vk -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" +deployment: + image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:latest + name: default-vk -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" +interlink: + URL: http://localhost + port: 5000 -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" +virtualNode: + CPUs: 100 + MemGiB: 1600 + Pods: 100 -podAnnotations: {} +OAUTH: + TokenURL: DUMMY + ClientID: DUMMY + ClientSecret: DUMMY + RefreshToken: DUMMY + Audience: DUMMY -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/main.go b/main.go index 67ddc514..f2b85846 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "net/http" + "strings" "github.com/sirupsen/logrus" "github.com/virtual-kubelet/virtual-kubelet/log" @@ -22,12 +23,11 @@ func main() { } logger := logrus.StandardLogger() + logger.SetLevel(logrus.InfoLevel) if interLinkConfig.VerboseLogging { logger.SetLevel(logrus.DebugLevel) } else if interLinkConfig.ErrorsOnlyLogging { logger.SetLevel(logrus.ErrorLevel) - } else { - logger.SetLevel(logrus.InfoLevel) } log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) @@ -36,9 +36,19 @@ func main() { log.G(ctx).Info(interLinkConfig) + sidecarEndpoint := "" + if strings.HasPrefix(interLinkConfig.Sidecarurl, "unix://") { + sidecarEndpoint = interLinkConfig.Sidecarurl + } else if strings.HasPrefix(interLinkConfig.Sidecarurl, "http://") { + sidecarEndpoint = interLinkConfig.Sidecarurl + ":" + interLinkConfig.Sidecarport + } else { + log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") + } + interLinkAPIs := api.InterLinkHandler{ - Config: interLinkConfig, - Ctx: ctx, + Config: interLinkConfig, + Ctx: ctx, + SidecarEndpoint: sidecarEndpoint, } mutex := http.NewServeMux() @@ -48,7 +58,17 @@ func main() { mutex.HandleFunc("/pinglink", interLinkAPIs.Ping) mutex.HandleFunc("/getLogs", interLinkAPIs.GetLogsHandler) mutex.HandleFunc("/updateCache", interLinkAPIs.UpdateCacheHandler) - err = http.ListenAndServe(":"+interLinkConfig.Interlinkport, mutex) + + interLinkEndpoint := "" + if strings.HasPrefix(interLinkConfig.InterlinkAddress, "unix://") { + interLinkEndpoint = interLinkConfig.InterlinkAddress + } else if strings.HasPrefix(interLinkConfig.Sidecarurl, "http://") { + interLinkEndpoint = interLinkConfig.InterlinkAddress + ":" + interLinkConfig.Interlinkport + } else { + log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") + } + + err = http.ListenAndServe(interLinkEndpoint, mutex) if err != nil { log.G(ctx).Fatal(err) diff --git a/pkg/interlink/api/create.go b/pkg/interlink/api/create.go index 4f569e50..908e4b3a 100644 --- a/pkg/interlink/api/create.go +++ b/pkg/interlink/api/create.go @@ -61,7 +61,7 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) reader := bytes.NewReader(bodyBytes) log.G(h.Ctx).Info(req) - req, err = http.NewRequest(http.MethodPost, h.Config.Sidecarurl+":"+h.Config.Sidecarport+"/create", reader) + req, err = http.NewRequest(http.MethodPost, h.SidecarEndpoint+"/create", reader) if err != nil { statusCode = http.StatusInternalServerError diff --git a/pkg/interlink/api/delete.go b/pkg/interlink/api/delete.go index 009836fc..864d9dbb 100644 --- a/pkg/interlink/api/delete.go +++ b/pkg/interlink/api/delete.go @@ -37,7 +37,7 @@ func (h *InterLinkHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) } deleteCachedStatus(string(pod.UID)) - req, err = http.NewRequest(http.MethodPost, h.Config.Sidecarurl+":"+h.Config.Sidecarport+"/delete", reader) + req, err = http.NewRequest(http.MethodPost, h.SidecarEndpoint+"/delete", reader) if err != nil { statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) diff --git a/pkg/interlink/api/handler.go b/pkg/interlink/api/handler.go index 0a7e7dab..54dcecd6 100644 --- a/pkg/interlink/api/handler.go +++ b/pkg/interlink/api/handler.go @@ -7,7 +7,8 @@ import ( ) type InterLinkHandler struct { - Config interlink.InterLinkConfig - Ctx context.Context + Config interlink.InterLinkConfig + Ctx context.Context + SidecarEndpoint string // TODO: http client with TLS } diff --git a/pkg/interlink/api/logs.go b/pkg/interlink/api/logs.go index 5a92422a..c53a0f1b 100644 --- a/pkg/interlink/api/logs.go +++ b/pkg/interlink/api/logs.go @@ -54,7 +54,7 @@ func (h *InterLinkHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request return } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodGet, h.Config.Sidecarurl+":"+h.Config.Sidecarport+"/getLogs", reader) + req, err := http.NewRequest(http.MethodGet, h.SidecarEndpoint+"/getLogs", reader) if err != nil { log.G(h.Ctx).Fatal(err) } diff --git a/pkg/interlink/api/status.go b/pkg/interlink/api/status.go index 6e7bf6e1..92dde171 100644 --- a/pkg/interlink/api/status.go +++ b/pkg/interlink/api/status.go @@ -49,7 +49,7 @@ func (h *InterLinkHandler) StatusHandler(w http.ResponseWriter, r *http.Request) } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodGet, h.Config.Sidecarurl+":"+h.Config.Sidecarport+"/status", reader) + req, err := http.NewRequest(http.MethodGet, h.SidecarEndpoint+"/status", reader) if err != nil { log.G(h.Ctx).Fatal(err) } diff --git a/pkg/virtualkubelet/execute.go b/pkg/virtualkubelet/execute.go index 37e220bb..bb7a7028 100644 --- a/pkg/virtualkubelet/execute.go +++ b/pkg/virtualkubelet/execute.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/containerd/containerd/log" @@ -19,11 +20,32 @@ import ( commonIL "github.com/intertwin-eu/interlink/pkg/interlink" ) +func doRequest(req *http.Request, token string) (*http.Response, error) { + + req.Header.Add("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + return http.DefaultClient.Do(req) + +} + +func getSidecarEndpoint(ctx context.Context, interLinkURL string, interLinkPort string) string { + interLinkEndpoint := "" + if strings.HasPrefix(interLinkURL, "unix://") { + interLinkEndpoint = interLinkURL + } else if strings.HasPrefix(interLinkURL, "http://") { + interLinkEndpoint = interLinkURL + ":" + interLinkPort + } else { + log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") + } + return interLinkEndpoint +} + // PingInterLink pings the InterLink API and returns true if there's an answer. The second return value is given by the answer provided by the API. func PingInterLink(ctx context.Context, config VirtualKubeletConfig) (bool, int, error) { - log.G(ctx).Info("Pinging: " + config.Interlinkurl + ":" + config.Interlinkport + "/pinglink") + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) + log.G(ctx).Info("Pinging: " + interLinkEndpoint + "/pinglink") retVal := -1 - req, err := http.NewRequest(http.MethodPost, config.Interlinkurl+":"+config.Interlinkport+"/pinglink", nil) + req, err := http.NewRequest(http.MethodPost, interLinkEndpoint+"/pinglink", nil) if err != nil { log.G(ctx).Error(err) @@ -59,11 +81,12 @@ func PingInterLink(ctx context.Context, config VirtualKubeletConfig) (bool, int, } // updateCacheRequest is called when the VK receives the status of a pod already deleted. It performs a REST call InterLink API to update the cache deleting that pod from the cached structure -func updateCacheRequest(config VirtualKubeletConfig, uid string, token string) error { +func updateCacheRequest(ctx context.Context, config VirtualKubeletConfig, uid string, token string) error { bodyBytes := []byte(uid) + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodPost, config.Interlinkurl+":"+config.Interlinkport+"/updateCache", reader) + req, err := http.NewRequest(http.MethodPost, interLinkEndpoint+"/updateCache", reader) if err != nil { log.L.Error(err) return err @@ -87,7 +110,8 @@ func updateCacheRequest(config VirtualKubeletConfig, uid string, token string) e // createRequest performs a REST call to the InterLink API when a Pod is registered to the VK. It Marshals the pod with already retrieved ConfigMaps and Secrets and sends it to InterLink. // Returns the call response expressed in bytes and/or the first encountered error -func createRequest(config VirtualKubeletConfig, pod commonIL.PodCreateRequests, token string) ([]byte, error) { +func createRequest(ctx context.Context, config VirtualKubeletConfig, pod commonIL.PodCreateRequests, token string) ([]byte, error) { + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) var returnValue, _ = json.Marshal(commonIL.PodStatus{}) bodyBytes, err := json.Marshal(pod) @@ -96,15 +120,13 @@ func createRequest(config VirtualKubeletConfig, pod commonIL.PodCreateRequests, return nil, err } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodPost, config.Interlinkurl+":"+config.Interlinkport+"/create", reader) + req, err := http.NewRequest(http.MethodPost, interLinkEndpoint+"/create", reader) if err != nil { log.L.Error(err) return nil, err } - req.Header.Add("Authorization", "Bearer "+token) - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := doRequest(req, token) if err != nil { log.L.Error(err) return nil, err @@ -126,22 +148,21 @@ func createRequest(config VirtualKubeletConfig, pod commonIL.PodCreateRequests, // deleteRequest performs a REST call to the InterLink API when a Pod is deleted from the VK. It Marshals the standard v1.Pod struct and sends it to InterLink. // Returns the call response expressed in bytes and/or the first encountered error -func deleteRequest(config VirtualKubeletConfig, pod *v1.Pod, token string) ([]byte, error) { +func deleteRequest(ctx context.Context, config VirtualKubeletConfig, pod *v1.Pod, token string) ([]byte, error) { + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) bodyBytes, err := json.Marshal(pod) if err != nil { log.G(context.Background()).Error(err) return nil, err } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodDelete, config.Interlinkurl+":"+config.Interlinkport+"/delete", reader) + req, err := http.NewRequest(http.MethodDelete, interLinkEndpoint+"/delete", reader) if err != nil { log.G(context.Background()).Error(err) return nil, err } - req.Header.Add("Authorization", "Bearer "+token) - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := doRequest(req, token) if err != nil { log.G(context.Background()).Error(err) return nil, err @@ -171,8 +192,9 @@ func deleteRequest(config VirtualKubeletConfig, pod *v1.Pod, token string) ([]by // statusRequest performs a REST call to the InterLink API when the VK needs an update on its Pods' status. A Marshalled slice of v1.Pod is sent to the InterLink API, // to query the below plugin for their status. // Returns the call response expressed in bytes and/or the first encountered error -func statusRequest(config VirtualKubeletConfig, podsList []*v1.Pod, token string) ([]byte, error) { +func statusRequest(ctx context.Context, config VirtualKubeletConfig, podsList []*v1.Pod, token string) ([]byte, error) { var returnValue []byte + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) bodyBytes, err := json.Marshal(podsList) if err != nil { @@ -180,7 +202,7 @@ func statusRequest(config VirtualKubeletConfig, podsList []*v1.Pod, token string return nil, err } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodGet, config.Interlinkurl+":"+config.Interlinkport+"/status", reader) + req, err := http.NewRequest(http.MethodGet, interLinkEndpoint+"/status", reader) if err != nil { log.L.Error(err) return nil, err @@ -188,10 +210,7 @@ func statusRequest(config VirtualKubeletConfig, podsList []*v1.Pod, token string //log.L.Println(string(bodyBytes)) - req.Header.Add("Authorization", "Bearer "+token) - - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := doRequest(req, token) if err != nil { return nil, err } @@ -213,6 +232,7 @@ func statusRequest(config VirtualKubeletConfig, podsList []*v1.Pod, token string // This struct only includes a minimum data set needed to identify the job/container to get the logs from. // Returns the call response and/or the first encountered error func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest commonIL.LogStruct) (io.ReadCloser, error) { + interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) b, err := os.ReadFile(config.VKTokenFile) // just pass the file name if err != nil { log.G(ctx).Fatal(err) @@ -225,18 +245,15 @@ func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest return nil, err } reader := bytes.NewReader(bodyBytes) - req, err := http.NewRequest(http.MethodGet, config.Interlinkurl+":"+config.Interlinkport+"/getLogs", reader) + req, err := http.NewRequest(http.MethodGet, interLinkEndpoint+"/getLogs", reader) if err != nil { log.G(ctx).Error(err) return nil, err } - log.G(ctx).Println(string(bodyBytes)) - - req.Header.Add("Authorization", "Bearer "+token) + //log.G(ctx).Println(string(bodyBytes)) - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := doRequest(req, token) if err != nil { log.G(ctx).Error(err) return nil, err @@ -329,7 +346,7 @@ func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *Virtua } } - returnVal, err := createRequest(config, req, token) + returnVal, err := createRequest(ctx, config, req, token) if err != nil { return err } @@ -338,7 +355,7 @@ func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *Virtua case DELETE: req := pod if pod.Status.Phase != "Initializing" { - returnVal, err := deleteRequest(config, req, token) + returnVal, err := deleteRequest(ctx, config, req, token) if err != nil { return err } @@ -358,7 +375,7 @@ func checkPodsStatus(ctx context.Context, p *VirtualKubeletProvider, podsList [] //log.G(ctx).Debug(p.pods) //commented out because it's too verbose. uncomment to see all registered pods - returnVal, err = statusRequest(config, podsList, token) + returnVal, err = statusRequest(ctx, config, podsList, token) if err != nil { return nil, err @@ -372,7 +389,7 @@ func checkPodsStatus(ctx context.Context, p *VirtualKubeletProvider, podsList [] pod, err := p.GetPod(ctx, podStatus.PodNamespace, podStatus.PodName) if err != nil { - updateCacheRequest(config, podStatus.PodUID, token) + updateCacheRequest(ctx, config, podStatus.PodUID, token) log.G(ctx).Warning("Error: " + err.Error() + "while getting statuses. Updating InterLink cache") return nil, err } @@ -436,11 +453,6 @@ func checkPodsStatus(ctx context.Context, p *VirtualKubeletProvider, podsList [] pod.Status.Reason = "Completed" } - err = p.UpdatePod(ctx, pod) - if err != nil { - log.G(ctx).Error(err) - return nil, err - } } } diff --git a/pkg/virtualkubelet/version.go b/pkg/virtualkubelet/version.go new file mode 100644 index 00000000..db339a26 --- /dev/null +++ b/pkg/virtualkubelet/version.go @@ -0,0 +1,5 @@ +package virtualkubelet + +var ( + KubeletVersion = "test" +) diff --git a/pkg/virtualkubelet/virtualkubelet.go b/pkg/virtualkubelet/virtualkubelet.go index efbdf333..f168fa68 100644 --- a/pkg/virtualkubelet/virtualkubelet.go +++ b/pkg/virtualkubelet/virtualkubelet.go @@ -37,11 +37,11 @@ const ( DELETE = 1 ) -func BuildKeyFromNames(namespace string, name string) (string, error) { +func buildKeyFromNames(namespace string, name string) (string, error) { return fmt.Sprintf("%s-%s", namespace, name), nil } -func BuildKey(pod *v1.Pod) (string, error) { +func buildKey(pod *v1.Pod) (string, error) { if pod.Namespace == "" { return "", fmt.Errorf("pod namespace not found") } @@ -50,9 +50,10 @@ func BuildKey(pod *v1.Pod) (string, error) { return "", fmt.Errorf("pod name not found") } - return BuildKeyFromNames(pod.Namespace, pod.Name) + return buildKeyFromNames(pod.Namespace, pod.Name) } +// VirtualKubeletProvider defines the properties of the virtual kubelet provider type VirtualKubeletProvider struct { nodeName string node *v1.Node @@ -67,9 +68,11 @@ type VirtualKubeletProvider struct { clientSet *kubernetes.Clientset } +// NewProviderConfig takes user-defined configuration and fills the Virtual Kubelet provider struct func NewProviderConfig( config VirtualKubeletConfig, nodeName string, + nodeVersion string, operatingSystem string, internalIP string, daemonEndpointPort int32, @@ -112,11 +115,11 @@ func NewProviderConfig( }}, }, Status: v1.NodeStatus{ - // NodeInfo: v1.NodeSystemInfo{ - // KubeletVersion: Version, - // Architecture: architecture, - // OperatingSystem: linuxos, - // }, + NodeInfo: v1.NodeSystemInfo{ + KubeletVersion: nodeVersion, + Architecture: "virtual-kubelet", + OperatingSystem: "linux", + }, Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: internalIP}}, DaemonEndpoints: v1.NodeDaemonEndpoints{KubeletEndpoint: v1.DaemonEndpoint{Port: int32(daemonEndpointPort)}}, Capacity: v1.ResourceList{ @@ -144,25 +147,21 @@ func NewProviderConfig( pods: make(map[string]*v1.Pod), config: config, startTime: time.Now(), - onNodeChangeCallback: func(node *v1.Node) { - }, - //notifier: func(p *v1.Pod) { - // }, } return &provider, nil } -// NewProvider creates a new Provider, which implements the PodNotifier interface -func NewProvider(providerConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, ctx context.Context) (*VirtualKubeletProvider, error) { +// NewProvider creates a new Provider, which implements the PodNotifier and other virtual-kubelet interfaces +func NewProvider(providerConfig, nodeName, nodeVersion, operatingSystem string, internalIP string, daemonEndpointPort int32, ctx context.Context) (*VirtualKubeletProvider, error) { config, err := LoadConfig(providerConfig, nodeName, ctx) if err != nil { return nil, err } - return NewProviderConfig(config, nodeName, operatingSystem, internalIP, daemonEndpointPort) + return NewProviderConfig(config, nodeName, nodeVersion, operatingSystem, internalIP, daemonEndpointPort) } -// loadConfig loads the given json configuration files and yaml to communicate with InterLink. +// LoadConfig loads the given json configuration files and return a VirtualKubeletConfig struct func LoadConfig(providerConfig, nodeName string, ctx context.Context) (config VirtualKubeletConfig, err error) { log.G(ctx).Info("Loading Virtual Kubelet config from " + providerConfig) @@ -207,28 +206,154 @@ func LoadConfig(providerConfig, nodeName string, ctx context.Context) (config Vi return config, nil } +// GetNode return the Node information at the initiation of a virtual node func (p *VirtualKubeletProvider) GetNode() *v1.Node { return p.node } +// NotifyNodeStatus runs once at initiation time and set the function to be used for node change notification (native of vk) +// it also starts a go routine for continously checking the node status and availability func (p *VirtualKubeletProvider) NotifyNodeStatus(ctx context.Context, f func(*v1.Node)) { p.onNodeChangeCallback = f + go p.nodeUpdate(ctx) +} + +// nodeUpdate continously checks for node status and availability +func (p *VirtualKubeletProvider) nodeUpdate(ctx context.Context) { + + t := time.NewTimer(5 * time.Second) + if !t.Stop() { + <-t.C + } + + log.G(ctx).Info("nodeLoop") + + _, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name + if err != nil { + log.G(context.Background()).Fatal(err) + } + + for { + t.Reset(30 * time.Second) + select { + case <-ctx.Done(): + return + case <-t.C: + } + bool, code, err := PingInterLink(ctx, p.config) + if err != nil || !bool { + p.node.Status.Conditions = []v1.NodeCondition{ + { + Type: "Ready", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletPending", + Message: "kubelet is pending.", + }, + { + Type: "OutOfDisk", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasSufficientDisk", + Message: "kubelet has sufficient disk space available", + }, + { + Type: "MemoryPressure", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasSufficientMemory", + Message: "kubelet has sufficient memory available", + }, + { + Type: "DiskPressure", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasNoDiskPressure", + Message: "kubelet has no disk pressure", + }, + { + Type: "NetworkUnavailable", + Status: v1.ConditionTrue, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "RouteCreated", + Message: "RouteController created a route", + }, + } + p.onNodeChangeCallback(p.node) + log.G(ctx).Error("Ping Failed with exit code: ", code) + } else { + + p.node.Status.Conditions = []v1.NodeCondition{ + { + Type: "Ready", + Status: v1.ConditionTrue, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletPending", + Message: "kubelet is pending.", + }, + { + Type: "OutOfDisk", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasSufficientDisk", + Message: "kubelet has sufficient disk space available", + }, + { + Type: "MemoryPressure", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasSufficientMemory", + Message: "kubelet has sufficient memory available", + }, + { + Type: "DiskPressure", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "KubeletHasNoDiskPressure", + Message: "kubelet has no disk pressure", + }, + { + Type: "NetworkUnavailable", + Status: v1.ConditionFalse, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: "RouteCreated", + Message: "RouteController created a route", + }, + } + + log.G(ctx).Info("Ping succeded with exit code: ", code) + p.onNodeChangeCallback(p.node) + } + log.G(ctx).Info("endNodeLoop") + } + } +// Ping the kubelet from the cluster, this will always be ok by design probably func (p *VirtualKubeletProvider) Ping(ctx context.Context) error { return nil } -// CreatePod accepts a Pod definition and stores it in memory. +// CreatePod accepts a Pod definition and stores it in memory in p.pods func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { ctx, span := trace.StartSpan(ctx, "CreatePod") var hasInitContainers = false var state v1.ContainerState defer span.End() - //distribution := "docker://" + // Add the pod's coordinates to the current span. ctx = addAttributes(ctx, span, NamespaceKey, pod.Namespace, NameKey, pod.Name) - key, err := BuildKey(pod) + key, err := buildKey(pod) if err != nil { return err } @@ -249,15 +374,8 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err if len(pod.Spec.InitContainers) > 0 { state = waitingState hasInitContainers = true - // run init container with remote execution enabled - /*for _, container := range pod.Spec.InitContainers { - // MUST TODO: Run init containers sequentialy and NOT all-together - err = RemoteExecution(p, ctx, CREATE, distribution+container.Image, pod, container) - if err != nil { - return err - } - }*/ + // we put the phase in running but initialization phase to false pod.Status = v1.PodStatus{ Phase: v1.PodRunning, HostIP: p.internalIP, @@ -279,6 +397,8 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err }, } } else { + + // if no init containers are there, go head and set phase to initialized pod.Status = v1.PodStatus{ Phase: v1.PodPending, HostIP: p.internalIP, @@ -301,8 +421,10 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err } } + // Create pod asynchronously on the remote plugin + // we don't care, the statusLoop will eventually reconcile the status go func() { - err = RemoteExecution(ctx, p.config, p, pod, CREATE) + err := RemoteExecution(ctx, p.config, p, pod, CREATE) if err != nil { if err.Error() == "Deleted pod before actual creation" { log.G(ctx).Warn(err) @@ -313,28 +435,20 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err } }() - // deploy main containers + // set pod containers status to notReady and waiting if there is an initContainer to be executed first for _, container := range pod.Spec.Containers { - //var err error - /*if !hasInitContainers { - err = RemoteExecution(p, ctx, CREATE, distribution+container.Image, pod, container) - if err != nil { - return err - } - }*/ pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, v1.ContainerStatus{ Name: container.Name, Image: container.Image, Ready: !hasInitContainers, - RestartCount: 1, + RestartCount: 0, State: state, }) } p.pods[key] = pod - p.notifier(pod) return nil } @@ -349,18 +463,12 @@ func (p *VirtualKubeletProvider) UpdatePod(ctx context.Context, pod *v1.Pod) err log.G(ctx).Infof("receive UpdatePod %q", pod.Name) - key, err := BuildKey(pod) - if err != nil { - return err - } - - p.pods[key] = pod p.notifier(pod) return nil } -// DeletePod deletes the specified pod out of memory. +// DeletePod deletes the specified pod and drops it out of p.pods func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (err error) { ctx, span := trace.StartSpan(ctx, "DeletePod") defer span.End() @@ -370,7 +478,7 @@ func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (er log.G(ctx).Infof("receive DeletePod %q", pod.Name) - key, err := BuildKey(pod) + key, err := buildKey(pod) if err != nil { return err } @@ -397,7 +505,6 @@ func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (er Message: "VK provider terminated container upon deletion", FinishedAt: now, Reason: "VKProviderPodContainerDeleted", - // StartedAt: pod.Status.ContainerStatuses[idx].State.Running.StartedAt, }, } } @@ -408,12 +515,14 @@ func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (er Message: "VK provider terminated container upon deletion", FinishedAt: now, Reason: "VKProviderPodContainerDeleted", - // StartedAt: pod.Status.InitContainerStatuses[idx].State.Running.StartedAt, }, } } - p.notifier(pod) + // tell k8s it's terminated + p.UpdatePod(ctx, pod) + + // delete from p.pods delete(p.pods, key) return nil @@ -421,6 +530,7 @@ func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (er // GetPod returns a pod by name that is stored in memory. func (p *VirtualKubeletProvider) GetPod(ctx context.Context, namespace, name string) (pod *v1.Pod, err error) { + ctx, span := trace.StartSpan(ctx, "GetPod") defer func() { span.SetStatus(err) @@ -432,7 +542,7 @@ func (p *VirtualKubeletProvider) GetPod(ctx context.Context, namespace, name str log.G(ctx).Infof("receive GetPod %q", name) - key, err := BuildKeyFromNames(namespace, name) + key, err := buildKeyFromNames(namespace, name) if err != nil { return nil, err } @@ -469,7 +579,7 @@ func (p *VirtualKubeletProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) log.G(ctx).Info("receive GetPods") - p.InitClientSet(ctx) + p.initClientSet(ctx) p.RetrievePodsFromInterlink(ctx) var pods []*v1.Pod @@ -483,8 +593,8 @@ func (p *VirtualKubeletProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) // NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status // within Kubernetes. +// TODO: use as var not function func nodeConditions() []v1.NodeCondition { - // TODO: Make this configurable return []v1.NodeCondition{ { Type: "Ready", @@ -530,27 +640,26 @@ func nodeConditions() []v1.NodeCondition { } -// NotifyPods is called to set a pod notifier callback function. This should be called before any operations are done -// within the provider. +// NotifyPods is called to set a pod notifier callback function. Also starts the go routine to monitor all vk pods func (p *VirtualKubeletProvider) NotifyPods(ctx context.Context, f func(*v1.Pod)) { p.notifier = f go p.statusLoop(ctx) } +// statusLoop preiodically monitoring the status of all the pods in p.pods func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { t := time.NewTimer(5 * time.Second) if !t.Stop() { <-t.C } - log.G(ctx).Info("statusLoop") - _, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name if err != nil { log.G(context.Background()).Fatal(err) } for { + log.G(ctx).Info("statusLoop") t.Reset(5 * time.Second) select { case <-ctx.Done(): @@ -567,6 +676,10 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { for _, pod := range p.pods { if pod.Status.Phase != "Initializing" { podsList = append(podsList, pod) + err = p.UpdatePod(ctx, pod) + if err != nil { + log.G(ctx).Error(err) + } } } @@ -575,6 +688,13 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { if err != nil { log.G(ctx).Error(err) } + for _, pod := range p.pods { + key, err := buildKey(pod) + if err != nil { + log.G(ctx).Error(err) + } + p.pods[key] = pod + } } log.G(ctx).Info("statusLoop=end") @@ -595,6 +715,7 @@ func addAttributes(ctx context.Context, span trace.Span, attrs ...string) contex return ctx } +// GetLogs implements the logic for interLink pod logs retrieval. func (p *VirtualKubeletProvider) GetLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) { var span trace.Span ctx, span = trace.StartSpan(ctx, "GetLogs") //nolint: ineffassign,staticcheck @@ -605,7 +726,7 @@ func (p *VirtualKubeletProvider) GetLogs(ctx context.Context, namespace, podName log.G(ctx).Infof("receive GetPodLogs %q", podName) - key, err := BuildKeyFromNames(namespace, podName) + key, err := buildKeyFromNames(namespace, podName) if err != nil { log.G(ctx).Error(err) } @@ -699,7 +820,8 @@ func (p *VirtualKubeletProvider) GetStatsSummary(ctx context.Context) (*stats.Su return res, nil } -// GetPods returns a list of all pods known to be "running". +// GetPods returns a list of all pods known to be "running" from the local disk cache info +// This will run at the initiation time only func (p *VirtualKubeletProvider) RetrievePodsFromInterlink(ctx context.Context) error { ctx, span := trace.StartSpan(ctx, "RetrievePodsFromInterlink") defer span.End() @@ -718,19 +840,19 @@ func (p *VirtualKubeletProvider) RetrievePodsFromInterlink(ctx context.Context) if err != nil { log.G(ctx).Warning("Unable to retrieve pod " + retrievedPod.Name + " from the cluster") } else { - key, err := BuildKey(retrievedPod) + key, err := buildKey(retrievedPod) if err != nil { log.G(ctx).Error(err) } p.pods[key] = retrievedPod - p.notifier(retrievedPod) + p.UpdatePod(ctx, retrievedPod) } } return err } -func (p *VirtualKubeletProvider) InitClientSet(ctx context.Context) error { +func (p *VirtualKubeletProvider) initClientSet(ctx context.Context) error { ctx, span := trace.StartSpan(ctx, "InitClientSet") defer span.End()