diff --git a/.DS_Store b/.DS_Store index 2a2dc9a3..16334541 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/workflows/build_docusaurus.yaml b/.github/workflows/build_docusaurus.yaml new file mode 100644 index 00000000..aea0b1a1 --- /dev/null +++ b/.github/workflows/build_docusaurus.yaml @@ -0,0 +1,50 @@ +name: Update Docusaurus + +on: + push: + branches: + - main + tags-ignore: + - "*" + workflow_call: + +jobs: + update-doc: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Remove existing openapi.json + run: rm ./docs/openapi/openapi.json + + - name: Download openapi.json + uses: robinraju/release-downloader@v1 + with: + latest: true + preRelease: false + fileName: 'openAPISpec' + out-file-path: 'docs/openapi' + + - name: Rename openAPISpec to openapi.json + run: mv ./docs/openapi/openAPISpec ./docs/openapi/openapi.json + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '22' # Ensure this matches the Node.js version required by Docusaurus + + - name: Install dependencies + working-directory: ./docs # Change to the directory where package.json is located + run: npm install + + - name: Build Docusaurus site + working-directory: ./docs # Change to the directory where package.json is located + run: npm run build + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/build # Ensure this matches the build output directory of Docusaurus# diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index 058fda16..a78a03c4 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -39,8 +39,8 @@ jobs: ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw:latest file: ./docker/Dockerfile.vk platforms: linux/amd64, linux/arm64 - build-args: - - VERSION=${{ env.RELEASE_VERSION }} + build-args: | + VERSION=${{ env.RELEASE_VERSION }} - name: Build container base image interlink uses: docker/build-push-action@v5 with: @@ -52,7 +52,7 @@ jobs: file: ./docker/Dockerfile.interlink platforms: linux/amd64, linux/arm64 - virtual-kubelet-refrest-token: + virtual-kubelet-refresh-token: runs-on: ubuntu-latest #env: # DOCKER_TARGET_PLATFORM: linux/arm64 diff --git a/.github/workflows/build_openapi.yaml b/.github/workflows/build_openapi.yaml new file mode 100644 index 00000000..a3c2a969 --- /dev/null +++ b/.github/workflows/build_openapi.yaml @@ -0,0 +1,40 @@ +name: Update OpenAPI + +on: + push: + tags: + - '*' + +jobs: + update-openapi: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # specify the Python version you need + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r example/requirements.txt # if you have a requirements file + + - name: Run script to generate OpenAPI JSON + run: python example/create_openapi.py + + - name: Upload json to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ./docs/openapi/openapi.json + asset_name: openAPISpec + tag: ${{ github.ref }} + overwrite: true + body: "OpenAPI spec for plugin REST" + + Trigger-Docusaurus-Update: + uses: ./.github/workflows/build_docusaurus.yaml \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0e42364..185062a2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,21 +12,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - #password: ${{ secrets.GITHUB_TOKEN }} - password: ${{ secrets.GH_CR_TOKEN }} - name: Get Repo Owner id: get_repo_owner run: echo ::set-output name=repo_owner::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') - name: Integration Test - uses: dagger/dagger-for-github@v5 + uses: dagger/dagger-for-github@v6 with: workdir: ci verb: call - args: -s build-images --source-folder ../ --virtual-kubelet-ref ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/virtual-kubelet-inttw:$GITHUB_SHA --interlink-ref ghcr.io/${{ steps.get_repo_owner.outputs.repo_owner }}/interlink/interlink:$GITHUB_SHA new-interlink --manifests $PWD/manifests load-plugin test stdout + args: -s --name slurm-test build-images new-interlink test stdout cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} - #dagger-flags: -d + version: "0.13.0" diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml deleted file mode 100644 index e55889d5..00000000 --- a/.github/workflows/pages.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Deploy to GitHub Pages - -on: - push: - branches: - - main - # Review gh actions docs if you want to further define triggers, paths, etc - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on - -permissions: - contents: write - -jobs: - deploy: - name: Deploy to GitHub Pages - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 18 - #cache: yarn - - - name: Install dependencies - run: cd docs && yarn install --frozen-lockfile - - name: Build website - run: cd docs && yarn build - - # Popular action to deploy to GitHub Pages: - # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - # Build output to publish to the `gh-pages` branch: - publish_dir: ./docs/build - # The following lines assign commit authorship to the official - # GH-Actions bot for deploys to `gh-pages` branch: - # https://github.com/actions/checkout/issues/13#issuecomment-724415212 - # The GH actions bot is used by default if you didn't specify the two fields. - # You can swap them out with your own user credentials. - user_name: github-actions[bot] - user_email: 41898282+github-actions[bot]@users.noreply.github.com diff --git a/.goreleaser.yaml b/.goreleaser.yaml index f25ebe14..096d9598 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -18,8 +18,10 @@ builds: - arm64 - amd64 main: ./cmd/virtual-kubelet - - id: "interlink" + - id: "interlink-api" binary: interlink + hooks: + pre: bash -c "KUBELET_VERSION={{.Version}} ./cmd/virtual-kubelet/set-version.sh" env: - CGO_ENABLED=0 goos: @@ -29,8 +31,9 @@ builds: - arm64 - amd64 - ppc64le - - id: "interlink-install" - binary: interlink-install + main: ./cmd/interlink + - id: "installer" + binary: interlink-installer env: - CGO_ENABLED=0 goos: @@ -41,6 +44,18 @@ builds: - amd64 - ppc64le main: ./cmd/installer + - id: "ssh-tunnel" + binary: ssh-tunnel + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + goarch: + - arm64 + - amd64 + - ppc64le + main: ./cmd/ssh-tunnel archives: - name_template: >- {{ .Binary }}_ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a8ed27c..26fa5667 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,7 @@ you submit the pull request). We have a 3 steps process for contributions. understanding and appreciating your pull request, please use the template to explain _why_ you are making this contribution, rather than just _what_ the contribution entails. +2. **Run E2E tests with success**. You can follow the steps described [here](https://intertwin-eu.github.io/interLink/docs/Developers) ### Code Review Process @@ -44,7 +45,7 @@ follows: 1. A maintainer will review your code and merge it if no changes are necessary. Your change will be merged into the repository's `main` branch. -1. If a maintainer has feedback or questions on your changes then they will set +2. If a maintainer has feedback or questions on your changes then they will set `request changes` in the review and provide an explanation. ## Using git diff --git a/Makefile b/Makefile index f9456082..431d953b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: interlink vk installer +all: interlink vk installer ssh-tunnel interlink: - CGO_ENABLED=0 OOS=linux go build -o bin/interlink + CGO_ENABLED=0 OOS=linux go build -o bin/interlink cmd/interlink/main.go vk: CGO_ENABLED=0 OOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go @@ -9,16 +9,16 @@ vk: installer: CGO_ENABLED=0 OOS=linux go build -o bin/installer cmd/installer/main.go +ssh-tunnel: + CGO_ENABLED=0 OOS=linux go build -o bin/ssh-tunnel cmd/ssh-tunnel/main.go + clean: rm -rf ./bin -dagger_registry_delete: - docker rm -fv registry || true - test: - dagger_registry_delete - docker run -d --rm --name registry -p 5432:5000 registry - cd ci - dagger go run go main.go k8s.go - cd - + dagger call -m ./ci \ + --name my-tests \ + build-images \ + new-interlink \ + test stdout diff --git a/README.md b/README.md index cbd5f584..67f9adde 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ +![GitHub License](https://img.shields.io/github/license/intertwin-eu/interlink) +![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/intertwin-eu/interlink/total) +![GitHub Repo stars](https://img.shields.io/github/stars/intertwin-eu/interlink) + +![GitHub Release](https://img.shields.io/github/v/release/intertwin-eu/interlink) +![Tested with Dagger](https://img.shields.io/badge/tested_with_dagger-v0.13.3-green) + +![Home Page](https://img.shields.io/badge/home_page-orange?link=https%3A%2F%2Fintertwin-eu.github.io%2Ft%2FinterLink%2F) +![Slack server](https://img.shields.io/badge/slack_server-8A2BE2?link=https%3A%2F%2Fjoin.slack.com%2Ft%2Fintertwin%2Fshared_invite%2Fzt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA) + ![Interlink logo](./docs/static/img/interlink_logo.png) ## :information_source: Overview diff --git a/ci/dagger.json b/ci/dagger.json index 44df161b..53fc83d3 100644 --- a/ci/dagger.json +++ b/ci/dagger.json @@ -1,6 +1,12 @@ { "name": "interlink", "sdk": "go", + "dependencies": [ + { + "name": "k3s", + "source": "github.com/marcosnils/daggerverse/k3s@ce8fe35d309bdb29f9983f7d90ea518e724534fe" + } + ], "source": ".", - "engineVersion": "v0.11.4" + "engineVersion": "v0.13.0" } diff --git a/ci/go.mod b/ci/go.mod index 8149d7ae..18ab4923 100644 --- a/ci/go.mod +++ b/ci/go.mod @@ -3,17 +3,22 @@ module dagger/interlink go 1.22.2 require ( - github.com/99designs/gqlgen v0.17.44 + github.com/99designs/gqlgen v0.17.49 github.com/Khan/genqlient v0.7.0 - github.com/vektah/gqlparser/v2 v2.5.11 - go.opentelemetry.io/otel v1.24.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 - go.opentelemetry.io/otel/sdk v1.24.0 - go.opentelemetry.io/otel/trace v1.24.0 + github.com/vektah/gqlparser/v2 v2.5.16 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/log v0.3.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/sdk/log v0.3.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.opentelemetry.io/proto/otlp v1.3.1 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.63.2 + google.golang.org/grpc v1.64.0 ) require ( @@ -21,16 +26,23 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect - github.com/sosodev/duration v1.2.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect - google.golang.org/protobuf v1.33.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sosodev/duration v1.3.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/protobuf v1.34.1 // indirect ) + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 + +replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 + +replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.3.0 + +replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.3.0 diff --git a/ci/go.sum b/ci/go.sum index 13039de9..6fea81b9 100644 --- a/ci/go.sum +++ b/ci/go.sum @@ -1,11 +1,12 @@ -github.com/99designs/gqlgen v0.17.44 h1:OS2wLk/67Y+vXM75XHbwRnNYJcbuJd4OBL76RX3NQQA= -github.com/99designs/gqlgen v0.17.44/go.mod h1:UTCu3xpK2mLI5qcMNw+HKDiEL77it/1XtAjisC4sLwM= +github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ= +github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0= github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -17,55 +18,70 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sosodev/duration v1.2.0 h1:pqK/FLSjsAADWY74SyWDCjOcd5l7H8GSnnOGEB9A1Us= -github.com/sosodev/duration v1.2.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= +github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8= -github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= +github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 h1:oM0GTNKGlc5qHctWeIGTVyda4iFFalOzMZ3Ehj5rwB4= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88/go.mod h1:JGG8ebaMO5nXOPnvKEl+DiA4MGwFjCbjsxT1WHIEBPY= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 h1:ccBrA8nCY5mM0y5uO7FT0ze4S0TuFcWdDB2FxGMTjkI= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0/go.mod h1:/9pb6634zi2Lk8LYg9Q0X8Ar6jka4dkFOylBLbVQPCE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/log v0.3.0 h1:kJRFkpUFYtny37NQzL386WbznUByZx186DpEMKhEGZs= +go.opentelemetry.io/otel/log v0.3.0/go.mod h1:ziCwqZr9soYDwGNbIL+6kAvQC+ANvjgG367HVcyR/ys= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= +go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa h1:RBgMaUMP+6soRkik4VoN8ojR2nex2TqZwjSSogic+eo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ci/k8s.go b/ci/k8s.go deleted file mode 100644 index a683d909..00000000 --- a/ci/k8s.go +++ /dev/null @@ -1,269 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - "time" -) - -// entrypoint to setup cgroup nesting since k3s only does it -// when running as PID 1. This doesn't happen in Dagger given that we're using -// our custom shim -const entrypoint = `#!/bin/sh - -set -o errexit -set -o nounset - -######################################################################################################################################### -# DISCLAIMER # -# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37 # -# Permission granted by Akihiro Suda (https://github.com/k3d-io/k3d/issues/493#issuecomment-827405962) # -# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE # -######################################################################################################################################### -if [ -f /sys/fs/cgroup/cgroup.controllers ]; then - echo "[$(date -Iseconds)] [CgroupV2 Fix] Evacuating Root Cgroup ..." - # move the processes from the root group to the /init group, - # otherwise writing subtree_control fails with EBUSY. - mkdir -p /sys/fs/cgroup/init - busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : - # enable controllers - sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control" - echo "[$(date -Iseconds)] [CgroupV2 Fix] Done" -fi - -exec "$@" -` - -func NewK8sInstance(ctx context.Context) *K8sInstance { - return &K8sInstance{ - KContainer: nil, - Registry: nil, - ConfigCache: dag.CacheVolume("k3s_config"), - ContainersCache: dag.CacheVolume("k3s_containers"), - } -} - -type K8sInstance struct { - KContainer *Container - K3s *Container - Registry *Service - ConfigCache *CacheVolume - ContainersCache *CacheVolume -} - -func (k *K8sInstance) start( - ctx context.Context, - manifests *Directory, - // +optional - bufferVK string, - // +optional - bufferIL string, - // +optional - kubeconfig *File, - // +optional - localCluster *Service) error { - - if kubeconfig == nil { - // create k3s service container - k.K3s = dag.Pipeline("k3s init").Container(). - From("rancher/k3s"). - WithNewFile("/usr/bin/entrypoint.sh", ContainerWithNewFileOpts{ - Contents: entrypoint, - Permissions: 0o755, - }). - WithEntrypoint([]string{"entrypoint.sh"}). - WithMountedCache("/etc/rancher/k3s", k.ConfigCache). - WithMountedTemp("/etc/lib/cni"). - WithMountedCache("/etc/lib/containers", k.ContainersCache). - WithMountedTemp("/var/lib/kubelet"). - WithMountedTemp("/var/lib/rancher/k3s"). - WithMountedTemp("/var/log"). - WithExec([]string{"sh", "-c", "k3s server --bind-address $(ip route | grep src | awk '{print $NF}') --disable traefik --disable metrics-server --egress-selector-mode=disabled"}, ContainerWithExecOpts{InsecureRootCapabilities: true}). - WithExposedPort(6443) - - k.KContainer = dag.Container(). - From("bitnami/kubectl"). - WithUser("root"). - WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithMountedCache("/cache/k3s", k.ConfigCache). - WithMountedDirectory("/manifests", manifests). - WithServiceBinding("k3s", k.K3s.AsService()). - WithEnvVariable("CACHE", time.Now().String()). - WithUser("root"). - WithExec([]string{"cp", "/cache/k3s/k3s.yaml", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithEntrypoint([]string{"sh", "-c"}) - - } else if localCluster != nil { - - // k.KContainer, err = dag.Container().From("ubuntu"). - // WithServiceBinding("localhost", localCluster). - // WithMountedDirectory("/manifests", manifests). - // WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). - // WithExec([]string{"apt", "install", "-y", "curl"}, ContainerWithExecOpts{SkipEntrypoint: true}). - // WithExec([]string{"curl", "-vvv", "localhost:59127"}).Sync(k.Ctx) - // if err != nil { - // return err - // } - - fileName, _ := kubeconfig.Name(ctx) - - k.KContainer = dag.Container(). - From("bitnami/kubectl"). - WithUser("root"). - WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithDirectory("/manifests", manifests). - WithServiceBinding("minikube", localCluster). - WithEnvVariable("CACHE", time.Now().String()). - WithUser("root"). - WithFile(fmt.Sprintf("/src/%s", fileName), kubeconfig). - WithExec([]string{"cp", fmt.Sprintf("/src/%s", fileName), "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithEntrypoint([]string{"sh", "-c"}) - } else if localCluster == nil { - - fileName, _ := kubeconfig.Name(ctx) - k.KContainer = dag.Container(). - From("bitnami/kubectl"). - WithUser("root"). - WithExec([]string{"apt", "update"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithDirectory("/manifests", manifests). - WithEnvVariable("CACHE", time.Now().String()). - WithUser("root"). - WithFile(fmt.Sprintf("/src/%s", fileName), kubeconfig). - WithExec([]string{"cp", fmt.Sprintf("/src/%s", fileName), "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"chown", "1001:0", "/.kube/config"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithUser("1001"). - WithEntrypoint([]string{"sh", "-c"}) - - } - - if bufferIL != "" { - k.KContainer = k.KContainer. - WithNewFile("/manifests/virtual-kubelet-merge.yaml", ContainerWithNewFileOpts{ - Contents: bufferVK, - Permissions: 0o755, - }) - } - if bufferIL != "" { - k.KContainer = k.KContainer. - WithNewFile("/manifests/interlink-merge.yaml", ContainerWithNewFileOpts{ - Contents: bufferIL, - Permissions: 0o755, - }) - } - return nil -} - -func (k *K8sInstance) kubectl(ctx context.Context, command string) (string, error) { - return k.exec(ctx, "kubectl", fmt.Sprintf("kubectl %v", command)) -} - -func (k *K8sInstance) exec(ctx context.Context, name, command string) (string, error) { - return k.KContainer.Pipeline(name).Pipeline(command). - WithEnvVariable("CACHE", time.Now().String()). - WithExec([]string{command}). - Stdout(ctx) -} - -func (k *K8sInstance) waitForNodes(ctx context.Context) (err error) { - maxRetries := 10 - retryBackoff := 30 * time.Second - for i := 0; i < maxRetries; i++ { - kubectlGetNodes, err := k.kubectl(ctx, "get nodes -o wide") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch nodes: %v", err)) - continue - } - if strings.Contains(kubectlGetNodes, "Ready") { - return nil - } - fmt.Println("waiting for k8s to start:", kubectlGetNodes) - time.Sleep(retryBackoff) - } - return fmt.Errorf("k8s took too long to start") -} - -func (k *K8sInstance) waitForVirtualKubelet(ctx context.Context) (err error) { - maxRetries := 10 - retryBackoff := 30 * time.Second - for i := 0; i < maxRetries; i++ { - time.Sleep(retryBackoff) - kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l nodeName=virtual-kubelet") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) - continue - } - if strings.Contains(kubectlGetPod, "1/1") || strings.Contains(kubectlGetPod, "2/2") { - return nil - } - fmt.Println("waiting for k8s to start:", kubectlGetPod) - describePod, err := k.kubectl(ctx, "logs -n interlink -l nodeName=virtual-kubelet -c inttw-vk") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) - continue - } - fmt.Println(describePod) - - } - return fmt.Errorf("k8s took too long to start") -} - -func (k *K8sInstance) waitForInterlink(ctx context.Context) (err error) { - maxRetries := 10 - retryBackoff := 30 * time.Second - for i := 0; i < maxRetries; i++ { - time.Sleep(retryBackoff) - kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l app=interlink") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) - continue - } - if strings.Contains(kubectlGetPod, "1/1") { - return nil - } - fmt.Println("waiting for k8s to start:", kubectlGetPod) - describePod, err := k.kubectl(ctx, "logs -n interlink -l app=interlink") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) - continue - } - fmt.Println(describePod) - - } - return fmt.Errorf("interlink took too long to start") -} - -func (k *K8sInstance) waitForPlugin(ctx context.Context) (err error) { - maxRetries := 10 - retryBackoff := 30 * time.Second - for i := 0; i < maxRetries; i++ { - time.Sleep(retryBackoff) - kubectlGetPod, err := k.kubectl(ctx, "get pod -n interlink -l app=plugin") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod: %v", err)) - continue - } - if strings.Contains(kubectlGetPod, "1/1") { - return nil - } - fmt.Println("waiting for k8s to start:", kubectlGetPod) - describePod, err := k.kubectl(ctx, "logs -n interlink -l app=plugin") - if err != nil { - fmt.Println(fmt.Errorf("could not fetch pod description: %v", err)) - continue - } - fmt.Println(describePod) - - } - return fmt.Errorf("plugin took too long to start") -} diff --git a/ci/main.go b/ci/main.go index 3040e2ef..1c724a77 100644 --- a/ci/main.go +++ b/ci/main.go @@ -1,29 +1,21 @@ +// A module to instantiate and tests interLink components +// +// Visit the interLink documentation for more info: https://intertwin-eu.github.io/interLink/docs/intro/ +// + package main import ( "bytes" "context" + "dagger/interlink/internal/dagger" "fmt" "html/template" "strings" - - "dagger/interlink/internal/dagger" + "time" ) var ( - interLinkPatch = ` -kind: Deployment -metadata: - name: interlink - namespace: interlink -spec: - template: - spec: - containers: - - name: interlink - image: "{{.InterLinkRef}}" - -` virtualKubeletPatch = ` kind: Deployment metadata: @@ -36,6 +28,11 @@ spec: - name: inttw-vk image: "{{.VirtualKubeletRef}}" ` + +// #- name: interlink +// # image: "{{.InterLinkRef}}" +// +// ` ) type patchSchema struct { @@ -43,114 +40,137 @@ type patchSchema struct { VirtualKubeletRef string } +// Interlink struct for initialization and internal variables type Interlink struct { - K8s *K8sInstance + Name string + Registry *dagger.Service + Manifests *dagger.Directory VirtualKubeletRef string InterlinkRef string - Manifests *Directory - // TODO: services on NodePort? - //virtualkubelet bool - //interlink bool - //plugin bool - CleanupCluster bool + PluginRef string + Kubectl *dagger.Container + KubeAPIs *dagger.Service + KubeConfig *dagger.File + // +private + KubeConfigHost *dagger.File + InterlinkContainer *dagger.Container + VKContainer *dagger.Container } -func (i *Interlink) BuildImages( - ctx context.Context, +// New initializes the Dagger module at each call +func New(name string, // +optional - // +default="ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw" - virtualKubeletRef string, + // +default="ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:0.3.1-rc1" + VirtualKubeletRef string, // +optional - // +default="ghcr.io/intertwin-eu/interlink/interlink" - interlinkRef string, + // +default="ghcr.io/intertwin-eu/interlink/interlink:0.3.1-rc1" + InterlinkRef string, // +optional - // +default="ghcr.io/intertwin-eu/interlink/plugin-test" + // +default="ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.3.2" pluginRef string, - sourceFolder *Directory, -) (*Interlink, error) { - - // TODO: get tag - - i.VirtualKubeletRef = virtualKubeletRef - i.InterlinkRef = interlinkRef - - workspace := dag.Container(). - WithDirectory("/src", sourceFolder). - WithWorkdir("/src"). - Directory("/src") - - vkVersionSplits := strings.Split(virtualKubeletRef, ":") +) *Interlink { - vkVersion := vkVersionSplits[len(vkVersionSplits)-1] - if vkVersion == "" { - return nil, fmt.Errorf("no tag specified on the image for VK") + return &Interlink{ + Name: name, + VirtualKubeletRef: VirtualKubeletRef, + InterlinkRef: InterlinkRef, + PluginRef: pluginRef, } - - modulesCache := dag.CacheVolume("go-mod-122") - - _, err := dag.Container(). - WithMountedCache("/go/pkg/mod", modulesCache). - WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). - Build(workspace, dagger.ContainerBuildOpts{ - Dockerfile: "docker/Dockerfile.vk", - BuildArgs: []dagger.BuildArg{ - {"VERSION", vkVersion}, - }, - }). - Publish(ctx, virtualKubeletRef) - if err != nil { - return nil, err - } - - _, err = dag.Container(). - WithMountedCache("/go/pkg/mod", modulesCache). - WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). - Build(workspace, dagger.ContainerBuildOpts{ - Dockerfile: "docker/Dockerfile.interlink", - }). - Publish(ctx, interlinkRef) - if err != nil { - return nil, err - } - - return i, nil } -func (i *Interlink) NewInterlink( +// Setup k8s e interlink components: +// virtual kubelet and interlink API server +func (m *Interlink) NewInterlink( ctx context.Context, - manifests *Directory, // +optional - kubeconfig *File, + // +defaultPath="./manifests" + manifests *dagger.Directory, // +optional - localCluster *Service, + kubeconfig *dagger.File, + // +optional + localRegistry *dagger.Service, + // +optional + localCluster *dagger.Service, + // +optional + interlinkEndpoint *dagger.Service, + // +optional + // +defaultPath="./manifests/interlink-config.yaml" + interlinkConfig *dagger.File, + // +optional + pluginEndpoint *dagger.Service, + // +optional + // +defaultPath="./manifests/plugin-config.yaml" + pluginConfig *dagger.File, ) (*Interlink, error) { - // create Kustomize patch for images to be used - patch := patchSchema{} - if i.InterlinkRef != "" && i.VirtualKubeletRef != "" { - patch = patchSchema{ - InterLinkRef: i.InterlinkRef, - VirtualKubeletRef: i.VirtualKubeletRef, + if localRegistry != nil { + m.Registry = localRegistry + } + + var err error + if pluginEndpoint == nil { + plugin := dag.Container().From(m.PluginRef). + WithFile("/etc/interlink/InterLinkConfig.yaml", pluginConfig). + WithEnvVariable("SLURMCONFIGPATH", "/etc/interlink/InterLinkConfig.yaml"). + WithEnvVariable("SHARED_FS", "true"). + WithExposedPort(4000). + WithExec([]string{}, dagger.ContainerWithExecOpts{UseEntrypoint: true, InsecureRootCapabilities: true}) + + pluginEndpoint, err = plugin.AsService().Start(ctx) + if err != nil { + return nil, err } - } else { - patch = patchSchema{ - InterLinkRef: "ghcr.io/intertwin-eu/interlink/interlink", - VirtualKubeletRef: "ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw", + } + + if interlinkEndpoint == nil { + interlink := m.InterlinkContainer. + WithFile("/etc/interlink/InterLinkConfig.yaml", interlinkConfig). + WithServiceBinding("plugin", pluginEndpoint). + WithEnvVariable("INTERLINKCONFIGPATH", "/etc/interlink/InterLinkConfig.yaml"). + WithExposedPort(3000). + WithExec([]string{}, dagger.ContainerWithExecOpts{UseEntrypoint: true, InsecureRootCapabilities: true}) + + interlinkEndpoint, err = interlink.AsService().Start(ctx) + if err != nil { + return nil, err } } - interLinkCompiler, err := template.New("interlink").Parse(interLinkPatch) + K3s := dag.K3S(m.Name).With(func(k *dagger.K3S) *dagger.K3S { + return k.WithContainer( + k.Container(). + WithEnvVariable("BUST", time.Now().String()). + WithExec([]string{"sh", "-c", ` +cat < /etc/rancher/k3s/registries.yaml +mirrors: + "registry:5000": + endpoint: + - "http://registry:5000" +EOF`}). + WithServiceBinding("registry", m.Registry). + WithServiceBinding("interlink", interlinkEndpoint), + ) + + }) + + _, err = K3s.Server().Start(ctx) if err != nil { return nil, err } - bufferIL := new(bytes.Buffer) + m.Manifests = manifests + m.KubeAPIs = K3s.Server() + m.KubeConfig = K3s.Config(dagger.K3SConfigOpts{Local: false}) + m.KubeConfigHost = K3s.Config(dagger.K3SConfigOpts{Local: true}) - err = interLinkCompiler.Execute(bufferIL, patch) - if err != nil { - return nil, err + // create Kustomize patch for images to be used + patch := patchSchema{ + InterLinkRef: m.InterlinkRef, + VirtualKubeletRef: m.VirtualKubeletRef, } + bufferIL := new(bytes.Buffer) + virtualKubeletCompiler, err := template.New("vk").Parse(virtualKubeletPatch) if err != nil { return nil, err @@ -163,131 +183,200 @@ func (i *Interlink) NewInterlink( return nil, err } - // use the manifest folder defined in the chain and install components - - if manifests != nil { - i.Manifests = manifests - } - fmt.Println(bufferVK.String()) - i.K8s = NewK8sInstance(ctx) - if err := i.K8s.start(ctx, i.Manifests, bufferVK.String(), bufferIL.String(), kubeconfig, localCluster); err != nil { - return nil, err - } + kubectl := dag.Container().From("bitnami/kubectl:1.29.7-debian-12-r3"). + WithServiceBinding("registry", m.Registry). + WithServiceBinding("plugin", pluginEndpoint). + WithServiceBinding("interlink", interlinkEndpoint). + WithUser("root"). + WithExec([]string{"mkdir", "-p", "/opt/user"}). + WithExec([]string{"chown", "-R", "1001:0", "/opt/user"}). + WithExec([]string{"apt", "update"}). + WithExec([]string{"apt", "update"}). + WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git", "vim"}). + WithMountedFile("/.kube/config", m.KubeConfig). + WithExec([]string{"chown", "1001:0", "/.kube/config"}). + WithUser("1001"). + WithDirectory("/manifests", m.Manifests). + WithNewFile("/manifests/virtual-kubelet-merge.yaml", bufferVK.String(), dagger.ContainerWithNewFileOpts{ + Permissions: 0o755, + }). + WithNewFile("/manifests/interlink-merge.yaml", bufferIL.String(), dagger.ContainerWithNewFileOpts{ + Permissions: 0o755, + }). + WithEntrypoint([]string{"kubectl"}) - err = i.K8s.waitForNodes(ctx) - if err != nil { - return nil, err - } + m.Kubectl = kubectl - ns, _ := i.K8s.kubectl(ctx, "create ns interlink") + ns, _ := kubectl.WithExec([]string{"create", "ns", "interlink"}, dagger.ContainerWithExecOpts{UseEntrypoint: true}).Stdout(ctx) fmt.Println(ns) - sa, err := i.K8s.kubectl(ctx, "apply -f /manifests/service-account.yaml") + sa, err := kubectl.WithExec([]string{"apply", "-f", "/manifests/service-account.yaml"}, dagger.ContainerWithExecOpts{UseEntrypoint: true}).Stdout(ctx) if err != nil { return nil, err } fmt.Println(sa) - vkConfig, err := i.K8s.kubectl(ctx, "apply -k /manifests/") + vkConfig, err := kubectl.WithExec([]string{"apply", "-k", "/manifests/"}, dagger.ContainerWithExecOpts{UseEntrypoint: true}).Stdout(ctx) if err != nil { return nil, err } fmt.Println(vkConfig) - return i, nil + return m, nil } -func (i *Interlink) LoadPlugin(ctx context.Context) (*Interlink, error) { - pluginConfig, err := i.K8s.kubectl(ctx, "apply -f /manifests/plugin-config.yaml") +// Returns the kubeconfig file of the k3s cluster +func (m *Interlink) Config() *dagger.File { + return dag.K3S(m.Name).Config(dagger.K3SConfigOpts{Local: true}) +} + +// Build interLink and virtual kubelet docker images from source +// and publish them in registry service +func (m *Interlink) BuildImages( + ctx context.Context, + // +optional + // +default="registry:5000/virtual-kubelet-inttw" + virtualKubeletRef string, + // +optional + // +default="registry:5000/interlink" + interlinkRef string, + // +optional + // +default="registry:5000/plugin-test" + pluginRef string, + // +optional + // +defaultPath="../" + sourceFolder *dagger.Directory, +) (*Interlink, error) { + + // TODO: get tag + m.Registry = dag.Container().From("registry"). + WithExposedPort(5000).AsService() + + m.VirtualKubeletRef = virtualKubeletRef + m.InterlinkRef = interlinkRef + + vkVersionSplits := strings.Split(virtualKubeletRef, ":") + + vkVersion := vkVersionSplits[len(vkVersionSplits)-1] + if vkVersion == "" { + return nil, fmt.Errorf("no tag specified on the image for VK") + } + + builder := dag.Container(). + From("golang:1.22"). + WithDirectory("/src", sourceFolder). + WithWorkdir("/src"). + WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod-122")). + WithEnvVariable("GOMODCACHE", "/go/pkg/mod"). + WithEnvVariable("VERSION", "local"). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). + WithEnvVariable("GOCACHE", "/go/build-cache"). + WithEnvVariable("CGO_ENABLED", "0"). + WithExec([]string{"bash", "-c", "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh"}). + WithExec([]string{"go", "build", "-o", "bin/interlink", "cmd/interlink/main.go"}) + + m.InterlinkContainer = dag.Container(). + From("alpine"). + WithFile("/bin/interlink", builder.File("/src/bin/interlink")). + WithEntrypoint([]string{"/bin/interlink"}) + + _, err := dag.Container().From("quay.io/skopeo/stable"). + WithServiceBinding("registry", m.Registry). + WithMountedFile("image.tar", m.InterlinkContainer.AsTarball()). + WithExec([]string{"copy", "--dest-tls-verify=false", "docker-archive:image.tar", "docker://" + m.InterlinkRef}, dagger.ContainerWithExecOpts{UseEntrypoint: true}). + Sync(ctx) if err != nil { return nil, err } - fmt.Println(pluginConfig) - plugin, err := i.K8s.kubectl(ctx, "apply -f /manifests/plugin.yaml") + builderVK := dag.Container(). + From("golang:1.22"). + WithDirectory("/src", sourceFolder). + WithWorkdir("/src"). + WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod-122")). + WithEnvVariable("GOMODCACHE", "/go/pkg/mod"). + WithEnvVariable("VERSION", "local"). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build-122")). + WithEnvVariable("GOCACHE", "/go/build-cache"). + WithEnvVariable("CGO_ENABLED", "0"). + WithExec([]string{"bash", "-c", "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh"}). + WithExec([]string{"go", "build", "-o", "bin/vk", "cmd/virtual-kubelet/main.go"}) + + m.VKContainer = dag.Container(). + From("alpine"). + WithFile("/bin/vk", builderVK.File("/src/bin/vk")). + WithEntrypoint([]string{"/bin/vk"}) + + _, err = dag.Container().From("quay.io/skopeo/stable"). + WithServiceBinding("registry", m.Registry). + WithMountedFile("image.tar", m.VKContainer.AsTarball()). + WithExec([]string{"copy", "--dest-tls-verify=false", "docker-archive:image.tar", "docker://" + m.VirtualKubeletRef}, dagger.ContainerWithExecOpts{UseEntrypoint: true}). + Sync(ctx) if err != nil { return nil, err } - fmt.Println(plugin) - - return i, nil + return m, nil } -func (i *Interlink) Cleanup(ctx context.Context) error { +// Wait for virtual node to be ready and expose the k8s endpoint as a service +func (m *Interlink) Kube( + ctx context.Context, +) (*dagger.Service, error) { - cleanup, err := i.K8s.kubectl(ctx, "delete -f /manifests/") - if err != nil { - return err - } - fmt.Println(cleanup) + return m.KubeAPIs, nil - return nil } -func (i *Interlink) Test( +// Wait for cluster to be ready, then setup the test container +func (m *Interlink) Run( ctx context.Context, // +optional - manifests *Directory, + // +defaultPath="./manifests" + manifests *dagger.Directory, +) (*dagger.Container, error) { + + return dag.Container().From("bitnami/kubectl:1.29.7-debian-12-r3"). + WithUser("root"). + WithExec([]string{"mkdir", "-p", "/opt/user"}). + WithExec([]string{"chown", "-R", "1001:0", "/opt/user"}). + WithExec([]string{"apt", "update"}). + WithExec([]string{"apt", "update"}). + WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git", "vim"}). + WithMountedFile("/.kube/config", dag.K3S(m.Name).Config(dagger.K3SConfigOpts{Local: false})). + WithExec([]string{"chown", "1001:0", "/.kube/config"}). + WithUser("1001"). + WithDirectory("/manifests", manifests). + WithEntrypoint([]string{"kubectl"}). + WithWorkdir("/opt/user"). + WithExec([]string{"bash", "-c", "git clone https://github.com/interTwin-eu/vk-test-set.git"}). + WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/user/vk-test-set/vktest_config.yaml"}). + WithWorkdir("/opt/user/vk-test-set"). + WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./ "}), nil + +} + +// Wait for cluster to be ready, setup the test container, run all tests +func (m *Interlink) Test( + ctx context.Context, // +optional - kubeconfig *File, + // +defaultPath="./manifests" + manifests *dagger.Directory, // +optional - localCluster *Service, + localCluster *dagger.Service, // +optional // +default false - cleanup bool, -) (*Container, error) { - - if manifests != nil { - i.Manifests = manifests - } + //cleanup bool, +) (*dagger.Container, error) { - if err := i.K8s.waitForVirtualKubelet(ctx); err != nil { - return nil, err - } - if err := i.K8s.waitForInterlink(ctx); err != nil { - return nil, err - } - if err := i.K8s.waitForPlugin(ctx); err != nil { + c, err := m.Run(ctx, manifests) + if err != nil { return nil, err } - result := i.K8s.KContainer. - WithWorkdir("/opt"). - WithExec([]string{"bash", "-c", "git clone https://github.com/interTwin-eu/vk-test-set.git"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/vk-test-set/vktest_config.yaml"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithWorkdir("/opt/vk-test-set"). - WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./ "}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config && pytest -vk 'not rclone' || echo OPS "}, ContainerWithExecOpts{SkipEntrypoint: true}) - - if i.CleanupCluster { - err := i.Cleanup(ctx) - if err != nil { - return nil, err - } - } + result := c.WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config && pytest -vk 'not rclone and not limits'"}) return result, nil } - -func (i *Interlink) Run( - ctx context.Context, -) (*Container, error) { - - if i.CleanupCluster { - err := i.Cleanup(ctx) - if err != nil { - return nil, err - } - } - - return i.K8s.KContainer. - WithWorkdir("/opt"). - WithExec([]string{"bash", "-c", "git clone https://github.com/interTwin-eu/vk-test-set.git"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/vk-test-set/vktest_config.yaml"}, ContainerWithExecOpts{SkipEntrypoint: true}). - WithWorkdir("/opt/vk-test-set"). - WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./ "}, ContainerWithExecOpts{SkipEntrypoint: true}), nil - -} diff --git a/ci/manifests/interlink-config.yaml b/ci/manifests/interlink-config.yaml index 7a0cb817..3b1a9417 100644 --- a/ci/manifests/interlink-config.yaml +++ b/ci/manifests/interlink-config.yaml @@ -1,15 +1,17 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: "interlink-config" - namespace: interlink -data: - InterLinkConfig.yaml: | - InterlinkURL: "http://localhost" - InterlinkPort: "3000" - SidecarURL: "http://plugin.interlink.svc.cluster.local" - SidecarPort: "4000" - VerboseLogging: true - ErrorsOnlyLogging: false - ExportPodData: true - DataRootFolder: "~/.interlink" +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: "interlink-config" +# namespace: interlink +# data: +# InterLinkConfig.yaml: | + #InterlinkAddress: "unix:///var/run/interlink.socket" +InterlinkAddress: "http://0.0.0.0" +InterlinkPort: "3000" +SidecarURL: "http://plugin" +#SidecarURL: "http://0.0.0.0" +SidecarPort: "4000" +VerboseLogging: true +ErrorsOnlyLogging: false +ExportPodData: true +DataRootFolder: "~/.interlink" diff --git a/ci/manifests/interlink.yaml b/ci/manifests/interlink.yaml deleted file mode 100644 index a11f6e7e..00000000 --- a/ci/manifests/interlink.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: interlink - namespace: interlink -spec: - selector: - app: interlink - ports: - - protocol: TCP - port: 3000 - targetPort: 3000 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: interlink - namespace: interlink - labels: - app: interlink -spec: - replicas: 1 - selector: - matchLabels: - app: interlink - template: - metadata: - labels: - app: interlink - spec: - containers: - - name: interlink - image: "ghcr.io/intertwin-eu/interlink/interlink" - imagePullPolicy: Always - env: - - name: CONFIGPATH - value: "/etc/interlink/InterLinkConfig.yaml" - volumeMounts: - - name: config - mountPath: /etc/interlink/InterLinkConfig.yaml - subPath: InterLinkConfig.yaml - volumes: - - name: config - configMap: - # Provide the name of the ConfigMap you want to mount. - name: interlink-config diff --git a/ci/manifests/kustomization.yaml b/ci/manifests/kustomization.yaml index 694260d7..8b98ba9d 100644 --- a/ci/manifests/kustomization.yaml +++ b/ci/manifests/kustomization.yaml @@ -1,15 +1,13 @@ resources: - virtual-kubelet-config.yaml - virtual-kubelet.yaml -- interlink-config.yaml -- interlink.yaml +#- interlink-config.yaml +#- interlink.yaml +#- plugin-k8s-config.yaml +#- plugin.yaml patches: - path: virtual-kubelet-merge.yaml target: kind: Deployment labelSelector: nodeName=virtual-kubelet -- path: interlink-merge.yaml - target: - kind: Deployment - labelSelector: app=interlink diff --git a/ci/manifests/plugin-config.yaml b/ci/manifests/plugin-config.yaml index e4aae3ba..61a766f9 100644 --- a/ci/manifests/plugin-config.yaml +++ b/ci/manifests/plugin-config.yaml @@ -1,25 +1,21 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: "plugin-config" - namespace: interlink -data: - InterLinkConfig.yaml: | - InterlinkURL: "http://localhost" - InterlinkPort: "3000" - SidecarURL: "http://0.0.0.0" - SidecarPort: "4000" - VerboseLogging: true - ErrorsOnlyLogging: false - ExportPodData: true - DataRootFolder: "~/.interlink" - SbatchPath: "/usr/bin/sbatch" - ScancelPath: "/usr/bin/scancel" - SqueuePath: "/usr/bin/squeue" - CommandPrefix: "" - SingularityPrefix: "" - Namespace: "vk" - Tsocks: false - TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" - TsocksLoginNode: "login01" - BashPath: /bin/bash +InterlinkURL: "http://interlink" +InterlinkPort: "3000" +SidecarURL: "http://0.0.0.0" +SidecarPort: "4000" +VerboseLogging: true +ErrorsOnlyLogging: false +ExportPodData: true +# NEEDED PATH FOR GITHUB ACTIONS +#DataRootFolder: "/home/runner/work/interLink/interLink/.interlink/" +# on your host use something like: +DataRootFolder: "/home/ubuntu/.interlink/" +SbatchPath: "/usr/bin/sbatch" +ScancelPath: "/usr/bin/scancel" +SqueuePath: "/usr/bin/squeue" +CommandPrefix: "" +SingularityPrefix: "" +Namespace: "vk" +Tsocks: false +TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" +TsocksLoginNode: "login01" +BashPath: /bin/bash diff --git a/ci/manifests/plugin-k8s-config.yaml b/ci/manifests/plugin-k8s-config.yaml new file mode 100644 index 00000000..8db723f4 --- /dev/null +++ b/ci/manifests/plugin-k8s-config.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "plugin-config" + namespace: interlink +data: + InterLinkConfig.yaml: | + InterlinkURL: "http://localhost" + InterlinkPort: "3000" + SidecarURL: "http://0.0.0.0" + SidecarPort: "4000" + VerboseLogging: true + ErrorsOnlyLogging: false + ExportPodData: true + DataRootFolder: "/home/runner/work/interLink/interLink/.interlink/" + SbatchPath: "/usr/bin/sbatch" + ScancelPath: "/usr/bin/scancel" + SqueuePath: "/usr/bin/squeue" + CommandPrefix: "" + SingularityPrefix: "" + Namespace: "vk" + Tsocks: false + TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so" + TsocksLoginNode: "login01" + BashPath: /bin/bash diff --git a/ci/manifests/plugin.yaml b/ci/manifests/plugin.yaml index 046f963d..082360e7 100644 --- a/ci/manifests/plugin.yaml +++ b/ci/manifests/plugin.yaml @@ -30,7 +30,7 @@ spec: spec: containers: - name: plugin - image: "ghcr.io/intertwin-eu/interlink-docker-plugin/docker-plugin:0.0.16-no-gpu" + image: "dciangot/docker-plugin:v1" #image: "ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.2.3" imagePullPolicy: Always command: diff --git a/ci/manifests/service-account.yaml b/ci/manifests/service-account.yaml index 8a0b3542..2169e592 100644 --- a/ci/manifests/service-account.yaml +++ b/ci/manifests/service-account.yaml @@ -28,6 +28,7 @@ rules: - secrets - services - serviceaccounts + - namespaces verbs: - get - list diff --git a/ci/manifests/virtual-kubelet-config.yaml b/ci/manifests/virtual-kubelet-config.yaml index 59fd6791..7feb69c2 100644 --- a/ci/manifests/virtual-kubelet-config.yaml +++ b/ci/manifests/virtual-kubelet-config.yaml @@ -5,14 +5,15 @@ metadata: namespace: interlink data: InterLinkConfig.yaml: | - InterlinkURL: http://interlink.interlink.svc.cluster.local - InterlinkPort: 3000 + #InterlinkURL: unix:///var/run/interlink.socket + InterlinkURL: "http://interlink" + InterlinkPort: "3000" ExportPodData: true VerboseLogging: true ErrorsOnlyLogging: false ServiceAccount: "virtual-kubelet" Namespace: interlink - VKTokenFile: /dev/null + VKTokenFile: "" CPU: "100" - Memory: "128GiB" + Memory: "128Gi" Pods: "100" diff --git a/ci/manifests/virtual-kubelet.yaml b/ci/manifests/virtual-kubelet.yaml index 51b34a88..8dc1c2e7 100644 --- a/ci/manifests/virtual-kubelet.yaml +++ b/ci/manifests/virtual-kubelet.yaml @@ -15,9 +15,22 @@ spec: labels: nodeName: virtual-kubelet spec: + hostNetwork: true automountServiceAccountToken: true serviceAccountName: virtual-kubelet containers: + # - name: interlink + # image: "ghcr.io/intertwin-eu/interlink/interlink" + # imagePullPolicy: Always + # env: + # - name: INTERLINKCONFIGPATH + # value: "/etc/interlink/InterLinkConfig.yaml" + # volumeMounts: + # - name: il-config + # mountPath: /etc/interlink/InterLinkConfig.yaml + # subPath: InterLinkConfig.yaml + # - name: sockets + # mountPath: /var/run/ - name: inttw-vk image: "ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw" imagePullPolicy: Always @@ -25,7 +38,7 @@ spec: - name: NODENAME value: virtual-kubelet - name: KUBELET_PORT - value: "10250" + value: "10255" - name: POD_IP valueFrom: fieldRef: @@ -36,8 +49,20 @@ spec: - name: config mountPath: /etc/interlink/InterLinkConfig.yaml subPath: InterLinkConfig.yaml + # - name: sockets + # mountPath: /var/run/ volumes: - name: config configMap: # Provide the name of the ConfigMap you want to mount. name: virtual-kubelet-config + - name: sockets + hostPath: + path: /var/run + type: Directory + # - name: il-config + # configMap: + # # Provide the name of the ConfigMap you want to mount. + # name: interlink-config + # - name: sockets + # emptyDir: {} diff --git a/ci/manifests/vktest_config.yaml b/ci/manifests/vktest_config.yaml index ab393912..4cca445e 100644 --- a/ci/manifests/vktest_config.yaml +++ b/ci/manifests/vktest_config.yaml @@ -6,12 +6,13 @@ required_namespaces: - kube-system - interlink -timeout_multiplier: 2. +timeout_multiplier: 10. values: namespace: interlink annotations: - slurm-job.vk.io/flags: "--job-name=test-pod-cfg -t 2800 --ntasks=8 --nodes=1 --mem-per-cpu=2000" + slurm-job.vk.io/flags: "--job-name=test-pod-cfg -t 2800" + slurm-job.vk.io/image-root: "docker://" tolerations: - key: virtual-node.interlink/no-schedule diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 8068b3dc..4a245ad2 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -39,11 +39,12 @@ type Resources struct { type oauthStruct struct { Provider string `yaml:"provider"` + GrantType string `default:"authorization_code" yaml:"grant_type"` Issuer string `yaml:"issuer,omitempty"` RefreshToken string `yaml:"refresh_token,omitempty"` Audience string `yaml:"audience,omitempty"` Group string `yaml:"group,omitempty"` - GroupClaim string `yaml:"groupClaim,omitempty"` + GroupClaim string `default:"groups" yaml:"group_claim"` Scopes []string `yaml:"scopes"` GitHUBUser string `yaml:"github_user"` TokenURL string `yaml:"token_url"` @@ -52,6 +53,7 @@ type oauthStruct struct { ClientSecret string `yaml:"client_secret"` } +// TODO: insert in-cluster and socket option e.g. --> no need OAUTH type dataStruct struct { InterLinkIP string `yaml:"interlink_ip"` InterLinkPort int `yaml:"interlink_port"` @@ -170,60 +172,52 @@ func root(cmd *cobra.Command, args []string) error { return err } - ctx := context.Background() - cfg := oauth2.Config{ - ClientID: configCLI.OAUTH.ClientID, - ClientSecret: configCLI.OAUTH.ClientSecret, - Endpoint: oauth2.Endpoint{ - TokenURL: configCLI.OAUTH.TokenURL, - DeviceAuthURL: configCLI.OAUTH.DeviceCodeURL, - }, - RedirectURL: "http://localhost:8080", - Scopes: configCLI.OAUTH.Scopes, - } + var token *oauth2.Token - response, err := cfg.DeviceAuth(ctx, oauth2.AccessTypeOffline) - if err != nil { - panic(err) - } + ctx := context.Background() + if configCLI.OAUTH.GrantType == "authorization_code" { + cfg := oauth2.Config{ + ClientID: configCLI.OAUTH.ClientID, + ClientSecret: configCLI.OAUTH.ClientSecret, + Endpoint: oauth2.Endpoint{ + TokenURL: configCLI.OAUTH.TokenURL, + DeviceAuthURL: configCLI.OAUTH.DeviceCodeURL, + }, + RedirectURL: "http://localhost:8080", + Scopes: configCLI.OAUTH.Scopes, + } - fmt.Printf("please enter code %s at %s\n", response.UserCode, response.VerificationURI) - token, err := cfg.DeviceAccessToken(ctx, response, oauth2.AccessTypeOffline) - if err != nil { - panic(err) - } - //fmt.Println(token.AccessToken) - //fmt.Println(token.RefreshToken) - //fmt.Println(token.Expiry) - //fmt.Println(token.TokenType) + response, err := cfg.DeviceAuth(ctx, oauth2.AccessTypeOffline) + if err != nil { + panic(err) + } - configCLI.OAUTH.RefreshToken = token.RefreshToken + fmt.Printf("please enter code %s at %s\n", response.UserCode, response.VerificationURI) + token, err = cfg.DeviceAccessToken(ctx, response, oauth2.AccessTypeOffline) + if err != nil { + panic(err) + } + //fmt.Println(token.AccessToken) + //fmt.Println(token.RefreshToken) + //fmt.Println(token.Expiry) + //fmt.Println(token.TokenType) + configCLI.OAUTH.RefreshToken = token.RefreshToken + } else if configCLI.OAUTH.GrantType == "client_credentials" { - namespaceYAML, err := evalManifest("templates/namespace.yaml", configCLI) - if err != nil { - panic(err) - } + fmt.Println("Client_credentials set, I won't try to get any refresh token.") - deploymentYAML, err := evalManifest("templates/deployment.yaml", configCLI) - if err != nil { - panic(err) - } + } else { - configYAML, err := evalManifest("templates/configs.yaml", configCLI) - if err != nil { - panic(err) + panic(fmt.Errorf("wrong grant type specified in the configuration. Only client_credentials and authorization_code are supported")) } - serviceaccountYAML, err := evalManifest("templates/service-account.yaml", configCLI) + valuesYAML, err := evalManifest("templates/values.yaml", configCLI) if err != nil { panic(err) } manifests := []string{ - namespaceYAML, - serviceaccountYAML, - configYAML, - deploymentYAML, + valuesYAML, } err = os.MkdirAll(outFolder, fs.ModePerm) @@ -231,7 +225,7 @@ func root(cmd *cobra.Command, args []string) error { panic(err) } // Create a file and use bufio.NewWriter. - f, err := os.Create(outFolder + "/interlink.yaml") + f, err := os.Create(outFolder + "/values.yaml") if err != nil { panic(err) } @@ -246,7 +240,7 @@ func root(cmd *cobra.Command, args []string) error { w.Flush() - fmt.Println("\n\n=== Deployment file written at: " + outFolder + "/interlink.yaml ===\n\n To deploy the virtual kubelet run:\n kubectl apply -f " + outFolder + "/interlink.yaml") + fmt.Println("\n\n=== Deployment file written at: " + outFolder + "/values.yaml ===\n\n To deploy the virtual kubelet run:\n helm --debug upgrade --install --create-namespace -n " + configCLI.Namespace + " " + configCLI.VKName + " oci://ghcr.io/intertwin-eu/interlink-helm-chart/interlink --values " + outFolder + "/values.yaml") // TODO: ilctl.sh templating tmpl, err := template.ParseFS(templates, "templates/interlink-install.sh") diff --git a/cmd/installer/templates/configs.yaml b/cmd/installer/templates/configs.yaml deleted file mode 100644 index c2de7e0c..00000000 --- a/cmd/installer/templates/configs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -data: - InterLinkConfig.yaml: | - InterlinkURL: https://{{.InterLinkIP}} - InterlinkPort: {{.InterLinkPort}} - CommandPrefix: "" - ExportPodData: true - ServiceAccount: "interlink" - Namespace: {{.Namespace}} - VKTokenFile: /opt/interlink/token - CPU: "{{.VKLimits.CPU}}" - Memory: "{{.VKLimits.Memory}}" - Pods: "{{.VKLimits.Pods}}" -kind: ConfigMap -metadata: - name: "{{.VKName}}-config" - namespace: {{.Namespace}} diff --git a/cmd/installer/templates/deployment.yaml b/cmd/installer/templates/deployment.yaml deleted file mode 100644 index b186dd9a..00000000 --- a/cmd/installer/templates/deployment.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{.VKName}} - namespace: {{.Namespace}} - labels: - nodeName: {{.VKName}} -spec: - replicas: 1 - selector: - matchLabels: - nodeName: {{.VKName}} - template: - metadata: - labels: - nodeName: {{.VKName}} - spec: - #hostNetwork: true - containers: - - name: jaeger - image: jaegertracing/all-in-one:1.51 - - name: inttw-vk - image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:latest - imagePullPolicy: Always - env: - - name: NODENAME - value: {{.VKName}} - - name: KUBELET_PORT - value: "10250" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: CONFIGPATH - value: "/etc/interlink/InterLinkConfig.yaml" - - name: VKTOKENFILE - value: "/opt/interlink/token" - volumeMounts: - - name: config - mountPath: /etc/interlink/InterLinkConfig.yaml - subPath: InterLinkConfig.yaml - - name: token - mountPath: /opt/interlink - resources: - limits: - cpu: 2000m - memory: 2Gi - requests: - cpu: 100m - memory: 200Mi - - name: refresh-token - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw-refresh:latest - imagePullPolicy: Always - env: - - name: IAM_TOKEN_ENDPOINT - value: {{.OAUTH.TokenURL}} - # TODO load env IAM client from secret - - name: IAM_CLIENT_ID - value: {{.OAUTH.ClientID}} - - name: IAM_CLIENT_SECRET - value: {{.OAUTH.ClientSecret}} - - name: IAM_REFRESH_TOKEN - value: {{.OAUTH.RefreshToken}} - - name: IAM_VK_AUD - value: {{.OAUTH.Audience}} - - name: TOKEN_PATH - value: /opt/interlink/token - command: - - python3 - - /opt/refresh.py - resources: - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 100m - memory: 300Mi - volumeMounts: - - name: token - mountPath: /opt/interlink - serviceAccountName: {{.VKName}} - volumes: - - name: config - configMap: - # Provide the name of the ConfigMap you want to mount. - name: {{.VKName}}-config - - name: token - hostPath: - path: /tmp - type: Directory diff --git a/cmd/installer/templates/interlink-install.sh b/cmd/installer/templates/interlink-install.sh index 88c883fc..f39441e9 100644 --- a/cmd/installer/templates/interlink-install.sh +++ b/cmd/installer/templates/interlink-install.sh @@ -24,20 +24,22 @@ install () { mkdir -p $HOME/.interlink/logs || exit 1 mkdir -p $HOME/.interlink/bin || exit 1 mkdir -p $HOME/.interlink/config || exit 1 - # set $HOME/.interlink/config/InterLinkConfig.yaml + + + # TODO download also service files for systemd cat <>$HOME/.interlink/config/InterLinkConfig.yaml -InterlinkURL: "http://localhost" -InterlinkPort: "30080" -SidecarURL: "http://localhost" -SidecarPort: "4000" +InterlinkAddress: "unix://${HOME}/.interlink/interlink.sock" +InterlinkPort: "0" +SidecarURL: "unix://${HOME}/.interlink/plugin.sock" +SidecarPort: "0" VerboseLogging: true ErrorsOnlyLogging: false ExportPodData: true DataRootFolder: "~/.interlink" EOF - echo "=== Configured to reach sidecar service on http://localhost:4000 . You can edit this behavior changing $HOME/.interlink/config/InterLinkConfig.yaml file. ===" + echo "=== Configured to reach sidecar service on unix://${HOME}/.interlink/plugin.sock. You can edit this behavior changing $HOME/.interlink/config/InterLinkConfig.yaml file. ===" ## Download binaries to $HOME/.local/interlink/ echo "curl --fail -L -o ${HOME}/.interlink/bin/interlink https://github.com/interTwin-eu/interLink/releases/download/{{.InterLinkVersion}}/interlink_$(uname -s)_$(uname -m)" @@ -87,10 +89,10 @@ start() { case "{{.OAUTH.Provider}}" in oidc) $HOME/.interlink/bin/oauth2-proxy \ - --client-id DUMMY \ - --client-secret DUMMY \ - --http-address 0.0.0.0:{{.InterLinkPort}} \ - --oidc-issuer-url {{.OAUTH.Issuer}} \ + --client-id "{{.OAUTH.ClientID}}" \ + --client-secret "\"{{.OAUTH.ClientSecret}}\"" \ + --http-address unix://${HOME}/.interlink/interlink.sock \ + --oidc-issuer-url "{{.OAUTH.Issuer}}" \ --pass-authorization-header true \ --provider oidc \ --redirect-url http://localhost:8081 \ @@ -112,10 +114,11 @@ start() { echo $! > $HOME/.interlink/oauth2-proxy.pid ;; github) + touch $HOME/.interlink/interlink.sock $HOME/.interlink/bin/oauth2-proxy \ --client-id {{.OAUTH.ClientID}} \ --client-secret {{.OAUTH.ClientSecret}} \ - --http-address 0.0.0.0:{{.InterLinkPort}} \ + --http-address unix://$HOME/.interlink/interlink.sock \ --pass-authorization-header true \ --provider github \ --redirect-url http://localhost:8081 \ @@ -137,9 +140,11 @@ start() { esac ## start interLink - export INTERLINKCONFIGPATH=$HOME/.interlink/config/InterLinkConfig.yaml - $HOME/.interlink/bin/interlink &> $HOME/.interlink/logs/interlink.log & - echo $! > $HOME/.interlink/interlink.pid + export INTERLINKCONFIGPATH=${HOME}/.interlink/config/InterLinkConfig.yaml + $HOME/.interlink/bin/interlink &> ${HOME}/.interlink/logs/interlink.log & + echo $! > ${HOME}/.interlink/interlink.pid + + ## TODO: if RUN_SLURM=1 then manage also slurm } diff --git a/cmd/installer/templates/namespace.yaml b/cmd/installer/templates/namespace.yaml deleted file mode 100644 index 337ae06c..00000000 --- a/cmd/installer/templates/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{.Namespace}} diff --git a/cmd/installer/templates/service-account.yaml b/cmd/installer/templates/service-account.yaml deleted file mode 100644 index 66825734..00000000 --- a/cmd/installer/templates/service-account.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{.VKName}} - namespace: {{.Namespace}} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{.VKName}} - namespace: {{.Namespace}} -rules: -- apiGroups: - - "" - resources: - - configmaps - - secrets - - services - - serviceaccounts - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - watch - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - create - - get -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{.VKName}} - namespace: {{.Namespace}} -subjects: -- kind: ServiceAccount - name: {{.VKName}} - namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{.VKName}} - diff --git a/cmd/installer/templates/values.yaml b/cmd/installer/templates/values.yaml new file mode 100644 index 00000000..1e0f02f9 --- /dev/null +++ b/cmd/installer/templates/values.yaml @@ -0,0 +1,21 @@ +nodeName: {{.VKName}} + +interlink: + address: https://{{.InterLinkIP}} + port: {{.InterLinkPort}} + +virtualNode: + CPUs: {{.VKLimits.CPU}} + MemGiB: {{.VKLimits.Memory}} + Pods: {{.VKLimits.Pods}} + HTTPProxies: + HTTP: null + HTTPs: null + +OAUTH: + TokenURL: {{.OAUTH.TokenURL}} + ClientID: {{.OAUTH.ClientID}} + ClientSecret: {{.OAUTH.ClientSecret}} + RefreshToken: {{.OAUTH.RefreshToken}} + GrantType: {{.OAUTH.GrantType}} + Audience: {{.OAUTH.Audience}} diff --git a/cmd/interlink/main.go b/cmd/interlink/main.go new file mode 100644 index 00000000..86769e2f --- /dev/null +++ b/cmd/interlink/main.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "github.com/virtual-kubelet/virtual-kubelet/log" + logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" + "github.com/virtual-kubelet/virtual-kubelet/trace" + "github.com/virtual-kubelet/virtual-kubelet/trace/opentelemetry" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + types "github.com/intertwin-eu/interlink/pkg/interlink" + "github.com/intertwin-eu/interlink/pkg/interlink/api" + "github.com/intertwin-eu/interlink/pkg/virtualkubelet" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" +) + +func initProvider(ctx context.Context) (func(context.Context) error, error) { + log.G(ctx).Info("Tracing is enabled, setting up the TracerProvider") + + // Get the TELEMETRY_UNIQUE_ID from the environment, if it is not set, use the hostname + uniqueID := os.Getenv("TELEMETRY_UNIQUE_ID") + if uniqueID == "" { + log.G(ctx).Info("No TELEMETRY_UNIQUE_ID set, generating a new one") + newUUID := uuid.New() + uniqueID = newUUID.String() + log.G(ctx).Info("Generated unique ID: ", uniqueID, " use InterLink-Plugin-"+uniqueID+" as service name from Grafana") + } + + serviceName := "InterLink-Plugin-" + uniqueID + + res, err := resource.New(ctx, + resource.WithAttributes( + // the service name used to display traces in backends + semconv.ServiceName(serviceName), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create resource: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + otlpEndpoint := os.Getenv("TELEMETRY_ENDPOINT") + + if otlpEndpoint == "" { + otlpEndpoint = "localhost:4317" + } + + log.G(ctx).Info("TELEMETRY_ENDPOINT: ", otlpEndpoint) + + caCrtFilePath := os.Getenv("TELEMETRY_CA_CRT_FILEPATH") + + conn := &grpc.ClientConn{} + if caCrtFilePath != "" { + + // if the CA certificate is provided, set up mutual TLS + + log.G(ctx).Info("CA certificate provided, setting up mutual TLS") + + caCert, err := ioutil.ReadFile(caCrtFilePath) + if err != nil { + return nil, fmt.Errorf("failed to load CA certificate: %w", err) + } + + clientKeyFilePath := os.Getenv("TELEMETRY_CLIENT_KEY_FILEPATH") + if clientKeyFilePath == "" { + return nil, fmt.Errorf("client key file path not provided. Since a CA certificate is provided, a client key is required for mutual TLS") + } + + clientCrtFilePath := os.Getenv("TELEMETRY_CLIENT_CRT_FILEPATH") + if clientCrtFilePath == "" { + return nil, fmt.Errorf("client certificate file path not provided. Since a CA certificate is provided, a client certificate is required for mutual TLS") + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to append CA certificate") + } + + cert, err := tls.LoadX509KeyPair(clientCrtFilePath, clientKeyFilePath) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate: %w", err) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: true, + } + creds := credentials.NewTLS(tlsConfig) + conn, err = grpc.NewClient(otlpEndpoint, grpc.WithTransportCredentials(creds), grpc.WithBlock()) + + } else { + conn, err = grpc.NewClient(otlpEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn.WaitForStateChange(ctx, connectivity.Ready) + + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) + } + + // Set up a trace exporter + traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("failed to create trace exporter: %w", err) + } + + // Register the trace exporter with a TracerProvider, using a batch + // span processor to aggregate spans before export. + bsp := sdktrace.NewBatchSpanProcessor(traceExporter) + tracerProvider := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(bsp), + ) + otel.SetTracerProvider(tracerProvider) + + // set global propagator to tracecontext (the default is no-op). + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return tracerProvider.Shutdown, nil +} + +func main() { + printVersion := flag.Bool("version", false, "show version") + flag.Parse() + + if *printVersion { + fmt.Println(virtualkubelet.KubeletVersion) + return + } + var cancel context.CancelFunc + api.PodStatuses.Statuses = make(map[string]types.PodStatus) + + interLinkConfig, err := types.NewInterLinkConfig() + if err != nil { + panic(err) + } + logger := logrus.StandardLogger() + + logger.SetLevel(logrus.InfoLevel) + if interLinkConfig.VerboseLogging { + logger.SetLevel(logrus.DebugLevel) + } else if interLinkConfig.ErrorsOnlyLogging { + logger.SetLevel(logrus.ErrorLevel) + } + + log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if os.Getenv("ENABLE_TRACING") == "1" { + shutdown, err := initProvider(ctx) + if err != nil { + log.G(ctx).Fatal(err) + } + defer func() { + if err = shutdown(ctx); err != nil { + log.G(ctx).Fatal("failed to shutdown TracerProvider: %w", err) + } + }() + + log.G(ctx).Info("Tracer setup succeeded") + + // TODO: disable this through options + trace.T = opentelemetry.Adapter{} + } + + log.G(ctx).Info(interLinkConfig) + + log.G(ctx).Info("interLink version: ", virtualkubelet.KubeletVersion) + + sidecarEndpoint := "" + if strings.HasPrefix(interLinkConfig.Sidecarurl, "unix://") { + sidecarEndpoint = interLinkConfig.Sidecarurl + // Dial the Unix socket + var conn net.Conn + for { + conn, err = net.Dial("unix", sidecarEndpoint) + if err != nil { + log.G(ctx).Error(err) + time.Sleep(30 * time.Second) + } else { + break + } + } + + http.DefaultTransport.(*http.Transport).DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return conn, nil + } + } else if strings.HasPrefix(interLinkConfig.Sidecarurl, "http://") { + sidecarEndpoint = interLinkConfig.Sidecarurl + ":" + interLinkConfig.Sidecarport + } else { + log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://: getting ", interLinkConfig.Sidecarurl) + } + + interLinkAPIs := api.InterLinkHandler{ + Config: interLinkConfig, + Ctx: ctx, + SidecarEndpoint: sidecarEndpoint, + } + + mutex := http.NewServeMux() + mutex.HandleFunc("/status", interLinkAPIs.StatusHandler) + mutex.HandleFunc("/create", interLinkAPIs.CreateHandler) + mutex.HandleFunc("/delete", interLinkAPIs.DeleteHandler) + mutex.HandleFunc("/pinglink", interLinkAPIs.Ping) + mutex.HandleFunc("/getLogs", interLinkAPIs.GetLogsHandler) + mutex.HandleFunc("/updateCache", interLinkAPIs.UpdateCacheHandler) + + interLinkEndpoint := "" + if strings.HasPrefix(interLinkConfig.InterlinkAddress, "unix://") { + interLinkEndpoint = interLinkConfig.InterlinkAddress + + // Create a Unix domain socket and listen for incoming connections. + socket, err := net.Listen("unix", strings.ReplaceAll(interLinkEndpoint, "unix://", "")) + if err != nil { + panic(err) + } + + // Cleanup the sockfile. + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + os.Remove(strings.ReplaceAll(interLinkEndpoint, "unix://", "")) + os.Exit(1) + }() + server := http.Server{ + Handler: mutex, + } + + log.G(ctx).Info(socket) + + if err := server.Serve(socket); err != nil { + log.G(ctx).Fatal(err) + } + } else if strings.HasPrefix(interLinkConfig.InterlinkAddress, "http://") { + interLinkEndpoint = strings.Replace(interLinkConfig.InterlinkAddress, "http://", "", -1) + ":" + interLinkConfig.Interlinkport + + err = http.ListenAndServe(interLinkEndpoint, mutex) + + if err != nil { + log.G(ctx).Fatal(err) + } + } else { + log.G(ctx).Fatal("Interlink URL should either start per unix:// or http://. Getting: ", interLinkConfig.InterlinkAddress) + } +} diff --git a/cmd/ssh-tunnel/main.go b/cmd/ssh-tunnel/main.go new file mode 100644 index 00000000..a4f08f8a --- /dev/null +++ b/cmd/ssh-tunnel/main.go @@ -0,0 +1,95 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "net" + "os" + + "golang.org/x/crypto/ssh" +) + +func runTunnel(local, remote net.Conn) { + defer local.Close() + defer remote.Close() + done := make(chan struct{}, 2) + + go func() { + io.Copy(local, remote) + done <- struct{}{} + }() + + go func() { + io.Copy(remote, local) + done <- struct{}{} + }() + + <-done +} + +func main() { + addr := flag.String("addr", "", "ssh server address to dial as :") + username := flag.String("user", "", "username for ssh") + keyFile := flag.String("keyfile", "", "file with private key for SSH authentication") + remotePort := flag.String("rport", "", "remote port for tunnel") + localSocket := flag.String("lsock", "", "local socket for tunnel") + flag.Parse() + + // Implement a HostKeyCallback to verify the server's host key + hostKeyCallback := ssh.InsecureIgnoreHostKey() // This is insecure and should be replaced with proper host key verification + + key, err := os.ReadFile(*keyFile) + if err != nil { + log.Fatalf("unable to read private key: %v", err) + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + log.Fatalf("unable to parse private key: %v", err) + } + // An SSH client is represented with a ClientConn. + // + // To authenticate with the remote server you must pass at least one + // implementation of AuthMethod via the Auth field in ClientConfig, + // and provide a HostKeyCallback. + config := &ssh.ClientConfig{ + User: *username, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: hostKeyCallback, + } + + client, err := ssh.Dial("tcp", *addr, config) + if err != nil { + log.Fatal("Failed to dial: ", err) + } + defer client.Close() + + listener, err := client.Listen("tcp", "localhost:"+*remotePort) + if err != nil { + log.Fatalf("Failed to listen on remote socket %s: %v", *remotePort, err) + } + defer listener.Close() + log.Printf("Listening on remote socket %s", *remotePort) + for { + remote, err := listener.Accept() + if err != nil { + log.Printf("Failed to accept connection on remote socket %s: %v", *remotePort, err) + continue + } + log.Printf("Accepted connection on remote socket %s", *remotePort) + go func() { + local, err := net.Dial("unix", *localSocket) + if err != nil { + log.Printf("Failed to dial local socket %s: %v", *localSocket, err) + remote.Close() + return + } + log.Printf("Connected to local socket %s", *localSocket) + fmt.Println("tunnel established with", local.LocalAddr()) + runTunnel(local, remote) + }() + } +} diff --git a/cmd/virtual-kubelet/main.go b/cmd/virtual-kubelet/main.go index fe5919ab..a615fbae 100644 --- a/cmd/virtual-kubelet/main.go +++ b/cmd/virtual-kubelet/main.go @@ -18,16 +18,21 @@ package main import ( "context" "crypto/tls" + "crypto/x509" "flag" "fmt" + "io/ioutil" "net" "os" "path" "strconv" + "strings" "time" // "k8s.io/client-go/rest" "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" @@ -64,6 +69,8 @@ import ( "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + + "github.com/google/uuid" ) func PodInformerFilter(node string) informers.SharedInformerOption { @@ -93,33 +100,98 @@ type Opts struct { ErrorsOnly bool } -func initProvider() (func(context.Context) error, error) { - ctx := context.Background() +func initProvider(ctx context.Context) (func(context.Context) error, error) { + + log.G(ctx).Info("Tracing is enabled, setting up the TracerProvider") + + // Get the TELEMETRY_UNIQUE_ID from the environment, if it is not set, use the hostname + uniqueID := os.Getenv("TELEMETRY_UNIQUE_ID") + if uniqueID == "" { + log.G(ctx).Info("No TELEMETRY_UNIQUE_ID set, generating a new one") + newUUID := uuid.New() + uniqueID = newUUID.String() + log.G(ctx).Info("Generated unique ID: ", uniqueID, " use VK-InterLink-"+uniqueID+" as service name from Grafana") + } + + // Create a new resource with the service name set to the TELEMETRY_UNIQUE_ID + // The nomenclature VK-InterLink- is used to identify the service in Grafana. + // VK-InterLink- means that the traces are coming from Virtual Kubelet + // and are related to the call that are made for the InterLink API service + + serviceName := "VK-InterLink-" + uniqueID res, err := resource.New(ctx, resource.WithAttributes( // the service name used to display traces in backends - semconv.ServiceName("InterLink-service"), + semconv.ServiceName(serviceName), ), ) if err != nil { return nil, fmt.Errorf("failed to create resource: %w", err) } - // TODO: disable is telemetry is disabled - - // If the OpenTelemetry Collector is running on a local cluster (minikube or - // microk8s), it should be accessible through the NodePort service at the - // `localhost:30080` endpoint. Otherwise, replace `localhost` with the - // endpoint of your cluster. If you run the app inside k8s, then you can - // probably connect directly to the service through dns. ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() - conn, err := grpc.DialContext(ctx, "localhost:4317", - // Note the use of insecure transport here. TLS is recommended in production. - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) + + otlpEndpoint := os.Getenv("TELEMETRY_ENDPOINT") + + if otlpEndpoint == "" { + otlpEndpoint = "localhost:4317" + } + + log.G(ctx).Info("TELEMETRY_ENDPOINT: ", otlpEndpoint) + + caCrtFilePath := os.Getenv("TELEMETRY_CA_CRT_FILEPATH") + + conn := &grpc.ClientConn{} + if caCrtFilePath != "" { + + // if the CA certificate is provided, set up mutual TLS + + log.G(ctx).Info("CA certificate provided, setting up mutual TLS") + + caCert, err := ioutil.ReadFile(caCrtFilePath) + if err != nil { + return nil, fmt.Errorf("failed to load CA certificate: %w", err) + } + + clientKeyFilePath := os.Getenv("TELEMETRY_CLIENT_KEY_FILEPATH") + if clientKeyFilePath == "" { + return nil, fmt.Errorf("client key file path not provided. Since a CA certificate is provided, a client key is required for mutual TLS") + } + + clientCrtFilePath := os.Getenv("TELEMETRY_CLIENT_CRT_FILEPATH") + if clientCrtFilePath == "" { + return nil, fmt.Errorf("client certificate file path not provided. Since a CA certificate is provided, a client certificate is required for mutual TLS") + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to append CA certificate") + } + + cert, err := tls.LoadX509KeyPair(clientCrtFilePath, clientKeyFilePath) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate: %w", err) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: true, + } + creds := credentials.NewTLS(tlsConfig) + conn, err = grpc.NewClient(otlpEndpoint, grpc.WithTransportCredentials(creds), grpc.WithBlock()) + + } else { + // if the CA certificate is not provided, use an insecure connection + // this means that the telemetry collector is not using a certificate, i.e. is inside the k8s cluster + conn, err = grpc.NewClient(otlpEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn.WaitForStateChange(ctx, connectivity.Ready) + if err != nil { return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err) } @@ -143,15 +215,9 @@ func initProvider() (func(context.Context) error, error) { // set global propagator to tracecontext (the default is no-op). otel.SetTextMapPropagator(propagation.TraceContext{}) - // Shutdown will flush any remaining spans and shut down the exporter. return tracerProvider.Shutdown, nil } -func tlsConfig(tls *tls.Config) error { - tls.InsecureSkipVerify = true - return nil -} - func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -192,8 +258,10 @@ func main() { } log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) + log.G(ctx).Info("Config dump", interLinkConfig) + if os.Getenv("ENABLE_TRACING") == "1" { - shutdown, err := initProvider() + shutdown, err := initProvider(ctx) if err != nil { log.G(ctx).Fatal(err) } @@ -214,6 +282,25 @@ func main() { http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + if strings.HasPrefix(interLinkConfig.InterlinkURL, "unix://") { + // Dial the Unix socket + interLinkEndpoint := strings.Replace(interLinkConfig.InterlinkURL, "unix://", "", -1) + var conn net.Conn + for { + conn, err = net.Dial("unix", interLinkEndpoint) + if err != nil { + log.G(ctx).Error(err) + time.Sleep(30 * time.Second) + } else { + break + } + } + + http.DefaultTransport.(*http.Transport).DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return conn, nil + } + } + dport, err := strconv.ParseInt(os.Getenv("KUBELET_PORT"), 10, 32) if err != nil { log.G(ctx).Fatal(err) @@ -301,6 +388,9 @@ func main() { resync, ) + scmInformer := scmInformerFactory.Core().V1().Secrets().Informer() + podInformer := podInformerFactory.Core().V1().Secrets().Informer() + podControllerConfig := node.PodControllerConfig{ PodClient: localClient.CoreV1(), Provider: nodeProvider, @@ -318,6 +408,8 @@ func main() { // start informers -> go podInformerFactory.Start(stopper) go scmInformerFactory.Start(stopper) + go scmInformer.Run(stopper) + go podInformer.Run(stopper) // start to sync and call list if !cache.WaitForCacheSync(stopper, podInformerFactory.Core().V1().Pods().Informer().HasSynced) { @@ -359,8 +451,10 @@ func main() { // TODO: create a csr auto approver https://github.com/liqotech/liqo/blob/master/cmd/liqo-controller-manager/main.go#L498 retriever := commonIL.NewSelfSignedCertificateRetriever(cfg.NodeName, net.ParseIP(cfg.InternalIP)) + kubeletPort := os.Getenv("KUBELET_PORT") + server := &http.Server{ - Addr: fmt.Sprintf("0.0.0.0:%d", 10250), + Addr: fmt.Sprintf("0.0.0.0:%s", kubeletPort), Handler: mux, ReadHeaderTimeout: 10 * time.Second, // Required to limit the effects of the Slowloris attack. TLSConfig: &tls.Config{ diff --git a/cmd/virtual-kubelet/set-version.sh b/cmd/virtual-kubelet/set-version.sh index 9af21950..d55fffa3 100755 --- a/cmd/virtual-kubelet/set-version.sh +++ b/cmd/virtual-kubelet/set-version.sh @@ -1,11 +1,11 @@ #!/bin/bash -if [ -z ${KUBELET_VERSION} ]; then +if [ -n ${KUBELET_VERSION} ]; then cat << EOF > pkg/virtualkubelet/version.go -package main +package virtualkubelet var ( - kubeletVersion = "$KUBELET_VERSION" + KubeletVersion = "$KUBELET_VERSION" ) EOF fi diff --git a/docker/Dockerfile.interlink b/docker/Dockerfile.interlink index a81767ee..8731532e 100644 --- a/docker/Dockerfile.interlink +++ b/docker/Dockerfile.interlink @@ -10,9 +10,11 @@ ENV GOCACHE="/go/build-cache" RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE +ARG VERSION +RUN bash -c "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh" RUN go mod tidy -RUN CGO_ENABLED=0 GOOS=linux go build -o bin/interlink +RUN CGO_ENABLED=0 GOOS=linux go build -o bin/interlink cmd/interlink/main.go # Deploy the application binary into a lean image FROM gcr.io/distroless/base-debian11:latest AS build-release-stage diff --git a/docker/Dockerfile.vk b/docker/Dockerfile.vk index 9a455a24..f0249d25 100644 --- a/docker/Dockerfile.vk +++ b/docker/Dockerfile.vk @@ -7,7 +7,6 @@ WORKDIR /app COPY .. . ARG VERSION -ENV KUBELET_VERSION=${VERSION} ENV GOMODCACHE="/go/pkg/mod" ENV GOCACHE="/go/build-cache" @@ -15,7 +14,7 @@ ENV GOCACHE="/go/build-cache" RUN mkdir -p $GOMODCACHE && mkdir -p $GOCACHE -RUN bash -c "./cmd/virtual-kubelet/set-version.sh" +RUN bash -c "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh" RUN go mod tidy RUN CGO_ENABLED=0 GOOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go diff --git a/docker/scripts/refresh.py b/docker/scripts/refresh.py index 1bf444dc..ab25b2cf 100644 --- a/docker/scripts/refresh.py +++ b/docker/scripts/refresh.py @@ -10,11 +10,19 @@ import requests from urllib import parse +logger = logging.getLogger(__name__) + if __name__ == '__main__': """ sync OIDC identities on user accounts """ try: + verbose = os.environ.get("VERBOSE") + if "True" == verbose: + logging.basicConfig(level=logging.DEBUG) + logger.info("Verbose mode: setting log to debug.") + + iam_grant_type = os.environ.get("IAM_GRANT_TYPE") iam_server = os.environ.get( "IAM_TOKEN_ENDPOINT", "https://cms-auth.web.cern.ch/token") iam_client_id = os.environ.get("IAM_CLIENT_ID") @@ -22,63 +30,120 @@ iam_refresh_token = os.environ.get("IAM_REFRESH_TOKEN") audience = os.environ.get("IAM_VK_AUD") output_file = os.environ.get("TOKEN_PATH", "/opt/interlink/token") - except Exception as ex: - print(ex) + except Exception: + logger.exception() exit(1) - try: - with open(output_file+"-refresh", "r") as text_file: - rt = text_file.readline() - if rt != "": - iam_refresh_token = rt - except: - logging.info("No cache for refresh token, starting from ENV value") - - print(iam_refresh_token) - token = None - while True: try: - request_data = { - "audience": audience, - "grant_type": "refresh_token", - "refresh_token": iam_refresh_token, - #"scope": "openid profile email address phone offline_access" - } - - from requests.auth import HTTPBasicAuth - auth = HTTPBasicAuth(iam_client_id, iam_client_secret) - - r = requests.post(iam_server, data=request_data, auth=auth) - print(r.text) + with open(output_file+"-refresh", "r") as text_file: + rt = text_file.readline() + if rt != "": + iam_refresh_token = rt + except: + logger.info("No cache for refresh token, starting from ENV value") + + logger.info("Current refresh token: %s", iam_refresh_token) + token = None + + if iam_grant_type == "client_credentials": try: - response = json.loads(r.text) - except: + request_data = { + "audience": audience, + "grant_type": iam_grant_type, + "client_id" : iam_client_id, + "client_secret": iam_client_secret + #"scope": "openid profile email address phone offline_access" + } + + from requests.auth import HTTPBasicAuth + auth = HTTPBasicAuth(iam_client_id, iam_client_secret) + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post(iam_server, data=request_data, auth=auth, headers=headers) + logger.debug("Raw response text: %s", r.text) try: - response = dict(parse.parse_qsl(r.text)) - print(response) + response = json.loads(r.text) except: - exit(1) - + try: + response = dict(parse.parse_qsl(r.text)) + logger.debug("Response text parsed: %s", response) + except: + exit(1) + - print(iam_client_id, iam_client_secret, response) - token = response['access_token'] - refresh_token = response['refresh_token'] + logger.debug("iam_client_id %s iam_client_secret %s response %s", iam_client_id, iam_client_secret, response) - print("Token retrieved") + token = response['access_token'] + try: + refresh_token = response['refresh_token'] + except: + refresh_token = iam_refresh_token - ## TODO: collect new refresh token and store it somewhere - with open(output_file+"-refresh", "w") as text_file: - text_file.write(refresh_token) - print(f"Refresh token written in {output_file+'-refresh'}") + logger.info("Token retrieved") - with open(output_file, "w") as text_file: - text_file.write(token) + ## TODO: collect new refresh token and store it somewhere + with open(output_file+"-refresh", "w") as text_file: + text_file.write(refresh_token) - print(f"Token written in {output_file}") + with open(output_file, "w") as text_file: + text_file.write(token) - except Exception as e: - logging.warn("ERROR oidc get token: {}".format(e)) - - time.sleep(1000) + logger.info(f"Refresh token written in {output_file+'-refresh'}") + + except Exception as e: + logger.warning("ERROR oidc get token: {}".format(e), exc_info=True) + logger.warning("Response if available: %s", response) + + elif iam_grant_type == "authorization_code": + + try: + request_data = { + "audience": audience, + "grant_type": "refresh_token", + "refresh_token": iam_refresh_token, + #"scope": "openid profile email address phone offline_access" + } + + from requests.auth import HTTPBasicAuth + auth = HTTPBasicAuth(iam_client_id, iam_client_secret) + + r = requests.post(iam_server, data=request_data, auth=auth) + print(r.text) + try: + response = json.loads(r.text) + except: + try: + response = dict(parse.parse_qsl(r.text)) + logger.debug(response) + except: + exit(1) + + logger.debug("iam_client_id %s iam_client_secret %s response %s", iam_client_id, iam_client_secret, response) + + token = response['access_token'] + try: + refresh_token = response['refresh_token'] + except: + refresh_token = iam_refresh_token + + + logger.info("Token retrieved") + + ## TODO: collect new refresh token and store it somewhere + with open(output_file+"-refresh", "w") as text_file: + text_file.write(refresh_token) + + + with open(output_file, "w") as text_file: + text_file.write(token) + + logger.info(f"Refresh token written in {output_file+'-refresh'}") + + except Exception as e: + logger.warning("ERROR oidc get token: {}".format(e), exc_info=True) + logger.warning("Response if available: %s", response) + else: + logger.error(f"Invalid grant type {iam_grant_type}", exc_info=True) + exit(1) + time.sleep(200) diff --git a/docs/README.md b/docs/README.md index 0c6c2c27..543ffc52 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,13 @@ This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator. +## Dagger module to site preview + +```bash +dagger -m github.com/levlaz/daggerverse/docusaurus@f073c72e0a7345bba2173a15269307df297c3c13 call \ (⎈|default:default) + --src ./ serve up +``` + ### Installation ``` @@ -11,7 +18,7 @@ $ yarn ### Local Development ``` -$ yarn start +$ yarn start --config docusaurus.config.local.ts ``` This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. diff --git a/docs/docs/Cookbook.mdx b/docs/docs/Cookbook.mdx new file mode 100644 index 00000000..09d85930 --- /dev/null +++ b/docs/docs/Cookbook.mdx @@ -0,0 +1,473 @@ +--- +sidebar_position: 3 +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import ThemedImage from '@theme/ThemedImage'; +import useBaseUrl from '@docusaurus/useBaseUrl'; + + +# Cookbook + +These are practical recipes for different deployment scenarios. + +Select here the tab with the scenario you want deploy: + + + + + + + + + + + + + +Select here the featured plugin you want to try: + + + + Offload your pods to a remote machine with Docker engine available + + + Offload your pods to an HPC SLURM based batch system + + + Offload your pods to a remote Kubernetes cluster: COMING SOON + For test instructions contact us! + + + +There are more 3rd-party plugins developed that you can get inspired by or even use out of the box. You can find some ref in the [quick start section](guides/deploy-interlink#attach-your-favorite-plugin-or-develop-one) + +## Install interLink + +### Deploy Remote components (if any) + +In general, starting from the deployment of the remote components is adviced. Since the kubernetes virtual node won't reach the `Ready` status until all the stack is successfully deployed. + +#### Interlink API server + + + + __For this deployment mode the remote host has to allow the kubernetes cluster to connect to the Oauth2 proxy service port (30443 if you use the automatic script for installation)__ + + - You first need to initialize an OIDC client with you Identity Provider (IdP). + - Different options. We have instructions ready for [GitHub](./guides/deploy-interlink#create-an-oauth-github-app), [EGI checkin](./guides/oidc-IAM), [INFN IAM](./guides/oidc-IAM). + - Any OIDC provider working with [OAuth2 Proxy](https://oauth2-proxy.github.io/oauth2-proxy/) tool will do the work though. + - Create the `install.sh` utility script through the [installation utility](./guides/deploy-interlink#configuring-your-virtual-kubelet-setup) + - __N.B.__ if your machine is shared with other users, you better indicate a socket as address to communicate with the plugin. Instead of a web URL is enough to insert something like `unix:///var/run/myplugin.socket` + - Install Oauth2-Proxy and interLink API server services as per [Quick start](./guides/deploy-interlink#deploy-the-interlink-core-components) + - by default logs are store in `~/.interlink/logs`, checkout there for any error before moving to the next step. + + + Go directly to ["Test and debugging tips"](Cookbook#test-and-debug). The selected scenario does not expect you to do anything here. + + + __For this installation you need to know which node port is open on the main kubernetes cluster, and that will be used to expose the ssh bastion for the tunnel.__ + + - Create utility folders: + + ```bash + mkdir -p $HOME/.interlink/logs + mkdir -p $HOME/.interlink/bin + mkdir -p $HOME/.interlink/config + ``` + - Generate a pair of password-less SSH keys: + ```bash + ssh-keygen -t ecdsa + ``` + - Download the ssh-tunnel binary [latest release](https://github.com/interTwin-eu/interLink/releases/latest) binary in `$HOME/.interlink/bin/ssh-tunnel` + - Start the tunnel + + ```bash + CLUSTER_PUBLIC_IP="IP of you cluster where SSH will be exposed" + SSH_TUNNEL_NODE_PORT="node port where the ssh service will be exposed" + PRIV_KEY_FILE="path the ssh priv key created above" + + $HOME/.interlink/bin/ssh-tunnel -addr $CLUSTER_PUBLIC_IP:$SSH_TUNNEL_NODE_PORT -keyfile $PRIV_KEY_FILE -user interlink -rport 3000 -lsock plugin.sock &> $HOME/.interlink/logs/ssh-tunnel.log & + echo $! > $HOME/.interlink/ssh-tunnel.pid + ``` + + - Check the logs in `$HOME/.interlink/logs/ssh-tunnel.log`. + - To kill and restart the process is enough: + + ```bash + # kill + kill $(cat $HOME/.interlink/ssh-tunnel.pid) + + # restart + $HOME/.interlink/bin/ssh-tunnel &> $HOME/.interlink/logs/ssh-tunnel.log & + echo $! > $HOME/.interlink/ssh-tunnel.pid + ``` + - at this stage __THIS WILL CORRECTLY FAIL__ until we setup all the stack. So let's go ahead + + + + +#### Plugin service + + + + + + + - Create a configuration file: + + ```bash title="./plugin-config.yaml" + ## Multi user host + # SidecarURL: "unix:///home/myusername/plugin.socket" + # InterlinkPort: "0" + # SidecarPort: "0" + + ## Dedicated edge node + # InterlinkURL: "http://127.0.0.1" + # SidecarURL: "http://127.0.0.1" + # InterlinkPort: "3000" + # SidecarPort: "4000" + + CommandPrefix: "" + ExportPodData: true + DataRootFolder: "/home/myusername/.interlink/jobs/" + BashPath: /bin/bash + VerboseLogging: true + ErrorsOnlyLogging: false + ``` + - __N.B.__ Depending on wheter you edge is single user or not, you should know by previous steps which section to uncomment here. + - More on configuration options at [official repo](https://github.com/interTwin-eu/interlink-docker-plugin/blob/main/README.md) + + - Create utility folders: + + ```bash + mkdir -p $HOME/.interlink/logs + mkdir -p $HOME/.interlink/bin + mkdir -p $HOME/.interlink/config + ``` + - Download the [latest release](https://github.com/interTwin-eu/interlink-docker-plugin/releases) binary in `$HOME/.interlink/bin/plugin` for either GPU host or CPU host (tags ending with `no-GPU`) + - Start the plugins passing the configuration that you have just created: + + ```bash + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + ``` + + - Check the logs in `$HOME/.interlink/logs/plugin.log`. + - To kill and restart the process is enough: + + ```bash + # kill + kill $(cat $HOME/.interlink/plugin.pid) + + # restart + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + + Almost there! Now it's time to add this virtual node into the Kubernetes cluster! + + + - Create a configuration file: + + ```bash title="./plugin-config.yaml" + ## Multi user host + # SidecarURL: "unix:///home/myusername/plugin.socket" + # InterlinkPort: "0" + # SidecarPort: "0" + + ## Dedicated edge node + # InterlinkURL: "http://127.0.0.1" + # SidecarURL: "http://127.0.0.1" + # InterlinkPort: "3000" + # SidecarPort: "4000" + + CommandPrefix: "" + ExportPodData: true + DataRootFolder: "/home/myusername/.interlink/jobs/" + BashPath: /bin/bash + VerboseLogging: true + ErrorsOnlyLogging: false + SbatchPath: "/usr/bin/sbatch" + ScancelPath: "/usr/bin/scancel" + SqueuePath: "/usr/bin/squeue" + SingularityPrefix: "" + ``` + - __N.B.__ Depending on wheter you edge is single user or not, you should know by previous steps which section to uncomment here. + - More on configuration options at [official repo](https://github.com/interTwin-eu/interlink-slurm-plugin/blob/main/README.md) + + - Create utility folders + + ```bash + mkdir -p $HOME/.interlink/logs + mkdir -p $HOME/.interlink/bin + mkdir -p $HOME/.interlink/config + ``` + - Download the [latest release](https://github.com/interTwin-eu/interlink-slurm-plugin/releases) binary in `$HOME/.interlink/bin/plugin` for either GPU host or CPU host (tags ending with `no-GPU`) + - Start the plugins passing the configuration that you have just created: + + ```bash + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + ``` + + - Check the logs in `$HOME/.interlink/logs/plugin.log`. + - To kill and restart the process is enough: + + ```bash + # kill + kill $(cat $HOME/.interlink/plugin.pid) + + # restart + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + + Almost there! Now it's time to add this virtual node into the Kubernetes cluster! + + + __KUBERNTES PLUGIN COMING SOOON... CONTACT US FOR TEST INSTRUCTIONS__ + + + + + + Go directly to ["Test and debugging tips"](Cookbook#test-and-debug). The selected scenario does not expect you to do anything here. + + + + + + - Create a configuration file: + + ```bash title="./plugin-config.yaml" + SidecarURL: "unix:///home/myusername/plugin.socket" + SidecarPort: "0" + + CommandPrefix: "" + ExportPodData: true + DataRootFolder: "/home/myusername/.interlink/jobs/" + BashPath: /bin/bash + VerboseLogging: true + ErrorsOnlyLogging: false + ``` + - __N.B.__ you should know by previous steps what to put in place of `myusername` here. + - More on configuration options at [official repo](https://github.com/interTwin-eu/interlink-docker-plugin/blob/main/README.md) + + - Create utility folders: + + ```bash + mkdir -p $HOME/.interlink/logs + mkdir -p $HOME/.interlink/bin + mkdir -p $HOME/.interlink/config + ``` + - Download the [latest release](https://github.com/interTwin-eu/interlink-docker-plugin/releases) binary in `$HOME/.interlink/bin/plugin` for either GPU host or CPU host (tags ending with `no-GPU`) + - Start the plugins passing the configuration that you have just created: + + ```bash + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + ``` + + - Check the logs in `$HOME/.interlink/logs/plugin.log`. + - To kill and restart the process is enough: + + ```bash + # kill + kill $(cat $HOME/.interlink/plugin.pid) + + # restart + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + + Almost there! Now it's time to add this virtual node into the Kubernetes cluster! + + + - Create a configuration file: + + ```bash title="./plugin-config.yaml" + SidecarURL: "unix:///home/myusername/plugin.socket" + SidecarPort: "0" + + CommandPrefix: "" + ExportPodData: true + DataRootFolder: "/home/myusername/.interlink/jobs/" + BashPath: /bin/bash + VerboseLogging: true + ErrorsOnlyLogging: false + SbatchPath: "/usr/bin/sbatch" + ScancelPath: "/usr/bin/scancel" + SqueuePath: "/usr/bin/squeue" + SingularityPrefix: "" + ``` + - __N.B.__ you should know by previous steps what to put in place of `myusername` here. + - More on configuration options at [official repo](https://github.com/interTwin-eu/interlink-slurm-plugin/blob/main/README.md) + - Create utility folders: + + ```bash + mkdir -p $HOME/.interlink/logs + mkdir -p $HOME/.interlink/bin + mkdir -p $HOME/.interlink/config + ``` + - Download the [latest release](https://github.com/interTwin-eu/interlink-slurm-plugin/releases) binary in `$HOME/.interlink/bin/plugin` for either GPU host or CPU host (tags ending with `no-GPU`) + - Start the plugins passing the configuration that you have just created: + + ```bash + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + ``` + + - Check the logs in `$HOME/.interlink/logs/plugin.log`. + - To kill and restart the process is enough: + + ```bash + # kill + kill $(cat $HOME/.interlink/plugin.pid) + + # restart + export INTERLINKCONFIGPATH=$PWD/plugin-config.yaml + $HOME/.interlink/bin/plugin &> $HOME/.interlink/logs/plugin.log & + echo $! > $HOME/.interlink/plugin.pid + + Almost there! Now it's time to add this virtual node into the Kubernetes cluster! + + + COMING SOOON... + + + + + + + +#### Test interLink stack health + +interLink comes with a call that can be used to monitor the overall status of both interlink server and plugins, at once. + +``` +curl -v $INTERLINK_SERVER_ADDRESS:$INTERLINK_PORT/pinginterlink +``` + +This call will return the status of the system and its readiness to submit jobs. + + +### Deploy Kubernetes components + +The deployment of the Kubernetes components are managed by the official [HELM chart](https://github.com/interTwin-eu/interlink-helm-chart). Depending on the scenario you selected, there might be additional operations to be done. + + + + __For this deployment mode the remote host has to allow the kubernetes cluster to connect to the Oauth2 proxy service port (30443 if you use the automatic script for installation)__ + + - Since you might already have followed the installation script steps, you can simply follow the [Guide](./guides/deploy-interlink#deploy-the-interlink-kubernetes-agent-kubeclt-host) + + __If the installation script is not what you are currently used, you can configure the virtual kubelet manually:__ + - Create an helm values file: + + ```yaml title="values.yaml" + nodeName: interlink-with-rest + + interlink: + address: https://remote_oauth2_proxy_endpoint + port: 30443 + + virtualNode: + CPUs: 1000 + MemGiB: 1600 + Pods: 100 + HTTPProxies: + HTTP: null + HTTPs: null + OAUTH: + image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw-refresh:latest + TokenURL: DUMMY + ClientID: DUMMY + ClientSecret: DUMMY + RefreshToken: DUMMY + GrantType: authorization_code + Audience: DUMMY + ``` + - Substitute the OAuth value accordingly as + + + - Create an helm values file: + + ```yaml title="values.yaml" + nodeName: interlink-with-socket + + plugin: + enabled: true + image: "plugin docker image here" + command: ["/bin/bash", "-c"] + args: ["/app/plugin"] + config: | + your plugin + configuration + goes here!!! + socket: unix:///var/run/plugin.socket + + interlink: + enabled: true + socket: unix:///var/run/interlink.socket + ``` + + + - Create an helm values file: + + ```yaml title="values.yaml" + nodeName: interlink-with-socket + + interlink: + enabled: true + socket: unix:///var/run/interlink.socket + + plugin: + address: http://localhost + + sshBastion: + enabled: true + clientKeys: + authorizedKey: | + ssh-rsa A..........MG0yNvbLfJT+37pw== + port: 31021 + ``` + - insert the plublic key generated when installing interlink and ssh tunnel service + + + +Eventually deploy the latest release of the official [helm chart](https://github.com/interTwin-eu/interlink-helm-chart): + +```bash +helm upgrade --install --create-namespace -n interlink my-virtual-node oci://ghcr.io/intertwin-eu/interlink-helm-chart/interlink --values ./values.yaml +``` + +Whenever you see the node ready, you are good to go! + +## Test the setup + +Please find a demo pod to test your setup [here](./guides/develop-a-plugin#lets-test-is-out). + + diff --git a/ci/README.md b/docs/docs/Developers.md similarity index 65% rename from ci/README.md rename to docs/docs/Developers.md index 25ff428a..61ef887f 100644 --- a/ci/README.md +++ b/docs/docs/Developers.md @@ -1,11 +1,15 @@ -# INTERLINK E2E INTEGRATION TESTS +--- +sidebar_position: 5 +--- + +# E2E integration tests Here you can find how to test a virtual kubelet implementation against the main pod use cases we mean to support. ## Requirements - [Docker engine](https://docs.docker.com/engine/install/) -- [Dagger CLI](https://docs.dagger.io/install/) +- [Dagger CLI v0.13.x](https://docs.dagger.io/install/) ## What's in the Dagger module @@ -22,6 +26,8 @@ That means you can test your code **before** any commit, discovering in advance ### Run e2e tests +The easiest way is to simply run `make test` from the root folder of interlink. But if you need to debug or understand further the test utility or a plugin, you should follow these instructions. + #### Edit manifests with your images - `service-account.yaml` is the default set of permission needed by the virtualkubelet. Do not touch unless you know what you are doing. @@ -29,12 +35,25 @@ That means you can test your code **before** any commit, discovering in advance - `virtual-kubelet.yaml` is the one that you should touch if you are pointing to different interlink endpoints or if you want to change the __virtual kubelet__ image to be tested. - `interlink-config.yaml` is the configuration mounted into the __interlink API__ component to determine its behaviour. - `interlink.yaml` is the one that you should touch if you are pointing to different plugin endpoints or if you want to change the __interlink API__ image to be tested. -- `plugin-config.yaml` is the configuration mounted into the __interLink plugin__ component to determine its behaviour. -- `plugin.yaml` is the one that you should touch if you are pointing to different plugin endpoints or if you want to change the __interlink plugin__ image to be tested. +- `plugin-config.yaml` is the configuration for the __interLink plugin__ component that you MUST TO START MANUALLY on your host. + - we do have solution to make it start inside dagger environment, but is not documented yet. + +#### Start the local docker plugin service +For a simple demonstration, you can use the plugin that we actually use in are Github Actions: + +```bash +wget https://github.com/interTwin-eu/interlink-docker-plugin/releases/download/0.0.24-no-gpu/docker-plugin_Linux_x86_64 -O docker-plugin \ + && chmod +x docker-plugin \ + && docker ps \ + && export INTERLINKCONFIGPATH=$PWD/ci/manifests/plugin-config.yaml \ + && ./docker-plugin +``` #### Run the tests +Then, in another terminal sessions you are ready to execute the e2e tests with Dagger. + First of all, in `ci/manifests/vktest_config.yaml` you will find the pytest configuration file. Please see the [test documentation](https://github.com/interTwin-eu/vk-test-set/tree/main) for understanding how to tweak it. The following instructions are thought for building docker images of the virtual-kubelet and interlink api server components at runtime and published on `virtual-kubelet-ref` and `interlink-ref` repositories (in this example it will be dockerHUB repository of the dciangot user). @@ -43,12 +62,12 @@ It basically consists on a chain of Dagger tasks for building core images (`buil To run the default tests you can move to `ci` folder and execute the Dagger pipeline with: ```bash -export YOUR_DOCKERHUB_USER=dciangot - -dagger call build-images --source-folder ../ --virtual-kubelet-ref ${YOUR_DOCKERHUB_USER}/vk --interlink-ref ${YOUR_DOCKERHUB_USER}/interlink \ - new-interlink --manifests $PWD/manifests \ - load-plugin \ - test stdout +dagger call \ + --name my-tests \ + build-images \ + new-interlink \ + --plugin-endpoint tcp://localhost:4000 \ + test stdout ``` :warning: by default the docker plugin is the one tested and to be referred to for any change as first thing. @@ -81,10 +100,13 @@ vktestset/basic_test.py::test_manifest[virtual-kubelet-060-init-container.yaml] In case something went wrong, you have the possibility to spawn a session inside the final step of the pipeline to debug things: ```bash -dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ - new-interlink --manifests $PWD/manifests \ - load-plugin \ - run terminal +dagger call \ + --name my-tests \ + build-images \ + new-interlink \ + --plugin-endpoint tcp://localhost:4000 \ + run terminal + ``` with this command (after some minutes) then you should be able to access a bash session doing the following commands: @@ -101,29 +123,45 @@ kubectl get pod -A pytest -vk 'not rclone' ``` +#### Debug from kubectl on your host -### Deploy on existing K8s cluster - -You might want to hijack the test machinery in order to have it instantiating the test environemnt on your own kubernetes cluster (e.g. to debug and develop plugins in a efficient way). We are introducing options for this purpose and it is expected to be extended even more in the future. - -If you have a kubernetes cluster **publically accessible**, you can pass your kubeconfig to the Dagger pipeline and use that instead of the internal one that is "one-shot" for the tests only. +You can get the Kubernetes service running with: ```bash -dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ - new-interlink --manifests $PWD/manifests --kubeconfig $PWD/kubeconfig.yaml \ - load-plugin \ - test stdout +dagger call \ + --name my-tests \ + build-images \ + new-interlink \ + --plugin-endpoint tcp://localhost:4000 \ + kube up ``` -If you have a *local* cluster (e.g. via MiniKube), you need to forward the local port of the Kubernetes API server (look inside the kubeconfig file) inside the Dagger runtime with the following: +and then from another session, you can get the kubeconfig with: ```bash -dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk --interlink-ref dciangot/interlink \ - new-interlink --manifests $PWD/manifests --kubeconfig $PWD/kubeconfig.yaml --local-cluster tcp://localhost:59127 \ - load-plugin \ - test stdout +dagger call \ + --name my-tests \ + config export --path ./kubeconfig.yaml ``` + +### Deploy on existing K8s cluster + +TBD + + + + + + + + + + + + + + ### Develop Virtual Kubelet code :warning: Coming soon @@ -133,3 +171,7 @@ dagger call build-images --source-folder ../ --virtual-kubelet-ref dciangot/vk :warning: Coming soon +### Develop your plugin + +:warning: Coming soon + diff --git a/docs/docs/tutorial-users/02-limitations.md b/docs/docs/Limitations.md similarity index 97% rename from docs/docs/tutorial-users/02-limitations.md rename to docs/docs/Limitations.md index c6111453..06b99e66 100644 --- a/docs/docs/tutorial-users/02-limitations.md +++ b/docs/docs/Limitations.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 6 --- # Current limitations diff --git a/docs/docs/arch.mdx b/docs/docs/arch.mdx new file mode 100644 index 00000000..5ad980e7 --- /dev/null +++ b/docs/docs/arch.mdx @@ -0,0 +1,24 @@ +--- +sidebar_position: 2 +--- +import ThemedImage from '@theme/ThemedImage'; +import useBaseUrl from '@docusaurus/useBaseUrl'; + +# Architecture + +InterLink aims to provide an abstraction for the execution of a Kubernetes pod on any remote resource capable of managing a Container execution lifecycle. + +The project consists of two main components: + +- __A Kubernetes Virtual Node:__ based on the [VirtualKubelet](https://virtual-kubelet.io/) technology. Translating request for a kubernetes pod execution into a remote call to the interLink API server. +- __The interLink API server:__ a modular and pluggable REST server where you can create your own Container manager plugin (called sidecars), or use the existing ones: remote docker execution on a remote host, singularity Container on a remote SLURM batch system. + +The project got inspired by the [KNoC](https://github.com/CARV-ICS-FORTH/knoc) and [Liqo](https://github.com/liqotech/liqo/tree/master) projects, enhancing that with the implemention a generic API layer b/w the virtual kubelet component and the provider logic for the container lifecycle management. + + diff --git a/docs/docs/tutorial-admins/01-deploy-interlink.mdx b/docs/docs/guides/01-deploy-interlink.mdx similarity index 84% rename from docs/docs/tutorial-admins/01-deploy-interlink.mdx rename to docs/docs/guides/01-deploy-interlink.mdx index 80277e42..f1dc7d85 100644 --- a/docs/docs/tutorial-admins/01-deploy-interlink.mdx +++ b/docs/docs/guides/01-deploy-interlink.mdx @@ -8,22 +8,21 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; Learn how to deploy interLink virtual nodes on your cluster. In this tutorial you are going to setup all the needed components to be able to either __develop__ or __deploy__ the plugin for container management on a **remote** host via a **local** kubernetes cluster. -The installation script that we are going to configure will take care of providing you with a complete Kubernetes manifest to instantiate the virtual node interface. Also you will get an installation bash script to be executed on the remote host where you want to delegate your container execution. That script is already configured to **automatically** authenticate the incoming request from the virtual node component, and forward the correct instructions to the openAPI interface of the [interLink plugin](./03-api-reference.mdx) (a.k.a. sidecar) of your choice. Thus you can use this setup also for directly [developing a plugin](./02-develop-a-plugin.md), without caring for anything else. +The installation script that we are going to configure will take care of providing you with a complete Kubernetes manifest to instantiate the virtual node interface. Also you will get an installation bash script to be executed on the remote host where you want to delegate your container execution. That script is already configured to **automatically** authenticate the incoming request from the virtual node component, and forward the correct instructions to the openAPI interface of the [interLink plugin](./api-reference) (a.k.a. sidecar) of your choice. Thus you can use this setup also for directly [developing a plugin](./develop-a-plugin), without caring for anything else. + +For a complete guide on all the possible scenarios, please refer to the [Cookbook](../cookbook). ## Requirements -- MiniKube +- __kubectl host__: an host with MiniKube installed and running - A GitHub account -- A "remote" machine with a port that is reachable by the MiniKube host +- __remote host__: A "remote" machine with a port that is reachable by the MiniKube host -:::danger -In this tutorial, we suppose the remote VM fully owned (not shared) by the user only. There is NO protection against call to the interLink services coming from the machine itself on the localhost. If you need to install it in a "multi user" environment, please refer to [this guide](./05-multi-user.md) -::: ## Create an OAuth GitHub app :::warning -In this tutorial GitHub tokens are just an example of authentication mechanism, any OpenID compliant identity provider is also supported with the very same deployment script, see [examples here](./04-oidc-IAM.md). +In this tutorial GitHub tokens are just an example of authentication mechanism, any OpenID compliant identity provider is also supported with the very same deployment script, see [examples here](./oidc-IAM). ::: As a first step, you need to create a GitHub OAuth application to allow interLink to make authentication between your Kubernetes cluster and the remote endpoint. @@ -66,9 +65,9 @@ You can click then on your application that should now appear at [https://github Now it's all set for the next steps. -## Configuring your virtual kubelet setup +## Configuring your virtual kubelet setup (remote host) -You can download the interLink **installer CLI** for your OS and processor architecture from the [release page](https://github.com/interTwin-eu/interLink/releases), looking for the binaries starting with `interlink-install`. For instance, if on a `Linux` platform with `x86_64` processor: +Login into the machine and and download the interLink **installer CLI** for your OS and processor architecture from the [release page](https://github.com/interTwin-eu/interLink/releases), looking for the binaries starting with `interlink-install`. For instance, if on a `Linux` platform with `x86_64` processor: ```bash export VERSION=0.2.3-pre6 @@ -90,7 +89,7 @@ Let's take the following as an example of a valid configuration file: see [release page](https://github.com/interTwin-eu/interLink/releases) to get the latest one! And change the value accordingly! ::: -```yaml +```yaml title="$HOME/.interlink.yaml" interlink_ip: 192.168.1.127 interlink_port: 30443 interlink_version: 0.2.1-patch2 @@ -103,6 +102,7 @@ node_limits: oauth: provider: github issuer: https://github.com/oauth + grant_type: authorization_code scopes: - "read:user" github_user: "dciangot" @@ -127,9 +127,10 @@ This config file has the following meaning: You are ready now to go ahead generating the needed manifests and script for the deployment. -## Deploy the interlink Kubernetes Agent -Generate the manifests and the automatic interlink installation script with: +## Deploy the interLink core components (remote host) + +Login into the machine and generate the manifests and the automatic interlink installation script with: ```bash ./interlink-install @@ -154,27 +155,8 @@ please enter code XXXX-XXXX at https://github.com/login/device "./interlink-remote.sh install" followed by "interlink-remote.sh start" ``` -We are almost there! Essentially you need to follow what suggested by the prompt. - -So go ahead and apply the produced manifest to your minikube/kubernetes instance with: - -```bash -kubectl apply -f $HOME/.interlink/interlink.yaml -``` - -Check that the node appears successfully after some time, or as soon as you see the pods in namespace `interlink` running. - -You are now ready to setup the second component on the remote host. - -## Deploy the interLink core components - -Copy the `$HOME/.interlink/interlink-remote.sh` file on the remote host: -```bash -scp -r $HOME/.interlink/interlink-remote.sh ubuntu@192.168.1.127:~ -``` - -Then login into the machine and start installing all the needed binaries and configurations: +Start installing all the needed binaries and configurations: ```bash chmod +x ./interlink-remote.sh @@ -202,9 +184,9 @@ To stop or restart the components you can use the dedicated commands: ./interlink-remote.sh restart ``` -## Attach your favorite plugin or develop one! +## Attach your favorite plugin or develop one! (remote host) -[Next chapter](./02-develop-a-plugin.md) will show the basics for developing a new plugin following the interLink openAPI spec. +[Next chapter](./develop-a-plugin) will show the basics for developing a new plugin following the interLink openAPI spec. In alterative you can start an already supported one. @@ -223,9 +205,9 @@ Note that the SLURM plugin repository is: [github.com/interTwin-eu/interlink-slu #### Configuration -Create a config file `.interlink/config/slurm.yaml`: +Create a config file `$HOME/.interlink/config/slurm.yaml`: -```yaml +```yaml title="$HOME/.interlink/config/slurm.yaml" # Plugin local endpoint SidecarPort: "4000" SidecarURL: "http://localhost" @@ -263,7 +245,7 @@ To get the latest version of the plugin, please visit the [release](https://gith Download the latest release with: ```bash -VERSION=0.2.2 +VERSION=0.3.1 wget -O $HOME/.interlink/bin/slurm-plugin https://github.com/interTwin-eu/interlink-slurm-plugin/releases/download/$VERSION/interlink-sidecar-slurm_Linux_x86_64 chmod +x $HOME/.interlink/bin/slurm-plugin ``` @@ -281,7 +263,7 @@ After=network.target [Service] Type=simple ExecStart=$HOME/.interlink/bin/slurm-plugin -Environment="INTERLINKCONFIGPATH=$HOME/.interlink/config/InterLinkConfig.yaml" +Environment="SLURMCONFIGPATH=$HOME/.interlink/config/slurm.yaml" Environment="SHARED_FS=true" StandardOutput=file:$HOME/.interlink/logs/plugin.log StandardError=file:$HOME/.interlink/logs/plugin.log @@ -313,7 +295,7 @@ Logs will be stored at `$HOME/.interlink/logs/plugin.log`. :::warning An mantained plugin will come soon... -In the meantime you can take a look at the ["developing a plugin"](./02-develop-a-plugin.md) example. +In the meantime you can take a look at the ["developing a plugin"](./develop-a-plugin) example. ::: - [Docker plugin repository](https://github.com/interTwin-eu/interlink-docker-plugin) @@ -327,3 +309,20 @@ Coming soon - [HTCondor plugin repository](https://github.com/interTwin-eu/interlink-htcondor-plugin) - [ARC plugin repository](https://github.com/interTwin-eu/interlink-arc-plugin) +## Deploy the interlink Kubernetes Agent (kubeclt host) + +We are almost there! Essentially you need to follow what suggested by the prompt of the installation script, so copy the generated `interlink.yaml` into you __kubectl host__. + +So go ahead and apply the produced manifest to your minikube/kubernetes instance with: + +```bash +kubectl apply -f $HOME/.interlink/interlink.yaml +``` + +Check that the node appears successfully after some time, or as soon as you see the pods in namespace `interlink` running. + +You are all setup, congratulations! + +## Test your setup + +Please find a demo pod to test your setup [here](./develop-a-plugin#lets-test-is-out). diff --git a/docs/docs/tutorial-admins/02-develop-a-plugin.md b/docs/docs/guides/02-develop-a-plugin.md similarity index 97% rename from docs/docs/tutorial-admins/02-develop-a-plugin.md rename to docs/docs/guides/02-develop-a-plugin.md index cc896969..30039512 100644 --- a/docs/docs/tutorial-admins/02-develop-a-plugin.md +++ b/docs/docs/guides/02-develop-a-plugin.md @@ -37,7 +37,7 @@ Then you are ready to install the python SDK with: #pip install "uvicorn[standard]" "git+https://github.com/interTwin-eu/interLink.git@${VERSION}#egg=interlink&subdirectory=example" # Or download the latest one with -pip install "uvicorn[standard]" "git+https://github.com/interTwin-eu/interLink.git#egg=interlink&subdirectory=example" +pip install "uvicorn[standard]" "git+https://baltig.infn.it/mgattari/interlink-plugin-sdk" ``` @@ -323,16 +323,20 @@ kind: Pod metadata: name: interlink-quickstart namespace: default + # # Uncomment this if you are using SLURM + # annotations: + # slurm-job.vk.io/image-root: "docker://" spec: nodeSelector: - kubernetes.io/hostname: my-civo-node + # The name of the virtual node HERE + kubernetes.io/hostname: civo-node automountServiceAccountToken: false containers: - args: - - -c - - 'sleep 600 && echo "FINISHED!"' + - sleep 100 && echo "FINISHED!" command: - /bin/sh + - -c image: busybox imagePullPolicy: Always name: my-container diff --git a/docs/docs/tutorial-admins/03-api-reference.mdx b/docs/docs/guides/03-api-reference.mdx similarity index 100% rename from docs/docs/tutorial-admins/03-api-reference.mdx rename to docs/docs/guides/03-api-reference.mdx diff --git a/docs/docs/tutorial-admins/04-oidc-IAM.md b/docs/docs/guides/04-oidc-IAM.md similarity index 97% rename from docs/docs/tutorial-admins/04-oidc-IAM.md rename to docs/docs/guides/04-oidc-IAM.md index de6758ca..39c9cc7c 100644 --- a/docs/docs/tutorial-admins/04-oidc-IAM.md +++ b/docs/docs/guides/04-oidc-IAM.md @@ -23,6 +23,7 @@ oauth: - "offline_access" - "profile" audience: interlink + grant_type: authorization_code group_claim: email group: "YOUR EMAIL HERE" token_url: "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" @@ -72,6 +73,7 @@ oauth: - "offline_access" - "profile" audience: users + grant_type: authorization_code group_claim: email group: "YOUR EMAIL HERE" token_url: "https://iam.cloud.infn.it/token" diff --git a/docs/docs/guides/05-monitoring.md b/docs/docs/guides/05-monitoring.md new file mode 100644 index 00000000..472beb1c --- /dev/null +++ b/docs/docs/guides/05-monitoring.md @@ -0,0 +1,101 @@ +--- +sidebar_position: 5 +--- + + +# InterLink Monitoring System + +## Description + +To monitor the InterLink stack, you can use an helm chart that deploys the entire stack and components needed for monitoring the Interlink components, in particular the Virtual Kubelet (VK) +of the Interlink project. In the near future, the monitoring system will be extended to monitor other components of the Interlink project. + +## Monitoring System Components + +The monitoring system is composed of the following two components: +1. [Grafana Tempo](https://grafana.com/docs/tempo/latest/) +2. [Grafana](https://grafana.com) + +### Grafana Tempo + +Grafana Tempo (or simply **Tempo**) is an open-source distributed tracing backend developed by Grafana Labs designed to handle high-scale and high-volume distributed tracing data. +The choice of Tempo was made because of its key features: +- **Scalability**: Tempo is designed to scale horizontally and handle millions of spans (a span is a single operation in a trace) per second. Moreover, it is capable of storing traces without requiring a database, instead leveraging object storage. +- **Simplicity**: Tempo does not index traces because it relies Grafana for querying and visualizing traces. This approach reduces complexity and operational overhead. +- **Cost**: Tempo is cost-effective because it uses object storage for storing traces, which is cheaper than traditional databases. +- **Integration**: Tempo integrates with popular tracing protocols. One of them is OpenTelemetry, which is used by the Virtual Kubelet (VK) of the Interlink project. +- **Querying**: Tempo provides a query language that allows users to filter and aggregate traces. It also supports distributed sampling, which allows users to sample traces across services. The powerful query language is a key feature that enables users to extract insights from traces and it is one of the main reasons why Tempo was chosen for the monitoring system. + +### Grafana + +Grafana is an open-source platform for monitoring and observability that allows users to query, visualize, alert on, and understand metrics no matter where they are stored. It is used to create, explore, and share dashboards with teams and stakeholders. Grafana supports a wide range of data sources, including Tempo. + + +## VK tracing + +The Virtual Kubelet (VK) of the Interlink project is instrumented with **OpenTelemetry** to generate traces. OpenTelemetry is an open-source observability framework that provides APIs, libraries, agents, and instrumentation to collect telemetry data from applications and services. The traces generated by the VK are sent to Tempo, where they are stored and queried. +Traces are generated by the VK when a request is made to the VK, and they contain information about the request, such as the details of the operation, the duration of the operation, and the services involved in the operation. A trace is a collection of spans, where each span represents a single operation in the trace. Spans are linked together to form a trace, which provides a complete view of the flow and performance of the operation. A span contains metadata, such as the name of the operation, the start and end time of the operation, and the service that generated the span. + +The flow is represented in the following image: + +![Monitoring flow](./img/vk_tracing.png) + +In the actual setup, Tempo is deployed in the same Kubernetes cluster as the VK. Thanks to a Kubernetes service, the VK can send traces to Tempo. +No TLS is used for the communication between the VK and Tempo, as the communication is internal to the cluster. +In the near future, the communication between the VK and Tempo will be secured using TLS if Tempo is deployed outside the cluster and a proxy will be used to authenticate the VK with Tempo. + +The following table is a list of spans generated by the VK: + +| Span Name | Description | Attributes | +| --- | --- | --- | +| CreateHttpCall | Span that represents the HTTP call made by the VK to the Interlink API to create a pod. | pod.name, pod.namespace, start.timestamp, end.timestamp, duration, exitc.code | +| DeleteHttpCall | Span that represents the HTTP call made by the VK to the Interlink API to delete a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration, exitc.code | +| StatusHttpCall | Span that represents the HTTP call made by the VK to the Interlink API to get the status of a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration, exitc.code | +| LogHttpCall | Span that represents the HTTP call made by the VK to the Interlink API to get the logs of a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration, exitc.code | +| PingHttpCall | Span that represents the HTTP call made by the VK to the Interlink API check if the API is alive. | start.timestamp, end.timestamp, duration, exitc.code | +| CreatePodVK | Span that represents the call made by the VK to the Kubernetes API to create a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration | +| DeletePodVK | Span that represents the call made by the VK to the Kubernetes API to delete a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration | +| UpdatePodVK | Span that represents the call made by the VK to the Kubernetes API to update a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration | +| GetPodVK | Span that represents the call made by the VK to the Kubernetes API to get a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration | +| GetPodStatusVK | Span that represents the call made by the VK to the Kubernetes API to get the status of a pod. | pod.name pod.namespace, start.timestamp, end.timestamp, duration | +| GetPodsVK | Span that represents the call made by the VK to the Kubernetes API to get all pods. | start.timestamp, end.timestamp, duration | + +## Installation + +:::warning +To Enable tracing for the Virtual Kubelet, you have to set the Environment Variable ```ENABLE_TRACING``` to 1 +::: + +A requirement for the monitoring system is to have Helm installed in your Kubernetes cluster or in your local machine. If you don't have Helm installed, you can follow the instructions in the following link: https://helm.sh/docs/intro/install/ + +This helm chart requires a NGINX Ingress Controller to be installed in the cluster in order to expose the services and access them from outside the cluster. +In particular, the Grafana service is exposed using an Ingress resource. + +To install the monitoring system, follow the steps below: + +1. Clone the repository: + +```bash +git clone https://github.com/interTwin-eu/interlink-monitoring-stack +``` + +2. Change the values in the `values.yaml` file to match your environment. +3. Create a namespace for the monitoring system using the following command: + +```bash +kubectl create namespace interlink-mon +``` +4. Install the monitoring system using the following command: + +```bash +helm upgrade --install helm-vk-monitoring-release interlink-monitoring-stack/ -n interlink-mon --debug +``` + +5. Deploy (or re-deploy) your Virtual Kubelet stack setting the ```TELEMETRY_ENDPOINT``` Environment Variable to your actual endpoint. +If not set, it defaults to ```localhost:4317```. Remember to enable the monitoring by also setting ```ENABLE_TRACING``` to 1. + +6. Access Grafana dashboard through the domain you defined in the `values.yaml` file with the credentials you defined in the `values.yaml` file. + +The following image shows the Grafana dashboard: + +![Grafana Dashboard](./img/dashboard.png) diff --git a/docs/docs/tutorial-admins/_category_.json b/docs/docs/guides/_category_.json similarity index 90% rename from docs/docs/tutorial-admins/_category_.json rename to docs/docs/guides/_category_.json index 7a6341cf..5d7733a7 100644 --- a/docs/docs/tutorial-admins/_category_.json +++ b/docs/docs/guides/_category_.json @@ -1,6 +1,6 @@ { "label": "Guides", - "position": 3, + "position": 4, "link": { "type": "generated-index", "description": "Learn how to deploy and adapt interLink plugins for your use case." diff --git a/docs/docs/guides/img/dashboard.png b/docs/docs/guides/img/dashboard.png new file mode 100644 index 00000000..fa7fa612 Binary files /dev/null and b/docs/docs/guides/img/dashboard.png differ diff --git a/docs/docs/tutorial-admins/img/docsVersionDropdown.png b/docs/docs/guides/img/docsVersionDropdown.png similarity index 100% rename from docs/docs/tutorial-admins/img/docsVersionDropdown.png rename to docs/docs/guides/img/docsVersionDropdown.png diff --git a/docs/docs/tutorial-admins/img/iam-client0.png b/docs/docs/guides/img/iam-client0.png similarity index 100% rename from docs/docs/tutorial-admins/img/iam-client0.png rename to docs/docs/guides/img/iam-client0.png diff --git a/docs/docs/tutorial-admins/img/iam-client1.png b/docs/docs/guides/img/iam-client1.png similarity index 100% rename from docs/docs/tutorial-admins/img/iam-client1.png rename to docs/docs/guides/img/iam-client1.png diff --git a/docs/docs/tutorial-admins/img/iam-client2.png b/docs/docs/guides/img/iam-client2.png similarity index 100% rename from docs/docs/tutorial-admins/img/iam-client2.png rename to docs/docs/guides/img/iam-client2.png diff --git a/docs/docs/tutorial-admins/img/localeDropdown.png b/docs/docs/guides/img/localeDropdown.png similarity index 100% rename from docs/docs/tutorial-admins/img/localeDropdown.png rename to docs/docs/guides/img/localeDropdown.png diff --git a/docs/docs/guides/img/vk_tracing.png b/docs/docs/guides/img/vk_tracing.png new file mode 100644 index 00000000..42a37443 Binary files /dev/null and b/docs/docs/guides/img/vk_tracing.png differ diff --git a/docs/docs/intro.mdx b/docs/docs/intro.mdx index e9d3eda2..446030b8 100644 --- a/docs/docs/intro.mdx +++ b/docs/docs/intro.mdx @@ -6,42 +6,82 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; # Introduction +![GitHub License](https://img.shields.io/github/license/intertwin-eu/interlink) +![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/intertwin-eu/interlink/total) +![GitHub Repo stars](https://img.shields.io/github/stars/intertwin-eu/interlink) + +![GitHub Release](https://img.shields.io/github/v/release/intertwin-eu/interlink) +![Tested with Dagger](https://img.shields.io/badge/tested_with_dagger-v0.13.3-green) + +![Slack server](https://img.shields.io/badge/slack_server-8A2BE2?link=https%3A%2F%2Fjoin.slack.com%2Ft%2Fintertwin%2Fshared_invite%2Fzt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA) + :::warning interLink is in early development phase, thus subject to breaking changes with no guarantee of backward compatibility. ::: -InterLink aims to provide an abstraction for the execution of a Kubernetes pod on any remote resource capable of managing a Container execution lifecycle. +## Targets + +- __K8s applications with tasks to be executed on HPC systems__: This target focuses on Kubernetes applications that require high-performance computing (HPC) resources for executing tasks. These tasks might involve complex computations, simulations, or data processing that benefit from the specialized hardware and optimized performance of HPC systems. + +- __Remote "runner"-like application for heavy payload execution requiring GPUs__: This target is designed for applications that need to execute heavy computational payloads, particularly those requiring GPU resources. These applications can be run remotely, leveraging powerful GPU hardware to handle tasks such as machine learning model training, data analysis, or rendering. + +- __Lambda-like functions calling on external resources__: This target involves running containers on demand with specific computing needs. Now these resources might also be outside of the Kubernetes cluster thanks to interLink functionality. + +## Target providers + +Our solution is designed to target a wide range of providers with container execution capabilities, including but not limited to: + +- __SLURM or HTCondor batch systems with Apptainer, Enroot, or Singularity__: These batch systems are widely used in high-performance computing environments to manage and schedule jobs. By integrating with container runtimes like Apptainer, Enroot, or Singularity, our solution can efficiently execute containerized tasks on these systems. +- __Remote/on-demand virtual machines with any container runtime__: This includes virtual machines that can be provisioned on-demand and support container runtimes such as Docker, Podman, or others. This flexibility allows for scalable and dynamic resource allocation based on workload requirements. +- __Remote Kubernetes clusters__: Our solution can extend the capabilities of existing Kubernetes clusters, enabling them to offload workloads to another remote cluster. This is particularly useful for distributing workloads across multiple clusters for better resource utilization and fault tolerance. +- __Lambda-like services__: These are serverless computing services that execute code in response to events and automatically manage the underlying compute resources. By targeting these services, our solution can leverage the scalability and efficiency of serverless architectures for containerized workloads. All of this, while exposing a bare Kubernetes API kind of orchestration. + +## NOT a target + +- __Long-running services__: Our solution is not designed for services that need to run continuously for extended periods. It is optimized for tasks that have a defined start and end, rather than persistent services exposing intra-cluster communication endpoints. +- __Kubernetes Federation__: We do not aim to support Kubernetes Federation, which involves managing multiple Kubernetes clusters as a single entity. Our focus is on enabling Kubernetes pods to execute on remote resources, not on federating all kind of resources on multiple clusters. + -The project consists of two main components: +## Deployment scenarios -- __A Kubernetes Virtual Node:__ based on the [VirtualKubelet](https://virtual-kubelet.io/) technology. Translating request for a kubernetes pod execution into a remote call to the interLink API server. -- __The interLink API server:__ a modular and pluggable REST server where you can create your own Container manager plugin (called sidecars), or use the existing ones: remote docker execution on a remote host, singularity Container on a remote SLURM batch system. +### In-cluster mode -The project got inspired by the [KNoC](https://github.com/CARV-ICS-FORTH/knoc) and [Liqo](https://github.com/liqotech/liqo/tree/master) projects, enhancing that with the implemention a generic API layer b/w the virtual kubelet component and the provider logic for the container lifecycle management. +This scenario involves deploying a Virtual Kubelet along with the interLink API server and the plugin to interact with a remote API. This setup allows Kubernetes pods to be executed on remote resources while all other components sits inside the Kubernetes cluster. -## Quick-start + -Let's discover [**interLink in less than 5 minutes**](./category/usage). -You need only a machine with [Docker](https://docs.docker.com/engine/install/) engine and git CLI installed. +### Service remote edge node -## Overview -# -InterLink aims to provide an abstraction for the execution of a Kubernetes pod on any remote resource capable of managing a Container execution lifecycle. +In this scenario, the Virtual Kubelet communicates with remote services deployed on a dedicate edge node exposing authenticated interLink APIs and its associated plugin. This setup is ideal for scenarios where edge computing resources are utilized for controlled communication b/w the Kubernetes cluster and the remote resources. -The project consists of two main components: + -- __A Kubernetes Virtual Node:__ based on the [VirtualKubelet](https://virtual-kubelet.io/) technology. Translating request for a kubernetes pod execution into a remote call to the interLink API server. -- __The interLink API server:__ a modular and pluggable REST server where you can create your own Container manager plugin (called sidecars), or use the existing ones: remote docker execution on a remote host, singularity Container on a remote SLURM batch system. +### Tunneled mode -The project got inspired by the [KNoC](https://github.com/CARV-ICS-FORTH/knoc) and [Liqo](https://github.com/liqotech/liqo/tree/master) projects, enhancing that with the implemention a generic API layer b/w the virtual kubelet component and the provider logic for the container lifecycle management. +This deployment involves the Virtual Kubelet connecting to a remote interLink API server and its plugin through a secure tunnel. This setup ensures secure communication between the Kubernetes cluster and the remote resources, making it suitable for environments with strict security requirements or to host services on a multi user host like a login node. + +For more information visit the [architecture page](arch) + diff --git a/docs/docs/tutorial-admins/05-multi-user.md b/docs/docs/tutorial-admins/05-multi-user.md deleted file mode 100644 index f79da230..00000000 --- a/docs/docs/tutorial-admins/05-multi-user.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -sidebar_position: 5 ---- - -# Multi-user setup - -We can rely on unix socket communication in order to restrict the access to the interLink API components. - -## Configuration - -:::warning -TBD -::: diff --git a/docs/docs/tutorial-users/01-quick-start.md b/docs/docs/tutorial-users/01-quick-start.md deleted file mode 100644 index ab0725c9..00000000 --- a/docs/docs/tutorial-users/01-quick-start.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -sidebar_position: 1 -toc_min_heading_level: 2 -toc_max_heading_level: 5 ---- - -# Quick-start: local environment - -:::danger - -__N.B.__ in the demo the oauth2 proxy authN/Z is disabled. DO NOT USE THIS IN PRODUCTION unless you know what you are doing. - -::: - -## Requirements - -- [Docker](https://docs.docker.com/engine/install/) -- [Minikube](https://minikube.sigs.k8s.io/docs/start/) (kubernetes-version 1.27.1) -- Clone interlink repo: - -```bash -git clone https://github.com/interTwin-eu/interLink.git -``` - -## Connect a remote machine with Docker - -Move to example location: - -```bash -cd interLink/example/interlink-docker -``` - -### Setup Kubernetes cluster - -```bash -minikube start --kubernetes-version=1.27.1 -``` - -### Deploy Interlink - -#### Configure interLink - -You need to provide the interLink IP address that should be reachable from the kubernetes pods. In case of this demo setup, that address __is the address of your machine__ - -```bash -export INTERLINK_IP_ADDRESS=XXX.XX.X.XXX - -sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' vk/InterLinkConfig.yaml - -sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/InterLinkConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/InterLinkConfig.yaml - -sed -i 's/InterlinkURL:.*/InterlinkURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/sidecarConfig.yaml | sed -i 's/SidecarURL:.*/SidecarURL: "http:\/\/'$INTERLINK_IP_ADDRESS'"/g' interlink/sidecarConfig.yaml -``` - -#### Deploy virtualKubelet - -Create the `vk` namespace: - -```bash -kubectl create ns vk -``` - -Deploy the vk resources on the cluster with: - -```bash -kubectl apply -n vk -k vk/ -``` - -Check that both the pods and the node are in ready status - -```bash -kubectl get pod -n vk - -kubectl get node -``` - -#### Deploy interLink via docker compose - -```bash -cd interlink - -docker compose up -d -``` - -Check logs for both interLink APIs and SLURM sidecar: - -```bash -docker logs interlink-interlink-1 - -docker logs interlink-docker-sidecar-1 -``` - -#### Deploy a sample application - -```bash -kubectl apply -f ../test_pod.yaml -``` - -Then observe the application running and eventually succeeding via: - -```bash -kubectl get pod -n vk --watch -``` - -When finished, interrupt the watch with `Ctrl+C` and retrieve the logs with: - -```bash -kubectl logs -n vk test-pod-cfg-cowsay-dciangot -``` - -Also you can see with `docker ps` the container appearing on the `interlink-docker-sidecar-1` container with: - -```bash -docker exec interlink-docker-sidecar-1 docker ps -``` diff --git a/docs/docs/tutorial-users/_category_.json b/docs/docs/tutorial-users/_category_.json deleted file mode 100644 index b2d0f184..00000000 --- a/docs/docs/tutorial-users/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Usage", - "position": 2, - "link": { - "type": "generated-index", - "description": "5 minutes to learn the most important interLink concepts." - } -} diff --git a/docs/docusaurus.config.local.ts b/docs/docusaurus.config.local.ts new file mode 100644 index 00000000..90cd2de0 --- /dev/null +++ b/docs/docusaurus.config.local.ts @@ -0,0 +1,136 @@ +import {themes as prismThemes} from 'prism-react-renderer'; +import type {Config} from '@docusaurus/types'; +import type * as Preset from '@docusaurus/preset-classic'; +import type * as Redocusaurus from 'redocusaurus'; + +const config: Config = { + title: 'interLink', + tagline: 'Your virtual kubelet ecosystem!', + favicon: 'img/favicon.ico', + + // Set the production url of your site here + url: 'https://intertwin-eu.github.io', + // Set the // pathname under which your site is served + // For GitHub pages deployment, it is often '//' + baseUrl: '/', + + // GitHub pages deployment config. + // If you aren't using GitHub pages, you don't need these. + organizationName: 'INFN', // Usually your GitHub org/user name. + projectName: 'interLink', // Usually your repo name. + + onBrokenLinks: 'throw', + onBrokenMarkdownLinks: 'warn', + + // Even if you don't use internationalization, you can use this field to set + // useful metadata like html lang. For example, if your site is Chinese, you + // may want to replace "en" with "zh-Hans". + i18n: { + defaultLocale: 'en', + locales: ['en'], + }, + + presets: [ + [ + 'classic', + { + docs: { + sidebarPath: './sidebars.ts', + // Please change this to your repo. + // Remove this to remove the "edit this page" links. + editUrl: + 'https://github.com/interTwin-eu/interLink', + }, + blog: false, + theme: { + customCss: './src/css/custom.css', + }, + } satisfies Preset.Options, + ], + [ + 'redocusaurus', + { + // Plugin Options for loading OpenAPI files + specs: [ + // Pass it a path to a local OpenAPI YAML file + { + // Redocusaurus will automatically bundle your spec into a single file during the build + id: 'using-single-yaml', + spec: 'openapi/openapi.json', + route: '/openapi/', + }, + ], + // Theme Options for modifying how redoc renders them + theme: { + // Change with your site colors + primaryColor: '#1890ff', + }, + }, + ], + + ], + + themeConfig: { + // Replace with your project's social card + image: 'img/img/interlink_logo.png', + navbar: { + title: 'Home', + logo: { + alt: 'interLink Logo', + src: 'img/interlink_logo.png', + }, + items: [ + { + type: 'docSidebar', + sidebarId: 'tutorialSidebar', + position: 'left', + label: 'Docs', + }, + { + href: 'https://github.com/interTwin-eu/interLink', + label: 'GitHub', + position: 'right', + }, + ], + }, + footer: { + style: 'dark', + links: [ + { + title: 'Docs', + items: [ + { + label: 'Docs', + to: '/docs/intro', + }, + ], + }, + { + title: 'Community', + items: [ + { + label: 'interTwin project Slack', + href: 'https://join.slack.com/t/intertwin/shared_invite/zt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA', + } + ], + }, + { + title: 'More', + items: [ + { + label: 'GitHub', + href: 'https://github.com/interTwin-eu/interLink', + }, + ], + }, + ], + copyright: `Copyright © ${new Date().getFullYear()} Istituto Nazionale di Fisica Nucleare (INFN) - Built with Docusaurus.`, + }, + prism: { + theme: prismThemes.github, + darkTheme: prismThemes.dracula, + }, + } satisfies Preset.ThemeConfig, +}; + +export default config; diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index dd2ae595..577bf35a 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -1 +1 @@ -{"openapi": "3.1.0", "info": {"title": "interLink sidecar", "description": "openapi spec for interLink apis <-> provider sidecar communication", "version": "v0.0.0"}, "paths": {"/create": {"post": {"summary": "Create Pod", "operationId": "create_pod_create_post", "requestBody": {"content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/Pod"}, "type": "array", "title": "Pods"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"type": "string", "title": "Response Create Pod Create Post"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/delete": {"post": {"summary": "Delete Pod", "operationId": "delete_pod_delete_post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/PodRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"type": "string", "title": "Response Delete Pod Delete Post"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/status": {"get": {"summary": "Status Pod", "operationId": "status_pod_status_get", "requestBody": {"content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/PodRequest"}, "type": "array", "title": "Pods"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/PodStatus"}, "type": "array", "title": "Response Status Pod Status Get"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/getLogs": {"get": {"summary": "Get Logs", "operationId": "get_logs_getLogs_get", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/LogRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"text/plain": {"schema": {"type": "string"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}}, "components": {"schemas": {"ConfigMap": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Data"}, "binaryData": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Binarydata"}, "type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Type"}, "immutable": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Immutable"}}, "type": "object", "required": ["metadata", "data"], "title": "ConfigMap"}, "ConfigMapKeySelector": {"properties": {"key": {"type": "string", "title": "Key"}, "name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}}, "type": "object", "required": ["key"], "title": "ConfigMapKeySelector"}, "ConfigMapVolumeSource": {"properties": {"name": {"type": "string", "title": "Name"}, "items": {"anyOf": [{"items": {"$ref": "#/components/schemas/KeyToPath"}, "type": "array"}, {"type": "null"}], "title": "Items", "default": []}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}, "defaultMode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Defaultmode"}}, "type": "object", "required": ["name"], "title": "ConfigMapVolumeSource"}, "Container": {"properties": {"name": {"type": "string", "title": "Name"}, "image": {"type": "string", "title": "Image"}, "tag": {"type": "string", "title": "Tag", "default": "latest"}, "command": {"items": {"type": "string"}, "type": "array", "title": "Command"}, "args": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Args", "default": []}, "resources": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Resources", "default": {}}, "volumeMounts": {"anyOf": [{"items": {"$ref": "#/components/schemas/VolumeMount"}, "type": "array"}, {"type": "null"}], "title": "Volumemounts", "default": []}, "env": {"anyOf": [{"items": {"$ref": "#/components/schemas/EnvVar"}, "type": "array"}, {"type": "null"}], "title": "Env"}, "securityContext": {"anyOf": [{"$ref": "#/components/schemas/SecurityContext"}, {"type": "null"}]}}, "type": "object", "required": ["name", "image", "command"], "title": "Container"}, "ContainerStates": {"properties": {"terminated": {"anyOf": [{"$ref": "#/components/schemas/StateTerminated"}, {"type": "null"}]}, "running": {"anyOf": [{"$ref": "#/components/schemas/StateRunning"}, {"type": "null"}]}, "waiting": {"anyOf": [{"$ref": "#/components/schemas/StateWaiting"}, {"type": "null"}]}}, "type": "object", "title": "ContainerStates"}, "ContainerStatus": {"properties": {"name": {"type": "string", "title": "Name"}, "state": {"$ref": "#/components/schemas/ContainerStates"}}, "type": "object", "required": ["name", "state"], "title": "ContainerStatus"}, "EnvVar": {"properties": {"name": {"type": "string", "title": "Name"}, "value": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Value"}, "valueFrom": {"anyOf": [{"$ref": "#/components/schemas/EnvVarSource"}, {"type": "null"}]}}, "type": "object", "required": ["name"], "title": "EnvVar"}, "EnvVarSource": {"properties": {"configMapKeyRef": {"anyOf": [{"$ref": "#/components/schemas/ConfigMapKeySelector"}, {"type": "null"}]}, "secretKeyRef": {"anyOf": [{"$ref": "#/components/schemas/SecretKeySelector"}, {"type": "null"}]}}, "type": "object", "title": "EnvVarSource"}, "HTTPValidationError": {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, "type": "array", "title": "Detail"}}, "type": "object", "title": "HTTPValidationError"}, "KeyToPath": {"properties": {"key": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Key"}, "path": {"type": "string", "title": "Path"}, "mode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Mode"}}, "type": "object", "required": ["key", "path"], "title": "KeyToPath"}, "LogOpts": {"properties": {"Tail": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Tail"}, "LimitBytes": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Limitbytes"}, "Timestamps": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Timestamps"}, "Previous": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Previous"}, "SinceSeconds": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Sinceseconds"}, "SinceTime": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Sincetime"}}, "type": "object", "title": "LogOpts"}, "LogRequest": {"properties": {"Namespace": {"type": "string", "title": "Namespace"}, "PodUID": {"type": "string", "title": "Poduid"}, "PodName": {"type": "string", "title": "Podname"}, "ContainerName": {"type": "string", "title": "Containername"}, "Opts": {"$ref": "#/components/schemas/LogOpts"}}, "type": "object", "required": ["Namespace", "PodUID", "PodName", "ContainerName", "Opts"], "title": "LogRequest"}, "Metadata": {"properties": {"name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "namespace": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Namespace"}, "uid": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Uid"}, "annotations": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Annotations", "default": {}}, "labels": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Labels", "default": {}}, "generateName": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Generatename"}}, "type": "object", "title": "Metadata"}, "Pod": {"properties": {"pod": {"$ref": "#/components/schemas/PodRequest"}, "container": {"items": {"$ref": "#/components/schemas/Volume"}, "type": "array", "title": "Container"}}, "type": "object", "required": ["pod", "container"], "title": "Pod"}, "PodRequest": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "spec": {"$ref": "#/components/schemas/PodSpec"}}, "type": "object", "required": ["metadata", "spec"], "title": "PodRequest"}, "PodSpec": {"properties": {"containers": {"items": {"$ref": "#/components/schemas/Container"}, "type": "array", "title": "Containers"}, "initContainers": {"anyOf": [{"items": {"$ref": "#/components/schemas/Container"}, "type": "array"}, {"type": "null"}], "title": "Initcontainers"}, "volumes": {"anyOf": [{"items": {"$ref": "#/components/schemas/PodVolume"}, "type": "array"}, {"type": "null"}], "title": "Volumes"}, "preemptionPolicy": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Preemptionpolicy"}, "priorityClassName": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Priorityclassname"}, "priority": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Priority"}, "restartPolicy": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Restartpolicy"}, "terminationGracePeriodSeconds": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Terminationgraceperiodseconds"}}, "type": "object", "required": ["containers"], "title": "PodSpec"}, "PodStatus": {"properties": {"name": {"type": "string", "title": "Name"}, "UID": {"type": "string", "title": "Uid"}, "namespace": {"type": "string", "title": "Namespace"}, "containers": {"items": {"$ref": "#/components/schemas/ContainerStatus"}, "type": "array", "title": "Containers"}}, "type": "object", "required": ["name", "UID", "namespace", "containers"], "title": "PodStatus"}, "PodVolume": {"properties": {"name": {"type": "string", "title": "Name"}, "emptyDir": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Emptydir"}, "secret": {"anyOf": [{"$ref": "#/components/schemas/SecretVolumeSource"}, {"type": "null"}]}, "configMap": {"anyOf": [{"$ref": "#/components/schemas/ConfigMapVolumeSource"}, {"type": "null"}]}}, "type": "object", "required": ["name"], "title": "PodVolume"}, "Secret": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Data"}, "stringData": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Stringdata"}, "type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Type"}, "immutable": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Immutable"}}, "type": "object", "required": ["metadata"], "title": "Secret"}, "SecretKeySelector": {"properties": {"key": {"type": "string", "title": "Key"}, "name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}}, "type": "object", "required": ["key"], "title": "SecretKeySelector"}, "SecretVolumeSource": {"properties": {"secretName": {"type": "string", "title": "Secretname"}, "items": {"anyOf": [{"items": {"$ref": "#/components/schemas/KeyToPath"}, "type": "array"}, {"type": "null"}], "title": "Items", "default": []}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}, "defaultMode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Defaultmode"}}, "type": "object", "required": ["secretName"], "title": "SecretVolumeSource"}, "SecurityContext": {"properties": {"allowPrivilegeEscalation": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Allowprivilegeescalation"}, "privileged": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Privileged"}, "procMount": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Procmount"}, "readOnlyFileSystem": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Readonlyfilesystem"}, "runAsGroup": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Runasgroup"}, "runAsNonRoot": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Runasnonroot"}, "runAsUser": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Runasuser"}}, "type": "object", "title": "SecurityContext"}, "StateRunning": {"properties": {"startedAt": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Startedat"}}, "type": "object", "title": "StateRunning"}, "StateTerminated": {"properties": {"exitCode": {"type": "integer", "title": "Exitcode"}, "reason": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Reason"}}, "type": "object", "required": ["exitCode"], "title": "StateTerminated"}, "StateWaiting": {"properties": {"message": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Message"}, "reason": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Reason"}}, "type": "object", "title": "StateWaiting"}, "ValidationError": {"properties": {"loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, "msg": {"type": "string", "title": "Message"}, "type": {"type": "string", "title": "Error Type"}}, "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"}, "Volume": {"properties": {"name": {"type": "string", "title": "Name"}, "configMaps": {"anyOf": [{"items": {"$ref": "#/components/schemas/ConfigMap"}, "type": "array"}, {"type": "null"}], "title": "Configmaps"}, "secrets": {"anyOf": [{"items": {"$ref": "#/components/schemas/Secret"}, "type": "array"}, {"type": "null"}], "title": "Secrets"}, "emptyDirs": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Emptydirs"}}, "type": "object", "required": ["name"], "title": "Volume"}, "VolumeMount": {"properties": {"name": {"type": "string", "title": "Name"}, "mountPath": {"type": "string", "title": "Mountpath"}, "subPath": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Subpath"}, "readOnly": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Readonly", "default": false}, "mountPropagation": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Mountpropagation"}}, "type": "object", "required": ["name", "mountPath"], "title": "VolumeMount"}}}} \ No newline at end of file +{"openapi": "3.1.0", "info": {"title": "interLink sidecar", "description": "openapi spec for interLink apis <-> provider sidecar communication", "version": "v0.0.0"}, "paths": {"/create": {"post": {"summary": "Create Pod", "operationId": "create_pod_create_post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Pod"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/CreateStruct"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/delete": {"post": {"summary": "Delete Pod", "operationId": "delete_pod_delete_post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/PodRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"type": "string", "title": "Response Delete Pod Delete Post"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/status": {"get": {"summary": "Status Pod", "operationId": "status_pod_status_get", "requestBody": {"content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/PodRequest"}, "type": "array", "title": "Pods"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"items": {"$ref": "#/components/schemas/PodStatus"}, "type": "array", "title": "Response Status Pod Status Get"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}, "/getLogs": {"get": {"summary": "Get Logs", "operationId": "get_logs_getLogs_get", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/LogRequest"}}}, "required": true}, "responses": {"200": {"description": "Successful Response", "content": {"text/plain": {"schema": {"type": "string"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}}}}, "components": {"schemas": {"ConfigMap": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Data"}, "binary_data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Binary Data"}, "type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Type"}, "immutable": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Immutable"}}, "type": "object", "required": ["metadata", "data"], "title": "ConfigMap"}, "ConfigMapKeySelector": {"properties": {"key": {"type": "string", "title": "Key"}, "name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}}, "type": "object", "required": ["key"], "title": "ConfigMapKeySelector"}, "ConfigMapVolumeSource": {"properties": {"name": {"type": "string", "title": "Name"}, "items": {"anyOf": [{"items": {"$ref": "#/components/schemas/KeyToPath"}, "type": "array"}, {"type": "null"}], "title": "Items", "default": []}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}, "default_mode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Default Mode"}}, "type": "object", "required": ["name"], "title": "ConfigMapVolumeSource"}, "Container": {"properties": {"name": {"type": "string", "title": "Name"}, "image": {"type": "string", "title": "Image"}, "tag": {"type": "string", "title": "Tag", "default": "latest"}, "command": {"items": {"type": "string"}, "type": "array", "title": "Command"}, "args": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Args", "default": []}, "resources": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Resources", "default": {}}, "volume_mounts": {"anyOf": [{"items": {"$ref": "#/components/schemas/VolumeMount"}, "type": "array"}, {"type": "null"}], "title": "Volume Mounts", "default": []}, "env": {"anyOf": [{"items": {"$ref": "#/components/schemas/EnvVar"}, "type": "array"}, {"type": "null"}], "title": "Env"}, "security_context": {"anyOf": [{"$ref": "#/components/schemas/SecurityContext"}, {"type": "null"}]}}, "type": "object", "required": ["name", "image", "command"], "title": "Container"}, "ContainerStates": {"properties": {"terminated": {"anyOf": [{"$ref": "#/components/schemas/StateTerminated"}, {"type": "null"}]}, "running": {"anyOf": [{"$ref": "#/components/schemas/StateRunning"}, {"type": "null"}]}, "waiting": {"anyOf": [{"$ref": "#/components/schemas/StateWaiting"}, {"type": "null"}]}}, "type": "object", "title": "ContainerStates"}, "ContainerStatus": {"properties": {"name": {"type": "string", "title": "Name"}, "state": {"$ref": "#/components/schemas/ContainerStates"}}, "type": "object", "required": ["name", "state"], "title": "ContainerStatus"}, "CreateStruct": {"properties": {"pod_uid": {"type": "string", "title": "Pod Uid"}, "pod_jid": {"type": "string", "title": "Pod Jid"}}, "type": "object", "required": ["pod_uid", "pod_jid"], "title": "CreateStruct"}, "EnvVar": {"properties": {"name": {"type": "string", "title": "Name"}, "value": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Value"}, "value_from": {"anyOf": [{"$ref": "#/components/schemas/EnvVarSource"}, {"type": "null"}]}}, "type": "object", "required": ["name"], "title": "EnvVar"}, "EnvVarSource": {"properties": {"config_map_key_ref": {"anyOf": [{"$ref": "#/components/schemas/ConfigMapKeySelector"}, {"type": "null"}]}, "secret_key_ref": {"anyOf": [{"$ref": "#/components/schemas/SecretKeySelector"}, {"type": "null"}]}}, "type": "object", "title": "EnvVarSource"}, "HTTPValidationError": {"properties": {"detail": {"items": {"$ref": "#/components/schemas/ValidationError"}, "type": "array", "title": "Detail"}}, "type": "object", "title": "HTTPValidationError"}, "KeyToPath": {"properties": {"key": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Key"}, "path": {"type": "string", "title": "Path"}, "mode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Mode"}}, "type": "object", "required": ["key", "path"], "title": "KeyToPath"}, "LogOpts": {"properties": {"tail": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Tail"}, "limit_bytes": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Limit Bytes"}, "timestamps": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Timestamps"}, "previous": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Previous"}, "since_seconds": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Since Seconds"}, "since_time": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}], "title": "Since Time"}}, "type": "object", "title": "LogOpts"}, "LogRequest": {"properties": {"namespace": {"type": "string", "title": "Namespace"}, "pod_uid": {"type": "string", "title": "Pod Uid"}, "pod_name": {"type": "string", "title": "Pod Name"}, "container_name": {"type": "string", "title": "Container Name"}, "opts": {"$ref": "#/components/schemas/LogOpts"}}, "type": "object", "required": ["namespace", "pod_uid", "pod_name", "container_name", "opts"], "title": "LogRequest"}, "Metadata": {"properties": {"name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "namespace": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Namespace"}, "uid": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Uid"}, "annotations": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Annotations", "default": {}}, "labels": {"anyOf": [{"additionalProperties": {"type": "string"}, "type": "object"}, {"type": "null"}], "title": "Labels", "default": {}}, "generate_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Generate Name"}}, "type": "object", "title": "Metadata"}, "Pod": {"properties": {"pod": {"$ref": "#/components/schemas/PodRequest"}, "container": {"items": {"$ref": "#/components/schemas/Volume"}, "type": "array", "title": "Container"}}, "type": "object", "required": ["pod", "container"], "title": "Pod"}, "PodRequest": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "spec": {"$ref": "#/components/schemas/PodSpec"}}, "type": "object", "required": ["metadata", "spec"], "title": "PodRequest"}, "PodSpec": {"properties": {"containers": {"items": {"$ref": "#/components/schemas/Container"}, "type": "array", "title": "Containers"}, "init_containers": {"anyOf": [{"items": {"$ref": "#/components/schemas/Container"}, "type": "array"}, {"type": "null"}], "title": "Init Containers"}, "volumes": {"anyOf": [{"items": {"$ref": "#/components/schemas/PodVolume"}, "type": "array"}, {"type": "null"}], "title": "Volumes"}, "preemption_policy": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Preemption Policy"}, "priority_class_name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Priority Class Name"}, "priority": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Priority"}, "restart_policy": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Restart Policy"}, "termination_grace_period_seconds": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Termination Grace Period Seconds"}}, "type": "object", "required": ["containers"], "title": "PodSpec"}, "PodStatus": {"properties": {"name": {"type": "string", "title": "Name"}, "uid": {"type": "string", "title": "Uid"}, "namespace": {"type": "string", "title": "Namespace"}, "containers": {"items": {"$ref": "#/components/schemas/ContainerStatus"}, "type": "array", "title": "Containers"}}, "type": "object", "required": ["name", "uid", "namespace", "containers"], "title": "PodStatus"}, "PodVolume": {"properties": {"name": {"type": "string", "title": "Name"}, "empty_dir": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Empty Dir"}, "secret": {"anyOf": [{"$ref": "#/components/schemas/SecretVolumeSource"}, {"type": "null"}]}, "config_map": {"anyOf": [{"$ref": "#/components/schemas/ConfigMapVolumeSource"}, {"type": "null"}]}}, "type": "object", "required": ["name"], "title": "PodVolume"}, "Secret": {"properties": {"metadata": {"$ref": "#/components/schemas/Metadata"}, "data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "Data"}, "string_data": {"anyOf": [{"type": "object"}, {"type": "null"}], "title": "String Data"}, "type": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Type"}, "immutable": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Immutable"}}, "type": "object", "required": ["metadata"], "title": "Secret"}, "SecretKeySelector": {"properties": {"key": {"type": "string", "title": "Key"}, "name": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Name"}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}}, "type": "object", "required": ["key"], "title": "SecretKeySelector"}, "SecretVolumeSource": {"properties": {"secret_name": {"type": "string", "title": "Secret Name"}, "items": {"anyOf": [{"items": {"$ref": "#/components/schemas/KeyToPath"}, "type": "array"}, {"type": "null"}], "title": "Items", "default": []}, "optional": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Optional"}, "default_mode": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Default Mode"}}, "type": "object", "required": ["secret_name"], "title": "SecretVolumeSource"}, "SecurityContext": {"properties": {"allow_privilege_escalation": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Allow Privilege Escalation"}, "privileged": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Privileged"}, "proc_mount": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Proc Mount"}, "read_only_file_system": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Read Only File System"}, "run_as_group": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Run As Group"}, "run_as_non_root": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Run As Non Root"}, "run_as_user": {"anyOf": [{"type": "integer"}, {"type": "null"}], "title": "Run As User"}}, "type": "object", "title": "SecurityContext"}, "StateRunning": {"properties": {"started_at": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Started At"}}, "type": "object", "title": "StateRunning"}, "StateTerminated": {"properties": {"exit_code": {"type": "integer", "title": "Exit Code"}, "reason": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Reason"}}, "type": "object", "required": ["exit_code"], "title": "StateTerminated"}, "StateWaiting": {"properties": {"message": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Message"}, "reason": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Reason"}}, "type": "object", "title": "StateWaiting"}, "ValidationError": {"properties": {"loc": {"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, "type": "array", "title": "Location"}, "msg": {"type": "string", "title": "Message"}, "type": {"type": "string", "title": "Error Type"}}, "type": "object", "required": ["loc", "msg", "type"], "title": "ValidationError"}, "Volume": {"properties": {"name": {"type": "string", "title": "Name"}, "config_maps": {"anyOf": [{"items": {"$ref": "#/components/schemas/ConfigMap"}, "type": "array"}, {"type": "null"}], "title": "Config Maps"}, "secrets": {"anyOf": [{"items": {"$ref": "#/components/schemas/Secret"}, "type": "array"}, {"type": "null"}], "title": "Secrets"}, "empty_dirs": {"anyOf": [{"items": {"type": "string"}, "type": "array"}, {"type": "null"}], "title": "Empty Dirs"}}, "type": "object", "required": ["name"], "title": "Volume"}, "VolumeMount": {"properties": {"name": {"type": "string", "title": "Name"}, "mount_path": {"type": "string", "title": "Mount Path"}, "sub_path": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Sub Path"}, "read_only": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "title": "Read Only", "default": false}, "mount_propagation": {"anyOf": [{"type": "string"}, {"type": "null"}], "title": "Mount Propagation"}}, "type": "object", "required": ["name", "mount_path"], "title": "VolumeMount"}}}} \ No newline at end of file diff --git a/docs/src/components/AdoptersFeatures/index.tsx b/docs/src/components/AdoptersFeatures/index.tsx new file mode 100644 index 00000000..ea028f14 --- /dev/null +++ b/docs/src/components/AdoptersFeatures/index.tsx @@ -0,0 +1,93 @@ +import clsx from 'clsx'; +import Heading from '@theme/Heading'; +import styles from './styles.module.css'; + +type FeatureItem = { + title: string; + Svg: React.ComponentType>; + description: JSX.Element; +}; + +const FeatureList: FeatureItem[] = [ + { + title: 'INFN', + Svg: require('@site/static/img/INFN_logo_sito.svg').default, + description: ( + <> + ... + + ), + }, + { + title: 'EGI', + Svg: require('@site/static/img/egi-logo.svg').default, + description: ( + <> + ... + + ), + }, + { + title: 'CERN', + Svg: require('@site/static/img/cern-logo.svg').default, + description: ( + <> + + ), + }, + { + title: 'UPV', + Svg: require('@site/static/img/cern-logo.svg').default, + description: ( + <> + + ), + }, + { + title: 'NuNet', + Svg: require('@site/static/img/cern-logo.svg').default, + description: ( + <> + + ), + }, + { + title: 'AOB', + Svg: require('@site/static/img/cern-logo.svg').default, + description: ( + <> + + ), + }, +]; + +function Feature({title, Svg, description}: FeatureItem) { + return ( +
+
+
+ +
+ {title} +

{description}

+
+
+ ); +} + +export default function AdoptersFeatures(): JSX.Element { + return ( +
+
+ + Evaluators and contributors + +
+ {FeatureList.map((props, idx) => ( + + ))} +
+
+
+ ); +} diff --git a/docs/src/components/AdoptersFeatures/styles.module.css b/docs/src/components/AdoptersFeatures/styles.module.css new file mode 100644 index 00000000..d83d9dac --- /dev/null +++ b/docs/src/components/AdoptersFeatures/styles.module.css @@ -0,0 +1,11 @@ +.features { + display: flex; + align-items: center; + padding: 2rem 0; + width: 100%; +} + +.featureSvg { + height: 300px; + width: 300px; +} diff --git a/docs/src/components/HomepageVideo/index.tsx b/docs/src/components/HomepageVideo/index.tsx index 62c0caab..a786fe09 100644 --- a/docs/src/components/HomepageVideo/index.tsx +++ b/docs/src/components/HomepageVideo/index.tsx @@ -6,14 +6,18 @@ export default function HomepageVideo(): JSX.Element { return (
-
- + + Video material + + +
+ Interlink overview at Kubecon colocated CloudNative AI Day
-
- +
+ SLURM at a EuroHPC is at your hand with interLink diff --git a/docs/src/pages/index.tsx b/docs/src/pages/index.tsx index c0525bc9..54f0ef04 100644 --- a/docs/src/pages/index.tsx +++ b/docs/src/pages/index.tsx @@ -10,6 +10,7 @@ import ThemedImage from '@theme/ThemedImage'; import useBaseUrl from '@docusaurus/useBaseUrl'; import styles from './index.module.css'; +import AdoptersFeatures from '../components/AdoptersFeatures'; function HomepageHeader() { const {siteConfig} = useDocusaurusContext(); @@ -19,11 +20,15 @@ function HomepageHeader() { + + + {siteConfig.tagline}
@@ -32,7 +37,12 @@ function HomepageHeader() { to="/docs/intro"> Try it out! 🚀 +
+ Stars +
+ Slack +
); @@ -46,7 +56,6 @@ export default function Home(): JSX.Element { description="Virtual Kubelets for everyone">
-
diff --git a/docs/static/img/37a0d3_bd169579737d47318ca1b1735db6e497~mv2.webp b/docs/static/img/37a0d3_bd169579737d47318ca1b1735db6e497~mv2.webp new file mode 100644 index 00000000..40145c64 Binary files /dev/null and b/docs/static/img/37a0d3_bd169579737d47318ca1b1735db6e497~mv2.webp differ diff --git a/docs/static/img/INFN_logo_sito.png b/docs/static/img/INFN_logo_sito.png new file mode 100644 index 00000000..7f95ce46 Binary files /dev/null and b/docs/static/img/INFN_logo_sito.png differ diff --git a/docs/static/img/INFN_logo_sito.svg b/docs/static/img/INFN_logo_sito.svg new file mode 100644 index 00000000..36adf9a2 --- /dev/null +++ b/docs/static/img/INFN_logo_sito.svg @@ -0,0 +1,43 @@ + + + + + + + + + + diff --git a/docs/static/img/cern-logo.png b/docs/static/img/cern-logo.png new file mode 100644 index 00000000..b6320c3c Binary files /dev/null and b/docs/static/img/cern-logo.png differ diff --git a/docs/static/img/cern-logo.svg b/docs/static/img/cern-logo.svg new file mode 100644 index 00000000..8633e877 --- /dev/null +++ b/docs/static/img/cern-logo.svg @@ -0,0 +1,24 @@ + + + + + + + + + diff --git a/docs/static/img/egi-logo.svg b/docs/static/img/egi-logo.svg new file mode 100644 index 00000000..1dbc7b9c --- /dev/null +++ b/docs/static/img/egi-logo.svg @@ -0,0 +1 @@ + Group 79 \ No newline at end of file diff --git a/docs/static/img/logo_infn b/docs/static/img/logo_infn new file mode 100644 index 00000000..83fdb01a Binary files /dev/null and b/docs/static/img/logo_infn differ diff --git a/docs/static/img/logo_infn.jpg b/docs/static/img/logo_infn.jpg new file mode 100644 index 00000000..14dec9ea Binary files /dev/null and b/docs/static/img/logo_infn.jpg differ diff --git a/docs/static/img/logo_infn.svg b/docs/static/img/logo_infn.svg new file mode 100644 index 00000000..9d1973b0 --- /dev/null +++ b/docs/static/img/logo_infn.svg @@ -0,0 +1,25 @@ + + + + + + + + + diff --git a/docs/static/img/nunet.webp b/docs/static/img/nunet.webp new file mode 100644 index 00000000..40145c64 Binary files /dev/null and b/docs/static/img/nunet.webp differ diff --git a/docs/static/img/scenario-1_dark.svg b/docs/static/img/scenario-1_dark.svg new file mode 100644 index 00000000..a23fe5db --- /dev/null +++ b/docs/static/img/scenario-1_dark.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19WVdcdTAwMWJZ0u17/1xulu/jLfI789BvXHUwMDE4T3jAlClcdTAwMWLbt79VS0hcdTAwMDJkXHUwMDA0kiUxuVf997t3MihPKkUmWGBcXGWqy+1CUigzT0TsXHUwMDFkcVwi4vz3X0tLjyZnw+6jfy896p62W/1eZ9Q6efRcdTAwMWJ/f9xcdTAwMWSNe4NDvKTy/1x1MDAxZVx1MDAwZo5G7fyde5PJcPzv//mf1nCYTT+VtVx1MDAwN1x1MDAwN+ef7Pa7XHUwMDA33cPJXHUwMDE47/1/+O+lpf/mf+KVXoefX/lcdTAwMWPl9mjzY+tl/9v41WdcdTAwMTFcdTAwMDabvZX8o/mbLi9o1G1PWoe7/e70pVP8XmrhM+GtMdaKqL2yVy+f8WWhVSZcXJB4i9JaiXD18kmvM9nDW4xwmbDGXHUwMDA1P/3oXre3uzfhzfqQOa2CVFbkP+7qPedcdTAwMTfz7yVx9ZvxZDTY765cdTAwMGX6g1x1MDAxMa/4/+yI0Fx1MDAxNmJ6vdut9v7uaHB02Cm8Z6fbjnH6np1ev785Ocsl4yHjYT4qyd+6uG5Z+v28T+FcdTAwMGJ391x1MDAwZbvjcfKZwbDV7k3OXHUwMDFm0fRcdTAwMGV4dcO1Tr5W/zu9plHroLvGxTo86vevft077HS5XHUwMDA0j7aLN5nf38XXXa70dFx1MDAxOfXFb/6aXny328lcdTAwMWa0j8Jf/XaqbtqF8m/XXHUwMDA3h7nqRVx1MDAxN72QXCJOr2j8XHUwMDA02jbJXHUwMDA17rT64+700fOqnpY1saiNibJNuqeTq1sq6OrXXHUwMDE3Zufj20396s1cdTAwMWZ7W8OTbW93O38+unrfX1x1MDAxN3+bPrqjYad1fj3SK2dcdTAwMWTuxoYwXYZ+73C//Fxc+4P2/vRcdTAwMTb+VXhcXCXLqb6aXHUwMDE5y0lu5sJonM2sV0E7ZZ0xKpSNxtdcdTAwMTiNXG5cdTAwMTFGXHUwMDEzo3BC2GCjmTVcdTAwMWQr7tNU9lrtvaNR92cwXHUwMDE2WW0sydsvrFx1MDAwMutcdTAwMTN9XGLGVlx1MDAxOIbyap5hSG9s8EZHf1x1MDAwYtNIruP79XeqjlRDPICnT54/XVp/++Tp0tv1pXdPN9++f7f6dOk/h1x1MDAxYu/eflh78vRdYYFcdTAwMDeHk83eN96RXHUwMDEyyW+ftVx1MDAwZXp9rodNxK/0e7t8NI/auJfu6FHx+Ux6gKarN0xcdTAwMDbD6attSGz1XHUwMDBlu6O1JoA0XHUwMDE49XZ7h63+XHUwMDFm197P7N20jiaDd93x+f1MRkfd4kPsvrg0XHUwMDFimSlbafPXXHUwMDAwYUEp7IyqXFwqhZIqOlx1MDAxN5yLVUrZetRAVc7djtNcdTAwMWJff4/t10+fd549V09aoydbrf5DQbBZV8NcdTAwMWK03mRKO22iMULA+yX+LsKZ4SVtZ9xd2U3JLv+53k21bCfs7Ezfc+kybTBcdTAwMTm+XHUwMDFiLvOcS0zv5MplXHUwMDFhpTLpI6iIifnP1K1eOFx1MDAwNKmMcV44q2/nhm6KzjO4eefQ1kjN9axHvPJ9VkvrXHUwMDFjnmCVmm83V/Ozky9n4fnqyccj8eXrwejV23cv/vzj4aBPtaL7IDItnHMyVii6XHUwMDA0z81cdTAwMDJ03FxiZaP0/k5cdTAwMTRdm5CBjmupKlxitTQ6s9piidS5jvuyilx1MDAwNyGM0bJw6X9TXHJPeVlBue1cXGB31kBcdTAwMDdC0FW63W6u299cdTAwMGVf/bn/XndG708/nJ6tvX+/uv3m7KHrtlVcdTAwMGWqXHUwMDEzXHUwMDEx5lx1MDAwNa1UXGIpaY1aZNFqXHUwMDExrPRU/ztRbe9xXHQ+XHUwMDA24aSJYL5xVsG1LSu0Q2iDy1Xx+1x1MDAxNTp5obHmeq29Nz7eirZcdTAwMTVJWEh+O4eE8Vx1MDAwMcpsaX3Q6S6d9CZ7S93T4WDc7SyNu6PjXru71JosTfa6S93ObndpsLP0YmN1qd0/XHUwMDFhJ4RtYWwuucUydfvuy7xcdTAwMTdOXHUwMDA3Zz1cdTAwMTfsonfgXHUwMDAzskj0p1x1MDAwZaHT3CFMTv54XHUwMDFkn339+Pq53Fx1MDAxZO+vXHUwMDE5uXvwaeuhO4QgdFx1MDAxNoFcdTAwMTTzwC7azPP52LtcdTAwMDQ7IbOYo8NcXFZn8Y6o5Fx1MDAxNakruKZzXHUwMDA34Vx1MDAxMY+bIOKP4nRXn7lRLmR39GJt/YWfXGLZds86wk6ef42PXHUwMDBiuZDfqsWef1j8vjFcdTAwMWO+23+zubJ3XCJWXHUwMDFlt7efb5x9Sb/l8vtbo9HgpKncOFx1MDAxY+ndk9PVXHUwMDBmXHUwMDFmV/Xvbbf1QlxmnjxuJvfib/eUu6l+elP/Nzd3XHUwMDAz8M+8hupbq51SPqZKXHUwMDBmrc+UR1x1MDAxOFxieEqU/lJhpVGZZuYmXG7rnSwkXHUwMDExpklPe43hzVx1MDAxOE/0sq3+XHUwMDE5mZvOh+apm2BUhF8yU6OeevRcdTAwMDJcdTAwMTKXXHUwMDFjelxiXHUwMDExzFx1MDAxN1x1MDAxZa2B577vxM2H3mhy1OovvTra5lx1MDAxNd1Lnuag1+lcdTAwMTRRsZSqqYGtMt7PvYNF4TjfXVx1MDAwNeEhinkrLoWSXCJE2GqVxnWbQ/hK7P/+bfJ4f//FRm/txfrxpndhXHUwMDFlp2+PXHUwMDA24/HyXmvS3vvxMO7g0aQwTntrlHCi5NGEXGaZkyD11oiofMGgbuOKtHLtrp3FcalcIoJWRFx1MDAxNyYo6YHHXHUwMDE1oauyWVx1MDAxNC5GJ61cZipcdTAwMTSw/jI7XHUwMDAzlu1EwJ1UupopNo9emr2dz19enk2Oevth6+zw9+HXzVx1MDAwMkw9ev9p7UtP7Dw9XHUwMDFjPH+3tr4+7i6//fb+XHUwMDEyrO4hZPguhzGetEaTx9Di3uFu+SPdw86cV/qt8WR1cHDQm+AyNlx1MDAwNr3DyYxnpdxcdTAwMTXC9l63NXPTkDz3tSHFpWsw/dvSVFx1MDAxZvP/uPr7//62gHcv61xm4ZcyJmqD/1x1MDAwNyq75PNcdTAwMTlcdTAwMDA9XHUwMDA246VcbkpHo0OtQFx1MDAxMzKrYjTRwcLhPXxRnlJQY6H4TS5cbm3cfYtcdTAwMDPbRaCOd9tcdTAwMTiksMIm4kxcdTAwMDaeXHUwMDBiXHUwMDAz0SriO52uXHUwMDE3XHUwMDA331x1MDAwMI5cdTAwMGaKLH1ApGNcdTAwMTJxIVPGwVx1MDAxZSFP4fJkrThn4MKtNNb5XHUwMDEwrDChKFx1MDAwZcG9U9xcdTAwMDdWIEfG+vqlWLw4rYP3XHUwMDExXGZcdTAwMDAuxiU3XHUwMDFiRCZcZlx1MDAxNMlcdTAwMDNMPJhdvTSdXHRcdTAwMWRcdTAwMTD9RKVMiMm1weXil1pcdTAwMTmEJVoqUy9MZVx1MDAxNjdiLdxekMH6kjSwSWu9tPiuqFW9UchMK0REykXtXHUwMDExwenkViXUzoPaXG6pJC7fqfqbXbhA6JYmf5POc+cg2pBcbrSZgMHihqHF1Pd6gSazXHUwMDAxXHUwMDE3J5jh10basrxcdTAwMTRb6uThXHUwMDAzWFRcdTAwMWI0kdNcblxcwzXy6rVl0eJcdTAwMTCFwKE556BcdTAwMGXBXHUwMDE1djKunlx1MDAxZb5cdTAwMDSaLGg8oVb/tMrghISDR/EyKjGzui5Kg6v28KPC1i5cdTAwMDZMw2BprcTHYFx1MDAwNr5sXHUwMDFiXHUwMDAxLMTSaytowP1cbrPgQlbCacDcYOW+5Fx1MDAwMoyWxiltXHUwMDAznLWOXHKkXHRcdTAwMDRcdTAwMTNG5Kym5Fx1MDAwM+ifYP5cdTAwMGWsXHRr5Vxc7bXZPN1cIlx1MDAwM/BcZmtcdTAwMDAjKvliXHUwMDE5oT7AMVx1MDAxOVx1MDAwMCbR3rc4qFxi9EPDg1x1MDAwMk9h5WXkQbSrqFwi8NRYqFpvbKDCXHUwMDBlrlx1MDAxOF+udVx1MDAxNCpcdTAwMTWHIDpKXHUwMDBidbNcdTAwMGXr4VxcrUWYTGJcdTAwMTGwfLhcdTAwMTdn4YJmaFx1MDAwMGJ6fFx1MDAxNcVhwWrtf9Hyllx0L3lcdTAwMWRcdTAwMTS8XHUwMDFl1EHBVyVcIpcl31x1MDAxMDT+XHUwMDA1XG6A5lx1MDAxYd1dlrX3PY8t/av4/zeNp2jrc1x1MDAwMyqm51x1MDAxZP4sxNDTgGqneUC1Nd58az6vrHftk6O1/uNcdTAwMGZxS/rfXHUwMDFm9j73eSil6T5KUcxFXY/JXHUwMDFjIFx1MDAxMvZOzS1cdTAwMTSz3S6rU5FcdTAwMGaNsLOgPVNcdTAwMWZcdTAwMGV+ulwiXHUwMDFkXHUwMDFhNbfBNXTXRi1cdTAwMTDQzURRcJTAXHUwMDA17avzoVcq9CtI+vtcdTAwMDRJMoPTXHUwMDAxXHUwMDEwwJFcdTAwMDZQZkRcYqk3i9abIGn2XHUwMDAwXHUwMDBiqWvBYFx1MDAxOYzAOiHhXHUwMDFkoWdw4TZhXGZUQVx1MDAwMWORWmisYmxcdTAwMTDX1Fx09Fx1MDAwZWCF+N/qgODGfv9cdTAwMTXeQiBcdTAwMThcdTAwMTfwSuJcdTAwMTNMXHUwMDAx61x1MDAwNO1cdTAwMWT4MXxi8Fx1MDAwMp6bYUlcdTAwMDN5XHUwMDAyolx1MDAxNFx1MDAxOD94XHUwMDFmiJxLXHUwMDE5q8wrajTJNCSbRlx1MDAwMkNcdTAwMDTiwzVcdTAwMTlng1x0psTiMjBPXHJcYqL940Lr4yWVRTxBxtX0dTYkSiPxXGKB9VwiXHUwMDA07lx1MDAwZeFp1EPgouX5LHAphGXhXHUwMDE1o+FEXHUwMDFl4lx1MDAwMU39g0gv4G/rgzA8XCJP1+pcdTAwMTGIg1x1MDAxOUWVXqAjQIPl4bvAXHUwMDFlXHUwMDAx0/VcdTAwMDJ9hlxiXHUwMDEwi8FlXHUwMDE03qlEXHUwMDFlo+tcdTAwMThBwLSJXHUwMDE2MUl9tK5kXHUwMDA2/uA9XHUwMDAyVFx1MDAxMC3Eglx0Z5JcdTAwMTm03XtcdTAwMTVcZt5cdTAwMDAu3MDoXHUwMDEwlYCGaIu3e1x1MDAxYUFyw+B0oNZcdTAwMGXvXHUwMDAwXlx1MDAwMHN0XHUwMDAzXHUwMDE1dJnBWihcdTAwMDYz/CMxXHUwMDExXHUwMDA1Qlxuhlxi/FxmXnKd6+PYhctcdTAwMGIsvFx1MDAwNVx1MDAwYtNcdTAwMDFLaZxXJYHeQo+MXHUwMDE2gbZZy/9cdTAwMTlcdTAwMTZD/Vx1MDAxOIGARHtn07hdIZ7weLjgXG6K+df68GRZI9ihXHUwMDE5QFx1MDAwMSNcdTAwMGItTEljXHUwMDAwh/BpzoKD6OjrNVx1MDAxMIxcdTAwMDQqK5RcdTAwMTDeSlx1MDAxMFx1MDAxY12Sx11cXFx1MDAwNlWRil+vMItcdTAwMTaHsNdaPCPpsFx1MDAxY16VomKPICBcIjRAXGKDh1hcdTAwMWZALWteXHUwMDAxQi1cdTAwMGJcdTAwMWZcdTAwMDJfXHUwMDE3y5dcdTAwMDcwXG4+391z+K5cdTAwMDZZj1x1MDAwMFx1MDAwYlx1MDAwMCaypFx1MDAwYkGh0eXVXHJcdTAwMGXPgVx1MDAxMZGF0tSL88yBw6CYQUFcYjVcdTAwMTPycE9cdTAwMGL0XHUwMDBlT1x1MDAxNo6tweU5hCFGXHUwMDAwyGCeuKdyalx1MDAxMXqEXGLFXHUwMDA3xoCx3t0vWlx1MDAxY8mwxa1cInSH70gzRvAtsFx1MDAxOVx1MDAwZqdcYu+CXHUwMDE1aSBOZDZcdTAwMDbGqnB+MIPUcl1cdTAwMTbPk6g+eOhMg4xcdTAwMWLkXHUwMDAxOCyzr1hcdHysnPo0kWVyXHUwMDExYahSoFx1MDAwZvW+JWZcYlxmLPmE4Fx1MDAxNqNIrzBm+DWWNk/XMDtz/1x1MDAwMpnlUVA9J8gpbOr8sFxcUWvCXHUwMDBiWFtskK7gXHUwMDAyXHUwMDAzi5jgZOZcdTAwMWQ+uCjOXGLWbYDZMJBcdTAwMDVcdTAwMTA3IFjwXHUwMDA2XHUwMDAy/iN4R1x1MDAwNyOmNWa5PM90XHUwMDAxeYtcdTAwMTZcdTAwMDZg1Uz/XHUwMDAy7ldJXHUwMDBi8lx1MDAxN1M2ZDXoYcRvLf7R4Fx0XHJu12Z4OFx1MDAwZYaOlVx1MDAwMFx1MDAxM3AleVx1MDAxMTZcdTAwMGLKQN9cIppkVFx1MDAxNyxcdTAwMGVcdTAwMTZcdTAwMDDXXHUwMDA28IdnM6VtXHUwMDA2azNcdTAwMTghOCvcMnOu9apiJK6OVFx1MDAwN1x1MDAwNo/lc2VxcHjMVbs8XdZgT8WAXGKBOVx1MDAxYlx1MDAwYl31pWSKdbg6UjXvoJyhyZ5cbu5cdTAwMDeoXHUwMDFm4JJcdTAwMWQuJHUtXHUwMDEwpyX4XHUwMDE5XoHHXG66kThW74NmXHUwMDAwybmd8LDEuYycXHUwMDFkWlx1MDAxZsBcdTAwMTWlL4tjvs1FwFx1MDAxMLyZaGC2sCTgoLdRkZKE0uV5KJ6B23agzVKqXHUwMDA2muKoqlxuulwisHygLElUJDJ8XHUwMDEzliiA/UaQrnovXHUwMDAweVx1MDAxZWZcdTAwMGJcYlx1MDAwN9JAXHUwMDFk0tQ2XHUwMDA0srBcdTAwMDaMXHUwMDE0QVx1MDAwMpG8wf2S5DJcdTAwMGU1XHUwMDAwXHJoWeL1XHUwMDFjXHUwMDE4NbyeZ1JcdTAwMDQoXHUwMDA0PGiiy4jxwPVcZjwv4CtccjsgkLG0cPhcIpZcdTAwMDE1iKWNJlx1MDAwYkLU44VcdTAwMDNcdPdcdTAwMGKQXHUwMDA3XHUwMDFmwC0xgII3KVx1MDAwYoI8wy1cdTAwMWNcdTAwMDV25FxyXHUwMDAy4HtcdTAwMTcnspzCsv1cdTAwMDJxbeluXHUwMDA1Q1x1MDAwZTiqXHUwMDE4sMxNnJ6Ak1x1MDAwNCZoR0qaXFyajYigmbBG+CVcdTAwMTHVNdg1Q8hcYj5cdTAwMDBcdTAwMDJcdDDzQpXFXHUwMDAx7ExeiFxiuFxiXHLCP5B5UEfPkmrNqNGV5OGJ4luAXHUwMDAyXGLLRYNgY8HiRO7iibdeXHUwMDAwMUp6XGarsZbFUiyE87ZBbOV5eXmRh45w9unKepF5QJ1UwYI/ilx1MDAwNslcdTAwMDdcdTAwMDU+YjVA32NBpDKJ3XpcdTAwMGI36jTjU4svbWBcdTAwMTWIJnBcdTAwMTeOXHUwMDA1YFDmKEqbNey3xCV7vlx1MDAwMupRL47RO/yQ5Vx1MDAwNibgq7yRJFx1MDAwMsBWMbp3scHu5eLFXHR4SFx1MDAxMFx1MDAxY2mZXHUwMDAz84muXHUwMDA0OD1EuVx1MDAwNtRcdTAwMWOeVtXfLENcdTAwMWRcdTAwMDNSzTRB1Mqb9PJcdTAwMWM3wllcdTAwMDCPP41s8PCUzFg/XHSyXHUwMDAy9Vx1MDAwYsGn5Cdwb1xuZMZr8DZymia6olxms302kq1cdTAwMTTqtS/kRTB4b1x1MDAxNPOAXGLXfoQ8aLCW2nBcdTAwMTfTWpXoXlx1MDAxNFx1MDAxOdiPZ1Ypgus1SUXy+nSQhmlcdTAwMDeWliS2XHUwMDExsbzkRSCQQlx1MDAwMklcdTAwMWLllXInXG6mrLHKJt3bi7h6qFx0iFx1MDAxZb1cdTAwMGY8c736xYx7z560XHUwMDA0qJbmqWY2KJqIY4aFgTO9m4ulq0NY6OG+glx1MDAwMjOI9Xeb21x1MDAxYaNPXHUwMDA27oJAU5JcdTAwMDdcXIJEZmRcdTAwMWLQKWYpoFx1MDAwYlx1MDAwZTZcdTAwMWJcdTAwMDTUXCItoYl5XHUwMDFhN8KLsu3K1u/EU1x1MDAxZfBcdTAwMDU+yFx1MDAxOWAu7F6XdFx1MDAwNTbBWDB6+opcdTAwMDZ3u2h5LlN0ucZcdTAwMGKQWyd92TaEXHUwMDA3k4FcdTAwMTbD3atcdTAwMDZJXGZcdE9OXHUwMDFkIWowV5tcdTAwMTb5XHUwMDA0wJBcdTAwMDJblVwi4iFcdTAwMDZX71x1MDAwYiSiUtyqNlx1MDAwZTQ7MFxylci7LMyJpCyhifJRXVx1MDAxOaRcdTAwMDH2aXBJlFx1MDAxYvJdXHUwMDA2p6nqtO56WJPc2Vx1MDAwNlW2bFx1MDAxYsLnSp5cdTAwMWVsjz5cdTAwMDVxKcHXNzC1XHUwMDA1y3NMSktunFBt4fNt+fHBXTPwXHUwMDBiXGJcdTAwMWbqWUaeZFx1MDAwMYYz8lR4RjJNOrB8XHUwMDAzIYOialx1MDAwMvbqQz9ET1x1MDAxZSvBbDlUzOlyMVxiy9jA0jRcdTAwMTbZ16dH8fAsXCJccnwxnLJQprS4N4RJaopDXHUwMDFjppiAXHUwMDA0ekRVVlx1MDAxNWOMY4FcdTAwMTmgw9dubyxanGHyXHUwMDEx/lODqMNLXHUwMDA1U3p0oOJcdTAwMWUqXGZQbrA3RC9cdTAwMDD8jsztsYYjlJ1cdTAwMDC+h/ZcdTAwMDK7cfXkjD5cbrzWXCLw057J7bKPMtGwXlx1MDAwMlx1MDAwMOVcdTAwMDFpXHLEcdGY+IFRXHUwMDFhacvwKIOh9eO1XHUwMDE461x1MDAxZtxipVx1MDAwMXxcdTAwMWO3tY2E3TqbljNFNo84zVx1MDAxYYKgPFx1MDAxNrdWXHUwMDFjaD9UXHUwMDFlXHUwMDExNlx1MDAxNlx1MDAxNYgrS9JcdTAwMDBuiFmA2lx1MDAxYV6+1rrAK/LNNISh+X6LKmNZhMIh9NCM3uqrS0lTVFx1MDAxZVx1MDAxZDpgs4oylMR5WF6+XHUwMDFiyLRzrZaQRSFYJ9GEuzUlXHUwMDFhwMZseECGRlx1MDAwZcpUX+CHi0NcdTAwMDTKWEVbLIYpaXBcdTAwMDDdRpjHfLipj1x1MDAxZMlojTReXHUwMDFhOFx1MDAxMbi5Mp81XGLJoOFSMni7X2HcpoieYU2e6k75XHTwXHUwMDE1hC4wxy1Y3lkrjpFcdTAwMDWAmISfg1x1MDAxMtIkbeB2XHUwMDE4XHUwMDFjNL5PcMur3lJcdTAwMDOor2JcdTAwMGVcdTAwMTbmRc+d+jjGPUBcZpZ0ws3VYjUsXHUwMDE1JJrzdlx1MDAxMLNyQzNxI1x1MDAxZbQ9ssVcdTAwMDG0k6n9elxy0YxcdGFWMGwymlx1MDAwNFlcdTAwMTGAqnxcdTAwMGZWM0Cuz2shOsbTxSWBlUCnXFxcdTAwMWG9IzpcdTAwMDZnUYhorVx1MDAwMWmvXHUwMDBmj9mNjfU0XGbg8bhcdTAwMTNcdTAwMTVB6K7PKalkVlXWXlx1MDAxYlx1MDAxM1x1MDAwYoF8XHTxO1widFFOLIC4gkDAJ5Fk1z42ZlG4v8Z6XHUwMDEzRIPBl6Q5XHUwMDEzXHUwMDFjyDzTbVxySklcdTAwMTcsjXv4IP6KSFx1MDAwM36vStL4KCXoXHUwMDFlaFx1MDAxNGhL7YMzXCLzTHv6PPflSntIXHUwMDBlludcdTAwMThcdTAwMTRcdTAwMTg+WPilWnFcdTAwMWFYXHUwMDAyMGXRSn5H5UxbXHUwMDFlQ1xi06CgXHUwMDFjothcdTAwMTJvmJdcdTAwMDVVU2VRltX0MpDJhXpAZYZcdTAwMTK4ZEHBqcNcIt2W/uHioO8we5BMai+QuiTNSXxcciRxmll9MMx0e1RMfXtcdTAwMGZklzPpZ1xcXHUwMDE20FlbuM36XG5cdTAwMDEm21xyq0SxXHUwMDEyivmYksLhXHUwMDBiQO1cdTAwMDRzdj7UX1x1MDAxYp2IZ/V40PSLqfrCl+Y0XHUwMDE2ZFx1MDAxME+j3ofk14bbXHUwMDAwkUecVEq0Q1x1MDAxYeuScLfUOF2/XHUwMDAzhedcdTAwMDb+o1x1MDAxMaXDVp3zvrzHXHUwMDAzfXQs32a6olx1MDAxZVDzLVx1MDAxZe7+R9hcdTAwMTVrZErSXHUwMDE0Y1x0XHUwMDE2doGh14c1xmTchKSSsOrLzew/XHQyaSCQZlx1MDAxN9w9S5N51YnO/Vx1MDAxMnxIqlwibFxmgJOHXHUwMDFlWptvNjRwcexxhatg9Mbdl5I8KXTePkSWXHUwMDE3XHUwMDFhgIPNPHwsXHUwMDFjUmCnS5zZp1x1MDAwNKfQeek3qGN9zVx1MDAwN3dlWUAnc8ftYiyJc3ntXHUwMDAzPD58fT2JW7g0PmjBXHUwMDFhXHJthU90jlx1MDAxYtCKSTXcK/NcdTAwMTH19WR5sVxmLJX7XHRCXHSrk5XgXHUwMDA2Ob9cYsCrgFx1MDAxZvWNaZy0XHUwMDE5rYeriIKJ9lx1MDAwNPN56dxcIlx1MDAxNFx1MDAwZUSFpVC1V6eAdqBcdNws9i5vjfstfVx1MDAxNdG9xoVcdTAwMDO+XCIoTj3Bidn5vEKEo6DTdqaVjFxcOzCjXG6X0qjDZZHiNGNITeT0hOJy8VxiXHUwMDFlXHUwMDE4O408XGY6NOivXCIpXHUwMDExrMN2mlmDtHeJpS2ej1x1MDAxNjhcdTAwMTFcdTAwMTD+Nrg4lzEkXHUwMDBm3OFcdTAwMTVsO0jEKXJcZsvxnWTc9b5Oe3a5cT+HwUu61a7yXCJWgUhcZmE0iXD9rS5WWsicol6x6NaKlGkya1x1MDAxY7xkpaFlJU99clx1MDAwZeK48ShVIFx1MDAxOEtVrveyzGmCTSmjXHUwMDE4YDS4V1x0lII+IVx1MDAxZVx1MDAwMekvV6MxQonMXHUwMDA0XHUwMDA1w4RQk2XlPqZEKFx1MDAxN50sSzMsMlQycphkXHUwMDAz60e8kT+XPE1cdTAwMDbMNiVxmuk1w399k1BcdTAwMWacP7K+UMKOYEVOllx1MDAxZZ2g+lx1MDAxYVx1MDAwNixcYlxy6q9cdTAwMGVcdTAwMTGEXHUwMDA36GBt8b9cdTAwMThT0OFKYJmwXHUwMDE2LDxiNfO9y1x1MDAwM1x1MDAwYmTZXHUwMDEwwMLAMl2YqeRj1pO7zlx1MDAxMm6iPmud15HCXFzhilx1MDAxMaSrUlx1MDAwNVxu60ixTkzZMeDWXHLSXHUwMDA3IE9s0Vx1MDAxMsRcdTAwMDOYhpwpdDVwXHUwMDAxwWp2nNbX4bKul1x1MDAxM3vgUaCnLHcsOSju+ELJZVxmLPSrXd6Fy2MtXHUwMDFmjImZZNZcdTAwMTOm4Vx0l4Pmj5dcdTAwMTDDwzHWiZMgNYicXHUwMDEx0ll6cZlmwWBcdTAwMWKKXHRwXHUwMDEzQ74xXa8tLFx1MDAxY3CAXHUwMDFlWFx1MDAxNFx1MDAxOVKaUpdAWrhcdTAwMDHL2lx1MDAwNYJ6PWtXXHUwMDE5XHUwMDAyXHUwMDFk4cGIXHUwMDEx3Vx1MDAwNcTRqSdApFx1MDAwNlx1MDAxZcBdeDZcdTAwMTVZWf/0XHUwMDAwV5aNylBcdTAwMTZcXKBIU+p5nT88LHxcdTAwMGJbXHUwMDAwXHUwMDFhXHUwMDE0XHUwMDFks1x1MDAxMYFxk2HdXHUwMDFkQq40imUjXHUwMDAyXFwse6xBK9naVy8v7+zlJrtjeiStXGLKuO1ic+uIXHJoQFx1MDAwNl9nmVx1MDAxOYHae5fuRuDJ4ZusXHUwMDAy9SNcdTAwMWZz9cVANeLob4Bzno1cdTAwMTdYkPtcdTAwMTYnXHUwMDE5XFwyUctcdTAwMWFqhP9pco1cdTAwMDWV3rOMXHRcdTAwMTRcdTAwMTTkvtHmXHUwMDBiq41xXHUwMDE5cEFYYDz1VKBgaSFdospcdTAwMDfYqfrtybtpN1x1MDAxNNPmnHK7oeYuk9aictzobvNuQ7H25WXrRfw2fNs7XG6Ptzs93Vx1MDAxN29+ivEtPmY5yOm84l+UZ4kzQuSANs9u1WC/r+dw7vhcdTAwMTZcdTAwMDTcLuQ7XHUwMDEziFx1MDAwMlx1MDAwYmOvptNb8Fx1MDAwZfg4XmTgSLiZuaOsXHUwMDFhXHUwMDE33Fx1MDAwN6+e03ilSr/aXHUwMDBlXHUwMDFmcNvhMmvG4LjzST6sIU0zz5d771xiVZhPdlxyulx1MDAwNCDQS3Juxao1IYy7RmCTMrSFXHUwMDBiRJCDXHUwMDAwh/28oGYstk3kldW+Vlx1MDAxYzNcdTAwMTaMpqzL9zXSjKC0eTFcdTAwMGLxzIo8bVxcK2+eafKHJW9cdTAwMWXLoMCMPGdPNCiTW7C4fFx1MDAwYofhXHUwMDEym1x1MDAwYpjOTSFSXCLgZl2g40LVJ9zZ4+hB5FgllVx1MDAwZnws9bGyRolcdTAwMTM72Y1cdTAwMDVQadSVL6jRjEhcdTAwMDNnX4RSqJFpXrxnUz6bc4ztLtdnfu5cbiXnXHUwMDBlKmVcdTAwMGJcdTAwMWN7XHUwMDE2K1x1MDAwN1x1MDAxN+/dYMjZYF1cdTAwMGZbQ386Ontcbu88Wn45ls9+XG6UXGZcdTAwMGVBXGJcdTAwMTSD8ZZcZq485FxmKFx0xlx1MDAwNFx1MDAwMo5w293RiDPH4fJcYvhYXG5cdTAwMDRmVXXeXHKtXHUwMDBiXHUwMDEwI1x1MDAxZLPG4FxcaqY7P3L2XHUwMDEwi/Z+oeRPi5I3e3fIXHUwMDEw5VwiLGRxPVvW07a7XGZOiYO5XHUwMDEwLmkjZH2VxHw15I9n8b3n+lx1MDAwNSvpk+vEMbaXrKtcdTAwMTA2L9FLxzRcdTAwMTGuhEMsbySTSPVcdTAwMDHYgsWpjL2wllxy3uxcdTAwMWMyaYp91tpqXHUwMDFmXHUwMDFln1x1MDAxZDtcdTAwMTPyYVtwqL50fVx1MDAxY/XD/Vx1MDAwZVx1MDAxN4I2XHKAXHUwMDAwj1txdJmWlmO9VLlFXVwijFx1MDAxN8wxcExCLVx1MDAxN7krXFxx5V9PXHUwMDBmNVx1MDAwMdJyP2/qy1xuwNJrXHUwMDBlLI/N6LNeW18zn5+dbmnn9ejN+NtPXHUwMDAxLKx/gHFyXHUwMDAziXOup375/PNcdTAwMGXmy1Q1O4FNYazXYqHF5p1cdTAwMWbWcC5QjD5WXGbGZ35cdTAwMWFuJJKZkazOjME2UjDh5uuGZ/5CloeLLKDTLJbVLs/tcVx1MDAwN7z4cWgqXGKQziekgqnWbyqUxKl0XHUwMDAzJWasi7OOlZeCRTb3LVx1MDAwZeTbh3ycUmRy15XzkoHdPNw0g8I06NCOmWPNv2c9l4JcdTAwMTmlaVOdXHUwMDFmRKiF87h812B6RlwiLpb3siCOL+JZXHUwMDA0mC1QoE7cMpP8Nlx1MDAxZsfBYZxGp002uXlcdTAwMWImsFlLXHUwMDE5mzR63oFAXHUwMDAz5DPRSIdbiyFtXHUwMDEzXHRcdTAwMWPVodhccqjyXHUwMDEyqlrtXHUwMDEzeY9cdTAwMTJiZsOJVqzoTOd7LF+8wbm8YVZAXHUwMDEzYn3O844gcm6CUlFcdTAwMDGMqVx1MDAwNMgvzVx1MDAwMXLydu3levfV+vHkZPLqcK+18+zDmftcdTAwMTlcdTAwMDDSXHUwMDBibp9rTpCVQU5cdTAwMWLFp/jInLbmiUWF+HXR6GiY01HCkNtOK49SdOShfWBg0HQlXFxcdTAwMDGoL/KTgvlcdTAwMDTj5S98/Hnx8Wbvhj9zOq9Gy4upfcr2ecpcdTAwMTbQhTWKkUXgXHJGUHJ2XG7HXHUwMDA3s6VKuVx1MDAxOXebKmCtu41MWSDky/fSvEvr3Vx0MDJcdTAwMWaOzFlfcEL1ua6SPFtcdTAwMDYsa9nEkHf43fjivk9cdTAwMTj3XHUwMDE4g1x1MDAwMK5Idlx1MDAxN0mRJpUlXHUwMDE4Nlx1MDAwN1x1MDAxZnNOXHUwMDEw9/DrxVx0zsjOx65cbqO1LfVcdTAwMWRg1Vx1MDAwMzdcXC1Zsoj1IzYz3CdcdTAwMGaYUd57mU9cZvktfVlz5lx1MDAxMVusjGLH1Fx1MDAwM9xGY9+wXG5zYGr/XHUwMDA2QzvXzPH65+fi+WhjXHUwMDEy14fH/YP+45WfXHUwMDAxplxcXHUwMDEwXHUwMDE5XHUwMDAyuHxcXFxidGMmQeiBU6BDjlx1MDAwN8DBgq85zeg7kCqyPYKDvKFKpEhcdTAwMTVAda6dzFpcdTAwMDQ2XHUwMDAziFmcUkFcdTAwMWFdPNjvXHUwMDE3TF1cbvtZYMqwKDxyXHUwMDFin4PrdFqvsizZXHUwMDEzQ1x1MDAxZOVEvnxSY3e5vnjNgyyrXGJcdTAwMWHEMnhR6qRhOVx1MDAxNTxcdTAwMTS8q7JcdTAwMWOeVJ9cdGNPv1x1MDAwZk5ohFx1MDAwZTzGqJRFZPmB5NQnlvHUQ6PKpOb2ST6rivtHXHQ2lnW+wbZS5lx1MDAwMydJxrzyl0nD6y6vyWRK2CNcdTAwMWJgOIuMvW/+mqf3g2Yyi/lnXHUwMDBmXHUwMDA3YFx1MDAxZNM+Vc6939y5byy/PJVH72T/zejZq4PuR1x1MDAxMJXD1k/h3KPJeD6E9uyJ1dPW1suT6kTGI1mDXG4sXHUwMDE1MndcdTAwMTSGuIxccuqsuYdcdTAwMWWDxVSFITxiXHUwMDEzYWye7VxyplwiXG7JT4VcdTAwMGZ6zvGsv9z7z+Deb/huTmDg1DbojVRsoUl9T9AsRDRcdTAwMWNcdTAwMTLrfGxcdTAwMDJcdTAwMDWSg1vzIXrkubI0+T9jgoZeU3BXvkGP5mKl8aRcdTAwMTA4WDZcdTAwMWJC/0tz7/LjX+GBTeR36fr9XHUwMDFhwVx1MDAxMYScXHUwMDExXHUwMDFh87NcdTAwMDLS3i+8XG4z01x1MDAxY1x1MDAwN1x1MDAxOVx1MDAxNXG2Vlx1MDAxZUCUK0CoYu2IK6dcYpnPwzJcdTAwMTjByspcdTAwMDbHarBPiO1fPCOA0VAqr+xcZlx1MDAxYcCUYCeJod157kCVQZ57TDykwbJcdTAwMTi/yWCaRVx1MDAwYuRgXHUwMDFmrIbK9zJcdTAwMTWbyZM7znKE95wugVWz9dUy5djLhbRcdTAwMDNU5IXqilEmJ1nZJp0qd1x1MDAwNc9zj0xwkYNOna/E54Pm+Hxw+nV15+1rtbr/eHly8Hx/b7S5t/tT4LOXXHUwMDE5Ylx1MDAxNlx1MDAxM/LxVXpakjXFZ6+jlGzjXGLBlq5rYfDMoeg8XydcdTAwMWakV9jJK8KzkVx1MDAxY6fIzm4mXHUwMDAzZ6tcdTAwMThpxlbNO236XHUwMDE3Pv/98Fx1MDAxOS7NKY7jNkbI4vnC5y+K4LxWcJF5o0f9XHUwMDA2heKwU4lcYoOzr1jYUfK4kufNeMVcdH+yPjdVJ1xywoLSPP9cdTAwMTJcdTAwMTDYwN/qTFx1MDAxOVx1MDAwZXdcdTAwMDD1tzntKFx1MDAwMVbJPO5dXHUwMDFlu1x1MDAwMjnJiMNcdTAwMWU5liCtp+AgNSNZ5sVTLFSDXHTknIqDR854UWm2dqUzudn2YjnCXHUwMDFmXHUwMDAxIFt56lx1MDAwNeazajxcdTAwMWJcdTAwMTelXHUwMDA07Fx1MDAwN1GGq2JFPoeQ/SCRgi1cdTAwMDaSmFx01GecP5PTLFxuxer9MFj15V9Pi1Mk29eT03ynuHrYXHUwMDFjV/Xmy1x1MDAwZk9Gf/a3J6Mvr56oP198XFx+uvNT4CrPI+JcdTAwMWWAZi4plnOa0WWSXHUwMDA3I1xuS2an7yjs9fmoalx1MDAwZeFcZiE/nbVcdTAwMDJXbcZBhEpyyIxjdVdcdTAwMTlXYdImn9v/XHUwMDBiVv8xsKqIc5bTTlx1MDAxMGolQa/gjOzI5njOXHUwMDE4N7FBs71iIGhcdTAwMTFSXHUwMDA2do2rNEqVRKFcYj/Hmf7a2/rJcDxcdTAwMGaR43VcdTAwMTD4IFxmjDrdQirJa3B6XuQ0XHLNMdMw11JDoeYkN1x1MDAwNKk8OU9yUnGdMJdxYpXFs8N1sb4jXHUwMDExZzi4N7KDgPN66vvFXHUwMDE3K43Za8lkLlx1MDAwMzLO6k9niM16glqBgadS5CV4nP0lXfnUII7K4dBSXHUwMDFlYGNcdTAwMWG0XHUwMDEwyHxcZlx1MDAxNDsnWVtSPE06XHUwMDE3mOVnI0SBgFx1MDAwNC6pyaTiRVx1MDAwYrxcdTAwMDBfJkLYoIJbL3U5sI7PcKOZM1x1MDAwZXlq6J1C86jbnpzDT1x1MDAwNT47N1x1MDAxN55cdTAwMDNcdTAwMTXAXHUwMDE2XHUwMDA2fFx1MDAxN9F50Fx1MDAxY52ft0++rTzb3zz9eLKy9fjLeGdrd3I4XHUwMDA3nVx1MDAxZsg5gTygJJOcvpXvOFx1MDAxNrO5XHUwMDE3yOxcdTAwMTGMWsEt9Civ22zs8p/rgbllO6HqqEAtJI/gcPnEcp6nMYvLlqrL7IqJ5z8z4a7lsCOvZHU7wo1cdTAwMDD3v1d6d6lal+mkv+bj8NVn/lvQ4Fx1MDAwYsXY31x1MDAwZSdPN072d7e0X373xT/92lx1MDAxNvJRUdWvdLhFXHUwMDAwfHT1yl+/Vcu9fPukezpJXHUwMDA1XSRghp2vR4/XOyudg9WVI2V2tvybUC/2/MNxONK7J6erXHUwMDFmPq7q39tu64VcdTAwMTg8edzscmf4zVxmKWFPdeHMznpSUmny193llZzKh3Su9oi9OD7WI2qwXCIpR8jVXHUwMDFlP+xcdTAwMDZcdTAwMDPWeJuo/Vx1MDAxNZeEXbDnntP7XCL3UmZVdrpcdTAwMTFZYXu3O2az245xjj/Za7X3jkbd+/QoN7Kvokf98KjS7FKmeG7WXHUwMDFj3MpjpqfXUoi45NxtRjh07WIxQTffd5ctObmMXHUwMDA1cupcXFx1MDAwYvFcdTAwMDDWXHUwMDBlJ91cdTAwMTE/t7Sysba02Vx1MDAxZOHaXHUwMDBiizo4nGz2vuVRo0h++6x10OtzXHJsXCJypd/bPczjvS5cdTAwMDUn0DfptVv9qzdcdTAwMWP0Op1cIja2IbSFaHa01lx1MDAwNL1cdTAwMDaj3m7vsNX/o9FttI4mg3fd8fmNTEZH3eJcdTAwMTPrvrhcbreyXHUwMDBiXHUwMDFhdDtIn19FJCNcdTAwMGaZUV5V9lx1MDAxOVx1MDAwZZtjuvy0vt33fz7prmyqg6OzdSPN8MmDx/R8k1lZnscpgyh1gsho89HZilVlXHUwMDBmXHUwMDFh081579ecWPtHYXpcdTAwMWT2rr99erjf+Wrfb783x1+/tb6JYFx1MDAwZppi7+KpwrncV69ON4/fvt86Xt1queHOh6efd96+Wlx1MDAxOKZHUTgj/LaYXlxy11NzL1xmQ6tIsIH/XHUwMDE1hy9cdTAwMTbN/Wtzc69eu1x1MDAwN1x1MDAwM7lzXGbe5EfiOael4llPsWTwiGAzn4+tV/Z6g79cdTAwMDVcdTAwMGK5YkScXHUwMDBlyTGYXHUwMDAx6mBcdTAwMGJcdTAwMTNhKlx1MDAxOdGlhXvuXHUwMDFmqEIgentcdTAwMGK/dZpcZv6lOLXlXHUwMDA2kH5jsOZcdTAwMTPcXHUwMDE4XHKOe53uaGnYP1x1MDAwMqI+St6wMDSvwa0yms+9qEVcdTAwMDF5p9c6XHUwMDE4XHUwMDFjdirt2s+NzPNdLOVcdTAwMGLH2lx1MDAxNO161Nyuo32z1Xr2Yvxhy+rj/oFcdTAwMWavhP2jh1xy4zYg9LbO8UxcZjFcdTAwMWKZS+8yRu8sXHUwMDE3u9ao1U7sXHUwMDFhc71Rb6tcdTAwMWS1vV1h1MFkmkea6Fx1MDAwYpCeXHUwMDEyrqlVS4vrNCxTnzHvwIlSvrjvsUBcdTAwMDC/rOBZOIDrx0P9VFx1MDAxZX9ag8KAXHUwMDA2XHUwMDBmzdGbSWxcbrTi943h8N3+m82VvVx1MDAxM7HyuL39fOPsy8KA1kc97Vx1MDAxY78zoJVzgfbirFx1MDAxNa8rgXZ8g52symf8wIHWXHUwMDAx41xcPmHe89ygUnGI4kBuZcBYpTbfa5HzYNbbjF3VLFnikbVVXHUwMDA2OWuHXHUwMDFjXHUwMDEw7n3hXHUwMDFjilx1MDAxZoCyt1x1MDAwZpxvh7KDztLgcOk/h8e90eSo1cffXHUwMDBlXHUwMDA3ne5cdTAwMWShbVxyvMyg7fnFXV5aemGLQtxz71Jh3oWTXFzK5m3ZUVx1MDAxY3yohNtJc+u+3lx0PlDr9vnZ3JGDKKtcdTAwMDBX8XC3vIttXHUwMDAxgDvXvDloXG5cdTAwMGIkVcVMXHUwMDFlXHUwMDFlgaZlXHUwMDA1i3ZaOVxc8N3kvlx1MDAxYsDstWbPaee3MfvSXHUwMDFldOHCuudf3cT0ckfRPuJVspRcdTAwMDTL53lKlfA8WlNcdTAwMTZKaFx1MDAxZe22hnz+XHUwMDE5p59cYmvz3kdfXHUwMDFjXVx1MDAwZa09Tbe1Z1x1MDAxZUyyNT7vgicnf7yOz75+fP1cXO6O99eM3D34tFV1wVwiXHUwMDBiPFx1MDAwZVxcWU5mt1x1MDAxY7Y7c7k6XHUwMDBiXHUwMDFjXHUwMDEy54pNL1xyrnPBXHUwMDFi9WVXs9C9+lxui+DP8qUxTD8/Q5+6/e3BSUMuNNdbhjCXXGZcdTAwMDWeMeVtdY7x6Fx1MDAwNsHJtfstXHUwMDBm1FtKKeAued5gYFnEbHyiLGt6YExG+KiKU15cdTAwMTboL5XhVFx1MDAxYimkdVVcdTAwMWXTslx1MDAxZsrlfNVFISuqZHliL4dcdTAwMDL8XHUwMDEzXedccjyR4CBKYUVcYkJcdTAwMDQvna/wnXfgK6/fk0idu+ah34El/YGzg9zsXHUwMDA1qlxmtDxcdTAwMTZcdTAwMDPVv523rLRcdTAwMDf+zFjCXfvNaObO6lx1MDAwMl/xYe7mzHFzx3l9svyhOk7L0lwizdH5MfIwpdIgXHUwMDEyyVx0kDxcdTAwMWLB1WzPfJffzMBcdTAwMThl4PGobGIqRFx1MDAwNFe+U9vMiXl5nehl4dC6f5DPvIlHXHUwMDEyivOlrFx1MDAxMTpgvYMsluZduCRE8/qG7qiR41x1MDAxY/75ubO2/XHtQ/fP7c/RPls5a9svlZfJONJcdTAwMDfHM5GsXHUwMDE3MMpZlsmhXHUwMDE3KnqEXHUwMDE4uFxybuqZXHUwMDFiXfBP5T/n2Vx1MDAwNX9cblx1MDAxNrEg7zkvXHUwMDAzV6hcdTAwMWSccZ7B563dlYNcdTAwMGWluFx1MDAwMevsL0/c+3efj4N5XHUwMDFin3bPjle/rdiH7jyDN5nXzLCxXHUwMDBlu1x1MDAxOFx1MDAwYp9/XrLNJLJcdTAwMDXvXHUwMDBld7qcyzxrgSWPVeGpwlXOs+wzXHUwMDFk+1x1MDAxNp2aXHUwMDFl4f7T5uBC8ttrcnBcdTAwMWYu8lvr35F4m1xmXG7FXHUwMDFhadYtue5yiq36u1x1MDAxN5Vbm5s59/Mz50p4Y4Wo7K2U8lx1MDAwNnbb/mNldfRUfz3+2N48/bTtn1xy9+f1gDxcdTAwMTi7ldJcdTAwMDUwUFx1MDAwMFxmZ8JcdTAwMDQly4ZcdTAwMWJC5jiHaVx1MDAxMbtZc1xyV+vM8oymXGJcdTAwMTdcIsX1NXuXs0hcdTAwMDNrv439XHUwMDA3pc5f/PHHxtL/XVo5muzdt9VWfvVdXHUwMDFirSo8mrLRSi9cdTAwMTEzXG5TXHUwMDE1qUjV3Giv52NcdTAwMGbVaK3iKXhcdTAwMWHhfj47YaZK1nEstHYzVbKLNNpcdTAwMTBByfKj83i8oqmaVVJYv1x1MDAwYquFg8nPMbpzsL3igYst5ZohlVx1MDAwYlxmdFxuPmHahFvnXHUwMDEzjlx1MDAwZXunS2Ncbpzct0+o/Or7KEux892C4IhcdTAwMTOO2a5yXHUwMDBiurlbeLuxt663X+iNfbv5rLO25tdcdTAwMDZ784bUPVx1MDAxY7dcdTAwMTCFzni2XHUwMDE3sFFcdTAwMWLtSyRcdTAwMWPsm51OnGdcdTAwMTiu3Vx1MDAwNt9cdTAwMTGhLcTtvILiXHSYilx1MDAxZHRVaV/Fw1xcXHUwMDEwISRJ4cuKcjbdXHUwMDE5VYywfoaalOOz8dFZODl96d5+tJ0tfTZ4/alx8eee/PrxaGUy7m7vvVl9evx6c+/L5PfFeaJcdTAwMWKyk9uB9DXpRMtYXGZxe6U1mubWWP2MXHUwMDFmvDV6kV10pzrHMTglZi1gjt5cdTAwMDetjHHujsxRwiNcYlx1MDAxZnnYK/v6XHUwMDBifaOFvZhcdTAwMTmUltFEXHUwMDExwC3+QeR6Y9D5z+HqJfqNb1x1MDAwYqY19Sg1uFJRj1J9UYuC2W6/31x1MDAxYo4rmziMmlx1MDAxYjJzWq9hW1+VXdvmdt16ufP+3dej/cngz3ZU4vOnlY1h+2FcdTAwMTd/Sm9cXMZDg/OzPGYxVtpcZuCnOWdcdTAwMTiWXThqcaFWrU1cdTAwMTaU0JXFKMZJXHUwMDFljeuqMJaRvNI2/FxcZZ9cdTAwMWRxNGhcdTAwMGI93lx1MDAxZO9MXuy8dJ3BaLNx38Z39Ff83aFb67n1Zo6lS1x1MDAxY0RdZeGuuYVXL91DR27vY6aByVx1MDAxMVGt8bGQO7yooJCg2dFcdTAwMDdcdTAwMTMkp3V+n5HPa9RyIYuW50g5XHUwMDFmWOldUVA6i9xeW1x1MDAwM6hfgH3/LLj9mHN0/nO4eTaedFx1MDAwZu5cYrVrcKqM2nMu6Xsxu4k/m7f9n6u1c5xJYpS3ynObpqTWSvGYcckjp1XknIxcdTAwMTmVXGaKXHUwMDEzolx1MDAxNefLKqOFqCCTQCZcdTAwMGLwXHUwMDEzQVlCjSx4kPpm69uko360XHUwMDBiuWWztfSPKs2uXHSsXvVocUZcblx1MDAxZXEsKvZVWDZ/w4NBNohK4XOLasK+fcfWQjeL5+opf5ZnVXQqb1x1MDAwNppcdTAwMWLukddv/F/f97VUrOny1lhOqrFSXHUwMDFh4YunXHUwMDBmX1x1MDAxNSdcdTAwMTgpNU9H5tlcdTAwMTR3U6hwvc9bSuspLM+LM96w1cIqN3vFOjOe03csy7it9DcrSfuuMoQm2/5NKN21rjVcdTAwMDRcdTAwMDTzWjIq4lx0m+UzjKR03GRT1lx1MDAxYlx1MDAwMe8qZl2rXHUwMDEzXHUwMDE5x0PCOyO0goHGqnQ6vlx1MDAwM9/EXHUwMDAzt6DXYPi/fGu1b1xy3+9bLciv40D/Kt+q5o43MJwuzDlNXHUwMDBid63cV/DTR/OjXFzrfD3lz/Ksit6Hb23sqzhcdTAwMGJaXHUwMDEzN420XlxijVxivYBcclx1MDAxN77KZSDcrKr1PN5dJtX9XHUwMDBicq7Xp4FS58r9vVx1MDAxMFxcfoaUYqXvzFx1MDAwNUvAXT5S3MhcdTAwMTBcclx1MDAxYilcdTAwMWWWd62LQ838YymktsKFxFx1MDAwZVx1MDAwYna+0jxcdTAwMTD9dHo8XHUwMDE5a3/0+Wjr1dh8XHUwMDFhXHUwMDFkXHUwMDFm91ovXHUwMDFljqebXHUwMDEziCrhODCex0JpeIBQXHUwMDFhz1x0JpeBx5h4l1x1MDAxNanWgG3E6GDM4Fx1MDAxY77Z/Fx1MDAwMCypU7bYgfFcdTAwMDP2eVx1MDAxZj/fer92uqnt5/3na+J99/OuOPlQsLJcdTAwMTlMv3rlLlI031x1MDAxN+a+XXuyet9cdTAwMWK86XcuJnxtPd/deTV+fTjc+PN16Fx1MDAwZVx1MDAwNjvL7f6XWY41k5c+r1WKnsdfgfhcdTAwMWGeXG6nSlx1MDAwMax2MfNcdTAwMWNjXHJi76MsbKJfXHK/MVx1MDAwMa5cdTAwMTLE2nM2p5BcdTAwMTXp08Cpolx1MDAxY1x1MDAwMK0ssMIpeY2N3mIuz2TUOlx1MDAxY1x1MDAwZlsjrNnD8T635VnDXHUwMDA18CzlgzB2tqw2t7S5uFx1MDAwMMdcdTAwMTg9z2u/zSCxW6ah945efzRHr99cdTAwMWXGzeXO6N2qfHXw7kvTdPHD8kXXXHUwMDE4aPVdzlx1MDAxOGjF6D74+0xcIvxcdTAwMDct5Im2urTfqTmZz4M4Rlx1MDAxM4wvXHUwMDBleZ7WXHUwMDBlaM64XHIusIDX2Yox0Deb3fdPM8dcdTAwMWLM74N3XHUwMDE0NrAgu8rw1ExKaWp4XHUwMDEyXHUwMDBiXHUwMDE0iyNcdTAwMGJcdTAwMWXOXHUwMDA0P1x1MDAwMtbSWlx1MDAwN9+Gh7p0OTqnsLA/bIZfXHLsVSHvdTeyXHUwMDE4LL7eJ12b71x1MDAwMHxmPO1cdTAwMTbkU1x1MDAxNE9HvVx1MDAxY4HDc9eFMjEoo+MsXHUwMDBlg1Naza5A5lx1MDAwYlx1MDAxOTBWWLpQXHUwMDE5h8GH/Fx1MDAxNIbiIM9fhn++hlPDXHUwMDFmfTdcdTAwMGUjXHUwMDFhhvFcdTAwMTXr5lx1MDAxYU6dQXxcdTAwMTaKx1x1MDAwNvzN0lx1MDAxZHNcdTAwMTWVPzMqOpU2g9lcdTAwMGLLdVxcXHUwMDFmyC5cdTAwMTVzXHUwMDFkynqPxbFcdTAwMWHWalxc8cjhi8SBIdX1VmvQMFx1MDAxNlx1MDAwZsaimFx1MDAwNaWRr3V8S0mmw1x1MDAxOCl54rKNXHUwMDBlf1xys9f7nVnjmdzFLVJcdTAwMWL/uljUR63hcHNcdTAwMDKdvbr3R8e97snjysCaPyR0+bXRy3Rz8/zrX3/9f1xmJ4VPIn0= + + + + + EDGE NODE ON RESOURCE PROVIDER1. Node with exposed service at the edge of HPC clusterVirtual KubeletInterlink API ServerProvider pluginPod on virtual nodeVirtual NodeHTTP + Authunix socketPodContainersBatchSystemOIDCOIDC Identity Provider \ No newline at end of file diff --git a/docs/static/img/scenario-1_light.svg b/docs/static/img/scenario-1_light.svg new file mode 100644 index 00000000..a517f3b1 --- /dev/null +++ b/docs/static/img/scenario-1_light.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19WVdcdTAwMWJZ0u17/1xulu/jLfI789BvXHUwMDE4T3jAlClcdTAwMWLbt79VS0hcdTAwMDJkXHUwMDA0kiUxuVf997t3MihPKkUmWGBcXGWqy+1CUigzT0TsXHUwMDFkcVwi4vz3X0tLjyZnw+6jfy896p62W/1eZ9Q6efRcdTAwMWJ/f9xcdTAwMWSNe4NDvKTy/1x1MDAxZVx1MDAwZo5G7fyde5PJcPzv//mf1nCYTT+VtVx1MDAwN1x1MDAwN+ef7Pa7XHUwMDA33cPJXHUwMDE47/1/+O+lpf/mf+KVXoefX/lcdTAwMWPl9mjzY+tl/9v41WdcdTAwMTFcdTAwMDabvZX8o/mbLi9o1G1PWoe7/e70pVP8XmrhM+GtMdaKqL2yVy+f8WWhVSZcXJB4i9JaiXD18kmvM9nDW4xwmbDGXHUwMDA1P/3oXre3uzfhzfqQOa2CVFbkP+7qPedcdTAwMTfz7yVx9ZvxZDTY765cdTAwMGX6g1x1MDAxMa/4/+yI0Fx1MDAxNmJ6vdut9v7uaHB02Cm8Z6fbjnH6np1ev785Ocsl4yHjYT4qyd+6uG5Z+v28T+FcdTAwMGJ391x1MDAwZbvjcfKZwbDV7k3OXHUwMDFm0fRcdTAwMGV4dcO1Tr5W/zu9plHroLvGxTo86vevft077HS5XHUwMDA0j7aLN5nf38XXXa70dFx1MDAxOfXFb/6aXny328lcdTAwMWa0j8Jf/XaqbtqF8m/XXHUwMDA3h7nqRVx1MDAxN72QXCJOr2j8XHUwMDA02jbJXHUwMDA17rT64+700fOqnpY1saiNibJNuqeTq1sq6OrXXHUwMDE3Zufj20396s1cdTAwMWZ7W8OTbW93O38+unrfX1x1MDAxN3+bPrqjYad1fj3SK2dcdTAwMWTuxoYwXYZ+73C//Fxc+4P2/vRcdTAwMTb+VXhcXCXLqb6aXHUwMDE5y0lu5sJonM2sV0E7ZZ0xKpSNxtdcdTAwMTiNXG5cdTAwMTFGXHUwMDEzo3BC2GCjmTVcdTAwMWQr7tNU9lrtvaNR92cwXHUwMDE2WW0sydsvrFx1MDAwMutcdTAwMTN9XGLGVlx1MDAxOIbyap5hSG9s8EZHf1x1MDAwYtNIruP79XeqjlRDPICnT54/XVp/++Tp0tv1pXdPN9++f7f6dOk/h1x1MDAxYu/eflh78vRdYYFcdTAwMDeHk83eN96RXHUwMDEyyW+ftVx1MDAwZXp9rodNxK/0e7t8NI/auJfu6FHx+Ux6gKarN0xcdTAwMDbD6attSGz1XHUwMDBlu6O1JoA0XHUwMDE49XZ7h63+XHUwMDFm197P7N20jiaDd93x+f1MRkfd4kPsvrg0XHUwMDFimSlbafPXXHUwMDAwYUEp7IyqXFwqhZIqOlx1MDAxN5yLVUrZetRAVc7djtNcdTAwMWJff4/t10+fd549V09aoydbrf5DQbBZV8NcdTAwMWK03mRKO22iMULA+yX+LsKZ4SVtZ9xd2U3JLv+53k21bCfs7Ezfc+kybTBcdTAwMTm+XHUwMDFiLvOcS0zv5MplXHUwMDFhpTLpI6iIifnP1K1eOFx1MDAwNKmMcV44q2/nhm6KzjO4eefQ1kjN9axHvPJ9VkvrXHUwMDFjnmCVmm83V/Ozky9n4fnqyccj8eXrwejV23cv/vzj4aBPtaL7IDItnHMyVii6XHUwMDA0z81cdTAwMDJ03FxiZaP0/k5cdTAwMTRdm5CBjmupKlxitTQ6s9piidS5jvuyilx1MDAwNyGM0bJw6X9TXHJPeVlBue1cXGB31kBcdTAwMDdC0FW63W6u299cdTAwMGVf/bn/XndG708/nJ6tvX+/uv3m7KHrtlVcdTAwMGWqXHUwMDEzXHUwMDEx5lx1MDAwNa1UXGIpaY1aZNFqXHUwMDExrPRU/ztRbe9xXHQ+XHUwMDA24aSJYL5xVsG1LSu0Q2iDy1Xx+1x1MDAxNTp5obHmeq29Nz7eirZcdTAwMTVJWEh+O4eE8Vx1MDAwMcpsaX3Q6S6d9CZ7S93T4WDc7SyNu6PjXru71JosTfa6S93ObndpsLP0YmN1qd0/XHUwMDFhJ4RtYWwuucUydfvuy7xcdTAwMTdOXHUwMDA3Zz1cdTAwMTfsonfgXHUwMDAzskj0p1x1MDAwZaHT3CFMTv54XHUwMDFkn339+Pq53Fx1MDAxZO+vXHUwMDE5uXvwaeuhO4QgdFx1MDAxNoFcdTAwMTTzwC7azPP52LtcdTAwMDQ7IbOYo8NcXFZn8Y6o5Fx1MDAxNakruKZzXHUwMDA34Vx1MDAxMY+bIOKP4nRXn7lRLmR39GJt/YWfXGLZds86wk6ef42PXHUwMDBiuZDfqsWef1j8vjFcdTAwMWO+23+zubJ3XCJWXHUwMDFlt7efb5x9Sb/l8vtbo9HgpKncOFx1MDAxY+ndk9PVXHUwMDBmXHUwMDFmV/Xvbbf1QlxmnjxuJvfib/eUu6l+elP/Nzd3XHUwMDAz8M+8hupbq51SPqZKXHUwMDBmrc+UR1x1MDAxOFxieEqU/lJhpVGZZuYmXG7rnSwkXHUwMDExpklPe43hzVx1MDAxOE/0sq3+XHUwMDE5mZvOh+apm2BUhF8yU6OeevRcdTAwMDJcdTAwMTKXXHUwMDFjelxiXHUwMDExzFx1MDAxN1x1MDAxZa2B577vxM2H3mhy1OovvTra5lx1MDAxNd1Lnuag1+lcdTAwMTRRsZSqqYGtMt7PvYNF4TjfXVx1MDAwNeEhinkrLoWSXCJE2GqVxnWbQ/hK7P/+bfJ4f//FRm/txfrxpndhXHUwMDFlp2+PXHUwMDA24/HyXmvS3vvxMO7g0aQwTntrlHCi5NGEXGaZkyD11oiofMGgbuOKtHLtrp3FcalcIoJWRFx1MDAxNyYo6YHHXHUwMDE1oauyWVx1MDAxNC5GJ61cZipcdTAwMTSw/jI7XHUwMDAzlu1EwJ1UupopNo9emr2dz19enk2Oevth6+zw9+HXzVx1MDAwMkw9ev9p7UtP7Dw9XHUwMDFjPH+3tr4+7i6//fb+XHUwMDEyrO4hZPguhzGetEaTx9Di3uFu+SPdw86cV/qt8WR1cHDQm+AyNlx1MDAwNr3DyYxnpdxcdTAwMTXC9l63NXPTkDz3tSHFpWsw/dvSVFx1MDAxZvP/uPr7//62gHcv61xm4ZcyJmqD/1x1MDAwNyq75PNcdTAwMTlcdTAwMDA9XHUwMDA246VcbkpHo0OtQFx1MDAxMzKrYjTRwcLhPXxRnlJQY6H4TS5cbm3cfYtcdTAwMDPbRaCOd9tcdTAwMTiksMIm4kxcdTAwMDaeXHUwMDBiXHUwMDAz0SriO52uXHUwMDE3XHUwMDA331x1MDAwMI5cdTAwMGaKLH1ApGNcdTAwMTJxIVPGwVx1MDAxZSFP4fJkrThn4MKtNNb5XHUwMDEwrDChKFx1MDAwZcG9U9xcdTAwMDdWIEfG+vqlWLw4rYP3XHUwMDExXGZcdTAwMDAuxiU3XHUwMDFiRCZcZlx1MDAxNMlcdTAwMDNMPJhdvTSdXHRcdTAwMWRcdTAwMTD9RKVMiMm1weXil1pcdTAwMTmEJVoqUy9MZVx1MDAxNjdiLdxekMH6kjSwSWu9tPiuqFW9UchMK0REykXtXHUwMDExwenkViXUzoPaXG6pJC7fqfqbXbhA6JYmf5POc+cg2pBcbrSZgMHihqHF1Pd6gSazXHUwMDAxXHUwMDE3J5jh10basrxcdTAwMTRb6uThXHUwMDAzWFRcdTAwMWI0kdNcblxcwzXy6rVl0eJcdTAwMTCFwKE556BcdTAwMGXBXHUwMDE1djKunlx1MDAxZb5cdTAwMDSaLGg8oVb/tMrghISDR/EyKjGzui5Kg6v28KPC1i5cdTAwMDZMw2BprcTHYFx1MDAwNr5sXHUwMDFiXHUwMDAxLMTSaytowP1cbrPgQlbCacDcYOW+5Fx1MDAwMoyWxiltXHUwMDAznLWOXHKkXHRcdTAwMDRcdTAwMTNG5Kym5Fx1MDAwM+ifYP5cdTAwMGWsXHRr5Vxc7bXZPN1cIlx1MDAwM/BcZmtcdTAwMDAjKvliXHUwMDE5oT7AMVx1MDAxOVx1MDAwMCbR3rc4qFxi9EPDg1x1MDAwMk9h5WXkQbSrqFwi8NRYqFpvbKDCXHUwMDBlrlx1MDAxOF+udVx1MDAxNCpcdTAwMTWHIDpKXHUwMDBidbNcdTAwMGXr4VxcrUWYTGJcdTAwMTGwfLhcdTAwMTdn4YJmaFx1MDAwMGJ6fFx1MDAxNcVhwWrtf9Hyllx0L3lcdTAwMWRcdTAwMTS8XHUwMDFl1EHBVyVcIpcl31x1MDAxMDT+XHUwMDA1XG6A5lx1MDAxYd1dlrX3PY8t/av4/zeNp2jrc1x1MDAwMyqm51x1MDAxZP4sxNDTgGqneUC1Nd58az6vrHftk6O1/uNcdTAwMGZxS/rfXHUwMDFm9j73eSil6T5KUcxFXY/JXHUwMDFjIFx1MDAxMvZOzS1cdTAwMTSz3S6rU5FcdTAwMGaNsLOgPVNcdTAwMWZcdTAwMGV+ulwiXHUwMDFkXHUwMDFhNbfBNXTXRi1cdTAwMTDQzURRcJTAXHUwMDA17avzoVcq9CtI+vtcdTAwMDRJMoPTXHUwMDAxXHUwMDEwwJFcdTAwMDZQZkRcYqk3i9abIGn2XHUwMDAwXHUwMDBiqWvBYFx1MDAxOYzAOiHhXHUwMDFkoWdw4TZhXGZUQVx1MDAwMWORWmisYmxcdTAwMTDX1Fx09Fx1MDAwZWCF+N/qgODGfv9cdTAwMTXeQiBcdTAwMThcdTAwMTfwSuJcdTAwMTNMXHUwMDAx61x1MDAwNO1cdTAwMWT4MXxi8Fx1MDAwMp6bYUlcdTAwMDN5XHUwMDAyolx1MDAxNFx1MDAxOD94XHUwMDFmiJxLXHUwMDE5q8wrajTJNCSbRlx1MDAwMkNcdTAwMDTiwzVcdTAwMTlng1x0psTiMjBPXHJcYqL940Lr4yWVRTxBxtX0dTYkSiPxXGKB9VwiXHUwMDA07lx1MDAwZeFp1EPgouX5LHAphGXhXHUwMDE1o+FEXHUwMDFl4lx1MDAwMU39g0gv4G/rgzA8XCJP1+pcdTAwMTGIg1x1MDAxOUWVXqAjQIPl4bvAXHUwMDFlXHUwMDAx0/VcdTAwMDJ9hlxiXHUwMDEwi8FlXHUwMDE03qlEXHUwMDFlo+tcdTAwMThBwLSJXHUwMDE2MUl9tK5kXHUwMDA2/uA9XHUwMDAyVFx1MDAxMC3Eglx0Z5JcdTAwMTm03XtcdTAwMTVcZt5cdTAwMDAu3MDoXHUwMDEwlYCGaIu3e1x1MDAxYUFyw+B0oNZcdTAwMGXvXHUwMDAwXlx1MDAwMHN0XHUwMDAzXHUwMDE1dJnBWihcdTAwMDYz/CMxXHUwMDExXHUwMDA1Qlxuhlxi/FxmXnKd6+PYhctcdTAwMGIsvFx1MDAwNVx1MDAwYtNcdTAwMDFLaZxXJYHeQo+MXHUwMDE2gbZZy/9cdTAwMTlcdTAwMTZD/Vx1MDAxOIGARHtn07hdIZ7weLjgXG6K+df68GRZI9ihXHUwMDE5QFx1MDAwMSNcdTAwMGItTEljXHUwMDAwh/BpzoKD6OjrNVx1MDAxMIxcdTAwMDQqK5RcdTAwMTDeSlx1MDAxMFx1MDAxY12Sx11cXFx1MDAwNlWRil+vMItcdTAwMTaHsNdaPCPpsFx1MDAxY16VomKPICBcIjRAXGKDh1hcdTAwMWZALWteXHUwMDAxQi1cdTAwMGJcdTAwMWZcdTAwMDJfXHUwMDE3y5dcdTAwMDcwXG4+391z+K5cdTAwMDZZj1x1MDAwMFx1MDAwYlx1MDAwMCaypFx1MDAwYkGh0eXVXHJcdTAwMGXPgVx1MDAxMZGF0tSL88yBw6CYQUFcYjVcdTAwMTPycE9cdTAwMGL0XHUwMDBlT1x1MDAxNo6tweU5hCFGXHUwMDAwyGCeuKdyalx1MDAxMXqEXGLFXHUwMDA3xoCx3t0vWlx1MDAxY8mwxa1cInSH70gzRvAtsFx1MDAxOVx1MDAwZqdcYu+CXHUwMDE1aSBOZDZcdTAwMDbGqnB+MIPUcl1cdTAwMTbPk6g+eOhMg4xcdTAwMWLkXHUwMDAxOCyzr1hcdHysnPo0kWVyXHUwMDExYahSoFx1MDAwZvW+JWZcYlxmLPmE4Fx1MDAxNqNIrzBm+DWWNk/XMDtz/1x1MDAwMpnlUVA9J8gpbOr8sFxcUWvCXHUwMDBiWFtskK7gXHUwMDAyXHUwMDAzi5jgZOZcdTAwMWQ+uCjOXGLWbYDZMJBcdTAwMDVcdTAwMTA3IFjwXHUwMDA2XHUwMDAy/iN4R1x1MDAwNyOmNWa5PM90XHUwMDAxeYtcdTAwMTZcdTAwMDZg1Uz/XHUwMDAy7ldJXHUwMDBi8lx1MDAxN1M2ZDXoYcRvLf7R4Fx0XHJu12Z4OFx1MDAwZYaOlVx1MDAwMFx1MDAxM3AleVx1MDAxMTZcdTAwMGLKQN9cIppkVFx1MDAxNyxcdTAwMGVcdTAwMTZcdTAwMDDXXHUwMDA28IdnM6VtXHUwMDA2azNcdTAwMTghOCvcMnOu9apiJK6OVFx1MDAwN1x1MDAwNo/lc2VxcHjMVbs8XdZgT8WAXGKBOVx1MDAxYlx1MDAwYl31pWSKdbg6UjXvoJyhyZ5cbu5cdTAwMDeoXHUwMDFm4JJcdTAwMWQuJHUtXHUwMDEwpyX4XHUwMDE5XoHHXG66kThW74NmXHUwMDAwybmd8LDEuYycXHUwMDFkWlx1MDAxZsBcdTAwMTWlL4tjvs1FwFx1MDAxMLyZaGC2sCTgoLdRkZKE0uV5KJ6B23agzVKqXHUwMDA2muKoqlxuulwisHygLElUJDJ8XHUwMDEzliiA/UaQrnovXHUwMDAweVx1MDAxZWZcdTAwMGJcYlx1MDAwN9JAXHUwMDFk0tQ2XHUwMDA0srBcdTAwMDaMXHUwMDE0QVx1MDAwMpG8wf2S5DJcdTAwMGU1XHUwMDAwXHJoWeL1XHUwMDFjXHUwMDE4NbyeZ1JcdTAwMDQoXHUwMDA0PGiiy4jxwPVcZjwv4CtccjsgkLG0cPhcIpZcdTAwMDE1iKWNJlx1MDAwYkLU44VcdTAwMDNcdPdcdTAwMGKQXHUwMDA3XHUwMDFmwC0xgII3KVx1MDAwYoI8wy1cdTAwMWNcdTAwMDV25FxyXHUwMDAy4HtcdTAwMTcnspzCsv1cdTAwMDJxbeluXHUwMDA1Q1x1MDAwZTiqXHUwMDE4sMxNnJ6Ak1x1MDAwNCZoR0qaXFyajYigmbBG+CVcdTAwMTHVNdg1Q8hcYj5cdTAwMDBcdTAwMDJcdDDzQpXFXHUwMDAx7ExeiFxiuFxiXHLCP5B5UEfPkmrNqNGV5OGJ4luAXHUwMDAyXGLLRYNgY8HiRO7iibdeXHUwMDAwMUp6XGarsZbFUiyE87ZBbOV5eXmRh45w9unKepF5QJ1UwYI/ilx1MDAwNslcdTAwMDdcdTAwMDU+YjVA32NBpDKJ3XpcdTAwMGI36jTjU4svbWBcdTAwMTWIJnBcdTAwMTeOXHUwMDA1YFDmKEqbNey3xCV7vlx1MDAwMupRL47RO/yQ5Vx1MDAwNibgq7yRJFx1MDAwMsBWMbp3scHu5eLFXHR4SFx1MDAxMFx1MDAxY2mZXHUwMDAz84muXHUwMDA0OD1EuVx1MDAwNtRcdTAwMWOeVtXfLENcdTAwMWRcdTAwMDNSzTRB1Mqb9PJcdTAwMWM3wllcdTAwMDCPP41s8PCUzFg/XHSyXHUwMDAy9Vx1MDAwYsGn5Cdwb1xuZMZr8DZymia6olxms302kq1cdTAwMTTqtS/kRTB4b1x1MDAxNPOAXGLXfoQ8aLCW2nBcdTAwMTfTWpXoXlx1MDAxNFx1MDAxOdiPZ1Ypgus1SUXy+nSQhmlcdTAwMDeWliS2XHUwMDExsbzkRSCQQlx1MDAwMklcdTAwMWLllXInXG6mrLHKJt3bi7h6qFx0iFx1MDAxZb1cdTAwMGY8c736xYx7z560XHUwMDA0qJbmqWY2KJqIY4aFgTO9m4ulq0NY6OG+glx1MDAwMjOI9Xeb21x1MDAxYaNPXHUwMDA27oJAU5JcdTAwMDdcXIJEZmRcdTAwMWLQKWYpoFx1MDAwYlx1MDAwZTZcdTAwMWJcdTAwMDTUXCItoYl5XHUwMDFhN8KLsu3K1u/EU1x1MDAxZfBcdTAwMDU+yFx1MDAxOWAu7F6XdFx1MDAwNTbBWDB6+opcdTAwMDZ3u2h5LlN0ucZcdTAwMGKQWyd92TaEXHUwMDA3k4FcdTAwMTbD3atcdTAwMDZJXGZcdE9OXHUwMDFkIWowV5tcdTAwMTb5XHUwMDA0wJBcdTAwMDJblVwi4iFcdTAwMDZX71x1MDAwYiSiUtyqNlx1MDAwZTQ7MFxylci7LMyJpCyhifJRXVx1MDAxOaRcdTAwMDH2aXBJlFx1MDAxYvJdXHUwMDA2p6nqtO56WJPc2Vx1MDAwNlW2bFx1MDAxYsLnSp5cdTAwMWVsjz5cdTAwMDVxKcHXNzC1XHUwMDA1y3NMSktunFBt4fNt+fHBXTPwXHUwMDBiXGJcdTAwMWbqWUaeZFx1MDAwMYYz8lR4RjJNOrB8XHUwMDAzIYOialx1MDAwMvbqQz9ET1x1MDAxZSvBbDlUzOlyMVxiy9jA0jRcdTAwMTbZ16dH8fAsXCJccnwxnLJQprS4N4RJaopDXHUwMDFjppiAXHUwMDA0ekRVVlx1MDAxNWOMY4FcdTAwMTmgw9dubyxanGHyXHUwMDEx/lODqMNLXHUwMDA1U3p0oOJcdTAwMWUqXGZQbrA3RC9cdTAwMDD8jsztsYYjlJ1cdTAwMDC+h/ZcdTAwMDK7cfXkjD5cbrzWXCLw057J7bKPMtGwXlx1MDAwMlx1MDAwMOVcdTAwMDFpXHLEcdGY+IFRXHUwMDFhacvwKIOh9eO1XHUwMDE461x1MDAxZtxipVx1MDAwMXxcdTAwMWO3tY2E3TqbljNFNo84zVx1MDAxYYKgPFx1MDAxNrdWXHUwMDFjaD9UXHUwMDFlXHUwMDExNlx1MDAxNlx1MDAxNYgrS9JcdTAwMDBuiFmA2lx1MDAxYV6+1rrAK/LNNISh+X6LKmNZhMIh9NCM3uqrS0lTVFx1MDAxZVx1MDAxZDpgs4oylMR5WF6+XHUwMDFiyLRzrZaQRSFYJ9GEuzUlXHUwMDFhwMZseECGRlx1MDAwZcpUX+CHi0NcdTAwMDTKWEVbLIYpaXBcdTAwMDDdRpjHfLipj1x1MDAxZMlojTReXHUwMDFhOFx1MDAxMbi5Mp81XGLJoOFSMni7X2HcpoieYU2e6k75XHTwXHUwMDE1hC4wxy1Y3lkrjpFcdTAwMDWAmISfg1x1MDAxMtIkbeB2XHUwMDE4XHUwMDFjNL5PcMur3lJcdTAwMDOor2JcdTAwMGVcdTAwMTbmRc+d+jjGPUBcZpZ0ws3VYjUsXHUwMDE1JJrzdlx1MDAxMLNyQzNxI1x1MDAxZbQ9ssVcdTAwMDG0k6n9elxy0YxcdGFWMGwymlx1MDAwNFlcdTAwMTGAqnxcdTAwMGZWM0Cuz2shOsbTxSWBlUCnXFxcdTAwMWG9IzpcdTAwMDZnUYhorVx1MDAwMWmvXHUwMDBmj9mNjfU0XGbg8bhcdTAwMTNcdTAwMTVB6K7PKalkVlXWXlx1MDAxYlx1MDAxM1x1MDAwYoF8XHTxO1widFFOLIC4gkDAJ5Fk1z42ZlG4v8Z6XHUwMDEzRIPBl6Q5XHUwMDEzXHUwMDFjyDzTbVxySklcdTAwMTcsjXv4IP6KSFx1MDAwM36vStL4KCXoXHUwMDFlaFx1MDAxNGhL7YMzXCLzTHv6PPflSntIXHUwMDBlludcdTAwMThcdTAwMTRcdTAwMTg+WPilWnFcdTAwMWFYXHUwMDAyMGXRSn5H5UxbXHUwMDFlQ1xi06CgXHUwMDFjothcdTAwMTJvmJdcdTAwMDVVU2VRltX0MpDJhXpAZYZcdTAwMTK4ZEHBqcNcIt2W/uHioO8we5BMai+QuiTNSXxcciRxmll9MMx0e1RMfXtcdTAwMGZklzPpZ1xcXHUwMDE20FlbuM36XG5cdTAwMDEm21xyq0SxXHUwMDEyivmYksLhXHUwMDBiQO1cdTAwMDRzdj7UX1x1MDAxYp2IZ/V40PSLqfrCl+Y0XHUwMDE2ZFx1MDAxME+j3ofk14bbXHUwMDAwkUecVEq0Q1x1MDAxYeuScLfUOF2/XHUwMDAzhedcdTAwMDb+o1x1MDAxMaXDVp3zvrzHXHUwMDAzfXQs32a6olx1MDAxZVDzLVx1MDAxZe7+R9hcdTAwMTVrZErSXHUwMDE0Y1x0XHUwMDE2doGh14c1xmTchKSSsOrLzew/XHQyaSCQZlx1MDAxN9w9S5N51YnO/Vx1MDAxMnxIqlwibFxmgJOHXHUwMDFlWptvNjRwcexxhatg9Mbdl5I8KXTePkSWXHUwMDE3XHUwMDFhgIPNPHwsXHUwMDFjUmCnS5zZp1x1MDAwNKfQeek3qGN9zVx1MDAwN3dlWUAnc8ftYiyJc3ntXHUwMDAzPD58fT2JW7g0PmjBXHUwMDFhXHJthU90jlx1MDAxYtCKSTXcK/NcdTAwMTH19WR5sVxmLJX7XHRCXHSrk5XgXHUwMDA2Ob9cYsCrgFx1MDAxZvWNaZy0XHUwMDE5rYeriIKJ9lx1MDAwNPN56dxcIlx1MDAxNFx1MDAwZUSFpVC1V6eAdqBcdNws9i5vjfstfVx1MDAxNdG9xoVcdTAwMDO+XCIoTj3Bidn5vEKEo6DTdqaVjFxcOzCjXG6X0qjDZZHiNGNITeT0hOJy8VxiXHUwMDFlXHUwMDE4O408XGY6NOivXCIpXHUwMDExrMN2mlmDtHeJpS2ej1x1MDAxNjhcdTAwMTFcdTAwMTD+Nrg4lzEkXHUwMDBm3OFcdTAwMTVsO0jEKXJcZsvxnWTc9b5Oe3a5cT+HwUu61a7yXCJWgUhcZmE0iXD9rS5WWsicol6x6NaKlGkya1x1MDAxY7xkpaFlJU99clx1MDAwZeK48ShVIFx1MDAxOEtVrveyzGmCTSmjXHUwMDE4YDS4V1x0lII+IVx1MDAxZVx1MDAwMekvV6MxQonMXHUwMDA0XHUwMDA1w4RQk2XlPqZEKFx1MDAxN50sSzMsMlQycphkXHUwMDAz60e8kT+XPE1cdTAwMDbMNiVxmuk1w399k1BcdTAwMWacP7K+UMKOYEVOllx1MDAxZZ2g+lx1MDAxYVx1MDAwNixcYlxy6q9cdTAwMGVcdTAwMTGEXHUwMDA36GBt8b9cdTAwMThT0OFKYJmwXHUwMDE2LDxiNfO9y1x1MDAwM1x1MDAwYmTZXHUwMDEwwMLAMl2YqeRj1pO7zlx1MDAxMm6iPmud15HCXFzhilx1MDAxMaSrUlx1MDAwNVxu60ixTkzZMeDWXHLSXHUwMDA3IE9s0Vx1MDAxMsRcdTAwMDOYhpwpdDVwXHUwMDAxwWp2nNbX4bKul1x1MDAxM3vgUaCnLHcsOSju+ELJZVxmLPSrXd6Fy2MtXHUwMDFmjImZZNZcdTAwMTOm4Vx0l4Pmj5dcdTAwMTDDwzHWiZMgNYicXHUwMDEx0ll6cZlmwWBcdTAwMWKKXHRwXHUwMDEzQ74xXa8tLFx1MDAxY3CAXHUwMDFlWFx1MDAxNFx1MDAxOVKaUpdAWrhcdTAwMDHL2lx1MDAwNYJ6PWtXXHUwMDE5XHUwMDAyXHUwMDFk4cGIXHUwMDEx3Vx1MDAwNcTRqSdApFx1MDAwNlx1MDAxZcBdeDZcdTAwMTVZWf/0XHUwMDAwV5aNylBcdTAwMTZcXKBIU+p5nT88LHxcdTAwMGJbXHUwMDAwXHUwMDFhXHUwMDE0XHUwMDFks1x1MDAxMYFxk2HdXHUwMDFkQq40imUjXHUwMDAyXFwse6xBK9naVy8v7+zlJrtjeiStXGLKuO1ic+uIXHJoQFx1MDAwNl9nmVx1MDAxOYHae5fuRuDJ4ZusXHUwMDAy9SNcdTAwMWZz9cVANeLob4Bzno1cdTAwMTdYkPtcdTAwMTYnXHUwMDE5XFwyUctcdTAwMWFqhP9pco1cdTAwMDWV3rOMXHRcdTAwMTRcdTAwMTTkvtHmXHUwMDBiq41xXHUwMDE5cEFYYDz1VKBgaSFdospcdTAwMDfYqfrtybtpN1x1MDAxNNPmnHK7oeYuk9aictzobvNuQ7H25WXrRfw2fNs7XG6Ptzs93Vx1MDAxN29+ivEtPmY5yOm84l+UZ4kzQuSANs9u1WC/r+dw7vhcdTAwMTZcdTAwMDTcLuQ7XHUwMDEziFx1MDAwMlx1MDAwYmOvptNb8Fx1MDAwZfg4XmTgSLiZuaOsXHUwMDFhXHUwMDE33Fx1MDAwN6+e03ilSr/aXHUwMDBlXHUwMDFmcNvhMmvG4LjzST6sIU0zz5d771xiVZhPdlxyulx1MDAwNCDQS3Juxao1IYy7RmCTMrSFXHUwMDBiRJCDXHUwMDAwh/28oGYstk3kldW+Vlx1MDAxYzNcdTAwMTaMpqzL9zXSjKC0eTFcdTAwMGLxzIo8bVxcK2+eafKHJW9cdTAwMWXLoMCMPGdPNCiTW7C4fFx1MDAwYofhXHUwMDEym1x1MDAwYpjOTSFSXCLgZl2g40LVJ9zZ4+hB5FgllVx1MDAwZnws9bGyRolcdTAwMTM72Y1cdTAwMDVQadSVL6jRjEhcdTAwMDNnX4RSqJFpXrxnUz6bc4ztLtdnfu5cbiXnXHUwMDBlKmVcdTAwMGJcdTAwMWN7XHUwMDE2K1x1MDAwN1x1MDAxN+/dYMjZYF1cdTAwMGZbQ386Ontcbu88Wn45ls9+XG6UXGZcdTAwMGVBXGJcdTAwMTSD8ZZcZq485FxmKFx0xlx1MDAwNFx1MDAwMo5w293RiDPH4fJcYvhYXG5cdTAwMDRmVXXeXHKtXHUwMDBiXHUwMDEwI1x1MDAxZLPG4FxcaqY7P3L2XHUwMDEwi/Z+oeRPi5I3e3fIXHUwMDEw5VwiLGRxPVvW07a7XGZOiYO5XHUwMDEwLmkjZH2VxHw15I9n8b3n+lx1MDAwNSvpk+vEMbaXrKtcdTAwMTA2L9FLxzRcdTAwMTGuhEMsbySTSPVcdTAwMDHYgsWpjL2wllxy3uxcdTAwMWMyaYp91tpqXHUwMDFmXHUwMDFln1x1MDAxZDtcdTAwMTPyYVtwqL50fVx1MDAxY/XD/Vx1MDAwZVx1MDAxN4I2XHKAXHUwMDAwj1txdJmWlmO9VLlFXVwijFx1MDAxN8wxcExCLVx1MDAxN7krXFxx5V9PXHUwMDBmNVx1MDAwMdJyP2/qy1xuwNJrXHUwMDBlLI/N6LNeW18zn5+dbmnn9ejN+NtPXHUwMDAxLKx/gHFyXHUwMDAziXOup375/PNcdTAwMGXmy1Q1O4FNYazXYqHF5p1cdTAwMWbWcC5QjD5WXGbGZ35cdTAwMWFuJJKZkazOjME2UjDh5uuGZ/5CloeLLKDTLJbVLs/tcVx1MDAwN7z4cWgqXGKQziekgqnWbyqUxKl0XHUwMDAzJWasi7OOlZeCRTb3LVx1MDAwZeTbh3ycUmRy15XzkoHdPNw0g8I06NCOmWPNv2c9l4JcdTAwMTmlaVOdXHUwMDFmRKiF87h812B6RlwiLpb3siCOL+JZXHUwMDA0mC1QoE7cMpP8Nlx1MDAxZsfBYZxGp002uXlcdTAwMWImsFlLXHUwMDE5mzR63oFAXHUwMDAz5DPRSIdbiyFtXHUwMDEzXHRcdTAwMWPVodhccqjyXHUwMDEyqlrtXHUwMDEzeY9cdTAwMTJiZsOJVqzoTOd7LF+8wbm8YVZAXHUwMDEzYn3O844gcm6CUlFcdTAwMDGMqVx1MDAwNMgvzVx1MDAwMXLydu3levfV+vHkZPLqcK+18+zDmftcdTAwMTlcdTAwMDDSXHUwMDBibp9rTpCVQU5cdTAwMWLFp/jInLbmiUWF+HXR6GiY01HCkNtOK49SdOShfWBg0HQlXFxcdTAwMDGoL/KTgvlcdTAwMDTj5S98/Hnx8Wbvhj9zOq9Gy4upfcr2ecpcdTAwMTbQhTWKkUXgXHJGUHJ2XG7HXHUwMDA3s6VKuVx1MDAxOXebKmCtu41MWSDky/fSvEvr3Vx0MDJcdTAwMWaOzFlfcEL1ua6SPFtcdTAwMDYsa9nEkHf43fjivk9cdTAwMTj3XHUwMDE4g1x1MDAwMK5Idlx1MDAxN0mRJpUlXHUwMDE4Nlx1MDAwN1x1MDAxZnNOXHUwMDEw9/DrxVx0zsjOx65cbqO1LfVcdTAwMWRg1Vx1MDAwMzdcXC1Zsoj1IzYz3CdcdTAwMGaYUd57mU9cZvktfVlz5lx1MDAxMVusjGLH1Fx1MDAwM9xGY9+wXG5zYGr/XHUwMDA2QzvXzPH65+fi+WhjXHUwMDEy14fH/YP+45WfXHUwMDAxplxcXHUwMDEwXHUwMDE5XHUwMDAyuHxcXFxidGMmQeiBU6BDjlx1MDAwN8DBgq85zeg7kCqyPYKDvKFKpEhcdTAwMTVAda6dzFpcdTAwMDQ2XHUwMDAziFmcUkFcdTAwMWFdPNjvXHUwMDE3TF1cbvtZYMqwKDxyXHUwMDFin4PrdFqvsizZXHUwMDEzQ1x1MDAxZOVEvnxSY3e5vnjNgyyrXGJcdTAwMWHEMnhR6qRhOVx1MDAxNTxcdTAwMTS8q7JcdTAwMWOeVJ9cdGNPv1x1MDAwZk5ohFx1MDAwZTzGqJRFZPmB5NQnlvHUQ6PKpOb2ST6rivtHXHQ2lnW+wbZS5lx1MDAwMydJxrzyl0nD6y6vyWRK2CNcdTAwMWJgOIuMvW/+mqf3g2Yyi/lnXHUwMDBmXHUwMDA3YFx1MDAxZNM+Vc6939y5byy/PJVH72T/zejZq4PuR1x1MDAxMJXD1k/h3KPJeD6E9uyJ1dPW1suT6kTGI1mDXG4sXHUwMDE1MndcdTAwMTSGuIxccuqsuYdcdTAwMWWDxVSFITxiXHUwMDEzYWye7VxyplwiXG7JT4VcdTAwMGZ6zvGsv9z7z+Deb/huTmDg1DbojVRsoUl9T9AsRDRcdTAwMWNcdTAwMTLrfGxcdTAwMDJcdTAwMDWSg1vzIXrkubI0+T9jgoZeU3BXvkGP5mKl8aRcdTAwMTA4WDZcdTAwMWJC/0tz7/LjX+GBTeR36fr9XHUwMDFhwVx1MDAxMYScXHUwMDExXHUwMDFh87NcdTAwMDLS3i+8XG4z01x1MDAxY1x1MDAwN1x1MDAxOVx1MDAxNXG2Vlx1MDAxZUCUK0CoYu2IK6dcYpnPwzJcdTAwMTjByspcdTAwMDbHarBPiO1fPCOA0VAqr+xcZlx1MDAxYcCUYCeJod157kCVQZ57TDykwbJcdTAwMTi/yWCaRVx1MDAwYuRgXHUwMDFmrIbK9zJcdTAwMTWbyZM7znKE95wugVWz9dUy5djLhbRcdTAwMDNU5IXqilEmJ1nZJp0qd1x1MDAwNc9zj0xwkYNOna/E54Pm+Hxw+nV15+1rtbr/eHly8Hx/b7S5t/tT4LOXXHUwMDE5Ylx1MDAxNlx1MDAxM/LxVXpakjXFZ6+jlGzjXGLBlq5rYfDMoeg8XydcdTAwMWakV9jJK8KzkVx1MDAxY6fIzm4mXHUwMDAzZ6tcdTAwMThpxlbNO236XHUwMDE3Pv/98Fx1MDAxOS7NKY7jNkbI4vnC5y+K4LxWcJF5o0f9XHUwMDA2heKwU4lcYoOzr1jYUfK4kufNeMVcdH+yPjdVJ1xywoLSPP9cdTAwMTJcdTAwMTDYwN/qTFx1MDAxOVx1MDAwZXdcdTAwMDD1tzntKFx1MDAwMVbJPO5dXHUwMDFlu1x1MDAwMjnJiMNcdTAwMWU5liCtp+AgNSNZ5sVTLFSDXHTknIqDR854UWm2dqUzudn2YjnCXHUwMDFmXHUwMDAxIFt56lx1MDAwNeazajxcdTAwMWJcdTAwMTelXHUwMDA07Fx1MDAwN1GGq2JFPoeQ/SCRgi1cdTAwMDaSmFx01GecP5PTLFxuxer9MFj15V9Pi1Mk29eT03ynuHrYXHUwMDFjV/Xmy1x1MDAwZk9Gf/a3J6Mvr56oP198XFx+uvNT4CrPI+JcdTAwMWWAZi4plnOa0WWSXHUwMDA3I1xuS2an7yjs9fmoalx1MDAwZeFcZiE/nbVcdTAwMDJXbcZBhEpyyIxjdVdcdTAwMTlXYdImn9v/XHUwMDBiVv8xsKqIc5bTTlx1MDAxMGolQa/gjOzI5njOXHUwMDE4N7FBs71iIGhcdTAwMTFSXHUwMDA2do2rNEqVRKFcYj/Hmf7a2/rJcDxcdTAwMGaR43VcdTAwMTD4IFxmjDrdQirJa3B6XuQ0XHLNMdMw11JDoeYkN1x1MDAwNKk8OU9yUnGdMJdxYpXFs8N1sb4jXHUwMDExZzi4N7KDgPN66vvFXHUwMDE3K43Za8lkLlx1MDAwMzLO6k9niM16glqBgadS5CV4nP0lXfnUII7K4dBSXHUwMDFlYGNcdTAwMWG0XHUwMDEwyHxcZlx1MDAxNDsnWVtSPE06XHUwMDE3mOVnI0SBgFx1MDAwNC6pyaTiRVx1MDAwYrxcdTAwMDBfJkLYoIJbL3U5sI7PcKOZM1x1MDAwZXlq6J1C86jbnpzDT1x1MDAwNT47N1x1MDAxN55cdTAwMDNcdTAwMTXAXHUwMDE2XHUwMDA2fFx1MDAxN9F50Fx1MDAxY52ft0++rTzb3zz9eLKy9fjLeGdrd3I4XHUwMDA3nVx1MDAxZsg5gTygJJOcvpXvOFx1MDAxNrO5XHUwMDE3yOxcdTAwMTGMWsEt9Civ22zs8p/rgbllO6HqqEAtJI/gcPnEcp6nMYvLlqrL7IqJ5z8z4a7lsCOvZHU7wo1cdTAwMDD3v1d6d6lal+mkv+bj8NVn/lvQ4Fx1MDAwYsXY31x1MDAwZSdPN072d7e0X373xT/92lx1MDAxNvJRUdWvdLhFXHUwMDAwfHT1yl+/Vcu9fPukezpJXHUwMDA1XSRghp2vR4/XOyudg9WVI2V2tvybUC/2/MNxONK7J6erXHUwMDFmPq7q39tu64VcdTAwMTg8edzscmf4zVxmKWFPdeHMznpSUmny193llZzKh3Su9oi9OD7WI2qwXCIpR8jVXHUwMDFlP+xcdTAwMDZcdTAwMDPWeJuo/Vx1MDAxNZeEXbDnntP7XCL3UmZVdrpcdTAwMTFZYXu3O2az245xjj/Za7X3jkbd+/QoN7Kvokf98KjS7FKmeG7WXHUwMDFj3MpjpqfXUoi45NxtRjh07WIxQTffd5ctObmMXHUwMDA1cupcXFx1MDAwYvFcdTAwMDDWXHUwMDBlJ91cdTAwMTE/t7Sysba02Vx1MDAxZOHaXHUwMDBiizo4nGz2vuVRo0h++6x10OtzXHJsXCJypd/bPczjvS5cdTAwMDUn0DfptVv9qzdcdTAwMWP0Op1cIja2IbSFaHa01lx1MDAwNL1cdTAwMDaj3m7vsNX/o9FttI4mg3fd8fmNTEZH3eJcdTAwMTPrvrhcbreyXHUwMDBiXHUwMDFhdDtIn19FJCNcdTAwMGaZUV5V9lx1MDAxOVx1MDAwZZtjuvy0vt33fz7prmyqg6OzdSPN8MmDx/R8k1lZnscpgyh1gsho89HZilVlXHUwMDBmXHUwMDFh081579ecWPtHYXpcdTAwMWT2rr99erjf+Wrfb783x1+/tb6JYFx1MDAwZppi7+KpwrncV69ON4/fvt86Xt1queHOh6efd96+Wlx1MDAxOKZHUTgj/LaYXlxy11NzL1xmQ6tIsIH/XHUwMDE1hy9cdTAwMTbN/Wtzc69eu1x1MDAwN1x1MDAwM7lzXGbe5EfiOael4llPsWTwiGAzn4+tV/Z6g79cdTAwMDVcdTAwMGK5YkScXHUwMDBlyTGYXHUwMDAx6mBcdTAwMGJcdTAwMTNhKlx1MDAxOdGlhXvuXHUwMDFmqEIgentcdTAwMGK/dZpcZv6lOLXlXHUwMDA2kH5jsOZcdTAwMTPcXHUwMDE4XHKOe53uaGnYP1x1MDAwMqI+St6wMDSvwa0yms+9qEVcdTAwMDF5p9c6XHUwMDE4XHUwMDFjdirt2s+NzPNdLOVcdTAwMGLH2lx1MDAxNO161Nyuo32z1Xr2Yvxhy+rj/oFcdTAwMWavhP2jh1xy4zYg9LbO8UxcZjFcdTAwMWKZS+8yRu8sXHUwMDE3u9ao1U7sXHUwMDFhc71Rb6tcdTAwMWS1vV1h1MFkmkea6Fx1MDAwYpCeXHUwMDEyrqlVS4vrNCxTnzHvwIlSvrjvsUBcdTAwMDC/rOBZOIDrx0P9VFx1MDAxZX9ag8KAXHUwMDA2XHUwMDBmzdGbSWxcbrTi943h8N3+m82VvVx1MDAxM7HyuL39fOPsy8KA1kc97Vx1MDAxY78zoJVzgfbirFx1MDAxNa8rgXZ8g52symf8wIHWXHUwMDAx41xcPmHe89ygUnGI4kBuZcBYpTbfa5HzYNbbjF3VLFnikbVVXHUwMDA2OWuHXHUwMDFjXHUwMDEw7n3hXHUwMDFjilx1MDAxZoCyt1x1MDAwZpxvh7KDztLgcOk/h8e90eSo1cffXHUwMDBlXHUwMDA3ne5cdTAwMWShbVxyvMyg7fnFXV5aemGLQtxz71Jh3oWTXFzK5m3ZUVx1MDAxY3yohNtJc+u+3lx0PlDr9vnZ3JGDKKtcdTAwMDBX8XC3vIttXHUwMDAxgDvXvDloXG5cdTAwMGIkVcVMXHUwMDFlXHUwMDFlgaZlXHUwMDA1i3ZaOVxc8N3kvlx1MDAxYsDstWbPaee3MfvSXHUwMDFldOHCuudf3cT0ckfRPuJVspRcdTAwMDTL53lKlfA8WlNcdTAwMTZKaFx1MDAxZe22hnz+XHUwMDE5p59cYmvz3kdfXHUwMDFjXVx1MDAwZa09Tbe1Z1x1MDAxZUyyNT7vgicnf7yOz75+fP1cXO6O99eM3D34tFV1wVwiXHUwMDBiPFx1MDAwZVxcWU5mt1x1MDAxY7Y7c7k6XHUwMDBiXHUwMDFjXHUwMDEy54pNL1xyrnPBXHUwMDFi9WVXs9C9+lxui+DP8qUxTD8/Q5+6/e3BSUMuNNdbhjCXXGZcdTAwMDWeMeVtdY7x6Fx1MDAwNsHJtfstXHUwMDBm1FtKKeAued5gYFnEbHyiLGt6YExG+KiKU15cdTAwMTboL5XhVFx1MDAxYimkdVVcdTAwMWXTslx1MDAxZsrlfNVFISuqZHliL4dcdTAwMDL8XHUwMDEzXedccjyR4CBKYUVcYkJcdTAwMDQvna/wnXfgK6/fk0idu+ah34El/YGzg9zsXHUwMDA1qlxmtDxcdTAwMTZcdTAwMDPVv523rLRcdTAwMDf+zFjCXfvNaObO6lx1MDAwMl/xYe7mzHFzx3l9svyhOk7L0lwizdH5MfIwpdIgXHUwMDEyyVx0kDxcdTAwMWLB1WzPfJffzMBcdTAwMThl4PGobGIqRFx1MDAwNFe+U9vMiXl5nehl4dC6f5DPvIlHXHUwMDEyivOlrFx1MDAxMTpgvYMsluZduCRE8/qG7qiR41x1MDAxY/75ubO2/XHtQ/fP7c/RPls5a9svlZfJONJcdTAwMDfHM5GsXHUwMDE3MMpZlsmhXHUwMDE3KnqEXHUwMDE4uFxybuqZXHUwMDFiXfBP5T/n2Vx1MDAwNX9cblx1MDAxNrEg7zkvXHUwMDAzV6hcdTAwMWSccZ7B563dlYNcdTAwMGWluFx1MDAwMevsL0/c+3efj4N5XHUwMDFin3bPjle/rdiH7jyDN5nXzLCxXHUwMDBlu1x1MDAxOFx1MDAwYp9/XrLNJLJcdTAwMDXvXHUwMDBld7qcyzxrgSWPVeGpwlXOs+wzXHUwMDFk+1x1MDAxNp2aXHUwMDFl4f7T5uBC8ttrcnBcdTAwMWYu8lvr35F4m1xmXG7FXHUwMDFhadYtue5yiq36u1x1MDAxN5Vbm5s59/Mz50p4Y4Wo7K2U8lx1MDAwNnbb/mNldfRUfz3+2N48/bTtn1xy9+f1gDxcdTAwMTi7ldJcdTAwMDUwUFx1MDAwMFxmZ8JcdTAwMDQly4ZcdTAwMWJC5jiHaVx1MDAxMbtZc1xyV+vM8oymXGJcdTAwMTdcIsX1NXuXs0hcdTAwMDNrv439XHUwMDA3pc5f/PHHxtL/XVo5muzdt9VWfvVdXHUwMDFirSo8mrLRSi9cdTAwMTEzXG5TXHUwMDE1qUjV3Giv52NcdTAwMGbVaK3iKXhcdTAwMWHhfj47YaZK1nEstHYzVbKLNNpcdTAwMTBByfKj83i8oqmaVVJYv1x1MDAwYquFg8nPMbpzsL3igYst5ZohlVx1MDAwYlxmdFxuPmHahFvnXHUwMDEzjlx1MDAwZXunS2Ncbpzct0+o/Or7KEux892C4IhcdTAwMTOO2a5yXHUwMDBiurlbeLuxt663X+iNfbv5rLO25tdcdTAwMDZ784bUPVx1MDAxY7dcdTAwMTCFzni2XHUwMDE3sFFcdTAwMWLtSyRcdTAwMWPsm51OnGdcdTAwMTiu3Vx1MDAwNt9cdTAwMTGhLcTtvILiXHSYilx1MDAxZHRVaV/Fw1xcXHUwMDEwISRJ4cuKcjbdXHUwMDE5VYywfoaalOOz8dFZODl96d5+tJ0tfTZ4/alx8eee/PrxaGUy7m7vvVl9evx6c+/L5PfFeaJcdTAwMWKyk9uB9DXpRMtYXGZxe6U1mubWWP2MXHUwMDFmvDV6kV10pzrHMTglZi1gjt5cdTAwMDetjHHujsxRwiNcYlx1MDAxZnnYK/v6XHUwMDBifaOFvZhcdTAwMTmUltFEXHUwMDExwC3+QeR6Y9D5z+HqJfqNb1x1MDAwYqY19Sg1uFJRj1J9UYuC2W6/31x1MDAxYo4rmziMmlx1MDAxYjJzWq9hW1+VXdvmdt16ufP+3dej/cngz3ZU4vOnlY1h+2FcdTAwMTd/Sm9cXMZDg/OzPGYxVtpcZuCnOWdcdTAwMTiWXThqcaFWrU1cdTAwMTaU0JXFKMZJXHUwMDFljeuqMJaRvNI2/FxcZZ9cdTAwMWRxNGhcdTAwMGI93lx1MDAxZO9MXuy8dJ3BaLNx38Z39Ff83aFb67n1Zo6lS1x1MDAxY0RdZeGuuYVXL91DR27vY6aByVx1MDAxMVGt8bGQO7yooJCg2dFcdTAwMDdcdTAwMTMkp3V+n5HPa9RyIYuW50g5XHUwMDFmWOldUVA6i9xeW1x1MDAwM6hfgH3/LLj9mHN0/nO4eTaedFx1MDAwZu5cYrVrcKqM2nMu6Xsxu4k/m7f9n6u1c5xJYpS3ynObpqTWSvGYcckjp1XknIxcdTAwMTmVXGaKXHUwMDEzolx1MDAxNefLKqOFqCCTQCZcdTAwMGLwXHUwMDEzQVlCjSx4kPpm69uko360XHUwMDBiuWWztfSPKs2uXHSsXvVocUZcblx1MDAxZXEsKvZVWDZ/w4NBNohK4XOLasK+fcfWQjeL5+opf5ZnVXQqb1x1MDAwNppcdTAwMWLukddv/F/f97VUrOny1lhOqrFSXHUwMDFh4YunXHUwMDBmX1x1MDAxNSdcdTAwMTgpNU9H5tlcdTAwMTR3U6hwvc9bSuspLM+LM96w1cIqN3vFOjOe03csy7it9DcrSfuuMoQm2/5NKN21rjVcdTAwMDRcdTAwMDTzWjIq4lx0m+UzjKR03GRT1lx1MDAxYlx1MDAwMe8qZl2rXHUwMDEzXHUwMDE5x0PCOyO0goHGqnQ6vlx1MDAwM9/EXHUwMDAzt6DXYPi/fGu1b1xy3+9bLciv40D/Kt+q5o43MJwuzDlNXHUwMDBid63cV/DTR/OjXFzrfD3lz/Ksit6Hb23sqzhcdTAwMGJaXHUwMDEzN420XlxijVxivYBcclx1MDAxN77KZSDcrKr1PN5dJtX9XHUwMDBicq7Xp4FS58r9vVx1MDAxMFxcfoaUYqXvzFx1MDAwNUvAXT5S3MhcdTAwMTBcclx1MDAxYilcdTAwMWWWd62LQ838YymktsKFxFx1MDAwZVx1MDAwYna+0jxcdTAwMTD9dHo8XHUwMDE5a3/0+Wjr1dh8XHUwMDFhXHUwMDFkXHUwMDFm91ovXHUwMDFljqebXHUwMDEziCrhODCex0JpeIBQXHUwMDFhz1x0JpeBx5h4l1x1MDAxNanWgG3E6GDM4Fx1MDAxY77Z/Fx1MDAwMCypU7bYgfFcdTAwMDP2eVx1MDAxZj/fer92uqnt5/3na+J99/OuOPlQsLJcdTAwMTlMv3rlLlI031x1MDAxN+a+XXuyet9cdTAwMWK86XcuJnxtPd/deTV+fTjc+PN16Fx1MDAwZVx1MDAwNjvL7f6XWY41k5c+r1WKnsdfgfhcdTAwMWGeXG6nSlx1MDAwMax2MfNcdTAwMWNjXHJi76MsbKJfXHK/MVx1MDAwMa5cdTAwMTLE2nM2p5BcdTAwMTXp08Cpolx1MDAxY1x1MDAwMK0ssMIpeY2N3mIuz2TUOlx1MDAxY1x1MDAwZlsjrNnD8T635VnDXHUwMDA18CzlgzB2tqw2t7S5uFx1MDAwMMdcdTAwMTg9z2u/zSCxW6ah945efzRHr99cdTAwMWXGzeXO6N2qfHXw7kvTdPHD8kXXXHUwMDE4aPVdzlx1MDAxOGjF6D74+0xcIvxcdTAwMDct5Im2urTfqTmZz4M4Rlx1MDAxM4wvXHUwMDBleZ7WXHUwMDBlaM64XHIusIDX2Yox0Deb3fdPM8dcdTAwMWLM74N3XHUwMDE0NrAgu8rw1ExKaWp4XHUwMDEyXHUwMDBiXHUwMDE0iyNcdTAwMGJcdTAwMWXOXHUwMDA0P1x1MDAwMtbSWlx1MDAwN9+Gh7p0OTqnsLA/bIZfXHLsVSHvdTeyXHUwMDE4LL7eJ12b71x1MDAwMHxmPO1cdTAwMTbkU1x1MDAxNE9HvVx1MDAxY4HDc9eFMjEoo+MsXHUwMDBlg1Naza5A5lx1MDAwYlx1MDAxOTBWWLpQXHUwMDE5h8GH/Fx1MDAxNIbiIM9fhn++hlPDXHUwMDFmfTdcdTAwMGUjXHUwMDFhhvFcdTAwMTXr5lx1MDAxYU6dQXxcdTAwMTaKx1x1MDAwNvzN0lx1MDAxZHNcdTAwMTWVPzMqOpU2g9lcdTAwMGLLdVxcXHUwMDFmyC5cdTAwMTVzXHUwMDFkynqPxbFcdTAwMWHWalxc8cjhi8SBIdX1VmvQMFx1MDAxNlx1MDAwZsaimFx1MDAwNaWRr3V8S0mmw1x1MDAxOCl54rKNXHUwMDBlf1xys9f7nVnjmdzFLVJcdTAwMWL/uljUR63hcHNcdTAwMDKdvbr3R8e97snjysCaPyR0+bXRy3Rz8/zrX3/9f1xmJ4VPIn0= + + + + + EDGE NODE ON RESOURCE PROVIDER1. Node with exposed service at the edge of HPC clusterVirtual KubeletInterlink API ServerProvider pluginPod on virtual nodeVirtual NodeHTTP + Authunix socketPodContainersBatchSystemOIDCOIDC Identity Provider \ No newline at end of file diff --git a/docs/static/img/scenario-2_dark.svg b/docs/static/img/scenario-2_dark.svg new file mode 100644 index 00000000..388f2042 --- /dev/null +++ b/docs/static/img/scenario-2_dark.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1da1dcdTAwMTTJsv0+v4LF/XiHuvl+nG/gXHUwMDEzXHUwMDE0RVx1MDAxMVDvucvV0FxytDTd2DQqzpr/fvcuXHUwMDFlXVldTVx1MDAxNdIgzFx1MDAxMUdH+1x1MDAxMZVVXHUwMDE5XHUwMDExO3ZkZORff8zNzY9Ojzrz/5qb73zfafW67WHr2/yffP1rZ3jcXHUwMDFk9PGWyv99PDhcdTAwMTnu5J/cXHUwMDFmjY6O//U//9M6OsrG38p2XHUwMDA2h2ff7PQ6h53+6Fx1MDAxOJ/9X/x7bu6v/E+8023z++/86dMt8TFuqM314Ubn63Cl92Mj/2r+oYtcdTAwMDFccjs7o1Z/r9dcdTAwMTm/9Vx1MDAxZK9bXHUwMDFmM+mDdDFcdTAwMDTjZZSX757iXVx1MDAxZEQmpNUqaCmFXHUwMDE0Vl++/a3bXHUwMDFl7VNAMJlcdTAwMTA6XG6nrMDPWMB+p7u3P8JHjFK4RnTBm5j/mMvPnI3oX3Pi8pXj0XBw0Hk06Fxyhlx1MDAxY/Z/yVx1MDAwZX+NXHUwMDA3vd3aOdhcdTAwMWJcdTAwMGVO+u3xZ1q2XHUwMDFkdnfHn9nt9nrro9NcXDKeNJ7ofEn+1vnoZen1ad/CXHUwMDA19/b7nePj5DuDo9ZOd8RcdTAwMDclxfhcdTAwMGU4uqPldj5h/zdcdTAwMWXTsHXYWeaM9U96vcuXu/12h/Mwvy1cXHK5fvv8clx1MDAxN9M9nkt9/srf48F3OpQspVHWXG5cdTAwMTPH0zTWO6kmXn016Oc6qITU0Vx1MDAxOevHn+hcdTAwMWU/huKNcrG7rd5xZzxcdTAwMDFcdTAwMWPbk7FSJrdzctRunX1JeuWs01x1MDAwMVxuZsfz3ev2XHUwMDBmyt/pXHJ2XHUwMDBlxtf5o3BnJU3f0+9aq5tcdTAwMWa3t8P6in2pl/qjl3t7jTXdXHUwMDA3mylcdTAwMTNMULpC041xXHUwMDE5XHUwMDFlXHUwMDFl3rduiqZrfERoKy00fVLJtdRZNFbLwpt3oN37rZ39k2HnIei3n4F+x6jgQ6Sv0m8np+q3UVpcdTAwMDZf+Np9VG/devtl7fDJalutb+mTha0vL/bWXHUwMDFmTar3qPN9VPLhwmTaXHUwMDA0LVx1MDAwMow4XHUwMDFhqU2i2lrILESt4Xg1noOXdkK1g5GZXHUwMDEy2rtgLb5cdTAwMWVdhX7/VutqtVx1MDAwZdVqnXz8XFx/XXRCeYBphfqq4Kapr6N3XHUwMDBlzsbZqq/XmPBcYpu6hvqOtZFaiPtX2dy7/c7csHM4XHUwMDE4deaOhoOv3XZnONf5fjQ47sy1+nOLa8tzo1x1MDAwMV7o7JzgXHUwMDEzO4P+qNXt41x1MDAwNlx1MDAwYjOP19a7P3JLXHLJq09bh91cdTAwMWUnyiZcdTAwMTde7HX3+Mzmd3CXneF88cGNulxioS4/MFx1MDAxYVx1MDAxY43fvbz25HxcdTAwMGWG3b1uv9V7N8Pbap2MXHUwMDA2bzvHZzc2XHUwMDFhnnSKz7nz/MKwYHn2XG6vMPSvX33e+No2neen4vny83c7XHUwMDBiq6Yx6Fx1MDAwNSmyqG2cXHUwMDAyelxi/oKV8GRhWnhH51x1MDAxMXNfNzW8s/hEVPIyulx1MDAxYuvTb0dRdFx1MDAxNIs3xz9N7Fx1MDAwM1xiVuGfXHUwMDE2U1x1MDAxZIiR0UVE8D+Df/97ObzxQC+V0/f1Zlh76T7vhD3/2qy+f7a2LC9vM1HQ1nA4+DZ/+c7ff14ld/vlM/3mQPdfLb3vf/ji9M6HJff8RnIr0bNwxa/t5d7Xb90t+7nlP6/8eN4+7q5cdTAwMGZcdTAwMGJiz/82nu9bXHJcdTAwMDWqR9MgXHUwMDE0XGJRZFx1MDAxZdGOhcmLXGKLTu09alx1MDAwNFxuXHUwMDFl/mBqkFx1MDAwYlx1MDAwNpHh/SiisN7h1ibt/XpcdTAwMDFu9HJHXW3gu7udnVx1MDAxOP9cdTAwMDFcdTAwMDa+2TxcdTAwMTSQ2jlcclx1MDAwMLZVpuzMNEtcdTAwMGVRXHUwMDFhfDHeg0h2XCJcdTAwMTTY7Fx1MDAwZUcnrd7ci5NtjqhcdTAwMWHfRfLqjfH9sNtuXHUwMDE3MS+F+DrwLKP+1Du4KZRfXHUwMDE4LD89XzHhPqppM25AKKVVXHUwMDA1clPQuFx1MDAwMj2Yrlx1MDAwN2cuZePtkWptrlx1MDAwZo9WzYr+0VrrvVnZmWZzO8PB8fHCfmu0s3+XdjfpTXibzqnMSYTHcGvOXHUwMDFinbo0XHUwMDFkTWaENFx1MDAwMn5cdTAwMDYxjPelcV3PXHUwMDEzaeV2Onb8mUuvqGJmjIaBXHUwMDA0Jb0u2OelV8Q0ZVG4XHUwMDE4nbQyXHUwMDE04thzg49CWyO8XHUwMDFkXHUwMDBmv+hmxjHA8sbeidj6rvufn784Pv34eFW4zV5cdTAwMDGx5vefPO/udlx1MDAwNt2WtCed02BcdTAwMGW3Py1+vUCqWnc11S9N4P4soa4wKa3haFx0XHUwMDFh3O3vlb/S6benvNNrXHUwMDFkj1x1MDAxZVxyXHUwMDBlXHUwMDBmuyNcZmNt0O2PJrwq5S4yXHUwMDE02O+0Jm5cdTAwMWGSp753RHHpXHUwMDFjjP82N9bF/Fx1MDAxZpd//78/Z/DpXHUwMDA1oLH2ylx1MDAwMJZccv4vxJh359/PQtCRYbtC+Fx1MDAxZY1cdTAwMGW1XHUwMDAyTcisilx1MDAxMVx1MDAwNFx1MDAxZVReqHHAx1x1MDAxZqWgw0LxSlx1MDAwZdpo3F2LXHUwMDAzUYAjw6dcdTAwMTGcIPhcdTAwMThcdTAwMDcnuTiTIWJcdFx1MDAwMEVcdTAwMTVxTafrxelMRuYz8Ds4L01cIi5kyoBvS8hTXHUwMDE4nqxcdTAwMTXnXGa8t5XGOlx1MDAxZoJcdTAwMTUmXHUwMDE0xXmXOeWCt1xugZGxvn4qZi9O6+Bhd87Bv7jkZpmlN1Akj/DPS+/rpelM6OAwuUqZXHUwMDEwk7FcdTAwMDWd4UWtXGZcdTAwMTidlsrUXHUwMDBiU5nFjVhcdTAwMWKEXGYyWF+ShkjSWi8trlx1MDAxNbWqN1xumWlcdTAwMDUyqUBZvESwk9yqhNp55aOQXG6oXHUwMDE4naq/2ZlcdTAwMGKEbmlcdTAwMTmNkc5cdTAwMDOSgFxuIVx1MDAxNWgzXHUwMDAxg8VcckOLqe/1XHUwMDAyTWZcdTAwMDNcdTAwMDYnuEqijbRleUVgXHUwMDExtaqML2BSbdDCOG1cdTAwMDXGcIW8em2ZtbhcYv1cbspcdTAwMDHXI3SwsFx1MDAxYXT59HBcdTAwMTFosqDxhFr90yqDXHUwMDEzXHUwMDEyIDRQsqjExOw6xMxcdTAwMTi1h1x1MDAxZlx1MDAxNbZ2MmBcdTAwMWFcdTAwMDZTayW+XHUwMDA2M/Bl21xiTkRLr62gXHUwMDAxdyvMZrhHXHSnXHUwMDAxc4OV+5JcdTAwMGIwXHUwMDFh3Fx1MDAwMNQvwFnr2ECacFx1MDAwZU8mXHUwMDBmaUo+gP5cdObvXHUwMDEwMmGunKtcdTAwMWRcdTAwMWJ8MVx1MDAxM1xmXHUwMDAxeIY5gFx1MDAxMZV8sYxQXHUwMDFm4JhcZlx1MDAwMJNo71pcdTAwMWNUXHUwMDA0+lx1MDAwMb6F+zSw8jLygOkqqlxiPDUmqtZcdTAwMWJcdTAwMWKosGNcYlx1MDAwNGvVUahUXHUwMDFjXGJ0lFx1MDAxNupmXHUwMDFk5sO5WoswmcQkYPpwL87CXHUwMDA1TYRcdTAwMDFBaFxciuIwYbX2P2t5XHUwMDBihFx1MDAxN2GNXHUwMDBi8HpQXHUwMDA3XHUwMDA1X5WIXFyQ/EDQ+FxyXHUwMDE0cFwiXHUwMDE43VmQtfc9LVr6o/j/61IpqVx1MDAwYkmPXHUwMDEyl4JLdjBcdTAwMGahXHUwMDBi9HlMplx1MDAxZTcnU1x1MDAxZjs7P37o5d1cdTAwMTf66LB/9OP15qfj9TCFTN35+vNVNIrTXHUwMDFkpVx1MDAwM7aPs7R5ZkjSfyhlnaUvMOpmPGq3mK+9IFFgaj5oXHUwMDFmoURcdTAwMGV+ulwik1x1MDAxYzVLXHQ0dNdGLZRcdTAwMWZTjIusSYCWOVx1MDAxM/149NUs6jdJ+seQJJnB6XCpXG6BNEJmMITUm0XrTaBiXHUwMDEzLKSuXHUwMDA1g1x1MDAwNdiBdULCO0LP4MJtXHUwMDEyMVBcdTAwMDVcdTAwMDVcdTAwMDBcdTAwMTR+RGNcdTAwMTZjXHUwMDAzXlMn0DuAlTfe6lx1MDAwMHJjbz7Cn1x1MDAxMIiIXHUwMDBi2ijxXHLlXHUwMDE1/ixcbnSIj8HI4Fx1MDAxOOG5SUtcdTAwMWHIy6uCXHUwMDEw8SPug2dxacQqM8bFmsE0JJtGXHUwMDAyQ1x1MDAwNOIr4Fx1MDAxMTy0XHSmXHUwMDE0xWWIPOHYXHJcdTAwMDIlkIxcdTAwMDZ8SWVcdTAwMTFPkLya6Vx1MDAxYVx1MDAxYlx1MDAxMqWReITAelx1MDAxMUI0QuBp1EPgrOX5LHAqXHUwMDA0XHUwMDEwyYUgUy4swVx1MDAwNzT1XHUwMDBmXCI9M1v1JFxmj8jTtXpcdTAwMTBxREZRpVx1MDAwM3RcdTAwMDRoeHZcXItcdTAwMTVCoV5lpM+AXHUwMDEymFxmTqPwTiXyyK5jRFx1MDAwMKZNtOAk9WxdyVxm8YP3IKhcYrTAXHUwMDA1k5hJZtB271Uw+Fx1MDAwMGLhXHUwMDA2Rlx1MDAwN1aCMETDy4L6wlxiklx1MDAxYkZMh9BcdTAwMWFIgfBcZnBcdTAwMGZuVS/QZVx1MDAwNnOhSGb4R2JcIlxuXHUwMDAxKVwiRJGXkXCe63nszOVcdTAwMDVcdTAwMDTsXHUwMDAxUZhcdTAwMGWYSuO8Klx09Fx1MDAxNnpktFxitM3a+J+0XHUwMDE46kdcdTAwMDaCINo7m/J2hXjA4+GCXHUwMDFiKFxc0tfTk1x1MDAwNVxyskMzgFx1MDAwMkZcdHdtSlx1MDAxYWNhIVI6q6XX0ddroDZcdTAwMTlUVighvJVcYnB0SZ5cdTAwMDR3XCKpilT8eoWZtTjQXmvxjFx1MDAxME1cdGhuiVx1MDAxNfuM9TZCgcLgIdZcdTAwMTOoXHUwMDA1zVx1MDAxMYBqWfhcdTAwMTD4ulhcdTAwMWVcdTAwMWXAKMD3wNRcdTAwMWOu1SDrXHUwMDExYFx1MDAwMcBEQCfXXHUwMDAzjS7PbnB4XHUwMDBlZERcdTAwMTZKUy/OM1x1MDAwMc7QS2Jm7STlgWdcdTAwMDaNZJkoXHUwMDFjW4PhOdBcdTAwMTAjXHUwMDAwZDBP3FM5tVxiPVx1MDAwMkPxgVx1MDAxYzDWu/tZi4P7MFx1MDAxNrdcbupcdTAwMGXfkWaM4FtgM1x1MDAxZU5cdTAwMTHeXHUwMDA1M9JAnMgs62ujg/ODXHUwMDE5pJbrwHXzJKpcdTAwMGY+ssCtiTxcdTAwMDCHZfZcdTAwMTUzga+VU59cdTAwMDY6JF1cdTAwMDRcclVcbuFDvW9hXHUwMDExsLWMJ4R1oMrpXGJjhpcxtXm6htmZu1x1MDAxN8gsj4LqsYBcboFH6vwwXVFrwlx1MDAwYqK22CBdwVx0XHUwMDA2XHUwMDE2McHJzDt8cFGcXHUwMDExLHlBZENcIlx1MDAwYiBuXHUwMDEwYMFcdTAwMWJcYviP4Fx1MDAxZFx1MDAxZIxcdTAwMTjXVOXyPNNcdTAwMDWMW7QwXHUwMDAwq2b6XHUwMDE3cL9KgpBhMpP5tVx1MDAxYeFhxKtcdTAwMTa/NOKEXHUwMDA2t2szPFx1MDAxY1x1MDAwN0PHTCBcdTAwMTJwJXlcdTAwMTE2i5CBvkU0yajOWFx1MDAxYyxcdTAwMDCuXHLgXHUwMDBmz2ZKy1xm1mYwQsSscMvMudaripFcdTAwMThcdTAwMWRDXHUwMDFkXHUwMDE4PKbPlcXB4TFX7fJ0WYM1XHUwMDE1g0BcYpGzsdBVX0qmWIfRMVTzXHUwMDBlylx1MDAxOZqsqeB+gPpcdTAwMDEu2WEgqWuBOC1cdTAwMTGf4Vx1MDAxZHgsUN0m4qAmXHUwMDExYVx1MDAwNpCcy1x090ucy1x1MDAxOLND61x1MDAwM2JF6cvimG9zXHUwMDExMFx1MDAwNG8mXHUwMDFhmC0sXHQ46G1UXGZJQml4XHUwMDFlimfgtlx1MDAxZMJmKVVcdTAwMDNNcVRVXHUwMDA1XVx1MDAxMZg+hCxcdCtcdTAwMTJcdTAwMTmuhClcboh+I4Kuei9cdTAwMDB5XHUwMDFlZlx1MDAwYlxi545cblx1MDAxN9LUNlx1MDAwNLKoXHUwMDA2XHUwMDExKUhcdTAwMDKRvMH9MsglXHUwMDBmNVx1MDAwMFxyaFni9Vx1MDAxY1wiang9sFx1MDAwMybSgeNccuRxr1x1MDAwNoJ6+FxywH/UKe2AQHJp4XAhllx1MDAwMDXg0kYzXG5cdTAwMDLr8cIhXGL3M5BcdTAwMDdcdTAwMWbAJTGAgjdpXHUwMDE0XHUwMDA0eYZLOFxu0ZE3IMB3Lk5keVxiyy0s4LWlu1x1MDAxNaRcdTAwMWNwVDFgmps4PVx1MDAwMSdcdEzQjiFpMjRcdTAwMWLBoJmwXHUwMDA2/ZJgdVxyVs1AXHUwMDE5XHUwMDExXHUwMDBmIIBknbtQZXFcdTAwMDA7k9dwXHUwMDAyLkJcdTAwMDP6h2BcdTAwMWWho4dZRlx1MDAxNlx1MDAxZlx1MDAwN1eShyeKq1x1MDAwMFx1MDAwNUDLRVx1MDAwM7IxY3FcInfxxFsvgFx1MDAxOCU9htVYXHUwMDBiplx1MDAwMUWHfdhcdTAwMDbcynN4eYWHjnD26cx61tQhWlHBSlbZ14tTLLrVXHUwMDAwfY9cdJHKJHbrLdyo0+SnXHUwMDE2XHUwMDE3bWBcdTAwMTVgXHUwMDEzuFx1MDAwYvhcdTAwMTM4XHUwMDE1XHUwMDAxh1BarIlZvizk+Vx1MDAwZUKPenFk7/BDllx1MDAwYpiAr/JCklxiXHUwMDAwW0V272KD1cvZi1x1MDAxM/CQXGJwpGVcdTAwMGXMJ7pcdTAwMTLg9MByXHJCc3haVX+zpDpcdTAwMDZBNdNcdTAwMDRRK2/S4XFLT8A7XGJFIbTBw1NcdTAwMTJeXFzgblxm1C9cdTAwMDSfXHUwMDA2P4FrU1xiZrxG3MaYpomuKMNsn42MVmT0JXlcdTAwMTFcdTAwMTG859akvJjhV8iDXHUwMDA2a6lcclcxrVWJ7kWRIfrxzCpFxHpNUpFcdTAwMWOfXHUwMDBl0jDtwNKSxDZcIqaXcVx1MDAxMVx1MDAwMkghgaSN8kq5XHUwMDEzRaSsMcsmXduLXHUwMDE4PdRcdTAwMDSBXHUwMDFlvVx1MDAwZjxzvfrFjGvPnmFcdFAtzVNNLFA0XHUwMDExx1xmXHUwMDBiiTO9m4ul0YFcdTAwMTZ6uK+gXHUwMDEwXHUwMDE5xPq7zW2N7JPEXVx1MDAxMGhK8oBLkMiMbINwillcbuiCg81cdTAwMDZcdTAwMDG1SEtoYp7GjfCisEdj61fiKVx1MDAwZvhcdTAwMDJcdTAwMWbkXGYwXHUwMDE3dq9Lulx1MDAwMpsgXHUwMDE3jJ6+osHdzlqey1x1MDAxNF2u8UKwpNKXbUN4RDLQYrh71SCJIeHJqSNEXHLmatNcIp9guaNcdTAwMTDxmYh4iMHV+1x1MDAwMlx0Vopb1cYhzFx1MDAwZUxDJfIuXG5zXCJDltBE+aiuJGmAfVx1MDAxYVxcwnJDvsrgNFWd1l1cdTAwMGZrkivbXGKVLbdcdTAwMGXheyVPj2iPPlx1MDAwNbyU4OtcdTAwMWKY2ozlOSalJVx1MDAxN06otvD5tvz44K5J/Fx1MDAwMuhDfZSRJ1mA4WSeilx1MDAxYqXSpFx1MDAwM8s3QFx1MDAxOVx1MDAxNFVcdTAwMTOwV0/9wJ48ZoLZcqiY0+ViXHUwMDEwlrEhStOYZF+fXHUwMDFlxcOzYFx1MDAxYbgwnLJQpjS514RJaopcdTAwMDNcdTAwMGZTTEBcdTAwMDI9oiqrijHGscBcZtDha5c3Zi3OMPlcYv+pXHUwMDExqMNLXHUwMDA1U3p0XGLFPVRcdTAwMTig3GBtiF5cdTAwMDD4XHUwMDFkmdtjXHJHKDtcdTAwMDFcXIf2XHUwMDAyu3H1wVx1MDAxOX1cdTAwMTTiWlx1MDAwYuKnPZPbZVx1MDAxZmWiYb1cdTAwMDRcdTAwMDDKXHUwMDAz0lx1MDAxYYjjpDHxXHUwMDAzozTSluFRXHUwMDA2Q+vHezHWP7jZSlx1MDAwM/iAX1x1MDAwMFx0uCGRlVx1MDAwMiXw4YKpUVQ5j8mtXHUwMDE1h7BcdTAwMWYqXHUwMDBmho1JXHUwMDA14sqSNIBcdTAwMWI4XHUwMDBiUFvDy9daXHUwMDE34op8MVxyNDRfb1FlLItQOFBcdTAwMGZN9lZfXcowReXs0Fx1MDAwMZtVlKEkzsPy8tVApp1rtYRRXHUwMDE0yDpcdTAwMDNNuFtTXG5cdTAwMDNcdTAwMTBEsTiJ1MhBmepcdTAwMGL8MDgwUHJcdTAwMTVti/t9+Vx1MDAwM1xyXHUwMDBlXGK3QfOYXHUwMDBmN/XckVx1MDAxMa2RxktcdTAwMDMnXHUwMDAyN1eOZ1xyKFx1MDAxOTRcXEqSt7tcdTAwMTXGZYroSWvyVHdcdTAwMWGfXHUwMDAwX1x1MDAxMdBcdTAwMDXmuFx1MDAwNcs7a8WRWVx1MDAwMIhcdTAwMTnwO1x1MDAxNlxmJpNcdTAwMTC4XHUwMDFjXHUwMDA2XHUwMDA3jetcdC551VtqQOirmIOFedFzpz6OvFx1MDAwN4jBkk64uVqshqVcIog2XHUwMDA2XHUwMDFj02ouaCZuxCNsx1NQXHUwMDFlYSdT+/VcdTAwMWGiyVx0YVYwbEY0XHSygoCqfFxyVpMg1+e1wI7xdDEkRCXQKZeyd7BjxCxcbozWXHUwMDFhXHUwMDA07fX0WDNcdTAwMGYmWF9cdTAwMWNcdTAwMTHTxURFQN31WUgqmVWVtWNjYiEwXuLWdqlEObGAwFx1MDAxNVx1MDAwMVx1MDAwNHxcdTAwMTKD7NrHxixcbtfXWG9cdTAwMDI2XHUwMDE4fEmaM8EhmGe6rUEp6YylcVxyXHUwMDFmgb9cItIgvlclaXyUXHUwMDEy4Vx1MDAxZcIohC21XHUwMDBmzojMM+3p89yXK60hOVieIykwfLDwS7XiNLBcdTAwMDRgyqKV/I7KmbacQ1xi06CgXHUwMDFjomBcbqw0jFxm1VRZXHUwMDE0O5xcdTAwMDRWglx0XHUwMDE36lx1MDAwMZVcdTAwMTlK4Fx1MDAxMrc8UodFuiz9y8VB32H2XGIyqb1A6pI0J3FcdTAwMTlIglOS9WSY6faomPr2XHUwMDFlyC4n0s9cdTAwMThcdTAwMTbQWVu4zfpcblx1MDAwMSbbXHKrRDFcdTAwMTOK+ZiSwuFcdTAwMDJcYu1cdTAwMDRzdj7Uj41OxLN6PGj6xVR94UvzMFx1MDAxNsEgnka9XHUwMDBmyceG20AgXHUwMDBmnlRKtENcdTAwMWHrknC31Dhdv1x1MDAwMsXeL5HF6Fx1MDAwNrbqnPflNVx1MDAxZeijY/k201x1MDAxNfWAmi/xcPU/wq5YI1OSpsglWNiFXGK9ntZcdTAwMTiTcVx1MDAxMZJKwqovN7H+JFx1MDAxOElcdTAwMDOBuL/srqXJvOpE535cdD4kVVx1MDAxMW5cZoCTh1x1MDAxZVqbLzY0cHHgb0w3kL25wma5c3lS6Hz7XHUwMDEwo7zQXHUwMDAwXHUwMDFjbObhY+GQXHUwMDAyd7rEiXVKxFx1MDAxNDov/UboWF/zwVVZXHUwMDE20MnccbtcdTAwMThL4lxcXvtcdTAwMDCPXHUwMDBmX19cdTAwMWbEzVxcXHUwMDFhXHUwMDFmtGCNhrbCJzrHXHUwMDA1aMWkXHUwMDFh7pX5iPp6srxYXHUwMDA2lsr1XHUwMDA0oYTVyUxwgZxcdTAwMTdcdTAwMDLwKuBH/cY0jeA8Wlx1MDAwZldcdTAwMTFcdTAwMDVcdTAwMTPtXHTmc+hcXFwiXHUwMDE0XHUwMDBlgVxuS6FqR6eAdlxiXHUwMDEzuFjsXb417s/0XbB7jYFcdTAwMDO+XCJCnPpcdTAwMDAnMryCL1x1MDAwM1x1MDAxZEU4bSe2kjHWXHUwMDBlzKjCpTTa4TJLcZpcdTAwMWNSXHUwMDEzOT2huFxcPIJcdTAwMDfGnUZcdTAwMWVcdTAwMDZcdTAwMWRcdTAwMWHsr2JQXCJYh+00s1x1MDAwNuneJZa2eD5a4ERcdTAwMDD9bTA4l5GSXHUwMDA3rvBcbm47SMQpxlx1MDAxOLD9vF5N1Ps67bnLjes5JC/pUrvKi1hcdTAwMDWYXHUwMDE4aDRcdTAwMDPh+ludrbSQOUW9YtEtfFRa71x1MDAwNThcZl6y0tCykqc+OVx1MDAwN3FcXHiUKlx1MDAxMIylKtd7WeY0XHUwMDExTSmjSDBcdTAwMWHcq1x1MDAwNEpBn8BHjInlajQylMhMXHUwMDEwXHUwMDFioKn6m8W0clx1MDAxZFOCykUny9JcZotcZpWEftvQwPrBN/LnkqfJgNmmJE4zvWb42zeh+oj5I+tcdTAwMGLZ7Y5bOmTp0Vx0qq8hYVx1MDAwMTWoXHUwMDFmXHUwMDFkXHUwMDE4hFx1MDAwN+hgbvFfjCnocCYwTZhcdTAwMGJcdTAwMTZcdTAwMWWxmvnO5SFcbmTZXHUwMDEwwMLAMl2YqORj1pOrzlx1MDAxMm6iPmud15HCXFzhikHSValcdTAwMDKFdaSYJ6bsSLh1g/RcdTAwMDFcdTAwMWLnXHUwMDAxXHUwMDA3XHUwMDA08Vx1MDAwMKYhJ1xuXVxyXFxAsJo7TuvrcFnXK1x1MDAwNLw3xqaYK3UlXHUwMDA3xVx1MDAxNV8ouYzcyONqp3fm8ljLXHUwMDA3Y2ImmfWEKT3hdND88Vx1MDAxNjg8XHUwMDFjY504iaBcdTAwMDbMOeQtW6DKaVx1MDAxNlxmtqGYXHUwMDAwNzHkXHUwMDBi0/XawsJcdTAwMDFcdTAwMDfogUUxQkpT6lx1MDAxMkhcdTAwMGI3YFm7QFCvj9pVXHUwMDA2oiM8XCJisLtcdTAwMDBcdTAwMWWdelx1MDAwMjA1xFx1MDAwMVxchVdcbuAu659cdTAwMWXgynKjMpRcdTAwMDVcdTAwMDNcdTAwMTRpSj2v84eHhW/hXHUwMDE2gFx1MDAwNkXH3IhA3mRYd1x1MDAwN8qVslhuRICL5Vx1MDAxZWuEldzaVy8v39nLRXbH9EhaXHUwMDExlHHZxebWXHUwMDExXHUwMDFihFx1MDAwMVx1MDAxOXydZWZcdTAwMDRq7126XHUwMDFhgSeHK1mF0I/xmKsvXHUwMDA2qlx1MDAxMUd/XHUwMDAznPPceIFcdLlrcZLkkola1lCD/qfJNVx1MDAxNlR6zzImhKBcYu5cdTAwMWItvrDaXHUwMDE4w4BcdTAwMGLCXHUwMDA046mnXHUwMDAyXHUwMDA1S1x1MDAwYulcdTAwMTJcdTAwMTXA1nM9vlZkXCJg/Ok/iv//++xDf52/WLvdMNrpXbc83FxySG1l55YnzTdcdTAwMWL+WDj5XHUwMDEyPveOT1e/v3q192bUXmw92n9cdTAwMTCdW1jGxEVcdTAwMDewXHUwMDEyXHUwMDAzrUjbUpp8XHUwMDAzQlx1MDAxNJhZwIeMwpRcdTAwMDY2o9Yt4Nsu5Fx1MDAwYlx1MDAxMyCBhf4w484t+Fx1MDAwNFxcXHUwMDFjeFxmgiNX9HFcdTAwMTfbXHUwMDBljeJSu7Hjx/B71+GFsIey63CBJWPw23lcdTAwMTdcdTAwMWaWkKaJ54uldzBcdTAwMTWmk12DTVx1MDAwMlx1MDAxMOglQ27FojUhjLtCYJMqtJlcdTAwMGJcdTAwMDTHXHUwMDAxv/HSsjSBtbaJvJLa15c7MGFBMmVdvqyRJlx1MDAwNKXNa1lcYmdW5FnjWnnTTJM/rHjzmFx1MDAwNoXAyLP1RIMquVx1MDAxOYvLV3DIlri3gNncXHUwMDE0ISX4NstcdTAwMDJcdTAwMWQnqj7fzi2OXHUwMDFlcVx1MDAxY4uk8laZpW2sLFFyzLmb6Fx1MDAwNcLRXHUwMDA22lwiXHUwMDA0NZqENLD1RSgxXHL2XHUwMDA0VojL8Tr35lx1MDAxONtZqE/83Fx1MDAxMkhObWjHjFx1MDAxZVx1MDAwYs2r+5s9bY6SO6cvn+283H/bsd+ePl56rtbDl+31XHUwMDA3gZLOkbpcdTAwMGKEg2Bcci6WWjaye1xylMZprqlDt29cciUlc1x1MDAxNyTiXHUwMDFhodW4XHUwMDAwNsVJgIx0TFx1MDAxYiPoXHUwMDFht6y72J1vJJRcclg5biv5XHUwMDFiJy+EPVx1MDAxNJy83qdDXHUwMDA2mss5Z2FcdTAwMGWZYkKGM7glduZcdTAwMDJf0kbI+jKJ6WrIXHUwMDFmz+p7z/lj0+JYX7xKci9ZWCFsXqOX9mlcImBcdMSoXFybtbFcdTAwMDFcdTAwMDObsTiVcTMs+5FwXHUwMDBmuDFpjn3C2lx1MDAxYVAw6fPTNfJuWyDRvjQ+9vrhgodcdTAwMGJBm1x1MDAwNlCAx63Yu0xLy75eqrxHXYLHXHUwMDBiJlx1MDAxOdgnoTZcdTAwMWG5JWRcdOVXx4daXHUwMDA0PDiur1chy7NrIMvG65VcdTAwMWarX+1g7VtrKJ6Fo6PXK/5BIItcdTAwMGXsWqDg7IJXflxccHh+4lx1MDAwNbuOIfpjrFx1MDAwNNJTaFx1MDAxZj1TZLH5zlx1MDAwZmvYXHUwMDE3KEZcdTAwMWarkMVzf6yNXGbNXHUwMDE4rY5n7KJxft4my/1cdTAwMDaWXHUwMDA3XGYsiKdZLKtdntvjXG548esxM0GA9jBFjFC1flGhJE6lXHUwMDBiKDFjXZx1rLxcdTAwMTQssrlrcYjafGBBXHUwMDA3kFwiKO3KecnA3TxcXDTT+Es9MMbMsebfs55LwYzStKnOXHUwMDFjj2hcdTAwMTLOY/iuQfeMRFxcLK9lQVx1MDAxY9/Es1xiMFuAQJ24hfzom7xcdTAwMWRcdTAwMDebcVx1MDAxYZ1ussnN2zCBzVrK2GSj5y1cYjRcdTAwMDA+no7icGsxpNtEXHUwMDAyW3Uo7lx1MDAwNlR5XHRVrfaJfI9cdTAwMTJIs2FHK1Z0pv09XHUwMDE2zj/gXFy+YVx1MDAxNvG7jPU5z9tByKlcdEo8PSfZn7Bw7NNcdTAwMTghnzdHyLcv3q98etp331x1MDAxZn/f2ttaXW/tvlr79iBcdTAwMTAyOHaPjojlJHd0jVx1MDAxZtUlQkZub/JcdTAwMTZBjr4l6lx1MDAwNYA0zOsoYVx1MDAxON2Oq6pTgDSWa1x1MDAwZtwmXHK2PJGilOxcdMlUxe9cdTAwMWPlw4XI631cdTAwMWEuzem8IC2vp/ZpvFx1MDAwZpeGXHUwMDE4n3t+eTafri/HN1x1MDAxOdunsIMwd1UpN+FxU1x1MDAwNaz1uMBM9nB1+XJcdTAwMWE4TJotZH1T3lx1MDAxZpntvpRv0ISyJM+WMYtHXrDAXHUwMDFlvv3ag7uZMC4zXHUwMDA2XHUwMDAxaJHcYCRFmliWmedmJG6RlVxcxq9cdTAwMTcn2CY777wqjNa2tPVcdTAwMDCzXHUwMDFluOZqXHUwMDE5KItY32Uzw33yeFx1MDAxZeW9l3nTkD/TtzXbXHUwMDFlcZeVUdw0dVx1MDAwZlfSJMvCXHUwMDEwjVTh1HJznPLt1rOjIPtvTlx1MDAxN9pra7td/+1t+82DwCnWY+f9v5Xi8S2llTTLdjFcXKQy3JMqVWlcXLPBqcj9XHUwMDEx7ORcckVijFRxXHUwMDA2wpluMmtcdTAwMTG4XHUwMDFiQEygVOC+X23s7/adXHUwMDBmXHUwMDE3pHjWoItcXMdn5zqdXHUwMDE2rCxIbopRJFwirOdmq8bOQn31mke0rFwigiDWwYvSVlx1MDAxYdZTwT/BtyrLNfX6TFx1MDAxODf1++CEXHUwMDA2d+BcdTAwMTlGpSxcIutcdTAwMGYk2z6xjqdcdTAwMWVcdTAwMThVJjVcdTAwMTdQ8mZVXFxBSpCxrPNcclx1MDAxNpYyXHUwMDFm2Eoy5qW/TFx1MDAxYV41vCatKWGP3Fx1MDAwMcNmZNz85q94er+mKXOcPOZo3JNZsVx1MDAxMNX4yp7MK82d+8uvvZMlp1x1MDAwNotvvVx1MDAxMlx1MDAxZsWPo6er2/1cdTAwMDfh3D1r8tllXYPRjutcXM58O1eHWLjpgYFs+WpL45rZ+lx1MDAwZneos+hcdTAwMWV6jFx1MDAxOGZMXHSL6z9cdTAwMTZmXHUwMDFm82xvMJNcdTAwMWMkP1ZM2d9cdTAwMTTkXHUwMDAxe/drfppcdTAwMWRcdTAwMTjYtVxyaiNcdTAwMTW30KSuJ2hcdTAwMTZcIlx1MDAxYTaJdT42QVx1MDAwMsnGrXlcdTAwMTM9XHUwMDA2ubLU+T9jgoZOU3BZvsFcdTAwMWXN2UrjSSHwr9xsXGL1L/W9g3V4PIBoXCKvpeuXa1x1MDAwNFtcdTAwMTCyR2jMz1xuSPd+4V1Ymc7PO+aKan1BOTCUM0CkYvGIK6dcYpnPwzRcdTAwMTjByspcdTAwMDbHanCfXHUwMDEwt3/xjFx1MDAwMFKhVF7ZXHUwMDE3NEApwZ0koFx1MDAxN0F6LkCVMZ5LTDykwbJcdTAwMTi/SWOaWVx1MDAwYmRjXHUwMDFmzIbKlzJcdTAwMTU3kyd3nOVcdTAwMDDv2V1cdTAwMDKzZuvLZcrEy4V0XHUwMDA3qMhcdTAwMGLVXHUwMDE1KSY7WdkmO1VuXHSdp5+YoFx1MDAxNUitnnL83Ivm6Nzb0kJcdTAwMGXk0L1cdTAwMTFuZNZW7PLSrn5cdTAwMTDobLnZiVx1MDAxM8iliUJ99iU8W8RWTrOvsFxm5pZcdTAwMTbRcJVg2H5M5Z30ZDU8gydbnjnEZeBiQHWOz0ww5rpYd1x1MDAwNt1vgP6nXHUwMDAwNHyaU+zHbZjmjukmQ1x0XHUwMDBm6jwsXFzofKdH/VxuhWK3U4Sjns2vWNhRcrmSXHUwMDA3znjFXHUwMDE2f7I+M1UnXHLCeG515MZcdTAwMDfVwOHqTFx1MDAxOXZ3XHUwMDEwztg87ighVsk87lxcXHUwMDFlt1x1MDAwNbKVXHUwMDExuz2yL0FaT8FOakZcbp67JOl2XHUwMDFiJFxuXHLPfHLki0pzb1falJv7Xix7+INcdTAwMDByL0+9wLxZjefORSmB+0GU8apYks8uZL9IpOBcdTAwMWVcdTAwMDNJ0Fx1MDAwNOyT509kNItCMXu/XG5XY/nVXHUwMDBiXFy1YPi6kJgoourL5qi6/MU9+jL6MVhd+mDfrbVbj3rrg96DQFV31lx1MDAxMIeN2KDxslT06CX3Zlx1MDAxYsVWpcq6UFx1MDAxYdeMQNXnjarZgjOwOMVVgarN2IZcdTAwMTBBkOAup+JS6cW5rkE69oj/ndL8z8FURZCz7HVcdTAwMDKilVBewVx1MDAwZdmRW+PZYdzEXHUwMDA2W+1cdTAwMTVpoFx1MDAwNaFcZtwzrlKOKlx0QVx1MDAxMU6OXHUwMDFk/XlaeP1e9izPp0nQXHUwMDFlkMCo09WjkrxcdTAwMDZn50X20tBsMlxy6CptJ9Ts41x1MDAwNorKc/Mk+1x1MDAxNNdcdHNcdTAwMTn7VcGkOS5WdyTiXGbb9kZuIGC3nvrd4rOVxtS1ZCaXdIyd+tNcdTAwMGViXHUwMDEznqA+MVx1MDAxYngmhVx1MDAwMZWU7PwlXfnMIDbKYctSXHUwMDFlX2NcdTAwMWHsIJB5XHUwMDEzKO6bZGVcdHG9NLU8XHUwMDE5IVxuj/DLuCZ9imct8Fx1MDAxY3mZXHUwMDA24f5cdTAwMTTcemmTg/CeXHUwMDA3XHUwMDA1RrY9YDv9WrC/XHUwMDExLlx1MDAwZjs7ozP4qVx1MDAwMGetp4Iz1MlcdTAwMDVbaFx1MDAxZllE59Xm6PzktTs+XHUwMDFkLlx1MDAxZC3ufu24XHUwMDFm611p7NBNQef91s7+ybDz65E5MFWUZ7HYv0LGlO/awMZiRkd22Fx1MDAxN4W+25PI3OGvq5G5Zduh6qRALSRP4HB5w3JcdTAwMWWnMVx0zJa6y+SKiWc/ZVxcholalt6NWXBcdTAwMTGXr1x1MDAwNbh/XerdhWpdJJP+no7Dl9/5q6DB54qx/fKZfnOg+6+W3vc/fHF658OSXHUwMDFil1slOtxcIlx1MDAwMM5fvvP3n1fJ/dJ5tvhi9Opg/WRl6dmnL892n348WbyR3IuPjzrfR6mgsyu+XHUwMDBiXHUwMDFmPo7ebu9+fbMqv+wtii39fG+jIHZcIr65eVBSafJXjeZSTuXNnCm9s1nkuVxybK/nQ1ooXHKOd9bVyzCdWNT5y0gy+oz77dm5L3JcdTAwMTllUl9cdTAwMGKh/qTd/dxcdTAwMTGbnZ1cdTAwMTjvjze5lnVcdTAwMTX96eZ8pdGlceJ5kVx1MDAxYk9ZgTuYXFxN5Fx1MDAxY16xxFx1MDAxOHlcYknxzNPprrtsyMk4Zlx1MDAxOFLnSkhcbtdcdTAwMWZ1hvze3OLa8tx6Z4jBXHUwMDE3ZnXQXHUwMDFmrXd/nO1nSF592jrs9nL1TEQu9rp7/ZzudSg4Qb5Rd6fVu/zAYbfdLkLjXHUwMDBlhLbAZIfLTcBrMOzudfut3rtGt9E6XHUwMDE5XHLedo7PbmQ0POlcdTAwMTSfWOf5JdvKzqOgn0P0gt2VXHUwMDExnUdcdTAwMGIyXHUwMDE2q9LAV81cdTAwMTE9XHUwMDFlfHuybLZPlzfeya/L71x1MDAwZV5snmxu3Vx1MDAxZlx1MDAxYpyC6II96GWsRnRcdTAwMTdExsS11fqeI7q0gWdcdTAwMWZcdTAwMTai1HtcdTAwMDTpz+KT0bfY0ivv9j9/P33U2txcdTAwMGYnXHUwMDFiXHUwMDBmXHUwMDEy0tvt5UX9ovu29/bLu07v8cHCyudHP35cdTAwMTmkV4+mXHSkXHUwMDA3bnJgfVJkqU0oaX00mWIxJPtcdTAwMDBXYzp7XHUwMDFisolj0Fx1MDAxMXHkuJ/Jb0yvxfTX14B0hedcIpRRXHUwMDEzbIyzNH3rgkBkb0PhRKd7XHUwMDA06WvDwdduuzOcO+qdXHUwMDAwJO9cdTAwMDOa11x1MDAwMFdcdTAwMTnNp97BTYH8zKhXeur7vvvwbct+XHUwMDE5nK682lLLh8ePJo260+t1j447qV3zXFzujFx1MDAwN9g4zap2Oa5LyFxyW/OsK3aNvzDssYJcXFq2NllQQktVsVx1MDAxNdGwhS+Pw3XXsu5dXHUwMDExdoS4gXWXrPT+2vabatueXHUwMDA01ItymjGgXpg8ezqrQjosieGmllwiXHUwMDAw/HlYn/qZIH5cbnTXQeHqs8dmY/Fwb/NH5/irfvmpc7D0etRcdTAwMTS5by1cIvj++dWXzd7S0/dLL/tb6t361vHn983k3i10Vz+9XHUwMDA20M3W7zyWi/1QXGLdhW2/uYtkk3imVVx1MDAwM2tkpZnEbvbyt0BtzbP/lKxqZmGvipp/xrrLke9Dxe6318DuwHaPslhcdTAwMWM0tmQ1ieiX4C25+9RGZ3/ClO9cbrz/3Vx1MDAwN5E9vlx1MDAwZthdg5bTsLt8XHUwMDAzs4Huz/7Rxt7K4fu1b5uPnz39+PrL3klrOGnU7W7rcNBvl+zaRJexoe1cdTAwMTToXHUwMDBl+elq3Os/XHK6XHUwMDE129wqLpRVYbdiyyZt5V1j91x1MDAwM7Lu9ZujN1x1MDAxZKrwXHUwMDE15p1bxlT4jmznaf3PxOs/id6LrXerg80l+2pvZ/HxxtHbXHUwMDBmj9vvn/zjULb6LpugrHXsKMeGh9E5X9xdd1bZXGJrzFx1MDAwZppzStvqQFx1MDAxYZG4XHUwMDAwRyZOi1x1MDAxOFxuq8O3iLP/XHUwMDE0S3zXXHUwMDFjZ1x1MDAxNVv5umJcdTAwMTg0trjCbtOJgFmwRuF+guyg/e/+o1x1MDAwYpi7XHUwMDE3KFtcdTAwMDNsXHUwMDEzKDvtXHUwMDBlZlx1MDAwM7NXk4eS60nNWnLTg46SR4xPgixocSZcdTAwMTG2xemJr6Cyqlx1MDAxZT08hkD6yJ3H1zBptVx1MDAxYjvG/GeY9MaNwVVcdTAwMWJcdTAwMWVcdTAwMWVcdTAwMTBkdUA91dKV4IG+qrBcdTAwMTdzhrbOht0/Y+uzbVx1MDAxZVRUSv4sXHUwMDE09XH87Vx07G1YgzauYCvMVufsITXJoOVOaueEt5tcdTAwMWZNrrTmSYiOlSSFXHUwMDAyrPm91lx1MDAxMac578BvtVT5Wkfx+Fx1MDAxMij/93SgXHUwMDEzupJcdTAwMTTcTVx1MDAxYu7VpCFcdTAwMTmuys/6MHBcdTAwMWPsXHUwMDA04Iu9XHUwMDEwzsfrMsRcdTAwMDc8XHUwMDAwOjBnnpxcdTAwMTjQYLzXKvIru7ROb3vwrVEgtH7w5MPrJ3ZnaeX49PWHx+tcdTAwMWZOWrvfXHUwMDFiMlx1MDAxM8uTk/OjMc58ZqnoxfnMs2nQ9Fx1MDAwMoBgMs2DfvX56lXF5nglbWatuVx1MDAwNf+5rXbV9vZDTyxOqVx1MDAwM7hcdTAwMGU10YH7rXSsKlx1MDAwZlBmXCJ8unCfrPlcdTAwMTO6cFx1MDAxMuHtc5ON5TdPhFo8tVx1MDAwYs+erfV0e/v9ilxcaMpNfF9vhrWX7vNO2POvzer7Z2vLclbcJHpb6DhyM25SfZdccrhcdFxiXHUwMDA1l6xZf4hh6ZD2ZWJfQ8mjhZ3hWVx1MDAxMVx1MDAxNfbobVx1MDAxNkFMuI1cdTAwMGZ/iipz/Fx1MDAxZMVMscOt5sQkalx1MDAxMVx1MDAxZNvnVFx1MDAxMZMgptlcdTAwMWK7tmmeRnRcdTAwMWbrcVx1MDAxMNjPXHL6c//uf+1cdTAwMGVHJ61cdTAwMWX+1lx1MDAxZrQ794Gi1CBcXFx1MDAwNUXhnVxc3Ed6XHUwMDE3s6EpV3uiq2iK1zJcdTAwMGJBqmhCXHUwMDA14jq2VLKFVGCVhcMvqFC5iFx1MDAxN13GY6WlXHUwMDExMVDVxlxu8dvUi6b+/saQXHUwMDBi3HRsjFnlXHUwMDAxlJq6fG/Zw8hcdTAwMTRcdTAwMWKF/8P4SoV68mdhQjPHgiageWbU5Wq/UeBcdTAwMDL51rqIXHUwMDAw38W8XHUwMDA3XGKPVJ5kLz5jO2hhbd5cbi5cdTAwMTRcdTAwMGZznFx1MDAxNXtcdTAwMTn6168+b3xtm87zU/F8+fm7nYVVM4VsOcvdoTyr0rJcdTAwMTlnXHUwMDA12VxuPDfjvPBMXHUwMDE0j8K+P+Tl6orwq1x1MDAxY2n0cKRcblPFsnwjS8xF68wqXHUwMDFlcDY125OF6LmxPVx1MDAwNLhkVzhcdTAwMDeqWFx1MDAxM1x1MDAwMaW9XHUwMDE15vJPcaRcdTAwMWZu7Ei5pDKlsFnJ6ZXNXG50XeooXG7Nk/5ZjnRhuoLyp6Cad+FJXHUwMDFi+yV6Uq9MhJ9cdTAwMDdcdLHsa1ZoT3LumWRcdTAwMDavZaMunnIwK1x1MDAwZnp1+XbiQYVyikfFXGJtrOCRdHZipIonXHUwMDFmhGuO845cXOfVXHUwMDE1sle7TsCHUS569lx1MDAwMLKl7r+Op1x1MDAxY1x1MDAwN8OzcmW171RcdTAwMTl3XHQqpWN+ILyoaNyrLXjsb9d5hev8ePOkucf8iThZOZZTzelVKEJcdTAwMTieWONvY4HsXvjOqfrJn4Jm3oXnbOyPXHUwMDE2zlx1MDAxY5JQmv2nvORhuZNcdTAwMTlpm92O27xGllx1MDAxZq5S5O3NlJeG7SYmXHUwMDA2aSZqVe6T35RcdTAwMDf90D+wJ62Pr9pcdTAwMDfD+PTTx1x1MDAxZlx1MDAxYofNKuvZ5FNcYqCx4YnupY5IXCIgJlfjZHlF3YBzmefmeclTiGWsOlx1MDAxMkRfx1/+J9XWf6v2l1XpOefy8/2qXHUwMDFko566migj29PEKG8jqLxpem7zPJv1ampOLiSv3jgnN1x1MDAxYVx1MDAxY01LyCXjLmffqlx1MDAwNzqbtNvTXHUwMDFk032z8Gr7udlaf3n4TD1pf3600sh0rcysZjdcdTAwMDQu9Mm0NFx1MDAwMPSZwTSmalx1MDAxYVlcZjFTLj+03rO5fKG/83iV6zr1Pv9Jcc735naLR0/qUGGi83l9x1TDtd475+y93Od60u9+nzvmZ0aVdjvu/vVr7bZynLPav5paZGFSp3eKMjy/geVcdTAwMDRVSnU632Cmz3zGXHUwMDBis6lcdTAwMGVcdTAwMGU/tX/Yg+7w82prZfR+ZdpcdTAwMTkt92frqvWZUWzgYuKEv3JcdTAwMDZvXHUwMDAy3kTtztWf8TM/5/MuXHUwMDBljPbEXmuqm0Ld925P17bKqXbzXHUwMDBmNlnlp/ph76JH0F/d3e1Hc5Ncclx1MDAwN1x1MDAxYm8+PVxub7fXh8Njv/rp+eNnXHUwMDA38r6brFx1MDAwNCXI2Nteapv3wihlVVxcYNmWm1xmMmZptFK4LLAy2VxurySMt2q3wITVSmZ70mWpWzLb8zcqqztcdTAwMGXWtpbF8FU/PDpuP+qPXHUwMDFlP9566T79ssrzmzmF5+/erc3999ziyWj/rp1C5aVnXHUwMDEzfl+9N+DK4lxcXHUwMDAzzJLGgfSGiuJcXGs1uyRoO3XZm9S7cslbhex3mnF6+C3jfKWZXmeFxmqedVvNpv30JVx1MDAxYWG4QqPkbVx1MDAwNOX3XCLNWNJJ/ixcXKrj+MtcdTAwMTPOaWa5xca1rswtXCKc5Fx1MDAxNrEgjJWI5yazdjHjOZBGXHUwMDBiNvf18nq1ro2SjFdvNSiN1/F8XHUwMDE0nlXGLuOxopjYZ3jbq2C0cizkvd5y/OyTjX+cz/F86+hofVx1MDAwNI29fFx1MDAxMvNfu51vS5VOhj9cdTAwMDSyfGB0Mp3cOv/+4+//XHUwMDA3bbKKhCJ9 + + + + + 2. The remote provider expose an API to execute containersVirtual KubeletInterlink API ServerProvider pluginProviderAPIsPodContainersPod on virtual nodeVirtual Nodeunix socketunix socketHTTP + Auth \ No newline at end of file diff --git a/docs/static/img/scenario-2_light.svg b/docs/static/img/scenario-2_light.svg new file mode 100644 index 00000000..be103dea --- /dev/null +++ b/docs/static/img/scenario-2_light.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1da1dcdTAwMTTJsv0+v4LF/XiHuvl+nG/gXHUwMDEzXHUwMDE0RVx1MDAxMVDvucvV0FxytDTd2DQqzpr/fvcuXHUwMDFlXVldTVx1MDAxNdIgzFx1MDAxMUdH+1x1MDAxMZVVXHUwMDE5XHUwMDExO3ZkZORff8zNzY9Ojzrz/5qb73zfafW67WHr2/yffP1rZ3jcXHUwMDFk9PGWyv99PDhcdTAwMTnu5J/cXHUwMDFmjY6O//U//9M6OsrG38p2XHUwMDA2h2ff7PQ6h53+6Fx1MDAxOJ/9X/x7bu6v/E+8023z++/86dMt8TFuqM314Ubn63Cl92Mj/2r+oYtcdTAwMDFccjs7o1Z/r9dcdTAwMTm/9Vx1MDAxZK9bXHUwMDFmM+mDdDFcdTAwMDTjZZSX757iXVx1MDAxZEQmpNUqaCmFXHUwMDE0Vl++/a3bXHUwMDFl7VNAMJlcdTAwMTA6XG6nrMDPWMB+p7u3P8JHjFK4RnTBm5j/mMvPnI3oX3Pi8pXj0XBw0Hk06Fxyhlx1MDAxY/Z/yVx1MDAwZX+NXHUwMDA3vd3aOdhcdTAwMWJcdTAwMGVO+u3xZ1q2XHUwMDFkdnfHn9nt9nrro9NcXDKeNJ7ofEn+1vnoZen1ad/CXHUwMDA19/b7nePj5DuDo9ZOd8RcdTAwMDclxfhcdTAwMGU4uqPldj5h/zdcdTAwMWXTsHXYWeaM9U96vcuXu/12h/Mwvy1cXHK5fvv8clx1MDAxN9M9nkt9/srf48F3OpQspVHWXG5cdTAwMTPH0zTWO6kmXn016Oc6qITU0Vx1MDAxOevHn+hcdTAwMWU/huKNcrG7rd5xZzxcdTAwMDFcdTAwMWPbk7FSJrdzctRunX1JeuWs01x1MDAwMVxuZsfz3ev2XHUwMDBmyt/pXHJ2XHUwMDBlxtf5o3BnJU3f0+9aq5tcdTAwMWa3t8P6in2pl/qjl3t7jTXdXHUwMDA3mylcdTAwMTNMULpC041xXHUwMDE5XHUwMDFlXHUwMDFl3rduiqZrfERoKy00fVLJtdRZNFbLwpt3oN37rZ39k2HnIei3n4F+x6jgQ6Sv0m8np+q3UVpcdTAwMDZf+Np9VG/devtl7fDJalutb+mTha0vL/bWXHUwMDFmTar3qPN9VPLhwmTaXHUwMDA0LVx1MDAwMow4XHUwMDFhqU2i2lrILESt4Xg1noOXdkK1g5GZXHUwMDEy2rtgLb5cdTAwMWVdhX7/VutqtVx1MDAwZdVqnXz8XFx/XXRCeYBphfqq4Kapr6N3XHUwMDBlzsbZqq/XmPBcYpu6hvqOtZFaiPtX2dy7/c7csHM4XHUwMDE4deaOhoOv3XZnONf5fjQ47sy1+nOLa8tzo1x1MDAwMV7o7JzgXHUwMDEzO4P+qNXt41x1MDAwNlx1MDAwYjOP19a7P3JLXHLJq09bh91cdTAwMWUnyiZcdTAwMTde7HX3+Mzmd3CXneF88cGNulxioS4/MFx1MDAxYVx1MDAxY43fvbz25HxcdTAwMGWG3b1uv9V7N8Pbap2MXHUwMDA2bzvHZzc2XHUwMDFhnnSKz7nz/MKwYHn2XG6vMPSvX33e+No2neen4vny83c7XHUwMDBiq6Yx6Fx1MDAwNSmyqG2cXHUwMDAyelxi/oKV8GRhWnhH51x1MDAxMXNfNzW8s/hEVPIyulx1MDAxYuvTb0dRdFx1MDAxNIs3xz9N7Fx1MDAwM1xiVuGfXHUwMDE2U1x1MDAxZIiR0UVE8D+Df/97ObzxQC+V0/f1Zlh76T7vhD3/2qy+f7a2LC9vM1HQ1nA4+DZ/+c7ff14ld/vlM/3mQPdfLb3vf/ji9M6HJff8RnIr0bNwxa/t5d7Xb90t+7nlP6/8eN4+7q5cdTAwMGZcdTAwMGJiz/82nu9bXHJcdTAwMDWqR9MgXHUwMDE0XGJRZFx1MDAxZdGOhcmLXGKLTu09alx1MDAwNFxuXHUwMDFl/mBqkFx1MDAwYlx1MDAwNpHh/SiisN7h1ibt/XpcdTAwMDFu9HJHXW3gu7udnVx1MDAxOP9cdTAwMDFcdTAwMDa+2TxcdTAwMTSQ2jlcclx1MDAwMLZVpuzMNEtcdTAwMGVRXHUwMDFhfDHeg0h2XCJcdTAwMTTY7Fx1MDAwZUcnrd7ci5NtjqhcdTAwMWHfRfLqjfH9sNtuXHUwMDE3MS+F+DrwLKP+1Du4KZRfXHUwMDE4LD89XzHhPqppM25AKKVVXHUwMDA1clPQuFx1MDAwMj2Yrlx1MDAwN2cuZePtkWptrlx1MDAwZo9WzYr+0VrrvVnZmWZzO8PB8fHCfmu0s3+XdjfpTXibzqnMSYTHcGvOXHUwMDFinbo0XHUwMDFkTWaENFx1MDAwMn5cdTAwMDYxjPelcV3PXHUwMDEzaeV2Onb8mUuvqGJmjIaBXHUwMDA0Jb0u2OelV8Q0ZVG4XHUwMDE4nbQyXHUwMDE04thzg49CWyO8XHUwMDFkXHUwMDBmv+hmxjHA8sbeidj6rvufn784Pv34eFW4zV5cdTAwMDGx5vefPO/udlx1MDAwNt2WtCed02BcdTAwMGW3Py1+vUCqWnc11S9N4P4soa4wKa3haFx0XHUwMDFh3O3vlb/S6benvNNrXHUwMDFkj1x1MDAxZVxyXHUwMDBlXHUwMDBmuyNcZmNt0O2PJrwq5S4yXHUwMDE02O+0Jm5cdTAwMWGSp753RHHpXHUwMDFjjP82N9bF/Fx1MDAxZpd//78/Z/DpXHUwMDA1oLH2ylx1MDAwMJZccv4vxJh359/PQtCRYbtC+Fx1MDAxZY1cdTAwMGW1XHUwMDAyTcisilx1MDAxMVx1MDAwNFx1MDAxZVReqHHAx1x1MDAxZqWgw0LxSlx1MDAwZdpo3F2LXHUwMDAzUYAjw6dcdTAwMTGcIPhcdTAwMThcdTAwMDcnuTiTIWJcdFx1MDAwMEVcdTAwMTVxTafrxelMRuYz8Ds4L01cIi5kyoBvS8hTXHUwMDE4nqxcdTAwMTXnXGa8t5XGOlx1MDAxZoJcdTAwMTUmXHUwMDE0xXmXOeWCt1xugZGxvn4qZi9O6+Bhd87Bv7jkZpmlN1Akj/DPS+/rpelM6OAwuUqZXHUwMDEwk7FcdTAwMDWd4UWtXGZcdTAwMTidlsrUXHUwMDBiU5nFjVhcdTAwMWKEXGYyWF+ShkjSWi8trlx1MDAxNbWqN1xumWlcdTAwMDUyqUBZvESwk9yqhNp55aOQXG6oXHUwMDE4naq/2ZlcdTAwMGKEbmlcdTAwMTmNkc5cdTAwMDOSgFxuIVx1MDAxNWgzXHUwMDAxg8VcckOLqe/1XHUwMDAyTWZcdTAwMDNcdTAwMDYnuEqijbRleUVgXHUwMDExtaqML2BSbdDCOG1cdTAwMDXGcIW8em2ZtbhcYv1cbspcdTAwMDHXI3SwsFx1MDAxYXT59HBcdTAwMTFosqDxhFr90yqDXHUwMDEzXHUwMDEyIDRQsqjExOw6xMxcdTAwMTi1h1x1MDAxZlx1MDAxNbZ2MmBcdTAwMWFcdTAwMDZTayW+XHUwMDA2M/Bl21xiTkRLr62gXHUwMDAxdyvMZrhHXHSnXHUwMDAxc4OV+5JcdTAwMGIwXHUwMDFh3Fx1MDAwMNQvwFnr2ECacFx1MDAwZU8mXHUwMDBmaUo+gP5cdObvXHUwMDEwMmGunKtcdTAwMWRcdTAwMWJ8MVx1MDAxM1xmXHUwMDAxeIY5gFx1MDAxMZV8sYxQXHUwMDFm4JhcZlx1MDAwMJNo71pcdTAwMWNUXHUwMDA0+lx1MDAwMb6F+zSw8jLygOkqqlxiPDUmqtZcdTAwMWJcdTAwMWKosGNcYlx1MDAwNGvVUahUXHUwMDFjXGJ0lFx1MDAxNupmXHUwMDFk5sO5WoswmcQkYPpwL87CXHUwMDA1TYRcdTAwMDFBaFxciuIwYbX2P2t5XHUwMDBihFx1MDAxN2GNXHUwMDBi8HpQXHUwMDA3XHUwMDA1X5WIXFyQ/EDQ+FxyXHUwMDE0cFwiXHUwMDE43VmQtfc9LVr6o/j/61IpqVx1MDAwYkmPXHUwMDEyl4JLdjBcdTAwMGahXHUwMDBi9HlMplx1MDAxZTcnU1x1MDAxZjs7P37o5d1cdTAwMTf66LB/9OP15qfj9TCFTN35+vNVNIrTXHUwMDFkpVx1MDAwM7aPs7R5ZkjSfyhlnaUvMOpmPGq3mK+9IFFgaj5oXHUwMDFmoURcdTAwMGV+ulwik1x1MDAxYzVLXHQ0dNdGLZRcdTAwMWZTjIusSYCWOVx1MDAxM/149NUs6jdJ+seQJJnB6XCpXG6BNEJmMITUm0XrTaBiXHUwMDEzLKSuXHUwMDA1g1x1MDAwNdiBdULCO0LP4MJtXHUwMDEyMVBcdTAwMDVcdTAwMDVcdTAwMDBcdTAwMTR+RGNcdTAwMTZjXHUwMDAzXlMn0DuAlTfe6lx1MDAwMHJjbz7Cn1x1MDAxMIiIXHUwMDBi2ijxXHLlXHUwMDE1/ixcbnSIj8HI4Fx1MDAxOOG5SUtcdTAwMWHIy6uCXHUwMDEw8SPug2dxacQqM8bFmsE0JJtGXHUwMDAyQ1x1MDAwNOIr4Fx1MDAxMTy0XHSmXHUwMDE0xWWIPOHYXHJcdTAwMDIlkIxcdTAwMDZ8SWVcdTAwMTFPkLya6Vx1MDAxYVx1MDAxYlx1MDAxMqWReITAelx1MDAxMUI0QuBp1EPgrOX5LHAqXHUwMDA0XHUwMDEwyYUgUy4swVx1MDAwNzT1XHUwMDBmXCI9M1v1JFxmj8jTtXpcdTAwMTBxREZRpVx1MDAwM3RcdTAwMDRoeHZcXItcdTAwMTVCoV5lpM+AXHUwMDEymFxmTqPwTiXyyK5jRFx1MDAwMKZNtOAk9WxdyVxm8YP3IKhcYrTAXHUwMDA1k5hJZtB271Uw+Fx1MDAwMGLhXHUwMDA2Rlx1MDAwN1aCMETDy4L6wlxiklx1MDAxYkZMh9BcdTAwMWFIgfBcZnBcdTAwMGZuVS/QZVx1MDAwNnOhSGb4R2JcIlxuXHUwMDAxKVwiRJGXkXCe63nszOVcdTAwMDVcdTAwMDTsXHUwMDAxUZhcdTAwMGWYSuO8Klx09Fx1MDAxNnpktFxitM3a+J+0XHUwMDE46kdcdTAwMDaCINo7m/J2hXjA4+GCXHUwMDFiKFxc0tfTk1x1MDAwNVxyskMzgFx1MDAwMkZcdHdtSlx1MDAxYWNhIVI6q6XX0ddroDZcdTAwMTlUVighvJVcYnB0SZ5cdTAwMDR3XCKpilT8eoWZtTjQXmvxjFx1MDAxME1cdGhuiVx1MDAxNfuM9TZCgcLgIdZcdTAwMTOoXHUwMDA1zVx1MDAxMYBqWfhcdTAwMTD4ulhcdTAwMWVcdTAwMWXAKMD3wNRcdTAwMWOu1SDrXHUwMDExYFx1MDAwMcBEQCfXXHUwMDAzjS7PbnB4XHUwMDBlZERcdTAwMTZKUy/OM1x1MDAwMc7QS2Jm7STlgWdcdTAwMDaNZJkoXHUwMDFjW4PhOdBcdTAwMTAjXHUwMDAwZDBP3FM5tVxiPVx1MDAwMkPxgVx1MDAxYzDWu/tZi4P7MFx1MDAxNrdcbupcdTAwMGXfkWaM4FtgM1x1MDAxZU5cdTAwMTHeXHUwMDA1M9JAnMgs62ujg/ODXHUwMDE5pJbrwHXzJKpcdTAwMGY+ssCtiTxcdTAwMDCHZfZcdTAwMTUzga+VU59cdTAwMDY6JF1cdTAwMDRcclVcbuFDvW9hXHUwMDExsLWMJ4R1oMrpXGJjhpcxtXm6htmZu1x1MDAxN8gsj4LqsYBcboFH6vwwXVFrwlx1MDAwYqK22CBdwVx0XHUwMDA2XHUwMDE2McHJzDt8cFGcXHUwMDExLHlBZENcIlx1MDAwYiBuXHUwMDEwYMFcdTAwMWJcYviP4Fx1MDAxZFx1MDAxZIxcdTAwMTjXVOXyPNNcdTAwMDWMW7QwXHUwMDAwq2b6XHUwMDE3cL9KgpBhMpP5tVx1MDAxYeFhxKtcdTAwMTa/NOKEXHUwMDA2t2szPFx1MDAxY1x1MDAwN0PHTCBcdTAwMTJwJXlcdTAwMTE2i5CBvkU0yajOWFx1MDAxYyxcdTAwMDCuXHLgXHUwMDBmz2ZKy1xm1mYwQsSscMvMudaripFcdTAwMThcdTAwMWRDXHUwMDFkXHUwMDE4PKbPlcXB4TFX7fJ0WYM1XHUwMDE1g0BcYpGzsdBVX0qmWIfRMVTzXHUwMDBlylx1MDAxOZqsqeB+gPpcdTAwMDEu2WEgqWuBOC1cdTAwMTGf4Vx1MDAxZHgsUN0m4qAmXHUwMDExYVx1MDAwNpCcy1x090ucy1x1MDAxOLND61x1MDAwM2JF6cvimG9zXHUwMDExMFx1MDAwNG8mXHUwMDFhmC0sXHQ46G1UXGZJQml4XHUwMDFlimfgtlx1MDAxZMJmKVVcdTAwMDNNcVRVXHUwMDA1XVx1MDAxMZg+hCxcdCtcdTAwMTJcdTAwMTmuhClcboh+I4Kuei9cdTAwMDB5XHUwMDFlZlx1MDAwYlxi545cblx1MDAxN9LUNlx1MDAwNLKoXHUwMDA2XHUwMDExKUhcdTAwMDKRvMH9MsglXHUwMDBmNVx1MDAwMFxyaFni9Vx1MDAxY1wiang9sFx1MDAwMybSgeNccuRxr1x1MDAwNoJ6+FxywH/UKe2AQHJp4XAhllx1MDAwMDXg0kYzXG5cdTAwMDLr8cIhXGL3M5BcdTAwMDdcdTAwMWbAJTGAgjdpXHUwMDE0XHUwMDA0eYZLOFxu0ZE3IMB3Lk5keVxiyy0s4LWlu1x1MDAxNaRcdTAwMWNwVDFgmps4PVx1MDAwMSdcdEzQjiFpMjRcdTAwMWLBoJmwXHUwMDA2/ZJgdVxyVs1AXHUwMDE5XHUwMDExXHUwMDBmIIBknbtQZXFcdTAwMDA7k9dwXHUwMDAyLkJcdTAwMDP6h2BcdTAwMWWho4dZRlx1MDAxNlx1MDAxZlx1MDAwN1eShyeKq1x1MDAwMFx1MDAwNUDLRVx1MDAwM7IxY3FcInfxxFsvgFx1MDAxOCU9htVYXHUwMDBiplx1MDAwMUWHfdhcdTAwMDbcynN4eYWHjnD26cx61tQhWlHBSlbZ14tTLLrVXHUwMDAwfY9cdJHKJHbrLdyo0+SnXHUwMDE2XHUwMDE3bWBcdTAwMTVgXHUwMDEzuFx1MDAwYvhcdTAwMTM4XHUwMDE1XHUwMDAxh1BarIlZvizk+Vx1MDAwZUKPenFk7/BDllx1MDAwYpiAr/JCklxiXHUwMDAwW0V272KD1cvZi1x1MDAxM/CQXGJwpGVcdTAwMGXMJ7pcdTAwMTLg9MByXHJCc3haVX+zpDpcdTAwMDZBNdNcdTAwMDRRK2/S4XFLT8A7XGJFIbTBw1NcdTAwMTJeXFzgblxm1C9cdTAwMDSfXHUwMDA2P4FrU1xiZrxG3MaYpomuKMNsn42MVmT0JXlcdTAwMTFcdTAwMTG859akvJjhV8iDXHUwMDA2a6lcclcxrVWJ7kWRIfrxzCpFxHpNUpFcdTAwMWOfXHUwMDBl0jDtwNKSxDZcIqaXcVx1MDAxMVx1MDAwMkghgaSN8kq5XHUwMDEzRaSsMcsmXduLXHUwMDE4PdRcdTAwMDSBXHUwMDFlvVx1MDAwZjxzvfrFjGvPnmFcdFAtzVNNLFA0XHUwMDExx1xmXHUwMDBiiTO9m4ul0YFcdTAwMTZ6uK+gXHUwMDEwXHUwMDE5xPq7zW2N7JPEXVx1MDAxMGhK8oBLkMiMbINwillcbuiCg81cdTAwMDZcdTAwMDG1SEtoYp7GjfCisEdj61fiKVx1MDAwZvhcdTAwMDJcdTAwMWbkXGYwXHUwMDE3dq9Lulx1MDAwMpsgXHUwMDE3jJ6+osHdzlqey1x1MDAxNF2u8UKwpNKXbUN4RDLQYrh71SCJIeHJqSNEXHLmatNcIp9guaNcdTAwMTDxmYh4iMHV+1x1MDAwMlx0Vopb1cYhzFx1MDAwZUxDJfIuXG5zXCJDltBE+aiuJGmAfVx1MDAxYVxcwnJDvsrgNFWd1l1cdTAwMGZrkivbXGKVLbdcdTAwMGXheyVPj2iPPlx1MDAwNbyU4OtcdTAwMWKY2ozlOSalJVx1MDAxN06otvD5tvz44K5J/Fx1MDAwMuhDfZSRJ1mA4WSeilx1MDAxYqXSpFx1MDAwM8s3QFx1MDAxOVx1MDAxNFVcdTAwMTOwV0/9wJ48ZoLZcqiY0+ViXHUwMDEwlrEhStOYZF+fXHUwMDFlxcOzYFx1MDAxYbgwnLJQpjS514RJaopcdTAwMDNcdTAwMGZTTEBcdTAwMDI9oiqrijHGscBcZtDha5c3Zi3OMPlcYv+pXHUwMDExqMNLXHUwMDA1U3p0XGLFPVRcdTAwMTig3GBtiF5cdTAwMDD4XHUwMDFkmdtjXHJHKDtcdTAwMDFcXIf2XHUwMDAyu3H1wVx1MDAxOX1cdTAwMTTiWlx1MDAwYuKnPZPbZVx1MDAxZmWiYb1cdTAwMDRcdTAwMDDKXHUwMDAz0lx1MDAxYYjjpDHxXHUwMDAzozTSluFRXHUwMDA2Q+vHezHWP7jZSlx1MDAwM/iAX1x1MDAwMFx0uCGRlVx1MDAwMiXw4YKpUVQ5j8mtXHUwMDE1h7BcdTAwMWYqXHUwMDBmho1JXHUwMDA14sqSNIBcdTAwMWI4XHUwMDBiUFvDy9daXHUwMDE34op8MVxyNDRfb1FlLItQOFBcdTAwMGZN9lZfXcowReXs0Fx1MDAwMZtVlKEkzsPy8tVApp1rtYRRXHUwMDE0yDpcdTAwMDNNuFtTXG5cdTAwMDNcdTAwMTBEsTiJ1MhBmepcdTAwMGL8MDgwUHJcdTAwMTVti/t9+Vx1MDAwM1xyXHUwMDBlXGK3QfOYXHUwMDBmN/XckVx1MDAxMa2RxktcdTAwMDMnXHUwMDAyN1eOZ1xyKFx1MDAxOTRcXEqSt7tcdTAwMTXGZYroSWvyVHdcdTAwMWGfXHUwMDAwX1x1MDAxMdBcdTAwMDXmuFx1MDAwNcs7a8WRWVx1MDAwMIhcdTAwMTnwO1x1MDAxNlxmJpNcdTAwMTC4XHUwMDFjXHUwMDA2XHUwMDA3jetcdC551VtqQOirmIOFedFzpz6OvFx1MDAwN4jBkk64uVqshqVcIog2XHUwMDA2XHUwMDFj02ouaCZuxCNsx1NQXHUwMDFlYSdT+/VcdTAwMWGiyVx0YVYwbEY0XHSygoCqfFxyVpMg1+e1wI7xdDEkRCXQKZeyd7BjxCxcbozWXHUwMDFhXHUwMDA07fX0WDNcdTAwMGYmWF9cdTAwMWNcdTAwMTHTxURFQN31WUgqmVWVtWNjYiEwXuLWdqlEObGAwFx1MDAxNVx1MDAwMVx1MDAwNHxcdTAwMTKD7NrHxixcbtfXWG9cdTAwMDI2XHUwMDE4fEmaM8EhmGe6rUEp6YylcVxyXHUwMDFmgb9cItIgvlclaXyUXHUwMDEy4Vx1MDAxZcIohC21XHUwMDBmzojMM+3p89yXK60hOVieIykwfLDwS7XiNLBcdTAwMDRgyqKV/I7KmbacQ1xi06CgXHUwMDFjomBcbqw0jFxm1VRZXHUwMDE0O5xcdTAwMDRWglx0XHUwMDE36lx1MDAwMZVcdTAwMTlK4Fx1MDAxMrc8UodFuiz9y8VB32H2XGIyqb1A6pI0J3FcdTAwMTlIglOS9WSY6faomPr2XHUwMDFlyC4n0s9cdTAwMThcdTAwMTbQWVu4zfpcblx1MDAwMSbbXHKrRDFcdTAwMTOK+ZiSwuFcdTAwMDJcYu1cdTAwMDRzdj7Uj41OxLN6PGj6xVR94UvzMFx1MDAxNsEgnka9XHUwMDBmyceG20AgXHUwMDBmnlRKtENcdTAwMWHrknC31Dhdv1x1MDAwMsXeL5HF6Fx1MDAwNrbqnPflNVx1MDAxZeijY/k201x1MDAxNfWAmi/xcPU/wq5YI1OSpsglWNiFXGK9ntZcdTAwMTiTcVx1MDAxMZJKwqovN7H+JFx1MDAxOElcdTAwMDOBuL/srqXJvOpE535cdD4kVVx1MDAxMW5cZoCTh1x1MDAxZVqbLzY0cHHgb0w3kL25wma5c3lS6Hz7XHUwMDEwo7zQXHUwMDAwXHUwMDFjbObhY+GQXHUwMDAyd7rEiXVKxFx1MDAxNDov/UboWF/zwVVZXHUwMDE20MnccbtcdTAwMThL4lxcXvtcdTAwMDCPXHUwMDBmX19cdTAwMWbEzVxcXHUwMDFhXHUwMDFmtGCNhrbCJzrHXHUwMDA1aMWkXHUwMDFh7pX5iPp6srxYXHUwMDA2lsr1XHUwMDA0oYTVyUxwgZxcdTAwMTdcdTAwMDLwKuBH/cY0jeA8Wlx1MDAwZldcdTAwMTFcdTAwMDVcdTAwMTPtXHTmc+hcXFwiXHUwMDE0XHUwMDBlgVxuS6FqR6eAdlxiXHUwMDEzuFjsXb417s/0XbB7jYFcdTAwMDO+XCJCnPpcdTAwMDAnMryCL1x1MDAwM1x1MDAxZEU4bSe2kjHWXHUwMDBlzKjCpTTa4TJLcZpcdTAwMWNSXHUwMDEzOT2huFxcPIJcdTAwMDfGnUZcdTAwMWVcdTAwMDZcdTAwMWRcdTAwMWHsr2JQXCJYh+00s1x1MDAwNuneJZa2eD5a4ERcdTAwMDD9bTA4l5GSXHUwMDA3rvBcbm47SMQpxlx1MDAxOLD9vF5N1Ps67bnLjes5JC/pUrvKi1hcdTAwMDWYXHUwMDE4aDRcdTAwMDPh+ludrbSQOUW9YtEtfFRa71x1MDAwNThcZl6y0tCykqc+OVx1MDAwN3FcXHiUKlx1MDAxMIylKtd7WeY0XHUwMDExTSmjSDBcdTAwMWHcq1x1MDAwNEpBn8BHjInlajQylMhMXHUwMDEwXHUwMDFioKn6m8W0clx1MDAxZFOCykUny9JcZotcZpWEftvQwPrBN/LnkqfJgNmmJE4zvWb42zeh+oj5I+tcdTAwMGLZ7Y5bOmTp0Vx0qq8hYVx1MDAwMTWoXHUwMDFmXHUwMDFkXHUwMDE4hFx1MDAwN+hgbvFfjCnocCYwTZhcdTAwMGJcdTAwMTZcdTAwMWWxmvnO5SFcbmTZXHUwMDEwwMLAMl2YqORj1pOrzlx1MDAxMm6iPmud15HCXFzhikHSValcdTAwMDKFdaSYJ6bsSLh1g/RcdTAwMDFcdTAwMWLnXHUwMDAxXHUwMDA3XHUwMDA08Vx1MDAwMKYhJ1xuXVxyXFxAsJo7TuvrcFnXK1x1MDAwNLw3xqaYK3UlXHUwMDA3xVx1MDAxNV8ouYzcyONqp3fm8ljLXHUwMDA3Y2ImmfWEKT3hdND88Vx1MDAxNjg8XHUwMDFjY504iaBcdTAwMDbMOeQtW6DKaVx1MDAxNlxmtqGYXHUwMDAwNzHkXHUwMDBi0/XawsJcdTAwMDFcdTAwMDfogUUxQkpT6lx1MDAxMkhcdTAwMGI3YFm7QFCvj9pVXHUwMDA2oiM8XCJisLtcdTAwMDBcdTAwMWWdelx1MDAwMjA1xFx1MDAwMVxchVdcbuAu659cdTAwMWXgynKjMpRcdTAwMDVcdTAwMDNcdTAwMTRpSj2v84eHhW/hXHUwMDE2gFx1MDAwNkXH3IhA3mRYd1x1MDAwN8qVslhuRICL5Vx1MDAxZWuEldzaVy8v39nLRXbH9EhaXHUwMDExlHHZxebWXHUwMDExXHUwMDFihFx1MDAwMVx1MDAxOXydZWZcdTAwMDRq7126XHUwMDFhgSeHK1mF0I/xmKsvXHUwMDA2qlx1MDAxMUd/XHUwMDAznPPceIFcdLlrcZLkkola1lCD/qfJNVx1MDAxNlR6zzImhKBcYu5cdTAwMWItvrDaXHUwMDE4w4BcdTAwMGLCXHUwMDA046mnXHUwMDAyXHUwMDA1S1x1MDAwYulcdTAwMTJcdTAwMTXA1nM9vlZkXCJg/Ok/iv//++xDf52/WLvdMNrpXbc83FxySG1l55YnzTdcdTAwMWL+WDj5XHUwMDEyPveOT1e/v3q192bUXmw92n9cdTAwMTCdW1jGxEVcdTAwMDewXHUwMDEyXHUwMDAzrUjbUpp8XHUwMDAzQlx1MDAxNJhZwIeMwpRcdTAwMDY2o9Yt4Nsu5Fx1MDAwYlx1MDAxMyCBhf4w484t+Fx1MDAwNFxcXHUwMDFjeFxmgiNX9HFcdTAwMTfbXHUwMDBljeJSu7Hjx/B71+GFsIey63CBJWPw23lcdTAwMTdcdTAwMWaWkKaJ54uldzBcdTAwMTWmk12DTVx1MDAwMlx1MDAxMOglQ27FojUhjLtCYJMqtJlcdTAwMGJcdTAwMDTHXHUwMDAxv/HSsjSBtbaJvJLa15c7MGFBMmVdvqyRJlx1MDAwNKXNa1lcYmdW5FnjWnnTTJM/rHjzmFx1MDAwNoXAyLP1RIMquVx1MDAxOYvLV3DIlri3gNncXHUwMDE0ISX4NstcdTAwMDJcdTAwMWQnqj7fzi2OXHUwMDFlcVx1MDAxY4uk8laZpW2sLFFyzLmb6Fx1MDAwNcLRXHUwMDA22lwiXHUwMDA0NZqENLD1RSgxXHL2XHUwMDA0VojL8Tr35lx1MDAxONtZqE/83Fx1MDAxMkhObWjHjFx1MDAxZVx1MDAwYs2r+5s9bY6SO6cvn+283H/bsd+ePl56rtbDl+31XHUwMDA3gZLOkbpcdTAwMGKEg2Bcci6WWjaye1xylMZprqlDt29cciUlc1x1MDAxNyTiXHUwMDFhodW4XHUwMDAwNsVJgIx0TFx1MDAxYiPoXHUwMDFht6y72J1vJJRcclg5biv5XHUwMDFiJy+EPVx1MDAxNJy83qdDXHUwMDA2mss5Z2FcdTAwMGWZYkKGM7glduZcdTAwMDJf0kbI+jKJ6WrIXHUwMDFmz+p7z/lj0+JYX7xKci9ZWCFsXqOX9mlcImBcdMSoXFybtbFcdTAwMDFcdTAwMDObsTiVcTMs+5FwXHUwMDBmuDFpjn3C2lx1MDAxYVAw6fPTNfJuWyDRvjQ+9vrhgodcdTAwMGJBm1x1MDAwNlCAx63Yu0xLy75eqrxHXYLHXHUwMDBiJlx1MDAxOdgnoTZcdTAwMWG5JWRcdOVXx4daXHUwMDA0PDiur1chy7NrIMvG65VcdTAwMWarX+1g7VtrKJ6Fo6PXK/5BIItcdTAwMGXsWqDg7IJXflxccHh+4lx1MDAwNbuOIfpjrFx1MDAwNNJTaFx1MDAxZj1TZLH5zlx1MDAwZmvYXHUwMDE3KEZcdTAwMWarkMVzf6yNXGbNXHUwMDE4rY5n7KJxft4my/1cdTAwMDaWXHUwMDA3XGYsiKdZLKtdntvjXG548esxM0GA9jBFjFC1flGhJE6lXHUwMDBiKDFjXZx1rLxcdTAwMTQssrlrcYjafGBBXHUwMDA3kFwiKO3KecnA3TxcXDTT+Es9MMbMsebfs55LwYzStKnOXHUwMDFjj2hcdTAwMTLOY/iuQfeMRFxcLK9lQVx1MDAxY9/Es1xiMFuAQJ24hfzom7xcdTAwMWRcdTAwMDebcVx1MDAxYZ1ussnN2zCBzVrK2GSj5y1cYjRcdTAwMDA+no7icGsxpNtEXHUwMDAyW3Uo7lx1MDAwNlR5XHRVrfaJfI9cdTAwMTJIs2FHK1Z0pv09XHUwMDE2zj/gXFy+YVx1MDAxNvG7jPU5z9tByKlcdEo8PSfZn7Bw7NNcdTAwMTghnzdHyLcv3q98etp331x1MDAxZn/f2ttaXW/tvlr79iBcdTAwMTAyOHaPjojlJHd0jVx1MDAxZtUlQkZub/JcdTAwMTZBjr4l6lx1MDAwNYA0zOsoYVx1MDAxON2Oq6pTgDSWa1x1MDAwZtwmXHK2PJGilOxcdMlUxe9cdTAwMWPlw4XI631cdTAwMWEuzem8IC2vp/ZpvFx1MDAwZpeGXHUwMDE4n3t+eTafri/HN1x1MDAxOdunsIMwd1UpN+FxU1x1MDAwNaz1uMBM9nB1+XJcdTAwMWE4TJotZH1T3lx1MDAxZpntvpRv0ISyJM+WMYtHXrDAXHUwMDFlvv3ag7uZMC4zXHUwMDA2XHUwMDAxaJHcYCRFmliWmedmJG6RlVxcxq9cdTAwMTcn2CY777wqjNa2tPVcdTAwMDCzXHUwMDFluOZqXHUwMDE5KItY32Uzw33yeFx1MDAxZeW9l3nTkD/TtzXbXHUwMDFlcZeVUdw0dVx1MDAwZlfSJMvCXHUwMDEwjVTh1HJznPLt1rOjIPtvTlx1MDAxN9pra7td/+1t+82DwCnWY+f9v5Xi8S2llTTLdjFcXKQy3JMqVWlcXLPBqcj9XHUwMDEx7ORcckVijFRxXHUwMDA2wpluMmtcdTAwMTG4XHUwMDFiQEygVOC+X23s7/adXHUwMDBmXHUwMDE3pHjWoItcXMdn5zqdXHUwMDE2rCxIbopRJFwirOdmq8bOQn31mke0rFwigiDWwYvSVlx1MDAxYdZTwT/BtyrLNfX6TFx1MDAxODf1++CEXHUwMDA2d+BcdTAwMTlGpSxcIutcdTAwMGYk2z6xjqdcdTAwMWVcdTAwMThVJjVcdTAwMTdQ8mZVXFxBSpCxrPNcclx1MDAxNpYyXHUwMDFm2Eoy5qW/TFx1MDAxYV41vCatKWGP3Fx1MDAwMcNmZNz85q94er+mKXOcPOZo3JNZsVx1MDAxMNX4yp7MK82d+8uvvZMlp1x1MDAwNotvvVx1MDAxMlx1MDAxZsWPo6er2/1cdTAwMDfh3D1r8tllXYPRjutcXM58O1eHWLjpgYFs+WpL45rZ+lx1MDAwZneos+hcdTAwMWV6jFx1MDAxOGZMXHSL6z9cdTAwMTZmXHUwMDFm82xvMJNcdTAwMWMkP1ZM2d9cdTAwMTTkXHUwMDAxe/drfppcdTAwMWRcdTAwMTjYtVxyaiNcdTAwMTW30KSuJ2hcdTAwMTZcIlx1MDAxYTaJdT42QVx1MDAwMsnGrXlcdTAwMTM9XHUwMDA2ubLU+T9jgoZOU3BZvsFcdTAwMWXN2UrjSSHwr9xsXGL1L/W9g3V4PIBoXCKvpeuXa1x1MDAwNFtcdTAwMTCyR2jMz1xuSPd+4V1Ymc7PO+aKan1BOTCUM0CkYvGIK6dcYpnPwzRcdTAwMTjByspcdTAwMDbHanCfXHUwMDEwt3/xjFx1MDAwMFKhVF7ZXHUwMDE3NEApwZ0koFx1MDAxN0F6LkCVMZ5LTDykwbJcdTAwMTi/SWOaWVx1MDAwYmRjXHUwMDFmzIbKlzJcdTAwMTU3kyd3nOVcdTAwMDDv2V1cdTAwMDKzZuvLZcrEy4V0XHUwMDA3qMhcdTAwMGLVXHUwMDE1KSY7WdkmO1VuXHSdp5+YoFx1MDAxNUitnnL83Ivm6Nzb0kJcdTAwMGXk0L1cdTAwMTFuZNZW7PLSrn5cdTAwMTDobLnZiVx1MDAxM8iliUJ99iU8W8RWTrOvsFxm5pZcdTAwMTbRcJVg2H5M5Z30ZDU8gydbnjnEZeBiQHWOz0ww5rpYd1x1MDAwNt1vgP6nXHUwMDAwNHyaU+zHbZjmjukmQ1x0XHUwMDBm6jwsXFzofKdH/VxuhWK3U4Sjns2vWNhRcrmSXHUwMDA3znjFXHUwMDE2f7I+M1UnXHLCeG515MZcdTAwMDfVwOHqTFx1MDAxOXZ3XHUwMDEwztg87ighVsk87lxcXHUwMDFlt1x1MDAwNbKVXHUwMDExuz2yL0FaT8FOakZcbp67JOl2XHUwMDFiJFxuXHLPfHLki0pzb1falJv7Xix7+INcdTAwMDByL0+9wLxZjefORSmB+0GU8apYks8uZL9IpOBcdTAwMWVcdTAwMDNJ0Fx1MDAwNOyT509kNItCMXu/XG5XY/nVXHUwMDBiXFy1YPi6kJgoourL5qi6/MU9+jL6MVhd+mDfrbVbj3rrg96DQFV31lx1MDAxMIeN2KDxslT06CX3Zlx1MDAxYsVWpcq6UFx1MDAxYdeMQNXnjarZgjOwOMVVgarN2IZcdTAwMTBBkOAup+JS6cW5rkE69oj/ndL8z8FURZCz7HVcdTAwMDKilVBewVx1MDAwZdmRW+PZYdzEXHUwMDA2W+1cdTAwMTVpoFx1MDAwNaFcZtwzrlKOKlx0QVx1MDAxMU6OXHUwMDFk/XlaeP1e9izPp0nQXHUwMDFlkMCo09WjkrxcdTAwMDZn50X20tBsMlxy6CptJ9Ts41x1MDAwNorKc/Mk+1x1MDAxNNdcdHNcdTAwMTn7VcGkOS5WdyTiXGbb9kZuIGC3nvrd4rOVxtS1ZCaXdIyd+tNcdTAwMGViXHUwMDEznqA+MVx1MDAxYngmhVx1MDAwMZWU7PwlXfnMIDbKYctSXHUwMDFlX2NcdTAwMWHsIJB5XHUwMDEzKO6bZGVcdHG9NLU8XHUwMDE5IVxuj/DLuCZ9imct8Fx1MDAxY3mZXHUwMDA24f5cdTAwMTTcemmTg/CeXHUwMDA3XHUwMDA1RrY9YDv9WrC/XHUwMDExLlx1MDAwZjs7ozP4qVx1MDAwMGetp4Iz1MlcdTAwMDVbaFx1MDAxZllE59Xm6PzktTs+XHUwMDFkLlx1MDAxZC3ufu24XHUwMDFm611p7NBNQef91s7+ybDz65E5MFWUZ7HYv0LGlO/awMZiRkd22Fx1MDAxN4W+25PI3OGvq5G5Zduh6qRALSRP4HB5w3JcdTAwMWWnMVx0zJa6y+SKiWc/ZVxcholalt6NWXBcdTAwMTGXr1x1MDAwNbh/XerdhWpdJJP+no7Dl9/5q6DB54qx/fKZfnOg+6+W3vc/fHF658OSXHUwMDFil1slOtxcIlx1MDAwMM5fvvP3n1fJ/dJ5tvhi9Opg/WRl6dmnL892n348WbyR3IuPjzrfR6mgsyu+XHUwMDBiXHUwMDFmPo7ebu9+fbMqv+wtii39fG+jIHZcIr65eVBSafJXjeZSTuXNnCm9s1nkuVxybK/nQ1ooXHKOd9bVyzCdWNT5y0gy+oz77dm5L3JcdTAwMTllUl9cdTAwMGKh/qTd/dxcdTAwMTGbnZ1cdTAwMTjvjze5lnVcdTAwMTX96eZ8pdGlceJ5kVx1MDAxYk9ZgTuYXFxN5Fx1MDAxY16xxFx1MDAxOHlcYknxzNPprrtsyMk4Zlx1MDAxOFLnSkhcbtdcdTAwMWZ1hvze3OLa8tx6Z4jBXHUwMDE3ZnXQXHUwMDFmrXd/nO1nSF592jrs9nL1TEQu9rp7/ZzudSg4Qb5Rd6fVu/zAYbfdLkLjXHUwMDBlhLbAZIfLTcBrMOzudfut3rtGt9E6XHUwMDE5XHLedo7PbmQ0POlcdTAwMTSfWOf5JdvKzqOgn0P0gt2VXHUwMDExnUdcdTAwMGIyXHUwMDE2q9LAV81cdTAwMTE9XHUwMDFlfHuybLZPlzfeya/L71x1MDAwZV5snmxu3Vx1MDAxZlx1MDAxYpyC6II96GWsRnRcdTAwMTdExsS11fqeI7q0gWdcdTAwMWZcdTAwMTai1HtcdTAwMDTpz+KT0bfY0ivv9j9/P33U2txcdTAwMGYnXHUwMDFiXHUwMDBmXHUwMDEy0tvt5UX9ovu29/bLu07v8cHCyudHP35cdTAwMTmkV4+mXHSkXHUwMDA3bnJgfVJkqU0oaX00mWIxJPtcdTAwMDBXYzp7XHUwMDFisolj0Fx1MDAxMXHkuJ/Jb0yvxfTX14B0hedcIpRRXHUwMDEzbIyzNH3rgkBkb0PhRKd7XHUwMDA06WvDwdduuzOcO+qdXHUwMDAwJO9cdTAwMDOa11x1MDAwMFdcdTAwMTnNp97BTYH8zKhXeur7vvvwbct+XHUwMDE5nK682lLLh8ePJo260+t1j447qV3zXFzujFx1MDAwN9g4zap2Oa5LyFxyW/OsK3aNvzDssYJcXFq2NllQQktVsVx1MDAxNdGwhS+Pw3XXsu5dXHUwMDExdoS4gXWXrPT+2vabatueXHUwMDA01ItymjGgXpg8ezqrQjosieGmllwiXHUwMDAw/HlYn/qZIH5cbnTXQeHqs8dmY/Fwb/NH5/irfvmpc7D0etRcdTAwMTS5by1cIvj++dWXzd7S0/dLL/tb6t361vHn983k3i10Vz+9XHUwMDA20M3W7zyWi/1QXGLdhW2/uYtkk3imVVx1MDAwM2tkpZnEbvbyt0BtzbP/lKxqZmGvipp/xrrLke9Dxe6318DuwHaPslhcdTAwMWM0tmQ1ieiX4C25+9RGZ3/ClO9cbrz/3Vx1MDAwN5E9vlx1MDAwZthdg5bTsLt8XHUwMDAzs4Huz/7Rxt7K4fu1b5uPnz39+PrL3klrOGnU7W7rcNBvl+zaRJexoe1cdTAwMTToXHUwMDBl+elq3Os/XHK6XHUwMDE129wqLpRVYbdiyyZt5V1j91x1MDAwM7Lu9ZujN1x1MDAxZKrwXHUwMDE15p1bxlT4jmznaf3PxOs/id6LrXerg80l+2pvZ/HxxtHbXHUwMDBmj9vvn/zjULb6LpugrHXsKMeGh9E5X9xdd1bZXGJrzFx1MDAwZppzStvqQFx1MDAxYZG4XHUwMDAwRyZOi1x1MDAxOFxuq8O3iLP/XHUwMDE0S3zXXHUwMDFjZ1x1MDAxNVv5umJcdTAwMTg0trjCbtOJgFmwRuF+guyg/e/+o1x1MDAwYpi7XHUwMDE3KFtcdTAwMDNsXHUwMDEzKDvtXHUwMDBlZlx1MDAwM7NXk4eS60nNWnLTg46SR4xPgixocSZcdTAwMTG2xemJr6Cyqlx1MDAxZT08hkD6yJ3H1zBptVx1MDAxYjvG/GeY9MaNwVVcdTAwMWJcdTAwMWVcdTAwMWVcdTAwMTBkdUA91dKV4IG+qrBcdTAwMTdzhrbOht0/Y+uzbVx1MDAxZVRUSv4sXHUwMDE09XH87Vx07G1YgzauYCvMVufsITXJoOVOaueEt5tcdTAwMWZNrrTmSYiOlSSFXHUwMDAyrPm91lx1MDAxMac578BvtVT5Wkfx+Fx1MDAxMij/93SgXHUwMDEzupJcdTAwMTTcTVx1MDAxYu7VpCFcdTAwMTmuys/6MHBcdTAwMWPsXHUwMDA04Iu9XHUwMDEwzsfrMsRcdTAwMDc8XHUwMDAwOjBnnpxcdTAwMTjQYLzXKvIru7ROb3vwrVEgtH7w5MPrJ3ZnaeX49PWHx+tcdTAwMWZOWrvfXHUwMDFiMlx1MDAxM8uTk/OjMc58ZqnoxfnMs2nQ9Fx1MDAwMoBgMs2DfvX56lXF5nglbWatuVx1MDAwNf+5rXbV9vZDTyxOqVx1MDAwM7hcdTAwMGU10YH7rXSsKlx1MDAwZlBmXCJ8unCfrPlcdTAwMTO6cFx1MDAxMuHtc5ON5TdPhFo8tVx1MDAwYs+erfV0e/v9ilxcaMpNfF9vhrWX7vNO2POvzer7Z2vLclbcJHpb6DhyM25SfZdccrhcdFxiXHUwMDA1l6xZf4hh6ZD2ZWJfQ8mjhZ3hWVx1MDAxMVx1MDAxNfbobVx1MDAxNkFMuI1cdTAwMGZ/iipz/Fx1MDAxZMVMscOt5sQkalx1MDAxMVx1MDAxZNvnVFx1MDAxMZMgptlcdTAwMWK7tmmeRnRcdTAwMWbrcVx1MDAxMNjPXHL6c//uf+1cdTAwMGVHJ61cdTAwMWX+1lx1MDAxZrQ794Gi1CBcXFx1MDAwNUXhnVxc3Ed6XHUwMDE3s6EpV3uiq2iK1zJcdTAwMGJBqmhCXHUwMDA14jq2VLKFVGCVhcMvqFC5iFx1MDAxN13GY6WlXHUwMDExMVDVxlxu8dvUi6b+/saQXHUwMDBi3HRsjFnlXHUwMDAxlJq6fG/Zw8hcdTAwMTRcdTAwMWKF/8P4SoV68mdhQjPHgiageWbU5Wq/UeBcdTAwMDL51rqIXHUwMDAw38W8XHUwMDA3XGKPVJ5kLz5jO2hhbd5cbi5cdTAwMTRcdTAwMGZznFx1MDAxNXtcdTAwMTn6168+b3xtm87zU/F8+fm7nYVVM4VsOcvdoTyr0rJcdTAwMTlnXHUwMDA12VxuPDfjvPBMXHUwMDE0j8K+P+Tl6orwq1x1MDAxY2n0cKRcblPFsnwjS8xF68wqXHUwMDFlcDY125OF6LmxPVx1MDAwNLhkVzhcdTAwMDeqWFx1MDAxM1x1MDAwMaW9XHUwMDE15vJPcaRcdTAwMWZu7Ei5pDKlsFnJ6ZXNXG50XeooXG7Nk/5ZjnRhuoLyp6Cad+FJXHUwMDFi+yV6Uq9MhJ9cdTAwMDdcdLHsa1ZoT3LumWRcdTAwMDavZaMunnIwK1x1MDAwZnp1+XbiQYVyikfFXGJtrOCRdHZipIonXHUwMDFmhGuO845cXOfVXHUwMDE1sle7TsCHUS569lx1MDAwMLKl7r+Op1x1MDAxY1x1MDAwN8OzcmW171RcdTAwMTl3XHQqpWN+ILyoaNyrLXjsb9d5hev8ePOkucf8iThZOZZTzelVKEJcdTAwMTieWONvY4HsXvjOqfrJn4Jm3oXnbOyPXHUwMDE2zlx1MDAxY5JQmv2nvORhuZNcdTAwMTlpm92O27xGllx1MDAxZq5S5O3NlJeG7SYmXHUwMDA2aSZqVe6T35RcdTAwMDf90D+wJ62Pr9pcdTAwMDfD+PTTx1x1MDAxZlx1MDAxYofNKuvZ5FNcYqCx4YnupY5IXCIgJlfjZHlF3YBzmefmeclTiGWsOlx1MDAxMkRfx1/+J9XWf6v2l1XpOefy8/2qXHUwMDFko566migj29PEKG8jqLxpem7zPJv1ampOLiSv3jgnN1x1MDAxYVx1MDAxY01LyCXjLmffqlx1MDAwNzqbtNvTXHUwMDFk032z8Gr7udlaf3n4TD1pf3600sh0rcysZjdcdTAwMDQu9Mm0NFx1MDAwMPSZwTSmalx1MDAxYVlcZjFTLj+03rO5fKG/83iV6zr1Pv9Jcc735naLR0/qUGGi83l9x1TDtd475+y93Od60u9+nzvmZ0aVdjvu/vVr7bZynLPav5paZGFSp3eKMjy/geVcdTAwMDRVSnU632Cmz3zGXHUwMDBis6lcdTAwMGVcdTAwMGU/tX/Yg+7w82prZfR+ZdpcdTAwMTkt92frqvWZUWzgYuKEv3JcdTAwMDZvXHUwMDAy3kTtztWf8TM/5/MuXHUwMDBljPbEXmuqm0Ld925P17bKqXbzXHUwMDBmNlnlp/ph76JH0F/d3e1Hc5Ncclx1MDAwN1x1MDAxYm8+PVxub7fXh8Njv/rp+eNnXHUwMDA38r6brFx1MDAwNCXI2Nteapv3wihlVVxcYNmWm1xmMmZptFK4LLAy2VxurySMt2q3wITVSmZ70mWpWzLb8zcqqztcdTAwMGXWtpbF8FU/PDpuP+qPXHUwMDFlP9566T79ssrzmzmF5+/erc3999ziyWj/rp1C5aVnXHUwMDEzfl+9N+DK4lxcXHUwMDAzzJLGgfSGiuJcXGs1uyRoO3XZm9S7cslbhex3mnF6+C3jfKWZXmeFxmqedVvNpv30JVx1MDAxYWG4QqPkbVx1MDAwNOX3XCLNWNJJ/ixcXKrj+MtcdTAwMTPOaWa5xca1rswtXCKc5Fx1MDAxNrEgjJWI5yazdjHjOZBGXHUwMDBiNvf18nq1ro2SjFdvNSiN1/F8XHUwMDE0nlXGLuOxopjYZ3jbq2C0cizkvd5y/OyTjX+cz/F86+hofVx1MDAwNI29fFx1MDAxMvNfu51vS5VOhj9cdTAwMDSyfGB0Mp3cOv/+4+//XHUwMDA3bbKKhCJ9 + + + + + 2. The remote provider expose an API to execute containersVirtual KubeletInterlink API ServerProvider pluginProviderAPIsPodContainersPod on virtual nodeVirtual Nodeunix socketunix socketHTTP + Auth \ No newline at end of file diff --git a/docs/static/img/scenario-3_dark.svg b/docs/static/img/scenario-3_dark.svg new file mode 100644 index 00000000..e2ce7009 --- /dev/null +++ b/docs/static/img/scenario-3_dark.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19bVtcdTAwMTPZ0vX38yu4fL5cdTAwMGV97/eX801cdTAwMTFcdTAwMTVfXHUwMDAwXHUwMDA1UXnOfXFcdTAwMDVcYlx1MDAxMFxyXHSGoMC55r/fazVIunc66UZcdTAwMDLGmWHOeJwkVHZ3V9VaVbuq9n//tbDwaHhx0n7074VH7fO9VrezP2h9f/RcdTAwMDdf/9ZcdTAwMWWcdvo9vKXy/z7tn1xy9vJPXHUwMDFlXHKHJ6f//p//aZ2cZKPfyvb6x1e/2e62j9u94Sk++//x31x1MDAwYlx1MDAwYv/N/yx816C9N2z1XHUwMDBlu+38XHUwMDE38rdGX2eCSF9d7ffyr9ZeSK1kiDdcdTAwMWbo9Pbb51x1MDAxNLkrn4yEdU6fYlx1MDAxMcP2Pt45aHVP26N3+NKjy8vn60cni8+fq/bX3eXtXHUwMDBmS4+3RXf061x1MDAwN51ud2N40c3XetrH9Y3eO1x1MDAxZFx1MDAwZfpf2lx1MDAxZjr7wyO8K5PXJ/3WoH92eNRrn56Wfqd/0trrXGYv+JpcdTAwMThd89WN+ffC6Fx1MDAxNV6g1EJmJiqpXFxQVpubN/Nfd0FmXG43x0vhoo9eJeta6nf7XHUwMDAzruv/XHUwMDFkiLAnxGhlu629L4dYXm+/8JmD9l6Mo898v75aI1xcJqxxwdub947ancOjIbXEh8xpXHUwMDE1pLJcIv9xozW08ychI17VVsnR4+M3n6zs56ryv6P7P2hcdTAwMWS3V/grvbNut3hcdTAwMTN7+9c38YdKjZRKX7/y5+jS+PnlVFx1MDAxOYtcbllSymH7fHhzzVx1MDAwNWVx51x1MDAxNyefzOKJOP9uV97IXHUwMDBmu+77jnh087k/r/82Wv7ZyX7rSvekV8467ZzVcvRIup3el/Tauv29LyN1/VfhQsaMp7TOgt3oKCfZjbfQmmhcXKXdLDW3m+pbUW03R629o7NBe1x1MDAxZSzHeqhttMqE6JyKqem4XHUwMDA3MFx1MDAxZFx1MDAxNU1cdTAwMTZlXGaRqzFFXHUwMDEzuDEgK1KDgeJcYi2NlHe3l9JcdTAwMWJjhjFL3Vx1MDAxZK2q31x1MDAxYm50LvlAlCi9+qx13Ony5o/cSK7UuIGv156vrC6srj1dXlhbXXi3vLH2/t3S8sJ/euvv1rZWni6/e1T6lcfdziF1/9FcdTAwMWUupT0omcWwXHUwMDAzaLr5wLB/Mnp3XHUwMDBmi2h1eu3BSlx1MDAxMzjoXHUwMDBmOoedXqu7OX2N4ytsnVxy++/ap1f3YDg4a1x1MDAxN29j+8WPXHUwMDA3XHUwMDBm122nWvxUuMRcdTAwMTOaZPZSSy+E915X2f3T5nZcdTAwMWZfxt7773ZxcP7t4PT8+bMtefbhy3zjpXWAI6uFXHUwMDE23jrATmr12mdOwiV4ay0+Z5OFjSxatvnPdKtv2f1wcDBu9TaYjN8t3Fx1MDAxNSiOLuXG6o1SmfRcdTAwMTGYamL+M0L2XHUwMDFmXkBcdTAwMWJcdTAwMTiicf5cdTAwMTeh5oNcdTAwMDHbdFZcdTAwMTgnskLp8Vx1MDAxNJ1cdTAwMTGqXHUwMDEy3pabq/nJY3/ycXHt7ZZ9/+JsZWWlfXm425t3ePNcdTAwMWU6XHUwMDE2RYxSiqBdwdbz3/dCZ0A3qJePMVx1MDAwNOvvRdG1iZmMUFx1MDAwNmltXGLKODOu6UrJLFxiXHUwMDAznVx0XllcdTAwMGaNXHUwMDFlI4hBeFx1MDAxZEw0f3lVn8ThTPRcdTAwMTO1XFxL61xcMLZKyZ81V/KLnS9cdTAwMTftJ8f7/c3X3fNXn9c/f758ujXvSm61y6BW0PVglFAyUXLrfWYl3GTUXG5cdTAwMGU1TOZwd/Lm4JFQ8GBMrlx1MDAxMqpcIlxiKuLItU5cdTAwMWJLblx0u/iFJC5IXHUwMDAxs7wriVx1MDAwYqVXp5A4nS2s9lx1MDAxN1ZWn6y9X326XHUwMDAwstWDZ+98w8NeXHUwMDE49lx1MDAxN4ZH7YVcdTAwMTfrS/dA5EpcdTAwMTeUsrZbLupBuJuVXHUwMDEzQzZqjFx1MDAwMqjFSu72vLm9XHUwMDFmbHefr2/KrZWzrTerXHUwMDFmXHUwMDA38lKYt+15t3dcdTAwMWZDpk3QRqsq9uZccqIpXHUwMDFmrFxuQdxcdTAwMWZ7Y8YlXpv6XHUwMDA09mbxiajkXHJ5XHUwMDFirfPa/K2Bv0DoOXqK85DyuNJcZiN6W+sve9sn8XCo5MXK5eDV7ogtlbS4NVx1MDAxOPS/XHUwMDE3clx1MDAxZX9Mk/tk/61+9mHz8OVy5+XJcvfpnuzu+DvJrUvRXFxcZl6ZNy/7Ty63vm9dtrut+PXg4qDpcodcdTAwMTeX21/P9O7qRf+FXHKrK4tvj49Xmy33+m+/nDYoodNXR7QhRlxy6iBHxKLgR17cgjdU3uM59yPBh0xa46VcdTAwMGLMO9qUXHUwMDFj25BcdTAwMTlvnFK1UWD0ck/9XFzuR1x1MDAxYZVpXHUwMDFiwdFcdTAwMDVoryxkmEbceJw3SG013Ir+XHUwMDFiJX+2OoPhWau78Opsl6r4s1xm4bizv1/E2yTbU1x1MDAwM4gpb5i4qFkxXHUwMDA0frrKqKHdKn35xqpcclwiLq2LRLhg1SvNrXr7y9n+6fKrlYtcdTAwMTX35Pmz7feL7U/mdIJV71xy+qeni0et4d7Rr7dsR8qP65eIaYNQIbFs53xcdTAwMTakXHUwMDBlKkTQditCsrDbWbZWbq9tKyxbxcxcdTAwMThcdTAwMWStXHRKel1cdTAwMTX1SmWzKFxcjFx1MDAwZT5cdTAwMTjLKbCIXHUwMDFmli6CxZMs+vCipY9Q/+jr25Xds4/m4Gzn85Otw71n5ycrb1x1MDAwYkD16PGTnVxycfj0eH+19bn3/ltr6/TLsf9cdTAwMDFX8+4xToetwfBcdLS40ztMf6Xd25/wTrd1Olxc6lx1MDAxZlx1MDAxZneGWMZ6v9NcdTAwMWKmn8jlPiZwXHUwMDFmtVtjXHUwMDE3XHLJXHUwMDEz3zuhuPIzXHUwMDE4/W1hpI/5f9z8/X//mMGnXHUwMDE3daa1V1x1MDAwNkZu8P9CRFf6/SxcdTAwMDRcdTAwMWRcdTAwMDNwXHKxro5Gh1qBJsBcXGI00XlcdTAwMDd7XHUwMDE50Vx1MDAwMf4oXHUwMDA1NVx1MDAxNorf5KLQxj20OPBoRDv4tI2MVYUtiTNZXHUwMDEwMTj4u4jvdLpenM4kkNMzXHUwMDAxXHUwMDFlnJemJC5kyjjYI+QpLE/WinNcdTAwMDY+3FwikndcdTAwMWXeRphQXHUwMDE0513mXHUwMDE0t0RcdTAwMTXw3Vhf/yhmL07r4H20MD5cdTAwMTlcXOlig8iEgVwi+Vx1MDAxMIWX3tdL05nQweHhKu6XldZcdTAwMTZ0hlx1MDAxN7UyXGJ4YOWmXpjKXHUwMDEwxVxuXHUwMDBiXHUwMDBmLYMsJFx1MDAxZq+lgVx1MDAxMME7S4vvilrVXHUwMDFihcy0QqylXFzUoHROly5VQu288lFIJbF8p+ovduZcdTAwMDKhW1pGY6TzSjtcdTAwMDBDKFx1MDAwYrSZgMHigqHF1Pd6gSazIbLuweNXjLSpvDK21MnDL+Ch2qCFcdpcbqxhirx6bZm1uJgxeee4SVx1MDAwYlx1MDAxZCzshdzcPXxcdDRZ0HhCrf5plcFcdFx0XHUwMDA3j+JlVGLs6booXHJW7eFHha19XHUwMDE4MFxyg0drJX5ccmbgU9tcYk4gwoCxKWjAw1xus1x1MDAxOa5cdTAwMTHhXGb8XG58sfeJXHUwMDBiMFpcIsjRNsBZ69hAmnBcdTAwMGV3Jmc1iVx1MDAwZqB/gvk7sCY8K+dq11x1MDAwNl9cZlcsXHUwMDAz8Fxmz1x1MDAwMEaU+GJcdTAwMTmhPsAxXHUwMDE5XHUwMDAwJtE+tDioXGL0Q8ODXHUwMDAyT2HlKfIgYFNUXHUwMDExeGo8qFpvbKDCXHUwMDBlrlx1MDAxOF+udVx1MDAxNKosXHUwMDBlcWBkslpyP1x1MDAwZnpea1x1MDAxMSaTeFxieHy4XHUwMDE2Z+GCxmhAXHUwMDEwXHUwMDFhX0VxeGC19j9reYuEl7wkXGJeXHUwMDBm6lx1MDAwMLqtS1wiXHUwMDE3JT9cdTAwMTA0/lx1MDAwNVxuOFx1MDAxMYxuL8ra657Elv5V/P9bXHUwMDA3VHbkPsdcdTAwMDIqXHUwMDA0XHROXHUwMDA34SvTJC+bXHUwMDA3VJsw6v3F02/d9vfOxsfXn7bbS1x1MDAxZk8mXHUwMDA0VHOyVZ6HUrBcdTAwMWJmQLxcdTAwMTd6dFx1MDAwYq5cdTAwMGJk4I2Mcd5cdTAwMDK6XG5cdTAwMDb3kzmSikRrhJlcdTAwMDXto2SJkqrKs0bNfXSEc8ZGLZRcdTAwMWbbJI9cZjzwR/XO4Y1cdTAwMDL9XHUwMDEzXCL9dUIkmcHlXHUwMDAwXHUwMDA28NBcdTAwMDNcYjPig7Ivi9abIFx08IBcbqlroWBcdTAwMTF8wDoh4Vx1MDAxYqFmcOC2xFx1MDAxN6iBXHUwMDAytiC10HiKsUFUUyfQO0BcdTAwMTWLXHUwMDE3dEBoY+++wp9cdTAwMTBcYr5cdTAwMDW0kvhccuVcdTAwMTX+LFxudGDHiMeCXHUwMDE38NtcZkpcdTAwMWHIXHUwMDEzXHUwMDEwpcD3wfpA41xcma/KjKxYk0pDsmkkMETgvVLc3VxyXHUwMDA2fqjM4TLwTlxyXHUwMDAwXHUwMDAyTWKRQ320pLKIO8iomvlcdTAwMWFcdTAwMWJKSiNxXHUwMDBigfRcIoRohMDdqFx1MDAwN8BZy/NZ4KNcdTAwMTBcdTAwMDBnXHUwMDE3XHUwMDE4XHUwMDBil+QhXHUwMDFh0NQ/iPRcdTAwMDLutj5cdTAwMDTDLfL0rFx1MDAxZWE4eFFU5Vx1MDAwNTrCMzhcdTAwMWW+XHUwMDBi3Fx1MDAxMSBdL9BniP/wMPhcdTAwMTiFd6okj7F1jKBfmrteIFx1MDAxM/U3UGZgXHUwMDBmwFx1MDAxZETrXGK2VElcdTAwMWXehLZ7r4LBXHUwMDA3wIRcdTAwMWJcdTAwMThcdTAwMWRiXHUwMDEykFx1MDAxMG3xcU8jKF0wXHUwMDE4XHUwMDFkiLXDJ1x1MDAxMEVcdTAwMDBydFx1MDAwM1x1MDAxNXSZwbNQXGZl+EfJRFx1MDAxNOgo+KHQUD7J51xcXHUwMDFmxc5cXF5cdTAwMDBdXHUwMDBm4GA64FFcdTAwMDKnVVwiXHUwMDEwwC2F0Vwi0DZr2T+DYqhcdTAwMWbjXHUwMDBmUGjvbDlqV4gmPG4uXCJcdTAwMDOFr/T1wcmiRqhDM4BcdTAwMDJGXHR3bVx1MDAxMo2xsFx1MDAxMCmJil5HX6+B2mRQWaGE8FaC3uhEnlx1MDAxNKz1xcOl4tcrzKzFIei1XHUwMDE290g6PFx1MDAwZa+SmNgjXHUwMDA0iJFcdGvexPrwaVFzXHUwMDA1gVVT0cDXxXR5XHUwMDAwo1x1MDAwMN9cdTAwMDNTc/iuXHUwMDA2OY+QcUddXHUwMDAzOlx1MDAwMZ/W6PTpXHUwMDA2MD3JeMhCaerFeWbAYVDMnyCAXHUwMDFhXHUwMDBieOCZXHUwMDExROq8Oq2Bw2ehXHUwMDBm9Fx1MDAxOEBcdTAwMDbzxDWliUXoXHUwMDEx4lx1MDAxM1x1MDAxZlx1MDAxOFx1MDAwMcZ6dz9rcXBcdTAwMWbG4lJcdTAwMTFLwHeU80WKhXjBwynCu+CJNFx1MDAxMCcyXHUwMDFiXHUwMDAzI1U4P1x1MDAxNs6VxDlEunlcbtVcdTAwMDdcdTAwMGadaZBvgzxcdTAwMTa6M/eKJ4FfS1x1MDAxM59cdTAwMDY6JF1EXHUwMDEwqlx1MDAxNOhDvW+JmWSFLPiEsFx1MDAwZYFyeYUxw8t4tHmyhrmZh1x1MDAxN8hcdTAwMWOPguo5QU5hy85cdTAwMGaPK2pNeFx1MDAwMWuLXHKSXHUwMDE1fMDAXCKmN5l3h1x1MDAwZi6KM4L1IGA2XGZjXHUwMDAxxFxyXGJcdTAwMTa8gYD/XGLe0cGIUd1/Ls8zWUDeolx1MDAwNYvKmulfwPUqiWBcclx1MDAwZrP0fK1cdTAwMDY9jHjV4lx1MDAxZlxyntDgcm2Gm+Ng6HhcdTAwMTJgXHUwMDAyLpFcdTAwMTdhs6BcZvQtokk+dcbiYFx1MDAwMXBtXHUwMDAwf3g2k2wyWJuxqlx1MDAwMY9cdTAwMWPGgaC7XlWMxOpIdWDweHwuXHUwMDE1XHUwMDA3h8dMtcuTZVxydlRcZohcdTAwMTCYs7HQVZ+kUqzD6kjVvJOs2mywo4Lr4W4mXFyyw0LKrlx1MDAwNeK0XHUwMDA0P8M78FhBN1x1MDAxMlx1MDAwNzWJoFx1MDAxOUBybibMlziXkbND61x1MDAwM7ii9Kk4ZttcXFx1MDAwNFxmwZuJXHUwMDA2Zlx1MDAwYktcdTAwMDJcdTAwMGV6XHUwMDFiXHUwMDE1KUlIlueheFx1MDAwNm7bgTZLqVx1MDAxYWiKo6oq6IrA4/Ou9GidyPBNeERcdTAwMDHsN4J01XtcdTAwMDHI8zBbQDiQxrFcdTAwMTUrXHUwMDExyMpcdTAwMTAwUlx1MDAwNFx0RPJcdTAwMDbXS5LLONRcdTAwMDA0oGUlr+fAqOH1XHUwMDEwXHUwMDFkMI1cdTAwMGVcdTAwMWNvII99XG4g9fBcdTAwMWKA/6jLYVx1MDAwN1x1MDAwNDKWXHUwMDE2XHUwMDBlX8Q6llx1MDAwNrG00WRBiHq8cCDhflx1MDAwNvLgXHUwMDAzuCFcdTAwMDZQYFlMXCLPcFx1MDAwM0eBXHUwMDFkeYNcdTAwMDD4wcWJLKewbN9AXFybXFytYMhcdTAwMDFHXHUwMDE1XHUwMDAzXHUwMDFlc1x1MDAxM6cn4CSBXHTakZKWlmYjXCJopqtcdTAwMTF+SUR1XHL2zFx1MDAxMDKCXHUwMDBmgEBcdTAwMDLMvFCpOICdyVx1MDAwYlx1MDAxY1x1MDAwMVx1MDAxN6FB+Fx1MDAwNzJcdTAwMGbq6GGWIDmIXHUwMDFhXVwiXHUwMDBmd1x1MDAxNN9cdTAwMDJcdTAwMTRAWC5cdTAwMWFcdTAwMDRcdTAwMWIzXHUwMDE2J3JcdTAwMTdPvPVcdTAwMDKIkegxrMZaRFx1MDAxYVB02IdtXHUwMDEwW3kuLy/x0Fx1MDAxMc6+/GS9yDygTqpgwVx1MDAxZkWD5INcdTAwMDJcdTAwMWaxXHUwMDFhoO/xQKQyJbv1XHUwMDE2btRpxqdcdTAwMTZf2sAqXHUwMDEwTVxctXbCqVxiOIRkqyZm+aaQ5zugXHUwMDFl9eJcdTAwMTi9w1x1MDAwZlluX1x1MDAwMr7SbSRcdTAwMTFcdTAwMDC2itG9i1xy9i5nL07AQ4LgSMtcdTAwMWOYL+lKgNNDlGtAzeFpVf3FMtQxINVMXHUwMDEzRK28KS/PcVx1MDAxYlx1MDAxY++AikJog5unJLy4wNVcdTAwMTioX1xivkx+XHUwMDAyd6ZcIlvxwNvIaZroijLM9tlItlwio0/kRTB4b1x1MDAxNPOAXGLXfoU8aLCW2nBcdTAwMGbTWlXSvSgysFx1MDAxZs+sUlx1MDAwNNdrkork+nSQhmlcdTAwMDdcdTAwMTaWlGwj4vGSXHUwMDE3gUBcblx0JG2UV8qdKJiyxlM25Z29iNVDTVj6qdlcdTAwMTjVwDhixp1nT1pcdTAwMDJUK+epxvYnmohjhoWBM72bi8nqXHUwMDEwXHUwMDE2enZtKDCDWH+1ua0x+mTgLlxiNIk84Fx1MDAxMiQyI9uATjFLXHUwMDAxXeB+UFx1MDAxMFCLclx1MDAwMU3M07hcdTAwMTFeXHUwMDE09sj69ibygC/wQc5cdTAwMDBzWe+W6FxubIKxYPT0XHUwMDE1XHKudtbyXFym6HKNXHUwMDE3ILdO+tQ2hFx1MDAwN5OBXHUwMDE2w92rXHUwMDA2SVxmXHRPTlx1MDAxZCFqMFdbLvFcdIAhXHUwMDA1tipFxE1cZq7eXHUwMDE3SESluFRtXHUwMDFj++uYhirJ+1GWXHUwMDEzSVlCXHUwMDEz5aO6Mkiz7C+PoVx1MDAxNOWGfJfBaao6rbtcdTAwMWXWJPe1QZVxTVxmO33i6cH26FNcdTAwMTCXXHUwMDEyfH1cdTAwMDNTm7E8x6S05MZcdNVcdTAwMTY+36a3XHUwMDBm7pqBX0D4UM8y8iRcdTAwMGIwnJGnwj2S5aRcdTAwMDO3S1x1MDAxMTIoqiZgrz70Q/Tk8SSYLYeKOZ2WgrCIXHIsTSs2wNU6Utw8i0hcdTAwMDNfXGanLJRJXHUwMDFl7i1hkpriXHUwMDEwhykmIIFcdTAwMWVRpapijHEsL1x1MDAwM3T42u2NWYszTD7Cf2pcdTAwMTB1eKlgkltcdTAwMDcq7qHCXHUwMDAw5Vx1MDAwNntD9Fx1MDAwMsDvyNxcdTAwMWUrOELqXHUwMDA08D20XHUwMDE32I2rJ2f0UZ7Fx1xi0DyT26mPMtGwWlx1MDAwMlx1MDAwMOVcdTAwMDFpXHLE8aEx8Vx1MDAwM6M00qbwKIOh9eO9XHUwMDE462/cbKVcdTAwMDF8XHUwMDEwX1x1MDAwMFx0JPfgbbmYKbL7wWmjqHJcdTAwMWVcdTAwMGa3Vlx1MDAxY2g/VFx1MDAxZVx1MDAxMTZcdTAwMWUqXHUwMDEwVybSXHUwMDAwbohZgNpcdTAwMWFevta6wCvyzTSEofl+i0qxLELhXHUwMDEwemhGb/W1paQpKo9cdTAwMGVcdTAwMWSwWUVcdTAwMTlcdTAwMTJxXHUwMDFllpfvXHUwMDA2Mu1cXKslZFFcYtZJNOFuTUJcdTAwMDNAoliaxNDIQZnqy/uwOESgjFW0xcMwiVx1MDAwNlx1MDAwN9BthHnMh5v62JGM1kjjpYFcdTAwMTOJUibCYFxiglx1MDAxYS4lg7eHXHUwMDE1xm2K6Fx1MDAxOdbkqe4yP1x1MDAwMb6C0Fx1MDAwNea4XHUwMDA1iztrxTGyXHUwMDAwXHUwMDEwk/A7llx1MDAwYpZcdTAwMWVC4HZcdTAwMThcdTAwMWM0vk9wy6veUlx1MDAwM6ivYlx1MDAwZVx1MDAxNuZFz132cYx7gFx1MDAxOCzohJurxWpYKlx1MDAxYupccmJMq7mhWXIjXHUwMDFltFx1MDAxZHdBedBOpvbrNUQzJoRZwbDJaErIilx1MDAwMFTle7CaXHUwMDAxcn1eXHUwMDBi0THuLpZcdTAwMDRWXHUwMDAynXLl6Fx1MDAxZNExhzIgorVcdTAwMDakvT481syDXHRWXHUwMDE3R3C6WFJcdTAwMTGE7vqKkkpmVWXt2phYXGLkS4jfXHUwMDExoYs0sVx1MDAwMOJcblx1MDAwMlx1MDAwMZ9Ekl1725hF4f5cdTAwMWHrTVx1MDAxMFxyXHUwMDA2n0hzJjiQeabbXHUwMDFhXHUwMDE0ks5YXHUwMDFh9/BB/Fx1MDAxNZFcdTAwMDb8XiXSeCsl6Fx1MDAxZWhcdTAwMTRoS+2NM1wi80x7+jz35ZI9JFx1MDAwN8tzXGZcblxmbyz8Uq04XHIsXHUwMDAxmLJoJb+iNNOWx1x1MDAxMMI0KCeHKJhcdTAwMDLrXGYjqZpKRVnW0stAJlx1MDAxN+pcdTAwMDGVXHUwMDE5SuCSXHUwMDA1XHUwMDA1p1x1MDAwZYvytvQvXHUwMDE3XHUwMDA3fedcdTAwMWNccs/dMGjpWIJX4msgXHROSdZcdTAwMDfDTLdHxdS390B2OZZ+xrKAztrCbdZXXGIw2W5YI4onoZiPSVx1MDAxNFx1MDAwZV/Apm3m7HyoX1x1MDAxYp2IZ+140PSLZfWFL81pLMgg7ka9XHUwMDBmydeGy1x1MDAwMJFHnJQk2iGNdUm4Wmqcrt+Bwn1cdTAwMDP/0YjSYavOeZ/u8UBcdTAwMWZcdTAwMWSLt5muqFx1MDAwN9R8i4e7/1x1MDAxMXbFXHUwMDFhmUSaYizBwi4w9PqwxpiMm5BUXHUwMDEyVn25sf0nQSZcclx1MDAwNIL/rVx1MDAwZmpmLE3mVSc690vwIWVcdTAwMTVhW1x1MDAwMJw89NDafLOhgYtD/MZ0XHUwMDAzozfuviTypNB581x1MDAxMFleaFx1MDAwMFx1MDAwZTbz8LFwSIF9LnFsn1x1MDAxMpxC54XfoI71NVx1MDAxZtyVZVx1MDAwMZ3MXHUwMDFkt4sxXHUwMDEx5/LaXHUwMDA3eHz4+npcdTAwMTI3c2m80YI1XHUwMDFh2lxuX9I5bkArJtVwrcxH1NeT5cUysFTuJ1xiJVxuLdO5PLhcdTAwMTJ+XHUwMDExgFdcdTAwMDE/6tvSNMh5tFx1MDAxZa5cIlxuJtpLmM+lc4tQOFx1MDAxMFx1MDAxNZZC1a5OXHUwMDAx7UBcdTAwMTO4Wexd3lx1MDAxOPdH+V1E91x1MDAxYVx1MDAwYlx1MDAwN3xFUJx6glx1MDAxM0mv4MtcdTAwMTCOgk7bsUYycu3AjCpcXEqj/pZZitOMITWR01x1MDAxM4rT4lx1MDAxMdww9lx1MDAxOXlcdTAwMTh0aNBdRVJcIliG7TSzXHUwMDA25c4llrZ43lrgRED422BxLmNIXHUwMDFluMMr2HRQXHUwMDEyp8gxYPt5vZqo93Xas8eN+zlcZl7KW+0qL2JcdTAwMTWIxFx1MDAxMEaTXGLXX+pspYXMKepcdTAwMTWLbq0oM01mjYOXrDS0rOSpT85BXHUwMDFjN1x1MDAxZaVcblx1MDAwNGOp0novy5wm2JQyilx1MDAwMUaDa5VAKehcdTAwMTPiXHUwMDExkP60XHUwMDFhjVx1MDAxMUpkJihcdTAwMTgmhJo8Vu5jSoRy0clUmmGRoZLQb1x1MDAxYlx1MDAxYVg/4o38vuRpMmC2ScRpptdcZv/1TUJ9cP7I+kJcdDuCXHUwMDE1OZncOkH1NVxmWFx1MDAxMFx1MDAxYdSvXHUwMDBlXHUwMDExhFx1MDAwN+jg2eJ/MZZBh09cdTAwMDKPXHTPgoVHrGZ+cHlggSxcdTAwMWJcdTAwMDJYXHUwMDE4WKZcdTAwMGJjlXzMenLXWcJN1Get8zpSmCtcXDGCdJVUoLCOXHUwMDE0z4kpO1x1MDAwNty6QfpcdTAwMDDkiVxyWoJ4XHUwMDEwOLY5kVx1MDAxN1xyXFxA4IxGVuTUirOMXHUwMDE14L2xNsVcXKlLXHUwMDFjXHUwMDE0d3yh5DJcdTAwMDZcdTAwMTb61T7emctjLVx1MDAxZoyJmWTWXHUwMDEzlsNcdTAwMTM+XHUwMDBlmj/eQlxmXHUwMDBmx1gnToLUIHJGSGfpxWU5XHUwMDBiXHUwMDA221BMgJtcdTAwMTjyjel6bWHhgFx1MDAwM/TAosiQyil1XHSkhVx1MDAxYrCsXSCo17N2lSHQ4WhRuHhcdTAwMWJcdTAwMTBHlz1cdTAwMDE7hnDPgD1KXHUwMDAx3GX93Vx1MDAwM1xccVx1MDAxYSfsM2CBopxSz+v84WHhW9hcdTAwMDLQoOiYjVxijJtcZuvuXHUwMDEwcpWjWDZcIsDFssNcdTAwMWG0ko199fLyvl5usjumR8pcdTAwMTVBXHUwMDE5t11sblx1MDAxZLFcdTAwMDFccsjg6ywzI1B778q7XHUwMDExuHP4JqtA/cjHXFx9MVCNOPpcdTAwMWLgnGfjXHUwMDA1XHUwMDFlyEOLk1xmLpmoZVxyNcL/cnKNXHUwMDA1lZ7DO0lBQe5cdTAwMWJtvrDaXHUwMDE4y4BcdTAwMGJSXHUwMDFj4SrLXHUwMDAyXHUwMDA1S1x1MDAwYulcdTAwMTKV58xLVb89eT/NhnLKXFxeXHUwMDA0UVx1MDAxMfGRrmw2fNW82fDZ5esvvuNfX7zVne72kjrbXSmMrprr6S3sRIWxsOTSy8LAxuuhpfB+itliJl7dlHGOd1x1MDAxOd6CgNuFfGdcdTAwMDJRYKHxczS7XHUwMDA1n4CPQyBcdTAwMDN2XHUwMDA0XHUwMDA1LrQ+Xo9uyfmJVLF6ov2NKv3TdjjHbYeLrFx1MDAxOYPjzuf4sIa0nHn+sfcuOHtcdTAwMTSMt0GXXHUwMDAwXHUwMDA0UqdcdTAwMDVJXHUwMDE0XHUwMDA3XHUwMDA3XHUwMDFhN0VgkzK0mVx1MDAwYkSQw2G70rI2gcW2JXmp2teKY8aC0ZR1+b5GOSMobV7MQjyzXCJPXHUwMDFi18qbZJr8Ycmbx2NQYEaekydcdTAwMWGUyc1YXFy+hcNwic1cdTAwMDVM55YhUlwi4GZdoOODqk+4s8fRg8ixSipcdTAwMWYkmfSxskaJ47/ZjSXAR1x1MDAxYmiLXHUwMDEw1GhGpIGTL0JcdTAwMTJqcLSmXHUwMDAyMcfrbM4xtr1Yn/m5L5Q06cujyYWCNVx1MDAwMLpyxNnr5iD5aXX18/vex/W17pH9uvX669fd7tdJg1x1MDAwYudcdTAwMGIkg8zYq5hP45GFTq1cdTAwMWYgaTJcdTAwMTagXHUwMDA0XHUwMDA1XHUwMDBlJFx1MDAwYlx1MDAxYlx1MDAwZbNGScnkXHUwMDA1kU6DW1VcdTAwMWT9QvtcdTAwMDLISMe8MViXXHUwMDFha8/nkFx1MDAxM3YyVVx1MDAwZjP8XHUwMDA3J39cdTAwMDecvN2nQ4Y4XHUwMDE3gSHL69m0Xm68y0j6nGfApI2Q9XVcdTAwMTKT1ZA/nuX3ns8vWEmvXFwnjtG9ZGWFsHmRXnlMXHUwMDEzXHUwMDAxSzhQdCOZRqpcdTAwMGbBZixOZeyGtWzxZu+QKSfZx62t9ubx3rE3IVx1MDAxZraFKNon6+OoXHUwMDFm7ni4XHUwMDEwtGlcdTAwMDBcdTAwMDVcdTAwMTknLlx1MDAwNlx1MDAxNlx1MDAwZlqO9VJpkzqYO1ttbD4ooZaN3Fx1MDAxN7KE9OVcdTAwMWJkXHUwMDExjEfDhFn6b5pDy+rmrvn6+vOGtp+OluS37id3tvjkt4BcdTAwMTbLXHRTLDFQV4N5XHUwMDEyaJE+i8xcdTAwMTMppWRcdTAwMTThnqZn2rz3w1x1MDAxYc5cdTAwMDVCLFx1MDAxOCtcdTAwMGVFYoZcdTAwMWFuJJKbka6OXHLYlpHFVsqHfya//L7QXHUwMDAyRs16We3y9Fx1MDAxZTfBi79cdTAwMWUzXHUwMDEzXHUwMDA0XHUwMDAyXHUwMDFmZolBVuv3XHUwMDE1XHUwMDEycaq8h1x1MDAxMjOWxlnH4kvBOpuHXHUwMDE2XHUwMDA3/u1cdTAwMDNrOoBcdTAwMTVcdTAwMWOplKYmXHUwMDAzXHUwMDFierhvpvGXemiMmWPZv2dJl4JcdTAwMWSVM6c6P5RPXHUwMDBi51x1MDAxZI9lqa9GLomL6XZcdTAwMTbE8U3ci1x1MDAwMLtcdTAwMDVcZtSJW2Se3+ZcdTAwMTM5OI3T6HKfTW7fhjlsllPGJr2e9yDQwDRNNNLh0mIod4pcdTAwMDRO61BsXGJUeVx1MDAxNVWt9om8TVx0YbPhTCtcdTAwMTZ1lkd8LF5/wLm8Z1ZAXHUwMDEzYn3a854wcnKOXHUwMDEyioSQw1VcdTAwMGZEW71cdTAwMDVGvjveePP5ZWtz8/Hi8+9cdTAwMTf7K+dcdTAwMDerr39cdTAwMDeM9Fx1MDAwMoYqhY4gYKylTY7c9Hk/XHUwMDFhXHUwMDE0MadWhc3GmWOkYW5HXHRDhjuqQCpjpLHcgGCvNMKs8YOVXGaLzZWy/+Qpf1+QvN2n4dSczqvS8qJqX+b8cGrg+Wz85eF0ur4m32ScocIhwmytUm7M55ZcdTAwMTWw1udcdTAwMDI1OcbV5XtqiGPKXHUwMDE5Q1x1MDAxNjnlI5I584uHc95Wnk1Ry1o2M+Sdfrde3N2Eca8xXGKAi2SXkVx1MDAxNOXkssw8O5LYJyu5l18vTnBSdj58VVx1MDAxOK1t0n+Ap1x1MDAxZbjxasmVRaxcdTAwMWa0meE6eYCN8t7LfHLIXHUwMDFm5bc1Z1x1MDAxZrHVyih2Ts3ldprncEpRXHUwMDFkzq01h6pcdTAwMTft/tn54cbjo+5B72Sw07nsXHUwMDFkfmz9XHUwMDBlUOXYOs5hiDIgmDIm3U5jaVx1MDAwN2sj2X2jw/0kXG4j2yQ4zlx1MDAxYqpEnlRcdTAwMDFUV9rJ3EVgU0DFXHTRtLtYrIj/XHUwMDA3p347nDKsXHUwMDBlj9zP51x1MDAwNDtdLlxcWZRsjlGMRljXzZGN7cX6KjZcdTAwMGaqpVwieFx1MDAxMOvhRdJSw7oquCjFYyU5Rak+Icbmflx1MDAxZpyAweRcdTAwMDfyJMlE1iFIjn9iPU89NqpMau6j5EOruJFUXHUwMDAyx1TpXHUwMDFi7C9lPnCkZMxLgJk7nLa8JiMqYZDshOFQMjbB+Sl371x1MDAxN41mloXJvWNn3Vh6XHUwMDE2U3m663pz7270MFx1MDAxY27CcHdfPH63Ozy+aPm132RcdTAwMWYoXHUwMDFmKs5yb8fAcZSNu/LumqNcdTAwMDRcdTAwMTDbO5OnoF2ysJntXHUwMDAzsVWd1fdQZPCYqkDEsa9JxjzrXHUwMDFizHhcdTAwMWOCxctcdTAwMTBcbvnsf9z7b+feb/lpjmLg+Dbqr2IvTdn3XHUwMDA0zYpEnmZlnY9NoEBygms+TY9EVyZcdTAwMDdcdTAwMDBkTNPQa1xubs83aNacrTRcdTAwMWVcdTAwMThcdTAwMDJcdTAwMDfLrkOofzJcdTAwMDBcdTAwMGbW4XFcdTAwMDOiifwuXb9tIziLkMNCY35kQLlcdFxm78LKNOdCRkWcrZVcdTAwMDdcdTAwMTDlXHUwMDEzIFSxiMSliUJm9VwiXHUwMDBm0maJZYPTNdgwxD4wXHUwMDFlXHUwMDE1wHCoLC/1XHUwMDA1XHJgSrClXHUwMDA0IUaQnlx1MDAxYlEpyHOriWc1WFblN5lQM2uBnPCDp6HyLU3FrvLSXHUwMDE1ZznCe46ZwFOz9WUzafDlQrlcdTAwMTVU5Fx1MDAxNeuKYSZHWtkmLSv3XHUwMDA1z5NPTpDgIJqLq8Lnt83x2YeLuNzuPIlftlx1MDAwZd7KtaXPvc9rj39cdTAwMGJ8djajXCLBj1x1MDAwNVuc8TnCZ4b/7OKmSdxbNSPno/OgnXymXuFkwFwiPlx1MDAxYsnJimzyjtH48YJGnr/AXHUwMDE2qbqz6P5B6L9cbkLDqTnFydzGQHVjud1QwoU6r1x1MDAxNZxk3vNRv1GhOPdUXCLG4Fx1MDAxOCxWeCQ+V/LgXHUwMDE5rzjsT9anp+qkQVhQgFS2QKhcdTAwMDZcdTAwMWVXZ8pwzoNwxubEI4GsxDxcdTAwMWVcXFx1MDAxZVx1MDAxYlx1MDAwNDnUiHNcdTAwMWY5oaBcXFjBmWpcdTAwMDbMm7PhXHUwMDExcjZcdTAwMThGzlx1MDAwMTm45YxcdTAwMTiVZpdXeTw3O2Asp/kjXHUwMDA0ZFdPvcB8bI1nXHUwMDBmIzNyKohcdTAwMTSwisX5nEf2i0RcbnZcdTAwMWJIoiZwn5H+WFqzKFx1MDAxNE/vl1x1MDAwMWtMX/5cdTAwMDGs4GHQfMB/XHUwMDE1rr5rjqvdzeHG6YuzXHUwMDBmbm+4bE9f2W3z9N3lb4GrPJeIdc+GZU7OpfWPnJFAospcdTAwMDJ8uIB7int9PrWa8zhcdTAwMDPrVFxcXHUwMDE1rrJbgVWYnDfjWOY1XHUwMDFl+EaeXG4m/slr/n1gVVx1MDAxMecsXHUwMDA3nyDYKoW9guOyI/vkOW7cxFx1MDAwNn33iqGghSVcdTAwMDQ2kKtynCqJQlx1MDAxMX6O4/1BxeuHxPFgRE7aQeiDQDDq8i5SXCKvwTF6kYM1NCdOXHUwMDAzvZLeQs2hblxiU3mEnuTQ4jphLuPwKot7h3WxzqMkznCGb2QzXHUwMDAxR/fUt47PVlx1MDAxYfPXkulcXIZkXHUwMDFj219cdTAwMWUnNu5cdGpcdTAwMDVcdTAwMDZcdTAwMWVQYVx1MDAxME5Kjlx1MDAwMZMuPUCIU3M4v5Rn2ZhcdTAwMDbdXHUwMDA0Mp9cYsUmStaYXHUwMDEw2pNHy2NcdTAwMTKiQFTijGsytHjWXHUwMDAyr8GXqVx1MDAxMPaq4NKThlx1MDAwN+E9z1xmjJyBwNn6tXh/J2hcdTAwMWW094ZX+FOBz4WjNFN45jhcdTAwMWb2XGZV1sdsNIfnb2eby0ut1feLT1x1MDAwZbcv11eevFh9sSTn+8BAuFxylVx1MDAxZv9cdTAwMWPY0ClEWlx1MDAxOZOPu+dcdTAwMTa1tTxT0CZcdTAwMGIrXHUwMDAwc5v/TFx1MDAwN+aW3Vx1MDAwZlWHXHUwMDA2aiF5XHUwMDFhh8uHl/NojXFcXLZUXeZXTLz6XHUwMDE5XHUwMDBid5nzo3uuXHUwMDBld29cdTAwMDW4/73Ru1x1MDAxZqr1I6H052RcdTAwMWO++Z3/XHUwMDE2NPhaMZ7sv9XPPmxcdTAwMWW+XFzuvDxZ7j7dk91cdTAwMWT/qKjqNzrcXCJcdTAwMDA+unnnzz+myV1ZP39+XHUwMDFlP7/8uipPn7WeXXRPn75avpPcXHUwMDFmXHUwMDFmXHUwMDFmts+HZUFX3/jx7UXn/Mh8PJHfnlx1MDAxZr7a9vJC6fWC2DF+M0ZKeMLHKHXcgJRMNfnSOlx1MDAwYtZegNXxknFcclx1MDAwMIbrl1Xmvtnc3KvvRbW5XHUwMDFmtfaOzlx1MDAwNu05MHgjXGaAyOm8078wROja4J3IXHUwMDE0Jzw4XHUwMDFl6jPV4Fx1MDAxYlx1MDAxZFx1MDAxMdrei7GCiUefcXhcdTAwMDHHIEZuRY1cdTAwMWK8sqmFa8TmXHUwMDFjwlrdeDTvjPqg31x1MDAxYm50LvlAlCi9+qx13Ony5o+uONdq2ndv2Fx1MDAxZfBLXHUwMDE2XHUwMDFlr68sbLRcdTAwMDdQ5UelTz3udlx1MDAwZXt5uNfmZ0umMOzstbo3XHUwMDFmOO7s71x1MDAxN5FxXHUwMDBmX91CLDtYaYJd/UHnsNNrdTdcdTAwMWKtrHU27L9rn15d7nBw1i7esfaLm2AruyZBP1x1MDAwN+iFZvLUxFx1MDAxMWpcbs4mrkxkv79cdTAwMDWih297345cdTAwMDavetvq07fDXHUwMDEwN+3up1x1MDAxN/OO6CbkXHUwMDAz+zlfc1x1MDAxY9FDPo7Qa3b4grj5KSnsX4zonFx1MDAwZodIY0Kj4a9cdTAwMDX0h1x1MDAwN97hwaZXXHUwMDE3x2+Gr9pLe+L1ivy0eLTUlCc8jlu9nUUnV15e7Fx1MDAwZbaWjlx1MDAwZd7uf1xcbbbcJoDuXHUwMDExXHLeN6BbMbm7mIdW5FFLlbVvNbf26ns894DOaXM8XHUwMDE2L1x1MDAwMrGVTFx1MDAwMD1YRqVcdTAwMWOcznNCptr7XVx1MDAwMJ1TXCI5XHUwMDBlM0BcdTAwMWRsYTLMNEA3PE9cdTAwMDEk41dcdTAwMDI654RcdTAwMTdSefdccujrg/63zn57sHDSPVx1MDAwM57eXHUwMDE3lk9HrVx1MDAxNMsnLmpWML7faVx1MDAxZPd7+1VmbcK0YjFcdTAwMGVcdTAwMWPU1Wb9oblZv3p89jp8Xn+1/aLvdlePdnaXvu5Mqlx1MDAxNptcdTAwMTNcdTAwMTCHXHRcdTAwMDE/NUdcdTAwMWZx/q5O9qGD4lxmbVx1MDAxZrldNZ2lq4PYNma6Ue+qXHUwMDAztbtbYdTBZJpHm+hrjFx1MDAxZXngkVVLm1lrWKaemnc+Vl+bqO9cdTAwMDXAf1x1MDAxNPDcXHUwMDFhwOuA9nJ7uL6+tvHa7Vx1MDAxZD6LavHL2lHncWhcbrRG9LbWX/a2T+LhUMmLlcvBq92GvKBcdTAwMDHQcqLnvUfOxZFcdTAwMGapRTqhjOExQlVcdTAwMTb5sblFVt/jOVx1MDAwN1rH6eyeh2yIwFx1MDAxNvuUWEfLijHrefzAnW1yXHUwMDEy0HqbsUuNNUs8vLbKJCssUVtcdTAwMWXBU7TRv3zovN7fX+j3XHUwMDE2/tP71lx1MDAxOVxmz1pd/K3X32/fXHUwMDEz4tZAzFx1MDAxOOJeLe7H0spcdTAwMGKbXHUwMDE16l45mFxuXHUwMDBij2LiOFx1MDAwNe5dWFx1MDAwZausMvBPtyjQnupcdTAwMDfn1MC9kjzeRlracFx1MDAwNeg6MGnYtVxus1x1MDAwMN2JXHUwMDA2zqFT1lx1MDAwN6kqpvPwODQtx5k0S1M8nIGq3pS+f6idaveuOFx1MDAxNPDnN6FcdTAwMGJcdTAwMGJrX311XHUwMDEz28s9xd5cdTAwMTlXyVpcdTAwMTIrjeeJVXDkjoe6XHUwMDE1tvxcdTAwMWVcdTAwMWS2TvhcdTAwMDAyXHUwMDBlQlx1MDAxMdbmXHKQoTjHXHUwMDFjente3thcdTAwMWW7M6XN8UkrPtjuPl/flFsrZ1tvVj9cdTAwMGXkpTBv21UrXHUwMDE2WeDZ4OCAQbNJv1hyfb1cXJ1cdTAwMDVOjLvOs4jiKTBccpY741x1MDAxZPvU58x0077CMviz+MMoRr8/RqXa3d3+94a8aKLblKqwYzA+hiZGLaSu3FPYbu44p+9cdTAwMTTNqePkyLJMXHUwMDE4nunjI1x1MDAwZqxcdTAwMWGJufacPIdQhFx1MDAxOJRj8czk/vq7eE7Fk69cdTAwMWTP0OXIXHUwMDBigFxcRVtL0Dz9XCJKXHUwMDFl0aVD4ajOXHUwMDFmntRLizjnnnZcdTAwMTHn3JGePn239aE92L1cXDp42Wpccndar5bWNqrdkmNLINM9+UkqRYO8dkuGXHUwMDA3McIsXHJwUkhXXHUwMDFhWzgrNzp9y6K0Xk5j1KxnMYrTp8e9vmJcbi0yjM3T1aXCl/vxo6mjeUhXOtlU+LM4ZiX371r9xJbwaFx1MDAwMpspbCUlbTX3rNNT9vPqWa2ImXXQSqgtz3lcdTAwMWRcdTAwMTG8K8/KwZGS53iImvqMO3nWTPGEVGZro+FhkuOOXHUwMDE1sbE1kocr8cxcdTAwMDbtx4NQXHUwMDA3/DSziUFn7Vl/Plx03NCzroWN3U9f3698XFxvvTk+2Vx1MDAxYrqT5fCiylMxqFx1MDAwMzO1UVx1MDAwNcHaND/uWSWPQlVwZIpHfUVZPIZtZp51alx1MDAwMrnkWVx1MDAwNYe2uXyghIpQgDi2YJPhyXPW/u0g4Heiplx1MDAxM1xyhD/jpjEjdzqx9sVP7PBiSp1cdTAwMDOIKp3pbnNnumaGw8fvPj7dWJXv1zaXulx1MDAxNzsvzt7MuzNcci7m4ywjjzDhdLCyL1x1MDAwNafJNNN78l5LX5yDx5YsXHUwMDE0hrLwdOJcbmc65jxcdTAwMDPIi1My/v5cdLxQenVKXHUwMDAyb+s6ObZ6h6zdsH8yKWVXWnean6v+7lkl5ibZrdNcdTAwMTNcdTAwMGJaXCKslmcqVpntXnOzPX1x/q2155Zff97zy+2lleVcdTAwMTX7+tm8m62UNlx1MDAxZp7tXGLUUlx1MDAxNKdcdTAwMDddV7T4LCqdXHUwMDBm7eWJcZNn4tyFXHUwMDAzSVx1MDAxMzNcdTAwMTVtXHUwMDBlcTz7p8DERptho4Vf221+tJdcdTAwMTFcdTAwMTN6MH8ju1x1MDAxZLXr1dntxsaLhferK1x1MDAxZlx1MDAxNzbWll4tbz606U78+vu2Xq0nZ4dcIqd88SDxKvvdv1x1MDAwNexO5ZHzar+WJ1XzXHUwMDAwXG5cdTAwMDZyxUNcdTAwMDCuzFx1MDAxN4RWXHUwMDFiXHUwMDFl3FBcdTAwMDe7dzHffO5Kflx1MDAxNlx1MDAxZs9rNFUjT8atV/KwZfaF3Lv53jDC2daajdHLX+1cdTAwMWPOep3zhVNcblx1MDAxYz60Y6j86oeocIlcIn31XHUwMDA21Vx1MDAxNc/LUqGSjbebu4WLwbu9/vOl81dq7WD1yfOnT95ePt+ee7dcdTAwMTCi5yHxWoFcdTAwMDOLdDdcdTAwMWRRMFx1MDAwZsqR7Gibvpt+IMKeXHUwMDEwP5nY4JGaSXBarG1hf5OV1lXsuLGPVatiWup3KG55/+3bZst9cFx1MDAwN1x1MDAxZt/tvPz+/FX3YnfTNC1uef763U7/cNA53926XFxZ3nizv/lhW/0yR/RzIFx1MDAxZCbnXHUwMDE5RVByUo/2QXNjrL7F826M0dmMLZ9Rq6uirbI5Olx1MDAxZZCez9OOwND7MkcpQFx1MDAxNHzk6bGcXHUwMDBlIKs4tlx1MDAxZEdpxcPOpbh/lJ6r6pb/9JZ+gN/pPVW11OBKRVVL9aJmhbLtbrdzclrZXGZi3USUXHUwMDA1sfQ8kKt6qtFhc8vePn6sLr7smFx1MDAwZi831Prxent1aWdz0vjzOakj5UCnTEhcdTAwMGXlsN6OgSyMXiBEhfbiXHUwMDEz02rD72TV2mSs462saDFO8qxdV4Wxxlx1MDAxOVx1MDAxMU10v2ov9ucg9uNJZ+n9Tu/l8f6X9+bbxc7J+atcdTAwMWT5XHUwMDAwjVx1MDAxYX916DZhooXz1F8/XHS6j5pcdTAwMWJ49aObd+j2XmdcdTAwMWONXHUwMDAz+1x1MDAwZaJ4yu91R6dcdTAwMTeZi5wmrXmC95SZZU2MfFLDl1x1MDAwYlx1MDAxObmBsY7t4pV1qePIrVx1MDAxNI+A8X+njs4nnMfzn97GxemwfXxPsF2DUylsT1jSvZehOjE5YeZDVFFcdTAwMTX7iFxuXHUwMDE23Wlu0dP96bxatLM289yp0lx1MDAxY1x1MDAxZVacS3GVMfMq46BcdTAwMGUpgOq2OM9xplx1MDAxOTOVRW1kkPAgweiqIYTaZkpFg5XyPC9cdTAwMTHGzopk/ixcdTAwMWZI93fc9L/FJrrnXGJSydM9gmGJY+FDV3voNjNS6nwupeaMU1dcdTAwMTQzmz3/6V6jsNy8jJYnuFx1MDAxOZBLxfGpxakvN2WpoJxcbkvVUGJbXHUwMDFlf/dcdTAwMTfb/J9oKPxZXHUwMDFjs5GRuDFmNZNaquJU0jG3aj1cdTAwMGZcdTAwMDGs3Eb83NyrTmeT8+pVXHUwMDAzXHUwMDFllMDzXHSeXHUwMDA36aixXHUwMDE5dFxihoxlxTc7eEJMXHUwMDE2Nlx1MDAxYqfqYOvGXHUwMDA2qVx1MDAwM7tkQciq9iF85oyHc4I/gFx1MDAwNVx1MDAxNY9cdTAwMDBcdTAwMTmdreGgcH/PKtXGfoojmbXkgVxi0nrB+FNcdTAwMTSmRl/7qXy8kfA8X4ddjFHfQ53q9DRK2bMqyYNYXHUwMDExiqvIXHUwMDAzIeT4iiW0OFx1MDAxZu1cclxyiFx1MDAwNu7X3GrFv5NnnWwt/FlcdTAwMWM3lFx1MDAxOfnWqWNH7OQ5Ylx1MDAxY1LKXHUwMDAx2pXbvN1blGlMLcSeU//qY4QtgbpysDrZSkJadci0YnVj1L5Y6TJvY0dgWFx1MDAxMDBfU8Tqck7761x1MDAxZs2q8rtvn5++uny69Xn96dPBt6a5oftcdTAwMWFONry43P56pndXL/ovbFhdWXx7fDyzoSNcdTAwMGaTc1J+ysHTXHUwMDE27Ffo6ilix82NvfrZzbmxR1x1MDAxOFx1MDAxYU9KUlx1MDAxNubmtU1LsmBlgafCXHUwMDBiXHUwMDE3plx1MDAxYvtcdTAwMWQqKeFvOIxbKcOuTGmrqNT4vi1blFx1MDAwMsKov1HOiSVRrUOs657yTTVwVVWhlSznrrmmJlx1MDAwZWdS5JSrs+GQXHUwMDFjXHUwMDA3XHUwMDAyyCnixSNcdTAwMTOuQoPoM+CW5NhojVx1MDAwMC+O6aLIjNLsb1x1MDAxMDpP2PhQkVx1MDAwMNU+XHUwMDBignvMwinlfXGCwLiVzVwi6vjVvuNWllXwnr1HlfbWJFT5kVlcdTAwMTY8XytGPTaSPfdcblPqbNlcdTAwMWVcXMxIT3bfqcWXrqY6LtJFvW5q8zOl1ItTNJU/4zo6hVM3jCVm2lx1MDAxMFxyVlx1MDAxN33keeZcdTAwMWW8PzppxztO7qFlu3FvZH5cdTAwMTKQ14xcdTAwMDRcdTAwMDLucjRKqfF8nsr81XQ+UFxiXHUwMDAzvnzLsPNOIdpMWkwmj5CQzFxyOqB/Ze6+35xcdTAwMTi9e7qzXHUwMDExNs3jo6VccnHUVp8/dva77+fHuU3oMdEx0/A9Klx1MDAwNJ7QKJLNuCBcdTAwMDJcdTAwMDfGa1x1MDAxN1x1MDAxY0fGXHUwMDE3SlZ+ebGrVuCyzD787sTob1SN+q9rb/yodXKyMcS9vPFgj7512t+fVOpcZn9cdTAwMThr5TZPM2rn4Prnv/78P1x1MDAwZn2PiyJ9 + + + + + LOGIN NODE ON RESOURCE PROVIDER3. No INBOUND connectivity to the HPCVirtual KubeletInterlink API ServerProvider pluginPod on virtual nodeVirtual NodeSSH UNIX SOCKETunix socketPodContainersBatchSystemSSH agentunix socket \ No newline at end of file diff --git a/docs/static/img/scenario-3_light.svg b/docs/static/img/scenario-3_light.svg new file mode 100644 index 00000000..64a87249 --- /dev/null +++ b/docs/static/img/scenario-3_light.svg @@ -0,0 +1,13 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19bVtcdTAwMTPZ0vX38yu4fL5cdTAwMGV97/eX801cdTAwMTFcdTAwMTVfXHUwMDAwXHUwMDA1UXnOfXFcdTAwMDVcYlx1MDAxMFxyXHSGoMC55r/fazVIunc66UZcdTAwMDLGmWHOeJwkVHZ3V9VaVbuq9n//tbDwaHhx0n7074VH7fO9VrezP2h9f/RcdTAwMDdf/9ZcdTAwMWWcdvo9vKXy/z7tn1xy9vJPXHUwMDFlXHKHJ6f//p//aZ2cZKPfyvb6x1e/2e62j9u94Sk++//x31x1MDAwYlx1MDAwYv/N/yx816C9N2z1XHUwMDBlu+38XHUwMDE38rdGX2eCSF9d7ffyr9ZeSK1kiDdcdTAwMWbo9Pbb51x1MDAxNLkrn4yEdU6fYlx1MDAxMcP2Pt45aHVP26N3+NKjy8vn60cni8+fq/bX3eXtXHUwMDBmS4+3RXf061x1MDAwN51ud2N40c3XetrH9Y3eO1x1MDAxZFx1MDAwZfpf2lx1MDAxZjr7wyO8K5PXJ/3WoH92eNRrn56Wfqd/0trrXGYv+JpcdTAwMThd89WN+ffC6Fx1MDAxNV6g1EJmJiqpXFxQVpubN/Nfd0FmXG43x0vhoo9eJeta6nf7XHUwMDAzruv/XHUwMDFkiLAnxGhlu629L4dYXm+/8JmD9l6Mo898v75aI1xcJqxxwdub947ancOjIbXEh8xpXHUwMDE1pLJcIv9xozW08ychI17VVsnR4+M3n6zs56ryv6P7P2hcdTAwMWS3V/grvbNut3hcdTAwMTN7+9c38YdKjZRKX7/y5+jS+PnlVFx1MDAxOYtcbllSymH7fHhzzVx1MDAwNWVx51x1MDAxNyefzOKJOP9uV97IXHUwMDBmu+77jnh087k/r/82Wv7ZyX7rSvekV8467ZzVcvRIup3el/Tauv29LyN1/VfhQsaMp7TOgt3oKCfZjbfQmmhcXKXdLDW3m+pbUW03R629o7NBe1x1MDAxZSzHeqhttMqE6JyKqem4XHUwMDA3MFx1MDAxZFx1MDAxNU1cdTAwMTZlXGaRqzFFXHUwMDEzuDEgK1KDgeJcYi2NlHe3l9JcdTAwMWJjhjFL3Vx1MDAxZK2q31x1MDAxYm50LvlAlCi9+qx13Ony5o/cSK7UuIGv156vrC6srj1dXlhbXXi3vLH2/t3S8sJ/euvv1rZWni6/e1T6lcfdziF1/9FcdTAwMWUupT0omcWwXHUwMDAzaLr5wLB/Mnp3XHUwMDBmi2h1eu3BSlx1MDAxMzjoXHUwMDBmOoedXqu7OX2N4ytsnVxy++/ap1f3YDg4a1x1MDAxN29j+8WPXHUwMDA3XHUwMDBm122nWvxUuMRcdTAwMTOaZPZSSy+E915X2f3T5nZcdTAwMWZfxt7773ZxcP7t4PT8+bMtefbhy3zjpXWAI6uFXHUwMDE23jrATmr12mdOwiV4ay0+Z5OFjSxatvnPdKtv2f1wcDBu9TaYjN8t3Fx1MDAxNSiOLuXG6o1SmfRcdTAwMTGYamL+M0L2XHUwMDFmXkBcdTAwMWJcdTAwMTiicf5cdTAwMTeh5oNcdTAwMDHbdFZcdTAwMTgnskLp8Vx1MDAxNJ1cdTAwMTGqXHUwMDEy3pabq/nJY3/ycXHt7ZZ9/+JsZWWlfXm425t3ePNcdTAwMWU6XHUwMDE2RYxSiqBdwdbz3/dCZ0A3qJePMVx1MDAwNOvvRdG1iZmMUFx1MDAwNmltXGLKODOu6UrJLFxiXHUwMDAznVx0XllcdTAwMGaNXHUwMDFlI4hBeFx1MDAxZEw0f3lVn8ThTPRcdTAwMTO1XFxL61xcMLZKyZ81V/KLnS9cdTAwMTftJ8f7/c3X3fNXn9c/f758ujXvSm61y6BW0PVglFAyUXLrfWYl3GTUXG5cdTAwMGU1TOZwd/Lm4JFQ8GBMrlx1MDAxMqpcIlxiKuLItU5cdTAwMWJLblx0u/iFJC5IXHUwMDAxs7wriVx1MDAwYqVXp5A4nS2s9lx1MDAxN1ZWn6y9X326XHUwMDAwstWDZ+98w8NeXHUwMDE49lx1MDAxN4ZH7YVcdTAwMTfrS/dA5EpcdTAwMTeUsrZbLupBuJuVXHUwMDEzQzZqjFx1MDAwMqjFSu72vLm9XHUwMDFmbHefr2/KrZWzrTerXHUwMDFmXHUwMDA38lKYt+15t3dcdTAwMWZDpk3QRqsq9uZccqIpXHUwMDFmrFxuQdxcdTAwMWZ7Y8YlXpv6XHUwMDA09mbxiajkXHJ5XHUwMDFirfPa/K2Bv0DoOXqK85DyuNJcZiN6W+sve9sn8XCo5MXK5eDV7ogtlbS4NVx1MDAxOPS/XHUwMDE3clx1MDAxZX9Mk/tk/61+9mHz8OVy5+XJcvfpnuzu+DvJrUvRXFxcZl6ZNy/7Ty63vm9dtrut+PXg4qDpcodcdTAwMTeX21/P9O7qRf+FXHKrK4tvj49Xmy33+m+/nDYoodNXR7QhRlxy6iBHxKLgR17cgjdU3uM59yPBh0xa46VcdTAwMGLMO9qUXHUwMDFj25BcdTAwMTlvnFK1UWD0ck/9XFzuR1x1MDAxYZVpXHUwMDFiwdFcdTAwMDVoryxkmEbceJw3SG013Ir+XHUwMDFiJX+2OoPhWau78Opsl6r4s1xm4bizv1/E2yTbU1x1MDAwM4gpb5i4qFkxXHUwMDA0frrKqKHdKn35xqpcclwiLq2LRLhg1SvNrXr7y9n+6fKrlYtcdTAwMTX35Pmz7feL7U/mdIJV71xy+qeni0et4d7Rr7dsR8qP65eIaYNQIbFs53xcdTAwMTakXHUwMDBlKkTQditCsrDbWbZWbq9tKyxbxcxcdTAwMThcdTAwMWStXHRKel1cdTAwMTX1SmWzKFxcjFx1MDAwZT5cdTAwMTjLKbCIXHUwMDFmli6CxZMs+vCipY9Q/+jr25Xds4/m4Gzn85Otw71n5ycrb1x1MDAwYkD16PGTnVxycfj0eH+19bn3/ltr6/TLsf9cdTAwMDFX8+4xToetwfBcdLS40ztMf6Xd25/wTrd1Olxc6lx1MDAxZlx1MDAxZneGWMZ6v9NcdTAwMWKmn8jlPiZwXHUwMDFmtVtjXHUwMDE3XHLJXHUwMDEz3zuhuPIzXHUwMDE4/W1hpI/5f9z8/X//mMGnXHUwMDE3daa1V1x1MDAwNkZu8P9CRFf6/SxcdTAwMDRcdTAwMWRcdTAwMDNwXHKxro5Gh1qBJsBcXGI00XlcdTAwMDd7XHUwMDE50Vx1MDAwMf4oXHUwMDA1NVx1MDAxNorf5KLQxj20OPBoRDv4tI2MVYUtiTNZXHUwMDEwMTj4u4jvdLpenM4kkNMzXHUwMDAxXHUwMDFlnJemJC5kyjjYI+QpLE/WinNcdTAwMDY+3FwikndcdTAwMWXeRphQXHUwMDE0513mXHUwMDE0t0RcdTAwMTXw3Vhf/yhmL07r4H20MD5cdTAwMTlcXOlig8iEgVwi+Vx1MDAxMIWX3tdL05nQweHhKu6XldZcdTAwMTZ0hlx1MDAxN7UyXGJ4YOWmXpjKXHUwMDEwxVxuXHUwMDBiXHUwMDBmLYMsJFx1MDAxZq+lgVx1MDAxMME7S4vvilrVXHUwMDFihcy0QqylXFzUoHROly5VQu288lFIJbF8p+ovduZcdTAwMDKhW1pGY6TzSjtcdTAwMDBDKFx1MDAwYrSZgMHigqHF1Pd6gSazIbLuweNXjLSpvDK21MnDL+Ch2qCFcdpcbqxhirx6bZm1uJgxeee4SVx1MDAwYlx1MDAxZCzshdzcPXxcdDRZ0HhCrf5plcFcdFx0XHUwMDA3j+JlVGLs6booXHJW7eFHha19XHUwMDE4MFxyg0drJX5ccmbgU9tcYk4gwoCxKWjAw1xus1x1MDAxOa5cdTAwMTHhXGb8XG58sfeJXHUwMDBiMFpcIsjRNsBZ69hAmnBcdTAwMGV3Jmc1iVx1MDAwZqB/gvk7sCY8K+dq11x1MDAwNl9cZlcsXHUwMDAz8Fxmz1x1MDAwMEaU+GJcdTAwMTmhPsAxXHUwMDE5XHUwMDAwJtE+tDioXGL0Q8ODXHUwMDAyT2HlKfIgYFNUXHUwMDExeGo8qFpvbKDCXHUwMDBlrlx1MDAxOF+udVx1MDAxNKosXHUwMDBlcWBkslpyP1x1MDAwZnpea1x1MDAxMSaTeFxieHy4XHUwMDE2Z+GCxmhAXHUwMDEwXHUwMDFhX0VxeGC19j9reYuEl7wkXGJeXHUwMDBm6lx1MDAwMLqtS1wiXHUwMDE3JT9cdTAwMTA0/lx1MDAwNVxuOFx1MDAxMYxuL8ra657Elv5V/P9bXHUwMDA3VHbkPsdcdTAwMDIqXHUwMDA0XHROXHUwMDA34SvTJC+bXHUwMDA3VJsw6v3F02/d9vfOxsfXn7bbS1x1MDAxZk8mXHUwMDA0VHOyVZ6HUrBcdTAwMWJmQLxcdTAwMTd6dFx1MDAwYq5cdTAwMGJk4I2Mcd5cdTAwMDK6XG5cdTAwMDb3kzmSikRrhJlcdTAwMDXto2SJkqrKs0bNfXSEc8ZGLZRcdTAwMWbbJI9cZjzwR/XO4Y1cdTAwMDL9XHUwMDEzXCL9dUIkmcHlXHUwMDAwXHUwMDA28NBcdTAwMDNcYjPig7Ivi9abIFx08IBcbqlroWBcdTAwMTF8wDoh4Vx1MDAxYqFmcOC2xFx1MDAxN6iBXHUwMDAytiC10HiKsUFUUyfQO0BcdTAwMTWLXHUwMDE3dEBoY+++wp9cdTAwMTBcYr5cdTAwMDW0kvhccuVcdTAwMTX+LFxudGDHiMeCXHUwMDE38NtcZkpcdTAwMWHIXHUwMDEzXHUwMDEwpcD3wfpA41xcma/KjKxYk0pDsmkkMETgvVLc3VxyXHUwMDA2fqjM4TLwTlxyXHUwMDAwXHUwMDAyTWKRQ320pLKIO8iomvlcdTAwMWFcdTAwMWJKSiNxXHUwMDBigfRcIoRohMDdqFx1MDAwN8BZy/NZ4KNcdTAwMTBcdTAwMDBnXHUwMDE3XHUwMDE4XHUwMDBil+QhXHUwMDFh0NQ/iPRcdTAwMDLutj5cdTAwMDTDLfL0rFx1MDAxZWE4eFFU5Vx1MDAwNTrCMzhcdTAwMWW+XHUwMDBi3Fx1MDAxMSBdL9BniP/wMPhcdTAwMTiFd6okj7F1jKBfmrteIFx1MDAxM/U3UGZgXHUwMDBmwFx1MDAxZETrXGK2VElcdTAwMWXehLZ7r4LBXHUwMDA3wIRcdTAwMWJcdTAwMThcdTAwMWRiXHUwMDEykFx1MDAxMG3xcU8jKF0wXHUwMDE4XHUwMDFkiLXDJ1x1MDAxMEVcdTAwMDBydFx1MDAwM1x1MDAxNXSZwbNQXGZl+EfJRFx1MDAxNOgo+KHQUD7J51xcXHUwMDFmxc5cXF5cdTAwMDBdXHUwMDBm4GA64FFcdTAwMDKnVVwiXHUwMDEwwC2F0Vwi0DZr2T+DYqhcdTAwMWbjXHUwMDBmUGjvbDlqV4gmPG4uXCJcdTAwMDOFr/T1wcmiRqhDM4BcdTAwMDJGXHR3bVx1MDAxMo2xsFx1MDAxMCmJil5HX6+B2mRQWaGE8FaC3uhEnlx1MDAxNKz1xcOl4tcrzKzFIei1XHUwMDE290g6PFx1MDAwZa+SmNgjXHUwMDA0iJFcdGvexPrwaVFzXHUwMDA1gVVT0cDXxXR5XHUwMDAwo1x1MDAwMN9cdTAwMDNTc/iuXHUwMDA2OY+QcUddXHUwMDAzOlx1MDAwMZ/W6PTpXHUwMDA2MD3JeMhCaerFeWbAYVDMnyCAXHUwMDFhXHUwMDBieOCZXHUwMDExROq8Oq2Bw2ehXHUwMDBm9Fx1MDAxOEBcdTAwMDbzxDWliUXoXHUwMDEx4lx1MDAxM1x1MDAxZlx1MDAxOFx1MDAwMcZ6dz9rcXBcdTAwMWbG4lJcdTAwMTFLwHeU80WKhXjBwynCu+CJNFx1MDAxMCcyXHUwMDFiXHUwMDAzI1U4P1x1MDAxNs6VxDlEunlcbtVcdTAwMDdcdTAwMGadaZBvgzxcdTAwMTa6M/eKJ4FfS1x1MDAxM59cdTAwMDY6JF1EXHUwMDEwqlx1MDAxNOhDvW+JmWSFLPiEsFx1MDAwZYFyeYUxw8t4tHmyhrmZh1x1MDAxN8hcdTAwMWOPguo5QU5hy85cdTAwMGaPK2pNeFx1MDAwMWuLXHKSXHUwMDE1fMDAXCKmN5l3h1x1MDAwZi6KM4L1IGA2XGZjXHUwMDAxxFxyXGJcdTAwMTa8gYD/XGLe0cGIUd1/Ls8zWUDeolx1MDAwNYvKmulfwPUqiWBcclx1MDAwZrP0fK1cdTAwMDY9jHjV4lx1MDAxZlxyntDgcm2Gm+Ng6HhcdTAwMTJgXHUwMDAyLpFcdTAwMTdhs6BcZvQtokk+dcbiYFx1MDAwMXBtXHUwMDAwf3g2k2wyWJuxqlx1MDAwMY9cdTAwMWPGgaC7XlWMxOpIdWDweHwuXHUwMDE1XHUwMDA3h8dMtcuTZVxydlRcZohcdTAwMTCYs7HQVZ+kUqzD6kjVvJOs2mywo4Lr4W4mXFyyw0LKrlx1MDAwNeK0XHUwMDA0P8M78FhBN1x1MDAxMlx1MDAwNzWJoFx1MDAxOUBybibMlziXkbND61x1MDAwM7ii9Kk4ZttcXFx1MDAwNFxmwZuJXHUwMDA2Zlx1MDAwYktcdTAwMDJcdTAwMGV6XHUwMDFiXHUwMDE1KUlIlueheFx1MDAwNm7bgTZLqVx1MDAxYWiKo6oq6IrA4/Ou9GidyPBNeERcdTAwMDHsN4J01XtcdTAwMDHI8zBbQDiQxrFcdTAwMTUrXHUwMDExyMpcdTAwMTAwUlx1MDAwNFx0RPJcdTAwMDbXS5LLONRcdTAwMDA0oGUlr+fAqOH1XHUwMDEwXHUwMDFkMI1cdTAwMGVcdTAwMWNvII99XG4g9fBcdTAwMWKA/6jLYVx1MDAwN1x1MDAwNDKWXHUwMDE2XHUwMDBlX8Q6llx1MDAwNrG00WRBiHq8cCDhflx1MDAwNvLgXHUwMDAzuCFcdTAwMDZQYFlMXCLPcFx1MDAwM0eBXHUwMDFkeYNcdTAwMDD4wcWJLKewbN9AXFybXFytYMhcdTAwMDFHXHUwMDE1XHUwMDAzXHUwMDFlc1x1MDAxM6cn4CSBXHTakZKWlmYjXCJopqtcdTAwMTF+SUR1XHL2zFx1MDAxMDKCXHUwMDBmgEBcdTAwMDLMvFCpOICdyVx1MDAwYlx1MDAxY1x1MDAwMVx1MDAxN6FB+Fx1MDAwNzJcdTAwMGbq6GGWIDmIXHUwMDFhXVwiXHUwMDBmd1x1MDAxNN9cdTAwMDJcdTAwMTRAWC5cdTAwMWFcdTAwMDRcdTAwMWIzXHUwMDE2J3JcdTAwMTdPvPVcdTAwMDKIkegxrMZaRFx1MDAxYVB02IdtXHUwMDEwW3kuLy/x0Fx1MDAxMc6+/GS9yDygTqpgwVx1MDAxZkWD5INcdTAwMDJcdTAwMWaxXHUwMDFhoO/xQKQyJbv1XHUwMDE2btRpxqdcdTAwMTZf2sAqXHUwMDEwTVxctXbCqVxiOIRkqyZm+aaQ5zugXHUwMDFl9eJcdTAwMTi9w1x1MDAwZlluX1x1MDAwMr7SbSRcdTAwMTFcdTAwMDC2itG9i1xy9i5nL07AQ4LgSMtcdTAwMWOYL+lKgNNDlGtAzeFpVf3FMtQxINVMXHUwMDEzRK28KS/PcVx1MDAxYlx1MDAxY++AikJog5unJLy4wNVcdTAwMTioX1xivkx+XHUwMDAyd6ZcIlvxwNvIaZroijLM9tlItlwio0/kRTB4b1x1MDAxNPOAXGLXfoU8aLCW2nBcdTAwMGbTWlXSvSgysFx1MDAxZs+sUlx1MDAwNNdrkork+nSQhmlcdTAwMDdcdTAwMTaWlGwj4vGSXHUwMDE3gUBcblx0JG2UV8qdKJiyxlM25Z29iNVDTVj6qdlcdTAwMTjVwDhixp1nT1pcdTAwMDJUK+epxvYnmohjhoWBM72bi8nqXHUwMDEwXHUwMDE2enZtKDCDWH+1ua0x+mTgLlxiNIk84Fx1MDAxMiQyI9uATjFLXHUwMDAxXeB+UFx1MDAxMFCLclx1MDAwMU3M07hcdTAwMTFeXHUwMDE09sj69ibygC/wQc5cdTAwMDBzWe+W6FxubIKxYPT0XHUwMDE1XHKudtbyXFym6HKNXHUwMDE3ILdO+tQ2hFx1MDAwN5OBXHUwMDE2w92rXHUwMDA2SVxmXHRPTlx1MDAxZCFqMFdbLvFcdIAhXHUwMDA1tipFxE1cZq7eXHUwMDE3SESluFRtXHUwMDFj++uYhirJ+1GWXHUwMDEzSVlCXHUwMDEz5aO6Mkiz7C+PoVx1MDAxNOWGfJfBaao6rbtcdTAwMWXWJPe1QZVxTVxmO33i6cH26FNcdTAwMTCXXHUwMDEyfH1cdTAwMDNTm7E8x6S05MZcdNVcdTAwMTY+36a3XHUwMDBm7pqBX0D4UM8y8iRcdTAwMGIwnJGnwj2S5aRcdTAwMDO3S1x1MDAxMTIoqiZgrz70Q/Tk8SSYLYeKOZ2WgrCIXHIsTSs2wNU6Utw8i0hcdTAwMDNfXGanLJRJXHUwMDFl7i1hkpriXHUwMDEwhykmIIFcdTAwMWVRpapijHEsL1x1MDAwM3T42u2NWYszTD7Cf2pcdTAwMTB1eKlgkltcdTAwMDcq7qHCXHUwMDAw5Vx1MDAwNntD9Fx1MDAwMsDvyNxcdTAwMWUrOELqXHUwMDA08D20XHUwMDE32I2rJ2f0UZ7Fx1xi0DyT26mPMtGwWlx1MDAwMlx1MDAwMOVcdTAwMDFpXHLE8aEx8Vx1MDAwM6M00qbwKIOh9eO9XHUwMDE462/cbKVcdTAwMDF8XHUwMDEwX1x1MDAwMFx0JPfgbbmYKbL7wWmjqHJcdTAwMWVcdTAwMGa3Vlx1MDAxY2g/VFx1MDAxZVx1MDAxMTZcdTAwMWUqXHUwMDEwVybSXHUwMDAwbohZgNpcdTAwMWFevta6wCvyzTSEofl+i0qxLELhXHUwMDEwemhGb/W1paQpKo9cdTAwMGVcdTAwMWSwWUVcdTAwMTlcdTAwMTJxXHUwMDFllpfvXHUwMDA2Mu1cXKslZFFcYtZJNOFuTUJcdTAwMDNAoliaxNDIQZnqy/uwOESgjFW0xcMwiVx1MDAwNlx1MDAwN9BthHnMh5v62JGM1kjjpYFcdTAwMTOJUibCYFxiglx1MDAxYS4lg7eHXHUwMDE1xm2K6Fx1MDAxOdbkqe4yP1x1MDAwMb6C0Fx1MDAwNea4XHUwMDA1iztrxTGyXHUwMDAwXHUwMDEwk/A7llx1MDAwYpZcdTAwMWVC4HZcdTAwMThcdTAwMWM0vk9wy6veUlx1MDAwM6ivYlx1MDAwZVx1MDAxNuZFz132cYx7gFx1MDAxOCzohJurxWpYKlx1MDAxYupccmJMq7mhWXIjXHUwMDFltFx1MDAxZHdBedBOpvbrNUQzJoRZwbDJaErIilx1MDAwMFTle7CaXHUwMDAxcn1eXHUwMDBi0THuLpZcdTAwMDRWXHUwMDAynXLl6Fx1MDAxZNExhzIgorVcdTAwMDakvT481syDXHRWXHUwMDE3R3C6WFJcdTAwMTGE7vqKkkpmVWXt2phYXGLkS4jfXHUwMDExoYs0sVx1MDAwMOJcblx1MDAwMlx1MDAwMZ9Ekl1725hF4f5cdTAwMWHrTVx1MDAxMFxyXHUwMDA2n0hzJjiQeabbXHUwMDFhXHUwMDE0ks5YXHUwMDFh9/BB/Fx1MDAxNZFcdTAwMDb8XiXSeCsl6Fx1MDAxZWhcdTAwMTRoS+2NM1wi80x7+jz35ZI9JFx1MDAwN8tzXGZcblxmbyz8Uq04XHIsXHUwMDAxmLJoJb+iNNOWx1x1MDAxMMI0KCeHKJhcdTAwMDLrXGYjqZpKRVnW0stAJlx1MDAxN+pcdTAwMDGVXHUwMDE5SuCSXHUwMDA1XHUwMDA1p1x1MDAwZYvytvQvXHUwMDE3XHUwMDA3fedcdTAwMWNccs/dMGjpWIJX4msgXHROSdZcdTAwMDfDTLdHxdS390B2OZZ+xrKAztrCbdZXXGIw2W5YI4onoZiPSVx1MDAxNFx1MDAwZV/Apm3m7HyoX1x1MDAxYp2IZ+140PSLZfWFL81pLMgg7ka9XHUwMDBmydeGy1x1MDAwMJFHnJQk2iGNdUm4Wmqcrt+Bwn1cdTAwMDP/0YjSYavOeZ/u8UBcdTAwMWZcdTAwMWSLt5muqFx1MDAwN9R8i4e7/1x1MDAxMXbFXHUwMDFhmUSaYizBwi4w9PqwxpiMm5BUXHUwMDEyVn25sf0nQSZcclx1MDAwNIL/rVx1MDAwZmpmLE3mVSc690vwIWVcdTAwMTVhW1x1MDAwMJw89NDafLOhgYtD/MZ0XHUwMDAzozfuviTypNB581x1MDAxMFleaFx1MDAwMFx1MDAwZTbz8LFwSIF9LnFsn1x1MDAxMpxC54XfoI71NVx1MDAxZtyVZVx1MDAwMZ3MXHUwMDFkt4sxXHUwMDEx5/LaXHUwMDA3eHz4+npcdTAwMTI3c2m80YI1XHUwMDFh2lxuX9I5bkArJtVwrcxH1NeT5cUysFTuJ1xiJVxuLdO5PLhcdTAwMTJ+XHUwMDExgFdcdTAwMDE/6tvSNMh5tFx1MDAxZa5cIlxuJtpLmM+lc4tQOFx1MDAxMFx1MDAxNZZC1a5OXHUwMDAx7UBcdTAwMTO4Wexd3lx1MDAxOPdH+V1E91x1MDAxYVx1MDAwYlx1MDAwN3xFUJx6glx1MDAxM0mv4MtcdTAwMTCOgk7bsUYycu3AjCpcXEqj/pZZitOMITWR01x1MDAxM4rT4lx1MDAxMdww9lx1MDAxOXlcdTAwMTh0aNBdRVJcIliG7TSzXHUwMDA25c4llrZ43lrgRED422BxLmNIXHUwMDFluMMr2HRQXHUwMDEyp8gxYPt5vZqo93Xas8eN+zlcZl7KW+0qL2JcdTAwMTWIxFx1MDAxMEaTXGLXX+pspYXMKepcdTAwMTWLbq0oM01mjYOXrDS0rOSpT85BXHUwMDFjN1x1MDAxZaVcblx1MDAwNGOp0novy5wm2JQyilx1MDAwMUaDa5VAKehcdTAwMTPiXHUwMDExkP60XHUwMDFhjVx1MDAxMUpkJihcdTAwMTgmhJo8Vu5jSoRy0clUmmGRoZLQb1x1MDAxYlx1MDAxYVg/4o38vuRpMmC2ScRpptdcZv/1TUJ9cP7I+kJcdDuCXHUwMDE1OZncOkH1NVxmWFx1MDAxMFx1MDAxYdSvXHUwMDBlXHUwMDExhFx1MDAwN+jg2eJ/MZZBh09cdTAwMDKPXHTPgoVHrGZ+cHlggSxcdTAwMWJcdTAwMDJYXHUwMDE4WKZcdTAwMGJjlXzMenLXWcJN1Get8zpSmCtcXDGCdJVUoLCOXHUwMDE0z4kpO1x1MDAwNty6QfpcdTAwMDDkiVxyWoJ4XHUwMDEwOLY5kVx1MDAxN1xyXFxA4IxGVuTUirOMXHUwMDE14L2xNsVcXKlLXHUwMDFjXHUwMDE0d3yh5DJcdTAwMDZcdTAwMTb61T7emctjLVx1MDAxZoyJmWTWXHUwMDEzlsNcdTAwMTM+XHUwMDBlmj/eQlxmXHUwMDBmx1gnToLUIHJGSGfpxWU5XHUwMDBiXHUwMDA221BMgJtcdTAwMTjyjel6bWHhgFx1MDAwM/TAosiQyil1XHSkhVx1MDAxYrCsXSCo17N2lSHQ4WhRuHhcdTAwMWJcdTAwMTBHlz1cdTAwMDE7hnDPgD1KXHUwMDAx3GX93Vx1MDAwM1xccVx1MDAxYSfsM2CBopxSz+v84WHhW9hcdTAwMDLQoOiYjVxijJtcZuvuXHUwMDEwcpWjWDZcIsDFssNcdTAwMWG0ko199fLyvl5usjumR8pcdTAwMTVBXHUwMDE5t11sblx1MDAxZLFcdTAwMDFccsjg6ywzI1B778q7XHUwMDExuHP4JqtA/cjHXFx9MVCNOPpcdTAwMWLgnGfjXHUwMDA1XHUwMDFlyEOLk1xmLpmoZVxyNcL/cnKNXHUwMDA1lZ7DO0lBQe5cdTAwMWJtvrDaXHUwMDE4y4BcdTAwMGJSXHUwMDFj4SrLXHUwMDAyXHUwMDA1S1x1MDAwYulcdTAwMTKV58xLVb89eT/NhnLKXFxeXHUwMDA0UVx1MDAxMfGRrmw2fNW82fDZ5esvvuNfX7zVne72kjrbXSmMrprr6S3sRIWxsOTSy8LAxuuhpfB+itliJl7dlHGOd1x1MDAxOd6CgNuFfGdcdTAwMDJRYKHxczS7XHUwMDA1n4CPQyBcdTAwMDN2XHUwMDA0XHUwMDA1LrQ+Xo9uyfmJVLF6ov2NKv3TdjjHbYeLrFx1MDAxOYPjzuf4sIa0nHn+sfcuOHtcdTAwMTSMt0GXXHUwMDAwXHUwMDA0UqdcdTAwMDVJXHUwMDE0XHUwMDA3XHUwMDA3XHUwMDFhN0VgkzK0mVx1MDAwYkSQw2G70rI2gcW2JXmp2teKY8aC0ZR1+b5GOSMobV7MQjyzXCJPXHUwMDFi18qbZJr8Ycmbx2NQYEaekydcdTAwMWGUyc1YXFy+hcNwic1cdTAwMDVM55YhUlwi4GZdoOODqk+4s8fRg8ixSipcdTAwMWYkmfSxskaJ47/ZjSXAR1x1MDAxYmiLXHUwMDEw1GhGpIGTL0JcdTAwMTJqcLSmXHUwMDAyMcfrbM4xtr1Yn/m5L5Q06cujyYWCNVx1MDAwMLpyxNnr5iD5aXX18/vex/W17pH9uvX669fd7tdJg1x1MDAwYudcdTAwMGIkg8zYq5hP45GFTq1cdTAwMWYgaTJcdTAwMTagXHUwMDA0XHUwMDA1XHUwMDBlJFx1MDAwYlx1MDAxYlx1MDAwZbNGScnkXHUwMDA1kU6DW1VcdTAwMWT9QvtcdTAwMDLISMe8MViXXHUwMDFha8/nkFx1MDAxM3YyVVx1MDAwZjP8XHUwMDA3J39cdTAwMDecvN2nQ4Y4XHUwMDE3gSHL69m0Xm68y0j6nGfApI2Q9XVcdTAwMTKT1ZA/nuX3ns8vWEmvXFwnjtG9ZGWFsHmRXnlMXHUwMDEzXHUwMDAxSzhQdCOZRqpcdTAwMGbBZixOZeyGtWzxZu+QKSfZx62t9ubx3rE3IVx1MDAxZraFKNon6+OoXHUwMDFm7ni4XHUwMDEwtGlcdTAwMDBcdTAwMDVcdTAwMTknLlx1MDAwNlx1MDAxNlx1MDAwZlqO9VJpkzqYO1ttbD4ooZaN3Fx1MDAxN7KE9OVcdTAwMWJkXHUwMDExjEfDhFn6b5pDy+rmrvn6+vOGtp+OluS37id3tvjkt4BcdTAwMTbLXHRTLDFQV4N5XHUwMDEyaJE+i8xcdTAwMTMppWRcdTAwMTThnqZn2rz3w1x1MDAxYc5cdTAwMDVCLFx1MDAxOCtcdTAwMGVFYoZcdTAwMWFuJJKbka6OXHLYlpHFVsqHfya//L7QXHUwMDAyRs16We3y9Fx1MDAxZTfBi79cdTAwMWUzXHUwMDEzXHUwMDA0XHUwMDAyXHUwMDFmZolBVuv3XHUwMDE1XHUwMDEycaq8h1x1MDAxMjOWxlnH4kvBOpuHXHUwMDE2XHUwMDA3/u1cdTAwMDNrOoBcdTAwMTVcdTAwMWOplKYmXHUwMDAzXHUwMDFierhvpvGXemiMmWPZv2dJl4JcdTAwMWSVM6c6P5RPXHUwMDBi51x1MDAxZI9lqa9GLomL6XZcdTAwMTbE8U3ci1x1MDAwMLtcdTAwMDVcZtSJW2Se3+ZcdTAwMTM5OI3T6HKfTW7fhjlsllPGJr2e9yDQwDRNNNLh0mIod4pcdTAwMDRO61BsXGJUeVx1MDAxNVWt9om8TVx0YbPhTCtcdTAwMTZ1lkd8LF5/wLm8Z1ZAXHUwMDEzYn3a854wcnKOXHUwMDEyioSQw1VcdTAwMGZEW71cdTAwMDVGvjveePP5ZWtz8/Hi8+9cdTAwMTf7K+dcdTAwMDerr39cdTAwMDeM9Fx1MDAwMoYqhY4gYKylTY7c9Hk/XHUwMDFhXHUwMDE0MadWhc3GmWOkYW5HXHRDhjuqQCpjpLHcgGCvNMKs8YOVXGaLzZWy/+Qpf1+QvN2n4dSczqvS8qJqX+b8cGrg+Wz85eF0ur4m32ScocIhwmytUm7M55ZcdTAwMTWw1udcdTAwMDI1OcbV5XtqiGPKXHUwMDE5Q1x1MDAxNjnlI5I584uHc95Wnk1Ry1o2M+Sdfrde3N2Eca8xXGKAi2SXkVx1MDAxNOXkssw8O5LYJyu5l18vTnBSdj58VVx1MDAxOK1t0n+Ap1x1MDAxZbjxasmVRaxcdTAwMWa0meE6eYCN8t7LfHLIXHUwMDFm5bc1Z1x1MDAxZrHVyih2Ts3ldprncEpRXHUwMDFkzq01h6pcdTAwMTft/tn54cbjo+5B72Sw07nsXHUwMDFkfmz9XHUwMDBlUOXYOs5hiDIgmDIm3U5jaVx1MDAwN2sj2X2jw/0kXG4j2yQ4zlx1MDAxYqpEnlRcdTAwMDFUV9rJ3EVgU0DFXHTRtLtYrIj/XHUwMDA3p347nDKsXHUwMDBlj9zP51x1MDAwNDtdLlxcWZRsjlGMRljXzZGN7cX6KjZcdTAwMGaqpVwieFx1MDAxMOvhRdJSw7oquCjFYyU5Rak+Icbmflx1MDAxZpyAweRcdTAwMDfyJMlE1iFIjn9iPU89NqpMau6j5EOruJFUXHUwMDAyx1TpXHUwMDFi7C9lPnCkZMxLgJk7nLa8JiMqYZDshOFQMjbB+Sl371x1MDAxN41mloXJvWNn3Vh6XHUwMDE2U3m663pz7270MFx1MDAxY27CcHdfPH63Ozy+aPm132RcdTAwMWYoXHUwMDFmKs5yb8fAcZSNu/LumqNcdTAwMDRcdTAwMTDbO5OnoF2ysJntXHUwMDAzsVWd1fdQZPCYqkDEsa9JxjzrXHUwMDFizHhcdTAwMWOCxctcdTAwMTBcbvnsf9z7b+feb/lpjmLg+Dbqr2IvTdn3XHUwMDA0zYpEnmZlnY9NoEBygms+TY9EVyZcdTAwMDdcdTAwMDBkTNPQa1xubs83aNacrTRcdTAwMWVcdTAwMThcdTAwMDJcdTAwMDfLrkOofzJcdTAwMDBcdTAwMGbW4XFcdTAwMDOiifwuXb9tIziLkMNCY35kQLlcdFxm78LKNOdCRkWcrZVcdTAwMDdcdTAwMTDlXHUwMDEzIFSxiMSliUJm9VwiXHUwMDBm0maJZYPTNdgwxD4wXHUwMDFlXHUwMDE1wHCoLC/1XHUwMDA1XHJgSrClXHUwMDA0IUaQnlx1MDAxYlEpyHOriWc1WFblN5lQM2uBnPCDp6HyLU3FrvLSXHUwMDE1ZznCe46ZwFOz9WUzafDlQrlcdTAwMTVU5Fx1MDAxNeuKYSZHWtkmLSv3XHUwMDA1z5NPTpDgIJqLq8Lnt83x2YeLuNzuPIlftlx1MDAwZd7KtaXPvc9rj39cdTAwMGJ8djajXCLBj1x1MDAwNVuc8TnCZ4b/7OKmSdxbNSPno/OgnXymXuFkwFwiPlx1MDAxYsnJimzyjtH48YJGnr/AXHUwMDE2qbqz6P5B6L9cbkLDqTnFydzGQHVjud1QwoU6r1x1MDAxNZxk3vNRv1GhOPdUXCLG4Fx1MDAxOCxWeCQ+V/LgXHUwMDE5rzjsT9anp+qkQVhQgFS2QKhcdTAwMDZcdTAwMWVXZ8pwzoNwxubEI4GsxDxcdTAwMWVcXFx1MDAxZVx1MDAxYlx1MDAwNDnUiHNcdTAwMWY5oaBcXFjBmWpcdTAwMDbMm7PhXHUwMDExcjZcdTAwMThGzlx1MDAwMTm45YxcdTAwMTiVZpdXeTw3O2Asp/kjXHUwMDA0ZFdPvcB8bI1nXHUwMDBmIzNyKohcdTAwMTSwisX5nEf2i0RcbnZcdTAwMWJIoiZwn5H+WFqzKFx1MDAxNE/vl1x1MDAwMWtMX/5cdTAwMDGs4GHQfMB/XHUwMDE1rr5rjqvdzeHG6YuzXHUwMDBmbm+4bE9f2W3z9N3lb4GrPJeIdc+GZU7OpfWPnJFAospcdTAwMDJ8uIB7int9PrWa8zhcdTAwMDPrVFxcXHUwMDE1rrJbgVWYnDfjWOY1XHUwMDFl+EaeXG4m/slr/n1gVVx1MDAxMecsXHUwMDA3nyDYKoW9guOyI/vkOW7cxFx1MDAwNn33iqGghSVcdTAwMDQ2kKtynCqJQlx1MDAxMX6O4/1BxeuHxPFgRE7aQeiDQDDq8i5SXCKvwTF6kYM1NCdOXHUwMDAzvZLeQs2hblxiU3mEnuTQ4jphLuPwKot7h3WxzqMkznCGb2QzXHUwMDAxR/fUt47PVlx1MDAxYfPXkulcXIZkXHUwMDFj219cdTAwMWUnNu5cdGpcdTAwMDVcdTAwMDZcdTAwMWVQYVx1MDAxME5Kjlx1MDAwMZMuPUCIU3M4v5Rn2ZhcdTAwMDbdXHUwMDA0Mp9cYsUmStaYXHUwMDEw2pNHy2NcdTAwMTKiQFTijGsytHjWXHUwMDAyr8GXqVx1MDAxMPaq4NKThlx1MDAwN+E9z1xmjJyBwNn6tXh/J2hcdTAwMWW094ZX+FOBz4WjNFN45jhcdTAwMWb2XGZV1sdsNIfnb2eby0ut1feLT1x1MDAwZbcv11eevFh9sSTn+8BAuFxylVx1MDAxZv9cdTAwMWPY0ClEWlx1MDAxOZOPu+dcdTAwMTa1tTxT0CZcdTAwMGIrXHUwMDAwc5v/TFx1MDAwN+aW3Vx1MDAwZlWHXHUwMDA2aiF5XHUwMDFhh8uHl/NojXFcXLZUXeZXTLz6XHUwMDE5XHUwMDBid5nzo3uuXHUwMDBld29cdTAwMDW4/73Ru1x1MDAxZqr1I6H052RcdTAwMWO++Z3/XHUwMDE2NPhaMZ7sv9XPPmxcdTAwMWW+XFzuvDxZ7j7dk91cdTAwMWT/qKjqNzrcXCJcdTAwMDA+unnnzz+myV1ZP39+XHUwMDFlP7/8uipPn7WeXXRPn75avpPcXHUwMDFmXHUwMDFmXHUwMDFmts+HZUFX3/jx7UXn/Mh8PJHfnlx1MDAxZr7a9vJC6fWC2DF+M0ZKeMLHKHXcgJRMNfnSOlx1MDAwYtZegNXxknFcclx1MDAwMIbrl1Xmvtnc3KvvRbW5XHUwMDFmtfaOzlx1MDAwNu05MHgjXGaAyOm8078wROja4J3IXHUwMDE0Jzw4XHUwMDFl6jPV4Fx1MDAxYlx1MDAxZFx1MDAxMdrei7GCiUefcXhcdTAwMDHHIEZuRY1cdTAwMWK8sqmFa8TmXHUwMDFjwlrdeDTvjPqg31x1MDAxYm50LvlAlCi9+qx13Ony5o+uONdq2ndv2Fx1MDAxZfBLXHUwMDE2XHUwMDFlr68sbLRcdTAwMDdQ5UelTz3udlx1MDAwZXt5uNfmZ0umMOzstbo3XHUwMDFmOO7s71x1MDAxN5FxXHUwMDBmX91CLDtYaYJd/UHnsNNrdTdcdTAwMWKtrHU27L9rn15d7nBw1i7esfaLm2AruyZBP1x1MDAwN+iFZvLUxFx1MDAxMWpcbs4mrkxkv79cdTAwMDWih297345cdTAwMDavetvq07fDXHUwMDEwN+3up1x1MDAxN/OO6CbkXHUwMDAz+zlfc1x1MDAxY9FDPo7Qa3b4grj5KSnsX4zonFx1MDAwZodIY0Kj4a9cdTAwMDX0h1x1MDAwN97hwaZXXHUwMDE3x2+Gr9pLe+L1ivy0eLTUlCc8jlu9nUUnV15e7Fx1MDAwZbaWjlx1MDAwZd7uf1xcbbbcJoDuXHUwMDExXHLeN6BbMbm7mIdW5FFLlbVvNbf26ns894DOaXM8XHUwMDE2L1x1MDAwMrGVTFx1MDAwMD1YRqVcdTAwMWOcznNCptr7XVx1MDAwMJ1TXCI5XHUwMDBlM0BcdTAwMWRsYTLMNEA3PE9cdTAwMDEk41dcdTAwMDI654RcdTAwMTdSefdccujrg/63zn57sHDSPVx1MDAwM57eXHUwMDE3lk9HrVx1MDAxNMsnLmpWML7faVx1MDAxZPd7+1VmbcK0YjFcdTAwMGVcdTAwMWPU1Wb9oblZv3p89jp8Xn+1/aLvdlePdnaXvu5Mqlx1MDAxNptcdTAwMTNcdTAwMTCHXHRcdTAwMDE/NUdcdTAwMWZx/q5O9qGD4lxmbVx1MDAxZrldNZ2lq4PYNma6Ue+qXHUwMDAztbtbYdTBZJpHm+hrjFx1MDAxZXngkVVLm1lrWKaemnc+Vl+bqO9cdTAwMDXAf1x1MDAxNPDcXHUwMDFhwOuA9nJ7uL6+tvHa7Vx1MDAxZD6LavHL2lHncWhcbrRG9LbWX/a2T+LhUMmLlcvBq92GvKBcdTAwMDHQcqLnvUfOxZFcdTAwMGapRTqhjOExQlVcdTAwMTb5sblFVt/jOVx1MDAwN1rH6eyeh2yIwFx1MDAxNvuUWEfLijHrefzAnW1yXHUwMDEy0HqbsUuNNUs8vLbKJCssUVtcdTAwMWXBU7TRv3zovN7fX+j3XHUwMDE2/tP71lx1MDAxOVxmz1pd/K3X32/fXHUwMDEz4tZAzFx1MDAxOOJeLe7H0spcdTAwMGKbXHUwMDE16l45mFxuXHUwMDBij2LiOFx1MDAwNe5dWFx1MDAwZausMvBPtyjQnupcdTAwMDfn1MC9kjzeRlracFx1MDAwNeg6MGnYtVxus1x1MDAwMN2JXHUwMDA2zqFT1lx1MDAwN6kqpvPwODQtx5k0S1M8nIGq3pS+f6idaveuOFx1MDAxNPDnN6FcdTAwMGJcdTAwMGJrX311XHUwMDEz28s9xd5cdTAwMTlXyVpcdTAwMTIrjeeJVXDkjoe6XHUwMDE1tvxcdTAwMWVcdTAwMWS2TvhcdTAwMDAyXHUwMDBlQlx1MDAxMdbmXHKQoTjHXHUwMDFjente3thcdTAwMWW7M6XN8UkrPtjuPl/flFsrZ1tvVj9cdTAwMGXkpTBv21UrXHUwMDE2WeDZ4OCAQbNJv1hyfb1cXJ1cdTAwMDVOjLvOs4jiKTBccpY741x1MDAxZPvU58x0077CMviz+MMoRr8/RqXa3d3+94a8aKLblKqwYzA+hiZGLaSu3FPYbu44p+9cdTAwMTTNqePkyLJMXHUwMDE4nunjI1x1MDAwZqxcdTAwMWGJufacPIdQhFx1MDAxOJRj8czk/vq7eE7Fk69cdTAwMWTP0OXIXHUwMDBigFxcRVtL0Dz9XCJKXHUwMDFl0aVD4ajOXHUwMDFmntRLizjnnnZcdTAwMTHn3JGePn239aE92L1cXDp42Wpccndar5bWNqrdkmNLINM9+UkqRYO8dkuGXHUwMDA3McIsXHJwUkhXXHUwMDFhWzgrNzp9y6K0Xk5j1KxnMYrTp8e9vmJcbi0yjM3T1aXCl/vxo6mjeUhXOtlU+LM4ZiX371r9xJbwaFx1MDAwMpspbCUlbTX3rNNT9vPqWa2ImXXQSqgtz3lcdTAwMWRcdTAwMTG8K8/KwZGS53iImvqMO3nWTPGEVGZro+FhkuOOXHUwMDE1sbE1kocr8cxcdTAwMDbtx4NQXHUwMDA3/DSziUFn7Vl/Plx03NCzroWN3U9f3698XFxvvTk+2Vx1MDAxYrqT5fCiylMxqFx1MDAwMzO1UVx1MDAwNcHaND/uWSWPQlVwZIpHfUVZPIZtZp51alx1MDAwMrnkWVx1MDAwNYe2uXyghIpQgDi2YJPhyXPW/u0g4Heiplx1MDAxM1xyhD/jpjEjdzqx9sVP7PBiSp1cdTAwMDOIKp3pbnNnumaGw8fvPj7dWJXv1zaXulx1MDAxNzsvzt7MuzNcci7m4ywjjzDhdLCyL1x1MDAwNafJNNN78l5LX5yDx5YsXHUwMDE0hrLwdOJcbmc65jxcdTAwMDPIi1My/v5cdLxQenVKXHUwMDAyb+s6ObZ6h6zdsH8yKWVXWnean6v+7lkl5ibZrdNcdTAwMTNcdTAwMGJaXCKslmcqVpntXnOzPX1x/q2155Zff97zy+2lleVcdTAwMTX7+tm8m62UNlx1MDAxZp7tXGLUUlx1MDAxNKdcdTAwMDddV7T4LCqdXHUwMDBm7eWJcZNn4tyFXHUwMDAzSVx1MDAxMzNcdTAwMTVtXHUwMDBlcTz7p8DERptho4Vf221+tJdcdTAwMTFcdTAwMTN6MH8ju1x1MDAxZLXr1dntxsaLhferK1x1MDAxZlx1MDAxNzbWll4tbz606U78+vu2Xq0nZ4dcIqd88SDxKvvdv1x1MDAwNexO5ZHzar+WJ1XzXHUwMDAwXG5cdTAwMDZyxUNcdTAwMDCuzFx1MDAxN4RWXHUwMDFiXHUwMDFl3FBcdTAwMDe7dzHffO5Kflx1MDAxNlx1MDAxZs9rNFUjT8atV/KwZfaF3Lv53jDC2daajdHLX+1cdTAwMWPOep3zhVNcblx1MDAxYz60Y6j86oeocIlcIn31XHUwMDA21Vx1MDAxNc/LUqGSjbebu4WLwbu9/vOl81dq7WD1yfOnT95ePt+ee7dcdTAwMTCi5yHxWoFcdTAwMDOLdDdcdTAwMWRRMFx1MDAwZsqR7Gibvpt+IMKeXHUwMDEwP5nY4JGaSXBarG1hf5OV1lXsuLGPVatiWup3KG55/+3bZst9cFx1MDAwN1x1MDAxZt/tvPz+/FX3YnfTNC1uef763U7/cNA53926XFxZ3nizv/lhW/0yR/RzIFx1MDAxZCbnXHUwMDE5RVByUo/2QXNjrL7F826M0dmMLZ9Rq6uirbI5Olx1MDAxZZCez9OOwND7MkcpQFx1MDAxNHzk6bGcXHUwMDBlIKs4tlx1MDAxZEdpxcPOpbh/lJ6r6pb/9JZ+gN/pPVW11OBKRVVL9aJmhbLtbrdzclrZXGZi3USUXHUwMDA1sfQ8kKt6qtFhc8vePn6sLr7smFx1MDAwZi831Prxent1aWdz0vjzOakj5UCnTEhcdTAwMGXlsN6OgSyMXiBEhfbiXHUwMDEz02rD72TV2mSs462saDFO8qxdV4Wxxlx1MDAxOVx1MDAxMU10v2ov9ucg9uNJZ+n9Tu/l8f6X9+bbxc7J+atcdTAwMWT5XHUwMDAwjVx1MDAxYX916DZhooXz1F8/XHS6j5pcdTAwMWJ49aObd+j2XmdcdTAwMWONXHUwMDAz+1x1MDAwZaJ4yu91R6dcdTAwMTeZi5wmrXmC95SZZU2MfFLDl1x1MDAwYlx1MDAxObmBsY7t4pV1qePIrVx1MDAxNI+A8X+njs4nnMfzn97GxemwfXxPsF2DUylsT1jSvZehOjE5YeZDVFFcdTAwMTX7iFxuXHUwMDE23Wlu0dP96bxatLM289yp0lx1MDAxY1x1MDAxZVacS3GVMfMq46BcdTAwMGUpgOq2OM9xplx1MDAxOTOVRW1kkPAgweiqIYTaZkpFg5XyPC9cdTAwMTHGzopk/ixcdTAwMWZI93fc9L/FJrrnXGJSydM9gmGJY+FDV3voNjNS6nwupeaMU1dcdTAwMTQzmz3/6V6jsNy8jJYnuFx1MDAxOZBLxfGpxakvN2WpoJxcbkvVUGJbXHUwMDFlf/dcdTAwMTfb/J9oKPxZXHUwMDFjs5GRuDFmNZNaquJU0jG3aj1cdTAwMGZcdTAwMDGs3Eb83NyrTmeT8+pVXHUwMDAzXHUwMDFllMDzXHSeXHUwMDA36aixXHUwMDE5dFxihoxlxTc7eEJMXHUwMDE2Nlx1MDAxYqfqYOvGXHUwMDA2qVx1MDAwM7tkQciq9iF85oyHc4I/gFx1MDAwNVx1MDAxNY9cdTAwMDBcdTAwMTmdreGgcH/PKtXGfoojmbXkgVxi0nrB+FNcdTAwMTSmRl/7qXy8kfA8X4ddjFHfQ53q9DRK2bMqyYNYXHUwMDExiqvIXHUwMDAzIeT4iiW0OFx1MDAxZu1cclxyiFx1MDAwNu7X3GrFv5NnnWwt/FlcdTAwMWM3lFx1MDAxOfnWqWNH7OQ5Ylx1MDAxY1LKXHUwMDAx2pXbvN1blGlMLcSeU//qY4QtgbpysDrZSkJadci0YnVj1L5Y6TJvY0dgWFx1MDAxMDBfU8Tqck7761x1MDAxZs2q8rtvn5++uny69Xn96dPBt6a5oftcdTAwMWFONry43P56pndXL/ovbFhdWXx7fDyzoSNcdTAwMGaTc1J+ysHTXHUwMDE27Ffo6ilix82NvfrZzbmxR1x1MDAxOFx1MDAxYU9KUlx1MDAxNubmtU1LsmBlgafCXHUwMDBiXHUwMDE3plx1MDAxYvtcdTAwMWQqKeFvOIxbKcOuTGmrqNT4vi1blFx1MDAwMsKov1HOiSVRrUOs657yTTVwVVWhlSznrrmmJlx1MDAwZWdS5JSrs+GQXHUwMDFjXHUwMDA3XHUwMDAyyCnixSNcdTAwMTOuQoPoM+CW5NhojVx1MDAwMC+O6aLIjNLsb1x1MDAxMDpP2PhQkVx1MDAwMNU+XHUwMDBignvMwinlfXGCwLiVzVwi6vjVvuNWllXwnr1HlfbWJFT5kVlcdTAwMTY8XytGPTaSPfdcblPqbNlcdTAwMWVcXMxIT3bfqcWXrqY6LtJFvW5q8zOl1ItTNJU/4zo6hVM3jCVm2lx1MDAxMFxyVlx1MDAxN33keeZcdTAwMWW8PzppxztO7qFlu3FvZH5cdTAwMTKQ14xcdTAwMDRcdTAwMDLucjRKqfF8nsr81XQ+UFxiXHUwMDAzvnzLsPNOIdpMWkwmj5CQzFxyOqB/Ze6+35xcdTAwMTi9e7qzXHUwMDExNs3jo6VccnHUVp8/dva77+fHuU3oMdEx0/A9Klx1MDAwNJ7QKJLNuCBcdTAwMDJcdTAwMDfGa1x1MDAxN1x1MDAxY0fGXHUwMDE3SlZ+ebGrVuCyzD787sTob1SN+q9rb/yodXKyMcS9vPFgj7512t+fVOpcZn9cdTAwMThr5TZPM2rn4Prnv/78P1x1MDAwZn2PiyJ9 + + + + + LOGIN NODE ON RESOURCE PROVIDER3. No INBOUND connectivity to the HPCVirtual KubeletInterlink API ServerProvider pluginPod on virtual nodeVirtual NodeSSH UNIX SOCKETunix socketPodContainersBatchSystemSSH agentunix socket \ No newline at end of file diff --git a/example/__pycache__/main.cpython-39.pyc b/example/__pycache__/main.cpython-39.pyc deleted file mode 100644 index b2cd2e50..00000000 Binary files a/example/__pycache__/main.cpython-39.pyc and /dev/null differ diff --git a/example/create_openapi.py b/example/create_openapi.py index 58e5c1c8..47a81b33 100644 --- a/example/create_openapi.py +++ b/example/create_openapi.py @@ -12,7 +12,7 @@ @app.post("/create") -async def create_pod(pods: List[interlink.Pod]) -> str: +async def create_pod(pod: List[interlink.Pod]) -> interlink.CreateStruct: raise NotImplementedError @@ -45,4 +45,4 @@ async def get_logs(req: interlink.LogRequest) -> bytes: routes=app.routes, ), f, - ) + ) \ No newline at end of file diff --git a/example/interlink-docker/interlink/InterLinkConfig.yaml b/example/interlink-docker/interlink/InterLinkConfig.yaml index f199890d..6738ee3c 100644 --- a/example/interlink-docker/interlink/InterLinkConfig.yaml +++ b/example/interlink-docker/interlink/InterLinkConfig.yaml @@ -1,5 +1,5 @@ VKTokenFile: "$HOME/interLink/token" -InterlinkURL: "http://XXX.XXX.XXX.XXX" +InterlinkAddress: "http://XXX.XXX.XXX.XXX" SidecarURL: "http://docker-sidecar" InterlinkPort: "3000" SidecarPort: "4000" diff --git a/example/interlink-docker/vk/InterLinkConfig.yaml b/example/interlink-docker/vk/InterLinkConfig.yaml index e1806b52..02f7f092 100644 --- a/example/interlink-docker/vk/InterLinkConfig.yaml +++ b/example/interlink-docker/vk/InterLinkConfig.yaml @@ -1,5 +1,5 @@ VKTokenFile: "$HOME/interLink/token" -InterlinkURL: "http://XXX.XXX.XXX.XXX" +InterlinkAddress: "http://XXX.XXX.XXX.XXX" CommandPrefix: "" ExportPodData: true ServiceAccount: "interlink" diff --git a/example/interlink-docker/vk/service-account.yaml b/example/interlink-docker/vk/service-account.yaml index b9cfb2f0..3dbaa7f2 100644 --- a/example/interlink-docker/vk/service-account.yaml +++ b/example/interlink-docker/vk/service-account.yaml @@ -10,6 +10,17 @@ metadata: name: interlink-role namespace: vk rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - update + - create + - get + - list + - watch + - patch - apiGroups: - "" resources: @@ -17,6 +28,7 @@ rules: - secrets - services - serviceaccounts + - namespaces verbs: - get - list diff --git a/example/interlink/provider.py b/example/interlink/provider.py index 401eb558..40378a6e 100644 --- a/example/interlink/provider.py +++ b/example/interlink/provider.py @@ -8,26 +8,24 @@ def __init__( self, docker_client, ): - self.DOCKER = docker_client - self.CONTAINER_POD_MAP = {} + self.docker = docker_client + self.container_pod_map = {} - def Create(self, pod: Pod) -> None: + def create(self, pod: Pod) -> CreateStruct: raise HTTPException(status_code=500, detail="NOT IMPLEMENTED YET") - def Delete(self, pod: PodRequest) -> None: + def delete(self, pod: PodRequest) -> None: raise HTTPException(status_code=500, detail="NOT IMPLEMENTED YET") - def Status(self, pod: PodRequest) -> PodStatus: + def status(self, pod: PodRequest) -> PodStatus: raise HTTPException(status_code=500, detail="NOT IMPLEMENTED YET") - def Logs(self, req: LogRequest) -> bytes: + def logs(self, req: LogRequest) -> bytes: raise HTTPException(status_code=500, detail="NOT IMPLEMENTED YET") - def create_pod(self, pods: List[Pod]) -> str: - pod = pods[0] - + def create_pod(self, pod: Pod) -> CreateStruct: try: - self.Create(pod) + self.create(pod) except Exception as ex: raise ex @@ -35,7 +33,7 @@ def create_pod(self, pods: List[Pod]) -> str: def delete_pod(self, pod: PodRequest) -> str: try: - self.Delete(pod) + self.delete(pod) except Exception as ex: raise ex @@ -52,4 +50,4 @@ def get_logs(self, req: LogRequest) -> bytes: except Exception as ex: raise ex - return logContent + return logContent \ No newline at end of file diff --git a/example/interlink/spec.py b/example/interlink/spec.py index cd3518e2..34f9a983 100644 --- a/example/interlink/spec.py +++ b/example/interlink/spec.py @@ -9,15 +9,15 @@ class Metadata(BaseModel): uid: Optional[str] = None annotations: Optional[Dict[str, str]] = Field({}) labels: Optional[Dict[str, str]] = Field({}) - generateName: Optional[str] = None + generate_name: Optional[str] = None class VolumeMount(BaseModel): name: str - mountPath: str - subPath: Optional[str] = None - readOnly: Optional[bool] = False - mountPropagation: Optional[str] = None + mount_path: str + sub_path: Optional[str] = None + read_only: Optional[bool] = False + mount_propagation: Optional[str] = None class ConfigMapKeySelector(BaseModel): @@ -33,24 +33,24 @@ class SecretKeySelector(BaseModel): class EnvVarSource(BaseModel): - configMapKeyRef: Optional[ConfigMapKeySelector] = None - secretKeyRef: Optional[SecretKeySelector] = None + config_map_key_ref: Optional[ConfigMapKeySelector] = None + secret_key_ref: Optional[SecretKeySelector] = None class EnvVar(BaseModel): name: str value: Optional[str] = None - valueFrom: Optional[EnvVarSource] = None + value_from: Optional[EnvVarSource] = None class SecurityContext(BaseModel): - allowPrivilegeEscalation: Optional[bool] = None + allow_privilege_escalation: Optional[bool] = None privileged: Optional[bool] = None - procMount: Optional[str] = None - readOnlyFileSystem: Optional[bool] = None - runAsGroup: Optional[int] = None - runAsNonRoot: Optional[bool] = None - runAsUser: Optional[int] = None + proc_mount: Optional[str] = None + read_only_file_system: Optional[bool] = None + run_as_group: Optional[int] = None + run_as_non_root: Optional[bool] = None + run_as_user: Optional[int] = None class Container(BaseModel): @@ -60,9 +60,9 @@ class Container(BaseModel): command: List[str] args: Optional[List[str]] = Field([]) resources: Optional[dict] = Field({}) - volumeMounts: Optional[List[VolumeMount]] = Field([]) + volume_mounts: Optional[List[VolumeMount]] = Field([]) env: Optional[List[EnvVar]] = None - securityContext: Optional[SecurityContext] = None + security_context: Optional[SecurityContext] = None class KeyToPath(BaseModel): @@ -72,17 +72,17 @@ class KeyToPath(BaseModel): class SecretVolumeSource(BaseModel): - secretName: str + secret_name: str items: Optional[List[KeyToPath]] = Field([]) optional: Optional[bool] = None - defaultMode: Optional[int] = None + default_mode: Optional[int] = None class ConfigMapVolumeSource(BaseModel): name: str items: Optional[List[KeyToPath]] = Field([]) optional: Optional[bool] = None - defaultMode: Optional[int] = None + default_mode: Optional[int] = None # class VolumeSource(BaseModel): @@ -94,20 +94,20 @@ class ConfigMapVolumeSource(BaseModel): class PodVolume(BaseModel): name: str # volumeSource: Optional[VolumeSource] = None - emptyDir: Optional[dict] = None + empty_dir: Optional[dict] = None secret: Optional[SecretVolumeSource] = None - configMap: Optional[ConfigMapVolumeSource] = None + config_map: Optional[ConfigMapVolumeSource] = None class PodSpec(BaseModel): containers: List[Container] - initContainers: Optional[List[Container]] = None + init_containers: Optional[List[Container]] = None volumes: Optional[List[PodVolume]] = None - preemptionPolicy: Optional[str] = None - priorityClassName: Optional[str] = None + preemption_policy: Optional[str] = None + priority_class_name: Optional[str] = None priority: Optional[int] = None - restartPolicy: Optional[str] = None - terminationGracePeriodSeconds: Optional[int] = None + restart_policy: Optional[str] = None + termination_grace_period_seconds: Optional[int] = None class PodRequest(BaseModel): @@ -118,7 +118,7 @@ class PodRequest(BaseModel): class ConfigMap(BaseModel): metadata: Metadata data: Optional[dict] - binaryData: Optional[dict] = None + binary_data: Optional[dict] = None type: Optional[str] = None immutable: Optional[bool] = None @@ -126,16 +126,16 @@ class ConfigMap(BaseModel): class Secret(BaseModel): metadata: Metadata data: Optional[dict] = None - stringData: Optional[dict] = None + string_data: Optional[dict] = None type: Optional[str] = None immutable: Optional[bool] = None class Volume(BaseModel): name: str - configMaps: Optional[List[ConfigMap]] = None + config_maps: Optional[List[ConfigMap]] = None secrets: Optional[List[Secret]] = None - emptyDirs: Optional[List[str]] = None + empty_dirs: Optional[List[str]] = None class Pod(BaseModel): @@ -144,12 +144,12 @@ class Pod(BaseModel): class StateTerminated(BaseModel): - exitCode: int + exit_code: int reason: Optional[str] = None class StateRunning(BaseModel): - startedAt: Optional[str] = None + started_at: Optional[str] = None class StateWaiting(BaseModel): @@ -170,23 +170,27 @@ class ContainerStatus(BaseModel): class PodStatus(BaseModel): name: str - UID: str + uid: str namespace: str containers: List[ContainerStatus] class LogOpts(BaseModel): - Tail: Optional[int] = None - LimitBytes: Optional[int] = None - Timestamps: Optional[bool] = None - Previous: Optional[bool] = None - SinceSeconds: Optional[int] = None - SinceTime: Optional[datetime.datetime] = None + tail: Optional[int] = None + limit_bytes: Optional[int] = None + timestamps: Optional[bool] = None + previous: Optional[bool] = None + since_seconds: Optional[int] = None + since_time: Optional[datetime.datetime] = None class LogRequest(BaseModel): - Namespace: str - PodUID: str - PodName: str - ContainerName: str - Opts: LogOpts + namespace: str + pod_uid: str + pod_name: str + container_name: str + opts: LogOpts + +class CreateStruct(BaseModel): + pod_uid: str + pod_jid: str \ No newline at end of file diff --git a/example/provider_demo.py b/example/provider_demo.py index 9621f847..83e6774a 100644 --- a/example/provider_demo.py +++ b/example/provider_demo.py @@ -8,27 +8,27 @@ import os -dockerCLI = docker.DockerClient() +docker_client = docker.DockerClient() # dockerCLI = docker.DockerClient(base_url="unix:///Users/dciangot/.docker/run/docker.sock") app = FastAPI() class MyProvider(interlink.provider.Provider): - def __init__(self, DOCKER): - super().__init__(DOCKER) + def __init__(self, docker): + super().__init__(docker) # Recover already running containers refs - self.CONTAINER_POD_MAP = {} - statuses = self.DOCKER.api.containers(all=True) + self.container_pod_map = {} + statuses = self.docker.api.containers(all=True) for status in statuses: name = status["Names"][0] if len(name.split("-")) > 1: uid = "-".join(name.split("-")[-5:]) - self.CONTAINER_POD_MAP.update({uid: [status["Id"]]}) - print(self.CONTAINER_POD_MAP) + self.container_pod_map.update({uid: [status["Id"]]}) + print(self.container_pod_map) - def DumpVolumes( + def dump_volumes( self, pods: List[interlink.PodVolume], volumes: List[interlink.Volume] ) -> List[str]: @@ -36,15 +36,15 @@ def DumpVolumes( # Match data source information (actual bytes) to the mount ref in pod description for v in volumes: - if v.configMaps: - for dataSource in v.configMaps: + if v.config_maps: + for data_source in v.config_maps: for ref in pods: - podMount = ref.volumeSource.configMap - if podMount: - if ref.name == dataSource.metadata.name: - for filename, content in dataSource.data.items(): + pod_mount = ref.volume_source.config_map + if pod_mount: + if ref.name == data_source.metadata.name: + for filename, content in data_source.data.items(): # write content to file - path = f"{dataSource.metadata.namespace}-{dataSource.metadata.name}/{filename}" + path = f"{data_source.metadata.namespace}-{data_source.metadata.name}/{filename}" try: os.makedirs( os.path.dirname(path), exist_ok=True @@ -60,32 +60,32 @@ def DumpVolumes( if v.secrets: pass - if v.emptyDirs: + if v.empty_dirs: pass return dataList - def Create(self, pod: interlink.Pod) -> None: + def create(self, pod: interlink.Pod) -> None: container = pod.pod.spec.containers[0] if pod.pod.spec.volumes: - _ = self.DumpVolumes(pod.pod.spec.volumes, pod.container) + _ = self.dump_volumes(pod.pod.spec.volumes, pod.container) volumes = [] - if container.volumeMounts: - for mount in container.volumeMounts: - if mount.subPath: + if container.volume_mounts: + for mount in container.volume_mounts: + if mount.sub_path: volumes.append( - f"{pod.pod.metadata.namespace}-{mount.name}/{mount.subPath}:{mount.mountPath}" + f"{pod.pod.metadata.namespace}-{mount.name}/{mount.sub_path}:{mount.mount_path}" ) else: volumes.append( - f"{pod.pod.metadata.namespace}-{mount.name}:{mount.mountPath}" + f"{pod.pod.metadata.namespace}-{mount.name}:{mount.mount_path}" ) try: cmds = " ".join(container.command) args = " ".join(container.args) - dockerContainer = self.DOCKER.containers.run( + docker_container = self.docker.containers.run( f"{container.image}:{container.tag}", f"{cmds} {args}", name=f"{container.name}-{pod.pod.metadata.uid}", @@ -95,35 +95,35 @@ def Create(self, pod: interlink.Pod) -> None: # device_requests=[ # docker.types.DeviceRequest(device_ids=["0"], capabilities=[['gpu']])] ) - print(dockerContainer) - docker_run_id = dockerContainer.id + print(docker_container) + docker_run_id = docker_container.id except Exception as ex: raise HTTPException(status_code=500, detail=ex) - self.CONTAINER_POD_MAP.update({pod.pod.metadata.uid: [docker_run_id]}) - print(self.CONTAINER_POD_MAP) + self.container_pod_map.update({pod.pod.metadata.uid: [docker_run_id]}) + print(self.container_pod_map) print(pod) - def Delete(self, pod: interlink.PodRequest) -> None: + def delete(self, pod: interlink.PodRequest) -> None: try: - print(f"docker rm -f {self.CONTAINER_POD_MAP[pod.metadata.uid][0]}") - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[pod.metadata.uid][0] + print(f"docker rm -f {self.container_pod_map[pod.metadata.uid][0]}") + container = self.docker.containers.get( + self.container_pod_map[pod.metadata.uid][0] ) container.remove(force=True) - self.CONTAINER_POD_MAP.pop(pod.metadata.uid) + self.container_pod_map.pop(pod.metadata.uid) except: raise HTTPException(status_code=404, detail="No containers found for UUID") print(pod) return - def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: - print(self.CONTAINER_POD_MAP) + def status(self, pod: interlink.PodRequest) -> interlink.PodStatus: + print(self.container_pod_map) print(pod.metadata.uid) try: - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[pod.metadata.uid][0] + container = self.docker.containers.get( + self.container_pod_map[pod.metadata.uid][0] ) status = container.status except: @@ -133,11 +133,11 @@ def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: if status == "running": try: - statuses = self.DOCKER.api.containers( + statuses = self.docker.api.containers( filters={"status": "running", "id": container.id} ) print(statuses) - startedAt = statuses[0]["Created"] + started_at = statuses[0]["Created"] except Exception as ex: raise HTTPException(status_code=500, detail=ex) @@ -149,7 +149,7 @@ def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: interlink.ContainerStatus( name=pod.spec.containers[0].name, state=interlink.ContainerStates( - running=interlink.StateRunning(startedAt=startedAt), + running=interlink.StateRunning(started_at=started_at), waiting=None, terminated=None, ), @@ -159,7 +159,7 @@ def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: elif status == "exited": try: - statuses = self.DOCKER.api.containers( + statuses = self.docker.api.containers( filters={"status": "exited", "id": container.id} ) print(statuses) @@ -211,11 +211,11 @@ def Status(self, pod: interlink.PodRequest) -> interlink.PodStatus: def Logs(self, req: interlink.LogRequest) -> bytes: # TODO: manage more complicated multi container pod # THIS IS ONLY FOR DEMONSTRATION - print(req.PodUID) - print(self.CONTAINER_POD_MAP[req.PodUID]) + print(req.pod_uid) + print(self.container_pod_map[req.pod_uid]) try: - container = self.DOCKER.containers.get( - self.CONTAINER_POD_MAP[req.PodUID][0] + container = self.docker.containers.get( + self.container_pod_map[req.pod_uid][0] ) # log = container.logs(timestamps=req.Opts.Timestamps, tail=req.Opts.Tail) log = container.logs() @@ -225,24 +225,24 @@ def Logs(self, req: interlink.LogRequest) -> bytes: return log -ProviderNew = MyProvider(dockerCLI) +provider_new = MyProvider(docker_client) @app.post("/create") async def create_pod(pods: List[interlink.Pod]) -> str: - return ProviderNew.create_pod(pods) + return provider_new.create_pod(pods) @app.post("/delete") async def delete_pod(pod: interlink.PodRequest) -> str: - return ProviderNew.delete_pod(pod) + return provider_new.delete_pod(pod) @app.get("/status") async def status_pod(pods: List[interlink.PodRequest]) -> List[interlink.PodStatus]: - return ProviderNew.get_status(pods) + return provider_new.get_status(pods) @app.get("/getLogs", response_class=PlainTextResponse) async def get_logs(req: interlink.LogRequest) -> bytes: - return ProviderNew.get_logs(req) + return provider_new.get_logs(req) \ No newline at end of file diff --git a/go.mod b/go.mod index 3453b20f..5a3c4063 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,17 @@ go 1.22 require ( github.com/containerd/containerd v1.7.6 + github.com/google/uuid v1.6.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/virtual-kubelet/virtual-kubelet v1.11.0 - go.opentelemetry.io/otel v1.22.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 - go.opentelemetry.io/otel/sdk v1.22.0 - golang.org/x/oauth2 v0.16.0 - google.golang.org/grpc v1.59.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + golang.org/x/crypto v0.23.0 + golang.org/x/oauth2 v0.20.0 + google.golang.org/grpc v1.64.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.29.1 @@ -20,10 +23,12 @@ require ( k8s.io/klog v1.0.0 ) +require golang.org/x/tools v0.18.0 // indirect + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -34,14 +39,13 @@ require ( github.com/go-openapi/swag v0.22.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -58,24 +62,20 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.9.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect - go.opentelemetry.io/otel/trace v1.22.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.16.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/apiserver v0.29.1 // indirect k8s.io/component-base v0.29.1 // indirect diff --git a/go.sum b/go.sum index f3935605..03e357e5 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -52,8 +52,6 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -67,10 +65,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -79,7 +75,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -89,15 +84,15 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -147,8 +142,8 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -170,36 +165,35 @@ github.com/virtual-kubelet/virtual-kubelet v1.11.0 h1:LOMcZQfP083xmYH9mYtyHAR+yb github.com/virtual-kubelet/virtual-kubelet v1.11.0/go.mod h1:WQfPHbIlzfhMNYkh6hFXF1ctGfNM8UJCYLYpLa/trxc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= -go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= -go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= -go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -209,43 +203,32 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -256,33 +239,28 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -292,10 +270,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/helm/interlink/.helmignore b/helm/interlink/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/helm/interlink/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/interlink/Chart.yaml b/helm/interlink/Chart.yaml deleted file mode 100644 index cc8bd96d..00000000 --- a/helm/interlink/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: interlink -description: A Helm chart for interLink virtual kubelet provider - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "0.2.3" diff --git a/helm/interlink/README.md b/helm/interlink/README.md deleted file mode 100644 index acb2b0e8..00000000 --- a/helm/interlink/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# interLink Helm Chart - -::: danger ::: - -Work in progress! - -::: danger ::: - diff --git a/helm/interlink/templates/NOTES.txt b/helm/interlink/templates/NOTES.txt deleted file mode 100644 index 6eb8cbc5..00000000 --- a/helm/interlink/templates/NOTES.txt +++ /dev/null @@ -1,5 +0,0 @@ -Check node status with: - -``` -kubectl get node {{ .Values.nodeName }} -``` diff --git a/helm/interlink/templates/_helpers.tpl b/helm/interlink/templates/_helpers.tpl deleted file mode 100644 index 38ea4b10..00000000 --- a/helm/interlink/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "interlink.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "interlink.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "interlink.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "interlink.labels" -}} -helm.sh/chart: {{ include "interlink.chart" . }} -{{ include "interlink.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "interlink.selectorLabels" -}} -app.kubernetes.io/name: {{ include "interlink.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "interlink.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "interlink.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/helm/interlink/templates/service-account.yaml b/helm/interlink/templates/service-account.yaml deleted file mode 100644 index 54111570..00000000 --- a/helm/interlink/templates/service-account.yaml +++ /dev/null @@ -1,86 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.nodeName }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: "{{ .Values.nodeName }}-role" - namespace: {{ .Release.Namespace }} -rules: -- apiGroups: - - "" - resources: - - configmaps - - secrets - - services - - serviceaccounts - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - watch - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - create - - get -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: "{{ .Values.nodeName }}-rolebinding" - namespace: {{ .Release.Namespace }} -subjects: -- kind: ServiceAccount - name: {{ .Values.nodeName }} - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "{{ .Values.nodeName }}-role" ---- -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.nodeName }}-secret" - namespace: "{{ .Release.Namespace }}" - annotations: - kubernetes.io/service-account.name: {{ .Values.nodeName }} - labels: - kubernetes.io/service-account.name: {{ .Values.nodeName }} -type: kubernetes.io/service-account-token diff --git a/helm/interlink/templates/virtual-kubelet-config.yaml b/helm/interlink/templates/virtual-kubelet-config.yaml deleted file mode 100644 index 4c32e3e9..00000000 --- a/helm/interlink/templates/virtual-kubelet-config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: "virtual-kubelet-config" - namespace: {{ .Release.Namespace }} -data: - InterLinkConfig.yaml: | - InterlinkURL: {{ .Values.interlink.URL }} - InterlinkPort: {{ .Values.interlink.port }} - ExportPodData: true - VerboseLogging: true - ErrorsOnlyLogging: false - ServiceAccount: "{{ .Values.nodeName }}" - Namespace: "" - VKTokenFile: /opt/interlink/token - CPU: "{{ .Values.virtualNode.CPUs }}" - Memory: "{{ .Values.virtualNode.MemGiB }}GiB" - Pods: "{{ .Values.virtualNode.Pods }}" diff --git a/helm/interlink/templates/virtual-kubelet.yaml b/helm/interlink/templates/virtual-kubelet.yaml deleted file mode 100644 index d4db5fb6..00000000 --- a/helm/interlink/templates/virtual-kubelet.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.deployment.name}} - namespace: {{ .Release.Namespace}} - labels: - nodeName: {{ .Values.nodeName }} -spec: - replicas: 1 - selector: - matchLabels: - nodeName: {{ .Values.nodeName }} - template: - metadata: - labels: - nodeName: {{ .Values.nodeName }} - spec: - automountServiceAccountToken: true - serviceAccountName: {{ .Values.deployment.name }} - containers: - - name: inttw-vk - image: {{ .Values.deployment.image }} - imagePullPolicy: Always - env: - - name: NODENAME - value: {{ .Values.nodeName }} - - name: KUBELET_PORT - value: "10250" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: CONFIGPATH - value: "/etc/interlink/InterLinkConfig.yaml" - - name: VKTOKENFILE - value: "/opt/interlink/token" - volumeMounts: - - name: config - mountPath: /etc/interlink/InterLinkConfig.yaml - subPath: InterLinkConfig.yaml - - name: token - mountPath: /opt/interlink - - name: jaeger - image: jaegertracing/all-in-one:1.51 - - name: refresh-token - image: ghcr.io/intertwin-eu/virtual-kubelet-inttw-refresh:latest - imagePullPolicy: Always - env: - - name: IAM_TOKEN_ENDPOINT - value: {{.Values.OAUTH.TokenURL}} - # TODO load env IAM client from secret - - name: IAM_CLIENT_ID - value: {{.Values.OAUTH.ClientID}} - - name: IAM_CLIENT_SECRET - value: {{.Values.OAUTH.ClientSecret}} - - name: IAM_REFRESH_TOKEN - value: {{.Values.OAUTH.RefreshToken}} - - name: IAM_VK_AUD - value: {{.Values.OAUTH.Audience}} - - name: TOKEN_PATH - value: /opt/interlink/token - command: - - python3 - - /opt/refresh.py - resources: - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 100m - memory: 300Mi - volumeMounts: - - name: token - mountPath: /opt/interlink - volumes: - - name: config - configMap: - name: virtual-kubelet-config - - name: token - hostPath: - path: /tmp - type: Directory diff --git a/helm/interlink/values.yaml b/helm/interlink/values.yaml deleted file mode 100644 index 6d53fd6c..00000000 --- a/helm/interlink/values.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Default values for interlink. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -nodeName: default-vk - -deployment: - image: ghcr.io/intertwin-eu/interlink/virtual-kubelet-inttw:latest - name: default-vk - -interlink: - URL: http://localhost - port: 5000 - -virtualNode: - CPUs: 100 - MemGiB: 1600 - Pods: 100 - -OAUTH: - TokenURL: DUMMY - ClientID: DUMMY - ClientSecret: DUMMY - RefreshToken: DUMMY - Audience: DUMMY - diff --git a/main.go b/main.go deleted file mode 100644 index f2b85846..00000000 --- a/main.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "context" - "net/http" - "strings" - - "github.com/sirupsen/logrus" - "github.com/virtual-kubelet/virtual-kubelet/log" - logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" - - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" - "github.com/intertwin-eu/interlink/pkg/interlink/api" -) - -func main() { - var cancel context.CancelFunc - api.PodStatuses.Statuses = make(map[string]commonIL.PodStatus) - - interLinkConfig, err := commonIL.NewInterLinkConfig() - if err != nil { - panic(err) - } - logger := logrus.StandardLogger() - - logger.SetLevel(logrus.InfoLevel) - if interLinkConfig.VerboseLogging { - logger.SetLevel(logrus.DebugLevel) - } else if interLinkConfig.ErrorsOnlyLogging { - logger.SetLevel(logrus.ErrorLevel) - } - - log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - log.G(ctx).Info(interLinkConfig) - - sidecarEndpoint := "" - if strings.HasPrefix(interLinkConfig.Sidecarurl, "unix://") { - sidecarEndpoint = interLinkConfig.Sidecarurl - } else if strings.HasPrefix(interLinkConfig.Sidecarurl, "http://") { - sidecarEndpoint = interLinkConfig.Sidecarurl + ":" + interLinkConfig.Sidecarport - } else { - log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") - } - - interLinkAPIs := api.InterLinkHandler{ - Config: interLinkConfig, - Ctx: ctx, - SidecarEndpoint: sidecarEndpoint, - } - - mutex := http.NewServeMux() - mutex.HandleFunc("/status", interLinkAPIs.StatusHandler) - mutex.HandleFunc("/create", interLinkAPIs.CreateHandler) - mutex.HandleFunc("/delete", interLinkAPIs.DeleteHandler) - mutex.HandleFunc("/pinglink", interLinkAPIs.Ping) - mutex.HandleFunc("/getLogs", interLinkAPIs.GetLogsHandler) - mutex.HandleFunc("/updateCache", interLinkAPIs.UpdateCacheHandler) - - interLinkEndpoint := "" - if strings.HasPrefix(interLinkConfig.InterlinkAddress, "unix://") { - interLinkEndpoint = interLinkConfig.InterlinkAddress - } else if strings.HasPrefix(interLinkConfig.Sidecarurl, "http://") { - interLinkEndpoint = interLinkConfig.InterlinkAddress + ":" + interLinkConfig.Interlinkport - } else { - log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") - } - - err = http.ListenAndServe(interLinkEndpoint, mutex) - - if err != nil { - log.G(ctx).Fatal(err) - } -} diff --git a/pkg/interlink/api/create.go b/pkg/interlink/api/create.go index 908e4b3a..c3afbf9b 100644 --- a/pkg/interlink/api/create.go +++ b/pkg/interlink/api/create.go @@ -5,14 +5,27 @@ import ( "encoding/json" "io" "net/http" + "time" "github.com/containerd/containerd/log" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" ) // CreateHandler collects and rearranges all needed ConfigMaps/Secrets/EmptyDirs to ship them to the sidecar, then sends a response to the client func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { + start := time.Now().UnixMicro() + tracer := otel.Tracer("interlink-API") + _, span := tracer.Start(h.Ctx, "CreateAPI", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) + log.G(h.Ctx).Info("InterLink: received Create call") statusCode := -1 @@ -21,28 +34,34 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) if err != nil { statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) - log.G(h.Ctx).Fatal(err) + log.G(h.Ctx).Error(err) return } - var req *http.Request //request to forward to sidecar - var pod commonIL.PodCreateRequests //request for interlink + var req *http.Request //request to forward to sidecar + var pod types.PodCreateRequests //request for interlink err = json.Unmarshal(bodyBytes, &pod) if err != nil { statusCode = http.StatusInternalServerError - log.G(h.Ctx).Fatal(err) + log.G(h.Ctx).Error(err) w.WriteHeader(statusCode) return } - var retrievedData []commonIL.RetrievedPodData + span.SetAttributes( + attribute.String("pod.name", pod.Pod.Name), + attribute.String("pod.namespace", pod.Pod.Namespace), + attribute.String("pod.uid", string(pod.Pod.UID)), + ) + + var retrievedData []types.RetrievedPodData - data := commonIL.RetrievedPodData{} + data := types.RetrievedPodData{} if h.Config.ExportPodData { - data, err = getData(h.Ctx, h.Config, pod) + data, err = getData(h.Ctx, h.Config, pod, span) if err != nil { statusCode = http.StatusInternalServerError - log.G(h.Ctx).Fatal(err) + log.G(h.Ctx).Error(err) w.WriteHeader(statusCode) return } @@ -54,7 +73,7 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) bodyBytes, err = json.Marshal(retrievedData) if err != nil { w.WriteHeader(http.StatusInternalServerError) - log.G(h.Ctx).Fatal(err) + log.G(h.Ctx).Error(err) return } log.G(h.Ctx).Debug(string(bodyBytes)) @@ -66,7 +85,7 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) if err != nil { statusCode = http.StatusInternalServerError w.WriteHeader(statusCode) - log.G(h.Ctx).Fatal(err) + log.G(h.Ctx).Error(err) return } @@ -82,17 +101,20 @@ func (h *InterLinkHandler) CreateHandler(w http.ResponseWriter, r *http.Request) return } - if resp.StatusCode == http.StatusOK { - statusCode = http.StatusOK - log.G(h.Ctx).Debug(statusCode) - } else { - statusCode = http.StatusInternalServerError - log.G(h.Ctx).Error(statusCode) + if resp != nil { + if resp.StatusCode == http.StatusOK { + statusCode = http.StatusOK + log.G(h.Ctx).Debug(statusCode) + } else { + statusCode = http.StatusInternalServerError + log.G(h.Ctx).Error(statusCode) + } + + returnValue, _ := io.ReadAll(resp.Body) + log.G(h.Ctx).Debug(string(returnValue)) + w.WriteHeader(statusCode) + types.SetDurationSpan(start, span, types.WithHTTPReturnCode(statusCode)) + w.Write(returnValue) } - - returnValue, _ := io.ReadAll(resp.Body) - log.G(h.Ctx).Debug(string(returnValue)) - w.WriteHeader(statusCode) - w.Write(returnValue) } } diff --git a/pkg/interlink/api/delete.go b/pkg/interlink/api/delete.go index 864d9dbb..cce503ad 100644 --- a/pkg/interlink/api/delete.go +++ b/pkg/interlink/api/delete.go @@ -5,15 +5,28 @@ import ( "encoding/json" "io" "net/http" + "time" "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" ) // DeleteHandler deletes the cached status for the provided Pod and forwards the request to the sidecar func (h *InterLinkHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { + start := time.Now().UnixMicro() + tracer := otel.Tracer("interlink-API") + _, span := tracer.Start(h.Ctx, "DeleteAPI", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) + log.G(h.Ctx).Info("InterLink: received Delete call") bodyBytes, err := io.ReadAll(r.Body) @@ -36,6 +49,12 @@ func (h *InterLinkHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) log.G(h.Ctx).Fatal(err) } + span.SetAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.String("pod.uid", string(pod.UID)), + ) + deleteCachedStatus(string(pod.UID)) req, err = http.NewRequest(http.MethodPost, h.SidecarEndpoint+"/delete", reader) if err != nil { @@ -55,24 +74,26 @@ func (h *InterLinkHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) return } - returnValue, _ := io.ReadAll(resp.Body) - statusCode = resp.StatusCode + if resp != nil { + returnValue, _ := io.ReadAll(resp.Body) + statusCode = resp.StatusCode - if statusCode != http.StatusOK { - w.WriteHeader(http.StatusInternalServerError) - } else { - w.WriteHeader(http.StatusOK) - } - log.G(h.Ctx).Debug("InterLink: " + string(returnValue)) - var returnJson []commonIL.PodStatus - returnJson = append(returnJson, commonIL.PodStatus{PodName: pod.Name, PodUID: string(pod.UID), PodNamespace: pod.Namespace}) + if statusCode != http.StatusOK { + w.WriteHeader(http.StatusInternalServerError) + } else { + w.WriteHeader(http.StatusOK) + } + log.G(h.Ctx).Debug("InterLink: " + string(returnValue)) + var returnJson []types.PodStatus + returnJson = append(returnJson, types.PodStatus{PodName: pod.Name, PodUID: string(pod.UID), PodNamespace: pod.Namespace}) - bodyBytes, err = json.Marshal(returnJson) - if err != nil { - log.G(h.Ctx).Error(err) - w.Write([]byte{}) - } else { - w.Write(bodyBytes) + bodyBytes, err = json.Marshal(returnJson) + if err != nil { + log.G(h.Ctx).Error(err) + w.Write([]byte{}) + } else { + types.SetDurationSpan(start, span, types.WithHTTPReturnCode(statusCode)) + w.Write(bodyBytes) + } } - } diff --git a/pkg/interlink/api/func.go b/pkg/interlink/api/func.go index 02c7de61..a9915273 100644 --- a/pkg/interlink/api/func.go +++ b/pkg/interlink/api/func.go @@ -4,16 +4,19 @@ import ( "context" "path/filepath" "sync" + "time" "github.com/containerd/containerd/log" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" v1 "k8s.io/api/core/v1" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" ) type MutexStatuses struct { mu sync.Mutex - Statuses map[string]commonIL.PodStatus + Statuses map[string]types.PodStatus } var PodStatuses MutexStatuses @@ -21,41 +24,57 @@ var PodStatuses MutexStatuses // getData retrieves ConfigMaps, Secrets and EmptyDirs from the provided pod by calling the retrieveData function. // The config is needed by the retrieveData function. // The function aggregates the return values of retrieveData function in a commonIL.RetrievedPodData variable and returns it, along with the first encountered error. -func getData(ctx context.Context, config commonIL.InterLinkConfig, pod commonIL.PodCreateRequests) (commonIL.RetrievedPodData, error) { +func getData(ctx context.Context, config types.InterLinkConfig, pod types.PodCreateRequests, span trace.Span) (types.RetrievedPodData, error) { + start := time.Now().UnixMicro() + span.AddEvent("Retrieving data for pod " + pod.Pod.Name) log.G(ctx).Debug(pod.ConfigMaps) - var retrievedData commonIL.RetrievedPodData + var retrievedData types.RetrievedPodData retrievedData.Pod = pod.Pod for _, container := range pod.Pod.Spec.InitContainers { + startContainer := time.Now().UnixMicro() log.G(ctx).Info("- Retrieving Secrets and ConfigMaps for the Docker Sidecar. InitContainer: " + container.Name) log.G(ctx).Debug(container.VolumeMounts) - data, err := retrieveData(ctx, config, pod, container) - if err != nil { - log.G(ctx).Error(err) - return commonIL.RetrievedPodData{}, err + data, InterlinkIP := retrieveData(ctx, config, pod, container) + if InterlinkIP != nil { + log.G(ctx).Error(InterlinkIP) + return types.RetrievedPodData{}, InterlinkIP } retrievedData.Containers = append(retrievedData.Containers, data) + + durationContainer := time.Now().UnixMicro() - startContainer + span.AddEvent("Init Container "+container.Name, trace.WithAttributes( + attribute.Int64("initcontainer.getdata.duration", durationContainer), + attribute.String("pod.name", pod.Pod.Name))) } for _, container := range pod.Pod.Spec.Containers { + startContainer := time.Now().UnixMicro() log.G(ctx).Info("- Retrieving Secrets and ConfigMaps for the Docker Sidecar. Container: " + container.Name) log.G(ctx).Debug(container.VolumeMounts) data, err := retrieveData(ctx, config, pod, container) if err != nil { log.G(ctx).Error(err) - return commonIL.RetrievedPodData{}, err + return types.RetrievedPodData{}, err } retrievedData.Containers = append(retrievedData.Containers, data) + + durationContainer := time.Now().UnixMicro() - startContainer + span.AddEvent("Container "+container.Name, trace.WithAttributes( + attribute.Int64("container.getdata.duration", durationContainer), + attribute.String("pod.name", pod.Pod.Name))) } + duration := time.Now().UnixMicro() - start + span.SetAttributes(attribute.Int64("getdata.duration", duration)) return retrievedData, nil } // retrieveData retrieves ConfigMaps, Secrets and EmptyDirs. // The config is needed to specify the EmptyDirs mounting point. // It returns the retrieved data in a variable of type commonIL.RetrievedContainer and the first encountered error. -func retrieveData(ctx context.Context, config commonIL.InterLinkConfig, pod commonIL.PodCreateRequests, container v1.Container) (commonIL.RetrievedContainer, error) { - retrievedData := commonIL.RetrievedContainer{} +func retrieveData(ctx context.Context, config types.InterLinkConfig, pod types.PodCreateRequests, container v1.Container) (types.RetrievedContainer, error) { + retrievedData := types.RetrievedContainer{} for _, mountVar := range container.VolumeMounts { log.G(ctx).Debug("-- Retrieving data for mountpoint " + mountVar.Name) @@ -114,7 +133,7 @@ func checkIfCached(uid string) bool { } // updateStatuses locks and updates the PodStatuses map with the statuses contained in the returnedStatuses slice -func updateStatuses(returnedStatuses []commonIL.PodStatus) { +func updateStatuses(returnedStatuses []types.PodStatus) { PodStatuses.mu.Lock() for _, new := range returnedStatuses { diff --git a/pkg/interlink/api/logs.go b/pkg/interlink/api/logs.go index c53a0f1b..63bd6f16 100644 --- a/pkg/interlink/api/logs.go +++ b/pkg/interlink/api/logs.go @@ -7,13 +7,26 @@ import ( "io" "net/http" "strconv" + "time" "github.com/containerd/containerd/log" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" ) func (h *InterLinkHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request) { + start := time.Now().UnixMicro() + tracer := otel.Tracer("interlink-API") + _, span := tracer.Start(h.Ctx, "GetLogsAPI", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) + statusCode := http.StatusOK log.G(h.Ctx).Info("InterLink: received GetLogs call") bodyBytes, err := io.ReadAll(r.Body) @@ -22,7 +35,7 @@ func (h *InterLinkHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request } log.G(h.Ctx).Info("InterLink: unmarshal GetLogs request") - var req2 commonIL.LogStruct //incoming request. To be used in interlink API. req is directly forwarded to sidecar + var req2 types.LogStruct //incoming request. To be used in interlink API. req is directly forwarded to sidecar err = json.Unmarshal(bodyBytes, &req2) if err != nil { statusCode = http.StatusInternalServerError @@ -31,6 +44,18 @@ func (h *InterLinkHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request return } + span.SetAttributes( + attribute.String("pod.name", req2.PodName), + attribute.String("pod.namespace", req2.Namespace), + attribute.Int("opts.limitbytes", req2.Opts.LimitBytes), + attribute.Int("opts.since", req2.Opts.SinceSeconds), + attribute.Int64("opts.sincetime", req2.Opts.SinceTime.UnixMicro()), + attribute.Int("opts.tail", req2.Opts.Tail), + attribute.Bool("opts.follow", req2.Opts.Follow), + attribute.Bool("opts.previous", req2.Opts.Previous), + attribute.Bool("opts.timestamps", req2.Opts.Timestamps), + ) + log.G(h.Ctx).Info("InterLink: new GetLogs podUID: now ", string(req2.PodUID)) if (req2.Opts.Tail != 0 && req2.Opts.LimitBytes != 0) || (req2.Opts.SinceSeconds != 0 && !req2.Opts.SinceTime.IsZero()) { statusCode = http.StatusInternalServerError @@ -69,14 +94,18 @@ func (h *InterLinkHandler) GetLogsHandler(w http.ResponseWriter, r *http.Request return } - if resp.StatusCode != http.StatusOK { - log.L.Error("Unexpected error occured. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check Sidecar's logs for further informations") - statusCode = http.StatusInternalServerError - } + if resp != nil { + if resp.StatusCode != http.StatusOK { + log.L.Error("Unexpected error occured. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check Sidecar's logs for further informations") + statusCode = http.StatusInternalServerError + } + + returnValue, _ := io.ReadAll(resp.Body) + log.G(h.Ctx).Debug("InterLink: logs " + string(returnValue)) - returnValue, _ := io.ReadAll(resp.Body) - log.G(h.Ctx).Debug("InterLink: logs " + string(returnValue)) + types.SetDurationSpan(start, span, types.WithHTTPReturnCode(statusCode)) - w.WriteHeader(statusCode) - w.Write(returnValue) + w.WriteHeader(statusCode) + w.Write(returnValue) + } } diff --git a/pkg/interlink/api/ping.go b/pkg/interlink/api/ping.go index bb7a4a0b..c0109031 100644 --- a/pkg/interlink/api/ping.go +++ b/pkg/interlink/api/ping.go @@ -1,22 +1,67 @@ package api import ( + "bytes" + "encoding/json" "net/http" - "os" + "strconv" + "time" "github.com/containerd/containerd/log" + types "github.com/intertwin-eu/interlink/pkg/interlink" + v1 "k8s.io/api/core/v1" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" ) // Ping is just a very basic Ping function func (h *InterLinkHandler) Ping(w http.ResponseWriter, r *http.Request) { + start := time.Now().UnixMicro() + tracer := otel.Tracer("interlink-API") + _, span := tracer.Start(h.Ctx, "PingAPI", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) + log.G(h.Ctx).Info("InterLink: received Ping call") - w.WriteHeader(http.StatusOK) - // 0 = KUBECONFIG already set - // 1 = KUBECONFIG not set - if os.Getenv("KUBECONFIG") != "" { + podsToBeChecked := []*v1.Pod{} + bodyBytes, err := json.Marshal(podsToBeChecked) + if err != nil { + log.G(h.Ctx).Error(err) + } + + reader := bytes.NewReader(bodyBytes) + req, err := http.NewRequest(http.MethodGet, h.SidecarEndpoint+"/status", reader) + if err != nil { + log.G(h.Ctx).Error(err) + } + + log.G(h.Ctx).Info("InterLink: forwarding GetStatus call to sidecar") + req.Header.Set("Content-Type", "application/json") + log.G(h.Ctx).Debug(req) + respPlugin, err := http.DefaultClient.Do(req) + if err != nil { + log.G(h.Ctx).Error(err) + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(strconv.Itoa(http.StatusServiceUnavailable))) + return + } + + if respPlugin != nil { + if respPlugin.StatusCode != http.StatusOK { + log.G(h.Ctx).Error("error pinging plugin") + w.WriteHeader(respPlugin.StatusCode) + w.Write([]byte(strconv.Itoa(http.StatusServiceUnavailable))) + return + } + + types.SetDurationSpan(start, span, types.WithHTTPReturnCode(respPlugin.StatusCode)) + + w.WriteHeader(http.StatusOK) w.Write([]byte("0")) - } else { - w.Write([]byte("1")) } } diff --git a/pkg/interlink/api/status.go b/pkg/interlink/api/status.go index 92dde171..9ce0f89d 100644 --- a/pkg/interlink/api/status.go +++ b/pkg/interlink/api/status.go @@ -6,14 +6,26 @@ import ( "io" "net/http" "strconv" + "time" "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" ) func (h *InterLinkHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { + start := time.Now().UnixMicro() + tracer := otel.Tracer("interlink-API") + _, span := tracer.Start(h.Ctx, "StatusAPI", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) statusCode := http.StatusOK var pods []*v1.Pod log.G(h.Ctx).Info("InterLink: received GetStatus call") @@ -28,15 +40,32 @@ func (h *InterLinkHandler) StatusHandler(w http.ResponseWriter, r *http.Request) log.G(h.Ctx).Error(err) } + span.SetAttributes( + attribute.Int("pods.count", len(pods)), + ) + var podsToBeChecked []*v1.Pod - var returnedStatuses []commonIL.PodStatus //returned from the query to the sidecar - var returnPods []commonIL.PodStatus //returned to the vk + var returnedStatuses []types.PodStatus //returned from the query to the sidecar + var returnPods []types.PodStatus //returned to the vk PodStatuses.mu.Lock() for _, pod := range pods { cached := checkIfCached(string(pod.UID)) if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodPending || !cached { + span.AddEvent("Pod "+pod.Name+" is not cached", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.String("pod.uid", string(pod.UID)), + attribute.String("pod.phase", string(pod.Status.Phase)), + )) podsToBeChecked = append(podsToBeChecked, pod) + } else if cached { + span.AddEvent("Pod "+pod.Name+" is cached", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.String("pod.uid", string(pod.UID)), + attribute.String("pod.phase", string(pod.Status.Phase)), + )) } } PodStatuses.mu.Unlock() @@ -65,29 +94,32 @@ func (h *InterLinkHandler) StatusHandler(w http.ResponseWriter, r *http.Request) return } - if resp.StatusCode != http.StatusOK { - log.L.Error("Unexpected error occured. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check Sidecar's logs for further informations") - statusCode = http.StatusInternalServerError - } + if resp != nil { + if resp.StatusCode != http.StatusOK { + log.L.Error("Unexpected error occured. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check Sidecar's logs for further informations") + statusCode = http.StatusInternalServerError + } - bodyBytes, err = io.ReadAll(resp.Body) - if err != nil { - statusCode = http.StatusInternalServerError - w.WriteHeader(statusCode) - log.G(h.Ctx).Error(err) - return - } + bodyBytes, err = io.ReadAll(resp.Body) + if err != nil { + statusCode = http.StatusInternalServerError + w.WriteHeader(statusCode) + log.G(h.Ctx).Error(err) + return + } - log.G(h.Ctx).Debug(string(bodyBytes)) - err = json.Unmarshal(bodyBytes, &returnedStatuses) - if err != nil { - statusCode = http.StatusInternalServerError - w.WriteHeader(statusCode) - log.G(h.Ctx).Error(err) - return - } + log.G(h.Ctx).Debug(string(bodyBytes)) + err = json.Unmarshal(bodyBytes, &returnedStatuses) + if err != nil { + statusCode = http.StatusInternalServerError + w.WriteHeader(statusCode) + log.G(h.Ctx).Error(err) + return + } - updateStatuses(returnedStatuses) + updateStatuses(returnedStatuses) + types.SetDurationSpan(start, span, types.WithHTTPReturnCode(statusCode)) + } } diff --git a/pkg/interlink/spans.go b/pkg/interlink/spans.go new file mode 100644 index 00000000..73469b5b --- /dev/null +++ b/pkg/interlink/spans.go @@ -0,0 +1,32 @@ +package interlink + +import ( + "time" + + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" +) + +func WithHTTPReturnCode(code int) SpanOption { + return func(cfg *SpanConfig) { + cfg.HTTPReturnCode = code + cfg.SetHTTPCode = true + } +} + +func SetDurationSpan(startTime int64, span trace.Span, opts ...SpanOption) { + endTime := time.Now().UnixMicro() + config := &SpanConfig{} + + for _, opt := range opts { + opt(config) + } + + duration := endTime - startTime + span.SetAttributes(attribute.Int64("end.timestamp", endTime), + attribute.Int64("duration", duration)) + + if config.SetHTTPCode { + span.SetAttributes(attribute.Int("exit.code", config.HTTPReturnCode)) + } +} diff --git a/pkg/interlink/types.go b/pkg/interlink/types.go index b09b0588..6ba742f4 100644 --- a/pkg/interlink/types.go +++ b/pkg/interlink/types.go @@ -18,9 +18,16 @@ type PodStatus struct { PodName string `json:"name"` PodUID string `json:"UID"` PodNamespace string `json:"namespace"` + JobID string `json:"JID"` Containers []v1.ContainerStatus `json:"containers"` } +// CreateStruct is the response to be received from interLink whenever asked to create a pod. It will allow for mapping remote ID with the pod UUID +type CreateStruct struct { + PodUID string `json:"PodUID"` + PodJID string `json:"PodJID"` +} + // RetrievedContainer is used in InterLink to rearrange data structure in a suitable way for the sidecar type RetrievedContainer struct { Name string `json:"name"` @@ -54,3 +61,10 @@ type LogStruct struct { ContainerName string `json:"ContainerName"` Opts ContainerLogOpts `json:"Opts"` } + +type SpanConfig struct { + HTTPReturnCode int + SetHTTPCode bool +} + +type SpanOption func(*SpanConfig) diff --git a/pkg/virtualkubelet/config.go b/pkg/virtualkubelet/config.go index ef8c2868..a7f68339 100644 --- a/pkg/virtualkubelet/config.go +++ b/pkg/virtualkubelet/config.go @@ -2,7 +2,7 @@ package virtualkubelet // VirtualKubeletConfig holds the whole configuration type VirtualKubeletConfig struct { - Interlinkurl string `yaml:"InterlinkURL"` + InterlinkURL string `yaml:"InterlinkURL"` Interlinkport string `yaml:"InterlinkPort"` VKConfigPath string `yaml:"VKConfigPath"` VKTokenFile string `yaml:"VKTokenFile"` @@ -11,8 +11,8 @@ type VirtualKubeletConfig struct { PodIP string `yaml:"PodIP"` VerboseLogging bool `yaml:"VerboseLogging"` ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` - CPU string `yaml:"cpu,omitempty"` - Memory string `yaml:"memory,omitempty"` - Pods string `yaml:"pods,omitempty"` + CPU string `yaml:"CPU,omitempty"` + Memory string `yaml:"Memory,omitempty"` + Pods string `yaml:"Pods,omitempty"` GPU string `yaml:"nvidia.com/gpu,omitempty"` } diff --git a/pkg/virtualkubelet/execute.go b/pkg/virtualkubelet/execute.go index bb7a7028..95a69990 100644 --- a/pkg/virtualkubelet/execute.go +++ b/pkg/virtualkubelet/execute.go @@ -14,15 +14,20 @@ import ( "time" "github.com/containerd/containerd/log" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" ) func doRequest(req *http.Request, token string) (*http.Response, error) { - req.Header.Add("Authorization", "Bearer "+token) + if token != "" { + req.Header.Add("Authorization", "Bearer "+token) + } req.Header.Set("Content-Type", "application/json") return http.DefaultClient.Do(req) @@ -30,19 +35,23 @@ func doRequest(req *http.Request, token string) (*http.Response, error) { func getSidecarEndpoint(ctx context.Context, interLinkURL string, interLinkPort string) string { interLinkEndpoint := "" + log.G(ctx).Info("InterlingURL: ", interLinkURL) if strings.HasPrefix(interLinkURL, "unix://") { - interLinkEndpoint = interLinkURL + interLinkEndpoint = "http://unix" } else if strings.HasPrefix(interLinkURL, "http://") { interLinkEndpoint = interLinkURL + ":" + interLinkPort + } else if strings.HasPrefix(interLinkURL, "https://") { + interLinkEndpoint = interLinkURL + ":" + interLinkPort } else { - log.G(ctx).Fatal("Sidecar URL should either start per unix:// or http://") + log.G(ctx).Fatal("InterLinkURL URL should either start per unix:// or http(s)://") } return interLinkEndpoint } // PingInterLink pings the InterLink API and returns true if there's an answer. The second return value is given by the answer provided by the API. func PingInterLink(ctx context.Context, config VirtualKubeletConfig) (bool, int, error) { - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) + tracer := otel.Tracer("interlink-service") + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) log.G(ctx).Info("Pinging: " + interLinkEndpoint + "/pinglink") retVal := -1 req, err := http.NewRequest(http.MethodPost, interLinkEndpoint+"/pinglink", nil) @@ -51,18 +60,31 @@ func PingInterLink(ctx context.Context, config VirtualKubeletConfig) (bool, int, log.G(ctx).Error(err) } - token, err := os.ReadFile(config.VKTokenFile) // just pass the file name - if err != nil { - log.G(ctx).Error(err) - return false, retVal, err + if config.VKTokenFile != "" { + token, err := os.ReadFile(config.VKTokenFile) // just pass the file name + if err != nil { + log.G(ctx).Error(err) + return false, retVal, err + } + req.Header.Add("Authorization", "Bearer "+string(token)) } - req.Header.Add("Authorization", "Bearer "+string(token)) + + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "PingHttpCall", trace.WithAttributes( + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := http.DefaultClient.Do(req) + if err != nil { + spanHttp.SetAttributes(attribute.Int("exit.code", http.StatusInternalServerError)) return false, retVal, err } - if resp.StatusCode == http.StatusOK { + if resp != nil { + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) retBytes, err := io.ReadAll(resp.Body) if err != nil { log.G(ctx).Error(err) @@ -73,18 +95,26 @@ func PingInterLink(ctx context.Context, config VirtualKubeletConfig) (bool, int, log.G(ctx).Error(err) return false, retVal, err } - return true, retVal, nil - } else { - log.G(ctx).Error("server error: " + fmt.Sprint(resp.StatusCode)) - return false, retVal, nil + + if resp.StatusCode != http.StatusOK { + log.G(ctx).Error("server error: " + fmt.Sprint(resp.StatusCode)) + return false, retVal, nil + } } + + return true, retVal, nil } // updateCacheRequest is called when the VK receives the status of a pod already deleted. It performs a REST call InterLink API to update the cache deleting that pod from the cached structure -func updateCacheRequest(ctx context.Context, config VirtualKubeletConfig, uid string, token string) error { - bodyBytes := []byte(uid) +func updateCacheRequest(ctx context.Context, config VirtualKubeletConfig, pod v1.Pod, token string) error { + tracer := otel.Tracer("interlink-service") + bodyBytes, err := json.Marshal(pod) + if err != nil { + log.L.Error(err) + return err + } - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) reader := bytes.NewReader(bodyBytes) req, err := http.NewRequest(http.MethodPost, interLinkEndpoint+"/updateCache", reader) if err != nil { @@ -92,17 +122,31 @@ func updateCacheRequest(ctx context.Context, config VirtualKubeletConfig, uid st return err } - req.Header.Add("Authorization", "Bearer "+token) + if token != "" { + req.Header.Add("Authorization", "Bearer "+token) + } req.Header.Set("Content-Type", "application/json") + + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "UpdateCacheHttpCall", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.String("pod.uid", string(pod.UID)), + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := http.DefaultClient.Do(req) if err != nil { log.L.Error(err) return err } - statusCode := resp.StatusCode - - if statusCode != http.StatusOK { - return errors.New("Unexpected error occured while updating InterLink cache. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + if resp != nil { + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) + if resp.StatusCode != http.StatusOK { + return errors.New("Unexpected error occured while updating InterLink cache. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + } } return err @@ -110,9 +154,10 @@ func updateCacheRequest(ctx context.Context, config VirtualKubeletConfig, uid st // createRequest performs a REST call to the InterLink API when a Pod is registered to the VK. It Marshals the pod with already retrieved ConfigMaps and Secrets and sends it to InterLink. // Returns the call response expressed in bytes and/or the first encountered error -func createRequest(ctx context.Context, config VirtualKubeletConfig, pod commonIL.PodCreateRequests, token string) ([]byte, error) { - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) - var returnValue, _ = json.Marshal(commonIL.PodStatus{}) +func createRequest(ctx context.Context, config VirtualKubeletConfig, pod types.PodCreateRequests, token string) ([]byte, error) { + tracer := otel.Tracer("interlink-service") + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) + var returnValue, _ = json.Marshal(types.CreateStruct{}) bodyBytes, err := json.Marshal(pod) if err != nil { @@ -126,20 +171,33 @@ func createRequest(ctx context.Context, config VirtualKubeletConfig, pod commonI return nil, err } + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "CreateHttpCall", trace.WithAttributes( + attribute.String("pod.name", pod.Pod.Name), + attribute.String("pod.namespace", pod.Pod.Namespace), + attribute.String("pod.uid", string(pod.Pod.UID)), + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := doRequest(req, token) if err != nil { log.L.Error(err) return nil, err } - statusCode := resp.StatusCode - if statusCode != http.StatusOK { - return nil, errors.New("Unexpected error occured while creating Pods. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") - } else { - returnValue, err = io.ReadAll(resp.Body) - if err != nil { - log.L.Error(err) - return nil, err + if resp != nil { + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Unexpected error occured while creating Pods. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + } else { + returnValue, err = io.ReadAll(resp.Body) + if err != nil { + log.L.Error(err) + return nil, err + } } } @@ -149,7 +207,9 @@ func createRequest(ctx context.Context, config VirtualKubeletConfig, pod commonI // deleteRequest performs a REST call to the InterLink API when a Pod is deleted from the VK. It Marshals the standard v1.Pod struct and sends it to InterLink. // Returns the call response expressed in bytes and/or the first encountered error func deleteRequest(ctx context.Context, config VirtualKubeletConfig, pod *v1.Pod, token string) ([]byte, error) { - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) + tracer := otel.Tracer("interlink-service") + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) + var returnValue []byte bodyBytes, err := json.Marshal(pod) if err != nil { log.G(context.Background()).Error(err) @@ -162,39 +222,53 @@ func deleteRequest(ctx context.Context, config VirtualKubeletConfig, pod *v1.Pod return nil, err } + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "DeleteHttpCall", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.String("pod.uid", string(pod.UID)), + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := doRequest(req, token) if err != nil { log.G(context.Background()).Error(err) return nil, err } - statusCode := resp.StatusCode - - if statusCode != http.StatusOK { - return nil, errors.New("Unexpected error occured while deleting Pods. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") - } else { - returnValue, err := io.ReadAll(resp.Body) - if err != nil { - log.G(context.Background()).Error(err) - return nil, err - } - log.G(context.Background()).Info(string(returnValue)) - var response []commonIL.PodStatus - err = json.Unmarshal(returnValue, &response) - if err != nil { - log.G(context.Background()).Error(err) - return nil, err + if resp != nil { + statusCode := resp.StatusCode + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) + if statusCode != http.StatusOK { + return nil, errors.New("Unexpected error occured while deleting Pods. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + } else { + returnValue, err := io.ReadAll(resp.Body) + if err != nil { + log.G(context.Background()).Error(err) + return nil, err + } + log.G(context.Background()).Info(string(returnValue)) + var response []types.PodStatus + err = json.Unmarshal(returnValue, &response) + if err != nil { + log.G(context.Background()).Error(err) + return nil, err + } } - return returnValue, nil } + + return returnValue, nil } // statusRequest performs a REST call to the InterLink API when the VK needs an update on its Pods' status. A Marshalled slice of v1.Pod is sent to the InterLink API, // to query the below plugin for their status. // Returns the call response expressed in bytes and/or the first encountered error func statusRequest(ctx context.Context, config VirtualKubeletConfig, podsList []*v1.Pod, token string) ([]byte, error) { - var returnValue []byte - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) + tracer := otel.Tracer("interlink-service") + returnValue, _ := json.Marshal(types.PodStatus{}) + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) bodyBytes, err := json.Marshal(podsList) if err != nil { @@ -210,18 +284,28 @@ func statusRequest(ctx context.Context, config VirtualKubeletConfig, podsList [] //log.L.Println(string(bodyBytes)) + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "StatusHttpCall", trace.WithAttributes( + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := doRequest(req, token) if err != nil { return nil, err } - if resp.StatusCode != http.StatusOK { - return nil, errors.New("Unexpected error occured while getting status. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") - } else { - returnValue, err = io.ReadAll(resp.Body) - if err != nil { - log.L.Error(err) - return nil, err + if resp != nil { + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Unexpected error occured while getting status. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + } else { + returnValue, err = io.ReadAll(resp.Body) + if err != nil { + log.L.Error(err) + return nil, err + } } } @@ -231,13 +315,20 @@ func statusRequest(ctx context.Context, config VirtualKubeletConfig, podsList [] // LogRetrieval performs a REST call to the InterLink API when the user ask for a log retrieval. Compared to create/delete/status request, a way smaller struct is marshalled and sent. // This struct only includes a minimum data set needed to identify the job/container to get the logs from. // Returns the call response and/or the first encountered error -func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest commonIL.LogStruct) (io.ReadCloser, error) { - interLinkEndpoint := getSidecarEndpoint(ctx, config.Interlinkurl, config.Interlinkport) - b, err := os.ReadFile(config.VKTokenFile) // just pass the file name - if err != nil { - log.G(ctx).Fatal(err) +func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest types.LogStruct) (io.ReadCloser, error) { + tracer := otel.Tracer("interlink-service") + var returnValue io.ReadCloser + interLinkEndpoint := getSidecarEndpoint(ctx, config.InterlinkURL, config.Interlinkport) + + token := "" + + if config.VKTokenFile != "" { + b, err := os.ReadFile(config.VKTokenFile) // just pass the file name + if err != nil { + log.G(ctx).Fatal(err) + } + token = string(b) } - token := string(b) bodyBytes, err := json.Marshal(logsRequest) if err != nil { @@ -253,18 +344,32 @@ func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest //log.G(ctx).Println(string(bodyBytes)) + startHttpCall := time.Now().UnixMicro() + _, spanHttp := tracer.Start(ctx, "LogHttpCall", trace.WithAttributes( + attribute.String("pod.name", logsRequest.PodName), + attribute.String("pod.namespace", logsRequest.Namespace), + attribute.String("pod.uid", logsRequest.PodUID), + attribute.Int64("start.timestamp", startHttpCall), + )) + defer spanHttp.End() + defer types.SetDurationSpan(startHttpCall, spanHttp) + resp, err := doRequest(req, token) if err != nil { log.G(ctx).Error(err) return nil, err } - if resp.StatusCode != http.StatusOK { - log.G(ctx).Info(resp.Body) - return nil, errors.New("Unexpected error occured while getting logs. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") - } else { - return resp.Body, nil + if resp != nil { + types.SetDurationSpan(startHttpCall, spanHttp, types.WithHTTPReturnCode(resp.StatusCode)) + if resp.StatusCode != http.StatusOK { + err = errors.New("Unexpected error occured while getting logs. Status code: " + strconv.Itoa(resp.StatusCode) + ". Check InterLink's logs for further informations") + } else { + returnValue = resp.Body + } } + + return returnValue, err } // RemoteExecution is called by the VK everytime a Pod is being registered or deleted to/from the VK. @@ -273,16 +378,20 @@ func LogRetrieval(ctx context.Context, config VirtualKubeletConfig, logsRequest // If after 5m they are not still available, the function errors out func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *VirtualKubeletProvider, pod *v1.Pod, mode int8) error { - b, err := os.ReadFile(config.VKTokenFile) // just pass the file name - if err != nil { - log.G(ctx).Fatal(err) - return err + token := "" + if config.VKTokenFile != "" { + b, err := os.ReadFile(config.VKTokenFile) // just pass the file name + if err != nil { + log.G(ctx).Fatal(err) + return err + } + token = string(b) } - token := string(b) - switch mode { case CREATE: - var req commonIL.PodCreateRequests + var req types.PodCreateRequests + var resp types.CreateStruct + req.Pod = *pod startTime := time.Now() @@ -337,7 +446,7 @@ func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *Virtua } else { pod.Status.Phase = v1.PodFailed pod.Status.Reason = "CFGMaps/Secrets not found" - for i, _ := range pod.Status.ContainerStatuses { + for i := range pod.Status.ContainerStatuses { pod.Status.ContainerStatuses[i].Ready = false } p.UpdatePod(ctx, pod) @@ -350,7 +459,27 @@ func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *Virtua if err != nil { return err } - log.G(ctx).Info(string(returnVal)) + + // get remote job ID and annotate it into the pod + err = json.Unmarshal(returnVal, &resp) + if err != nil { + return err + } + + if string(pod.UID) == resp.PodUID { + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations["JobID"] = resp.PodJID + } + + err = p.UpdatePod(ctx, pod) + if err != nil { + return err + } + + log.G(ctx).Info("Pod " + pod.Name + " created successfully and with Job ID " + resp.PodJID) + log.G(ctx).Debug(string(returnVal)) case DELETE: req := pod @@ -368,99 +497,129 @@ func RemoteExecution(ctx context.Context, config VirtualKubeletConfig, p *Virtua // checkPodsStatus is regularly called by the VK itself at regular intervals of time to query InterLink for Pods' status. // It basically append all available pods registered to the VK to a slice and passes this slice to the statusRequest function. // After the statusRequest returns a response, this function uses that response to update every Pod and Container status. -func checkPodsStatus(ctx context.Context, p *VirtualKubeletProvider, podsList []*v1.Pod, token string, config VirtualKubeletConfig) ([]commonIL.PodStatus, error) { - var returnVal []byte - var ret []commonIL.PodStatus - var err error - - //log.G(ctx).Debug(p.pods) //commented out because it's too verbose. uncomment to see all registered pods - - returnVal, err = statusRequest(ctx, config, podsList, token) +func checkPodsStatus(ctx context.Context, p *VirtualKubeletProvider, podsList []*v1.Pod, token string, config VirtualKubeletConfig) ([]types.PodStatus, error) { + var ret []types.PodStatus + //commented out because it's too verbose. uncomment to see all registered pods + //log.G(ctx).Debug(p.pods) + // retrieve pod status from remote interlink + returnVal, err := statusRequest(ctx, config, podsList, token) if err != nil { return nil, err - } else if returnVal != nil { + } + + if returnVal != nil { + err = json.Unmarshal(returnVal, &ret) if err != nil { return nil, err } + + // if there is a pod status available go ahead to match with the latest state available in etcd if podsList != nil { - for _, podStatus := range ret { + for _, podRemoteStatus := range ret { - pod, err := p.GetPod(ctx, podStatus.PodNamespace, podStatus.PodName) - if err != nil { - updateCacheRequest(ctx, config, podStatus.PodUID, token) - log.G(ctx).Warning("Error: " + err.Error() + "while getting statuses. Updating InterLink cache") - return nil, err - } + log.G(ctx).Debug(fmt.Sprintln("Get status from remote status len: ", len(podRemoteStatus.Containers))) + // avoid asking for status too early, when etcd as not been updated + if podRemoteStatus.PodName != "" { - if podStatus.PodUID == string(pod.UID) { - podRunning := false - podErrored := false - podCompleted := false - failedReason := "" - terminatedContainers := 0 - for _, containerStatus := range podStatus.Containers { - index := 0 - foundCt := false - - for i, checkedContainer := range pod.Status.ContainerStatuses { - if checkedContainer.Name == containerStatus.Name { - foundCt = true - index = i + // get pod reference from cluster etcd + podRefInCluster, err := p.GetPod(ctx, podRemoteStatus.PodNamespace, podRemoteStatus.PodName) + if err != nil { + log.G(ctx).Warning(err) + continue + } + log.G(ctx).Debug(fmt.Sprintln("Get pod from k8s cluster status: ", podRefInCluster.Status.ContainerStatuses)) + + // if the PodUID match with the one in etcd we are talking of the same thing. GOOD + if podRemoteStatus.PodUID == string(podRefInCluster.UID) { + podRunning := false + podErrored := false + podCompleted := false + failedReason := "" + + // For each container of the pod we check if there is a previous state known by K8s + for _, containerRemoteStatus := range podRemoteStatus.Containers { + index := 0 + foundCt := false + + for i, checkedContainer := range podRefInCluster.Status.ContainerStatuses { + if checkedContainer.Name == containerRemoteStatus.Name { + foundCt = true + index = i + } } - } - if !foundCt { - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, containerStatus) - } else { - pod.Status.ContainerStatuses[index] = containerStatus - } + // if it is the first time checking the container, append it to the pod containers, otherwise just update the correct item + if !foundCt { + podRefInCluster.Status.ContainerStatuses = append(podRefInCluster.Status.ContainerStatuses, containerRemoteStatus) + } else { + podRefInCluster.Status.ContainerStatuses[index] = containerRemoteStatus + } - if containerStatus.State.Terminated != nil { - log.G(ctx).Debug("Pod " + podStatus.PodName + ": Service " + containerStatus.Name + " is not running on Sidecar") - terminatedContainers++ - pod.Status.ContainerStatuses[index].State.Terminated.Reason = "Completed" - if containerStatus.State.Terminated.ExitCode != 0 { - podErrored = true - failedReason = "Error: " + string(containerStatus.State.Terminated.ExitCode) - pod.Status.ContainerStatuses[index].State.Terminated.Reason = failedReason - log.G(ctx).Error("Container " + containerStatus.Name + " exited with error: " + string(containerStatus.State.Terminated.ExitCode)) + log.G(ctx).Debug(containerRemoteStatus.State.Running) + + // if plugin cannot return any non-terminated container set the status to terminated + // if the exit code is != 0 get the error and set error reason + rememeber to set pod to failed + if containerRemoteStatus.State.Terminated != nil { + log.G(ctx).Debug("Pod " + podRemoteStatus.PodName + ": Service " + containerRemoteStatus.Name + " is not running on Plugin side") + podCompleted = true + podRefInCluster.Status.ContainerStatuses[index].State.Terminated.Reason = "Completed" + if containerRemoteStatus.State.Terminated.ExitCode != 0 { + podErrored = true + failedReason = "Error: " + strconv.Itoa(int(containerRemoteStatus.State.Terminated.ExitCode)) + podRefInCluster.Status.ContainerStatuses[index].State.Terminated.Reason = failedReason + log.G(ctx).Error("Container " + containerRemoteStatus.Name + " exited with error: " + strconv.Itoa(int(containerRemoteStatus.State.Terminated.ExitCode))) + } + } else if containerRemoteStatus.State.Waiting != nil { + log.G(ctx).Info("Pod " + podRemoteStatus.PodName + ": Service " + containerRemoteStatus.Name + " is setting up on Sidecar") + podRunning = true + } else if containerRemoteStatus.State.Running != nil { + podRunning = true + log.G(ctx).Debug("Pod " + podRemoteStatus.PodName + ": Service " + containerRemoteStatus.Name + " is running on Sidecar") } - } else if containerStatus.State.Waiting != nil { - log.G(ctx).Info("Pod " + podStatus.PodName + ": Service " + containerStatus.Name + " is setting up on Sidecar") - podRunning = true - } else if containerStatus.State.Running != nil { - podRunning = true - log.G(ctx).Debug("Pod " + podStatus.PodName + ": Service " + containerStatus.Name + " is running on Sidecar") + + // if this is the first time you see a container running/errored/completed, update the status of the pod. + if podRunning && podRefInCluster.Status.Phase != v1.PodRunning { + podRefInCluster.Status.Phase = v1.PodRunning + podRefInCluster.Status.Conditions = append(podRefInCluster.Status.Conditions, v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue}) + } else if podErrored && podRefInCluster.Status.Phase != v1.PodFailed { + podRefInCluster.Status.Phase = v1.PodFailed + podRefInCluster.Status.Reason = failedReason + } else if podCompleted && podRefInCluster.Status.Phase != v1.PodSucceeded { + podRefInCluster.Status.Conditions = append(podRefInCluster.Status.Conditions, v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionFalse}) + podRefInCluster.Status.Phase = v1.PodSucceeded + podRefInCluster.Status.Reason = "Completed" + } + } + } else { - } + // if you don't now any UID yet, collect the status and updated the status cache + list, err := p.clientSet.CoreV1().Pods(podRemoteStatus.PodNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + log.G(ctx).Error(err) + return nil, err + } - if terminatedContainers == len(podStatus.Containers) { - podCompleted = true - } + pods := list.Items - if podRunning && pod.Status.Phase != v1.PodRunning { - pod.Status.Phase = v1.PodRunning - pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue}) - } else if podErrored && pod.Status.Phase != v1.PodFailed { - pod.Status.Phase = v1.PodFailed - pod.Status.Reason = failedReason - } else if podCompleted && pod.Status.Phase != v1.PodSucceeded { - pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionFalse}) - pod.Status.Phase = v1.PodSucceeded - pod.Status.Reason = "Completed" - } + for _, pod := range pods { + if string(pod.UID) == podRemoteStatus.PodUID { + err = updateCacheRequest(ctx, config, pod, token) + if err != nil { + log.G(ctx).Error(err) + continue + } + } + } + } } } - log.G(ctx).Info("No errors while getting statuses") log.G(ctx).Debug(ret) return nil, nil - } else { - return ret, err } } diff --git a/pkg/virtualkubelet/virtualkubelet.go b/pkg/virtualkubelet/virtualkubelet.go index f168fa68..aedb799e 100644 --- a/pkg/virtualkubelet/virtualkubelet.go +++ b/pkg/virtualkubelet/virtualkubelet.go @@ -15,14 +15,16 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/node/api" stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" - "github.com/virtual-kubelet/virtual-kubelet/trace" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - commonIL "github.com/intertwin-eu/interlink/pkg/interlink" + types "github.com/intertwin-eu/interlink/pkg/interlink" ) const ( @@ -158,6 +160,7 @@ func NewProvider(providerConfig, nodeName, nodeVersion, operatingSystem string, if err != nil { return nil, err } + log.G(ctx).Info("Init server with config:", config) return NewProviderConfig(config, nodeName, nodeVersion, operatingSystem, internalIP, daemonEndpointPort) } @@ -174,6 +177,7 @@ func LoadConfig(providerConfig, nodeName string, ctx context.Context) (config Vi err = yaml.Unmarshal(data, &config) if err != nil { + log.G(ctx).Fatal(err) return config, err } @@ -228,9 +232,11 @@ func (p *VirtualKubeletProvider) nodeUpdate(ctx context.Context) { log.G(ctx).Info("nodeLoop") - _, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name - if err != nil { - log.G(context.Background()).Fatal(err) + if p.config.VKTokenFile != "" { + _, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name + if err != nil { + log.G(context.Background()).Fatal(err) + } } for { @@ -346,13 +352,19 @@ func (p *VirtualKubeletProvider) Ping(ctx context.Context) error { // CreatePod accepts a Pod definition and stores it in memory in p.pods func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { - ctx, span := trace.StartSpan(ctx, "CreatePod") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "CreatePodVK", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) + var hasInitContainers = false var state v1.ContainerState - defer span.End() - // Add the pod's coordinates to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, pod.Namespace, NameKey, pod.Name) key, err := buildKey(pod) if err != nil { return err @@ -429,7 +441,31 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err if err.Error() == "Deleted pod before actual creation" { log.G(ctx).Warn(err) } else { + // TODO if node in NotReady put it to Unknown/pending? log.G(ctx).Error(err) + pod.Status = v1.PodStatus{ + Phase: v1.PodFailed, + HostIP: p.internalIP, + PodIP: p.internalIP, + StartTime: &now, + Conditions: []v1.PodCondition{ + { + Type: v1.PodInitialized, + Status: v1.ConditionFalse, + }, + { + Type: v1.PodReady, + Status: v1.ConditionFalse, + }, + { + Type: v1.PodScheduled, + Status: v1.ConditionFalse, + }, + }, + } + + p.UpdatePod(ctx, pod) + } return } @@ -455,11 +491,15 @@ func (p *VirtualKubeletProvider) CreatePod(ctx context.Context, pod *v1.Pod) err // UpdatePod accepts a Pod definition and updates its reference. func (p *VirtualKubeletProvider) UpdatePod(ctx context.Context, pod *v1.Pod) error { - ctx, span := trace.StartSpan(ctx, "UpdatePod") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "UpdatePodVK", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.Int64("start.timestamp", start), + )) defer span.End() - - // Add the pod's coordinates to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, pod.Namespace, NameKey, pod.Name) + defer types.SetDurationSpan(start, span) log.G(ctx).Infof("receive UpdatePod %q", pod.Name) @@ -470,11 +510,15 @@ func (p *VirtualKubeletProvider) UpdatePod(ctx context.Context, pod *v1.Pod) err // DeletePod deletes the specified pod and drops it out of p.pods func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (err error) { - ctx, span := trace.StartSpan(ctx, "DeletePod") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "DeletePodVK", trace.WithAttributes( + attribute.String("pod.name", pod.Name), + attribute.String("pod.namespace", pod.Namespace), + attribute.Int64("start.timestamp", start), + )) defer span.End() - - // Add the pod's coordinates to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, pod.Namespace, NameKey, pod.Name) + defer types.SetDurationSpan(start, span) log.G(ctx).Infof("receive DeletePod %q", pod.Name) @@ -530,15 +574,15 @@ func (p *VirtualKubeletProvider) DeletePod(ctx context.Context, pod *v1.Pod) (er // GetPod returns a pod by name that is stored in memory. func (p *VirtualKubeletProvider) GetPod(ctx context.Context, namespace, name string) (pod *v1.Pod, err error) { - - ctx, span := trace.StartSpan(ctx, "GetPod") - defer func() { - span.SetStatus(err) - span.End() - }() - - // Add the pod's coordinates to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, namespace, NameKey, name) + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "GetPodVK", trace.WithAttributes( + attribute.String("pod.name", name), + attribute.String("pod.namespace", namespace), + attribute.Int64("start.timestamp", start), + )) + defer span.End() + defer types.SetDurationSpan(start, span) log.G(ctx).Infof("receive GetPod %q", name) @@ -550,17 +594,22 @@ func (p *VirtualKubeletProvider) GetPod(ctx context.Context, namespace, name str if pod, ok := p.pods[key]; ok { return pod, nil } + return nil, errdefs.NotFoundf("pod \"%s/%s\" is not known to the provider", namespace, name) } // GetPodStatus returns the status of a pod by name that is "running". // returns nil if a pod by that name is not found. func (p *VirtualKubeletProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) { - ctx, span := trace.StartSpan(ctx, "GetPodStatus") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "GetPodStatusVK", trace.WithAttributes( + attribute.String("pod.name", name), + attribute.String("pod.namespace", namespace), + attribute.Int64("start.timestamp", start), + )) defer span.End() - - // Add namespace and name as attributes to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, namespace, NameKey, name) + defer types.SetDurationSpan(start, span) log.G(ctx).Infof("receive GetPodStatus %q", name) @@ -574,13 +623,18 @@ func (p *VirtualKubeletProvider) GetPodStatus(ctx context.Context, namespace, na // GetPods returns a list of all pods known to be "running". func (p *VirtualKubeletProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) { - ctx, span := trace.StartSpan(ctx, "GetPods") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "GetPodsVK", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) defer span.End() + defer types.SetDurationSpan(start, span) log.G(ctx).Info("receive GetPods") p.initClientSet(ctx) - p.RetrievePodsFromInterlink(ctx) + p.RetrievePodsFromCluster(ctx) var pods []*v1.Pod @@ -598,7 +652,7 @@ func nodeConditions() []v1.NodeCondition { return []v1.NodeCondition{ { Type: "Ready", - Status: v1.ConditionTrue, + Status: v1.ConditionFalse, LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletPending", @@ -630,7 +684,7 @@ func nodeConditions() []v1.NodeCondition { }, { Type: "NetworkUnavailable", - Status: v1.ConditionFalse, + Status: v1.ConditionTrue, LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "RouteCreated", @@ -653,11 +707,6 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { <-t.C } - _, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name - if err != nil { - log.G(context.Background()).Fatal(err) - } - for { log.G(ctx).Info("statusLoop") t.Reset(5 * time.Second) @@ -667,24 +716,28 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { case <-t.C: } - b, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name - if err != nil { - fmt.Print(err) + token := "" + if p.config.VKTokenFile != "" { + b, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name + if err != nil { + fmt.Print(err) + } + token = string(b) } var podsList []*v1.Pod for _, pod := range p.pods { if pod.Status.Phase != "Initializing" { podsList = append(podsList, pod) - err = p.UpdatePod(ctx, pod) + err := p.UpdatePod(ctx, pod) if err != nil { log.G(ctx).Error(err) } } } - if podsList != nil { - _, err = checkPodsStatus(ctx, p, podsList, string(b), p.config) + if len(podsList) > 0 { + _, err := checkPodsStatus(ctx, p, podsList, token, p.config) if err != nil { log.G(ctx).Error(err) } @@ -695,34 +748,23 @@ func (p *VirtualKubeletProvider) statusLoop(ctx context.Context) { } p.pods[key] = pod } + } else { + log.G(ctx).Info("No pods to monitor, waiting for the next loop to start") } log.G(ctx).Info("statusLoop=end") } } -// addAttributes adds the specified attributes to the provided span. -// attrs must be an even-sized list of string arguments. -// Otherwise, the span won't be modified. -// TODO: Refactor and move to a "tracing utilities" package. -func addAttributes(ctx context.Context, span trace.Span, attrs ...string) context.Context { - if len(attrs)%2 == 1 { - return ctx - } - for i := 0; i < len(attrs); i += 2 { - ctx = span.WithField(ctx, attrs[i], attrs[i+1]) - } - return ctx -} - // GetLogs implements the logic for interLink pod logs retrieval. func (p *VirtualKubeletProvider) GetLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) { - var span trace.Span - ctx, span = trace.StartSpan(ctx, "GetLogs") //nolint: ineffassign,staticcheck + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "GetLogsVK", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) defer span.End() - - // Add namespace and name as attributes to the current span. - ctx = addAttributes(ctx, span, NamespaceKey, namespace, NameKey, podName) + defer types.SetDurationSpan(start, span) log.G(ctx).Infof("receive GetPodLogs %q", podName) @@ -731,12 +773,12 @@ func (p *VirtualKubeletProvider) GetLogs(ctx context.Context, namespace, podName log.G(ctx).Error(err) } - logsRequest := commonIL.LogStruct{ + logsRequest := types.LogStruct{ Namespace: namespace, PodUID: string(p.pods[key].UID), PodName: podName, ContainerName: containerName, - Opts: commonIL.ContainerLogOpts(opts), + Opts: types.ContainerLogOpts(opts), } return LogRetrieval(ctx, p.config, logsRequest) @@ -744,9 +786,13 @@ func (p *VirtualKubeletProvider) GetLogs(ctx context.Context, namespace, podName // GetStatsSummary returns dummy stats for all pods known by this provider. func (p *VirtualKubeletProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, error) { - var span trace.Span - _, span = trace.StartSpan(ctx, "GetStatsSummary") //nolint: ineffassign,staticcheck + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + _, span := tracer.Start(ctx, "GetStatsSummaryVK", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) defer span.End() + defer types.SetDurationSpan(start, span) // Grab the current timestamp so we can report it as the time the stats were generated. time := metav1.NewTime(time.Now()) @@ -820,41 +866,66 @@ func (p *VirtualKubeletProvider) GetStatsSummary(ctx context.Context) (*stats.Su return res, nil } -// GetPods returns a list of all pods known to be "running" from the local disk cache info +// RetrievePodsFromCluster scans all pods registered to the K8S cluster and re-assigns the ones with a valid JobID to the Virtual Kubelet. // This will run at the initiation time only -func (p *VirtualKubeletProvider) RetrievePodsFromInterlink(ctx context.Context) error { - ctx, span := trace.StartSpan(ctx, "RetrievePodsFromInterlink") +func (p *VirtualKubeletProvider) RetrievePodsFromCluster(ctx context.Context) error { + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "RetrievePodsFromCluster", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) defer span.End() + defer types.SetDurationSpan(start, span) - log.G(ctx).Info("Retrieving ALL cached InterLink Pods") + log.G(ctx).Info("Retrieving ALL Pods registered to the cluster and owned by VK") - b, err := os.ReadFile(p.config.VKTokenFile) // just pass the file name + namespaces, err := p.clientSet.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - log.G(ctx).Error(err) + log.G(ctx).Error("Unable to retrieve all namespaces available in the cluster") + return err } - cached_pods, err := checkPodsStatus(ctx, p, nil, string(b), p.config) - - for _, pod := range cached_pods { - retrievedPod, err := p.clientSet.CoreV1().Pods(pod.PodNamespace).Get(ctx, pod.PodName, metav1.GetOptions{}) + for _, ns := range namespaces.Items { + podsList, err := p.clientSet.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{}) if err != nil { - log.G(ctx).Warning("Unable to retrieve pod " + retrievedPod.Name + " from the cluster") - } else { - key, err := buildKey(retrievedPod) - if err != nil { - log.G(ctx).Error(err) + log.G(ctx).Warning("Unable to retrieve pods from the namespace " + ns.Name) + } + for _, pod := range podsList.Items { + if CheckIfAnnotationExists(&pod, "JobID") && p.nodeName == pod.Spec.NodeName { + key, err := buildKeyFromNames(pod.Namespace, pod.Name) + if err != nil { + log.G(ctx).Error(err) + return err + } + p.pods[key] = &pod + p.notifier(&pod) } - p.pods[key] = retrievedPod - p.UpdatePod(ctx, retrievedPod) } + } return err } +// CheckIfAnnotationExists checks if a specific annotation (key) is available between the annotation of a pod +func CheckIfAnnotationExists(pod *v1.Pod, key string) bool { + _, ok := pod.Annotations[key] + + if ok { + return true + } else { + return false + } +} + func (p *VirtualKubeletProvider) initClientSet(ctx context.Context) error { - ctx, span := trace.StartSpan(ctx, "InitClientSet") + start := time.Now().Unix() + tracer := otel.Tracer("interlink-service") + ctx, span := tracer.Start(ctx, "InitClientSet", trace.WithAttributes( + attribute.Int64("start.timestamp", start), + )) defer span.End() + defer types.SetDurationSpan(start, span) if p.clientSet == nil { kubeconfig := os.Getenv("KUBECONFIG") diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..fb57ccd1 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,4 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +