From 8180876e601aaab3320a8b21bb09ff3ce4590fd8 Mon Sep 17 00:00:00 2001 From: arueth Date: Fri, 29 Mar 2024 19:06:14 +0000 Subject: [PATCH] Restructured folder --- .../ml-platform}/README.md | 2 + .../ml-platform}/docs/images/configsync.png | Bin .../images/ray-dataprocessing-workflow.png | Bin .../examples/platform/sandbox/README.md | 334 ++++++++++++++++++ .../examples/platform/sandbox}/backend.tf | 0 .../examples/platform/sandbox}/main.tf | 0 .../platform/sandbox}/mlp.auto.tfvars | 0 .../examples/platform/sandbox}/outputs.tf | 0 .../sandbox}/scripts/create_cluster_yamls.sh | 0 .../sandbox}/scripts/create_git_cred.sh | 0 .../sandbox}/scripts/create_namespace.sh | 0 .../scripts/install_kuberay_operator.sh | 0 .../sandbox}/scripts/install_ray_cluster.sh | 0 .../sandbox}/scripts/manage_ray_ns.sh | 0 .../acm-template/manifests/apps/.gitkeep | 0 .../acm-template/manifests/clusters/.gitkeep | 0 .../templates/_cluster_template/cluster.yaml | 0 .../_cluster_template/config-selector.yaml | 0 .../kuberay/kustomization.yaml | 0 .../kuberay/rayclusters.yaml | 0 .../_cluster_template/kuberay/rayjobs.yaml | 0 .../kuberay/rayservices.yaml | 0 .../_cluster_template/kuberay/rbac.yaml | 0 .../_cluster_template/kuberay/values.yaml | 0 .../_cluster_template/kustomization.yaml | 0 .../templates/_cluster_template/selector.yaml | 0 .../_cluster_template/team/kustomization.yaml | 0 .../_cluster_template/team/namespace.yaml | 0 .../team/network-policy.yaml | 0 .../_cluster_template/team/rbac.yaml | 0 .../_cluster_template/team/reposync.yaml | 0 .../app/fluentd_config.yaml | 0 .../app/kustomization.yaml | 0 .../app/serviceaccount_ray_head.yaml | 0 .../app/serviceaccount_ray_worker.yaml | 0 .../_namespace_template/app/values.yaml | 0 .../examples/platform/sandbox}/variables.tf | 0 .../examples/platform/sandbox}/versions.tf | 0 .../examples/ray-dataprocessing/CONVERSION.md | 0 .../examples/ray-dataprocessing/README.md | 0 .../examples/ray-dataprocessing/job.yaml | 0 .../ray-dataprocessing/src/Dockerfile | 0 .../ray-dataprocessing/src/preprocessing.py | 0 .../ray-dataprocessing/src/requirements.txt | 0 .../terraform/features}/initialize/backend.tf | 0 .../features}/initialize/backend.tf.bucket | 0 .../initialize/initialize.auto.tfvars | 0 .../terraform/features}/initialize/main.tf | 0 .../features/initialize/state/default.tfstate | 21 ++ .../initialize/state/default.tfstate.backup | 197 +++++++++++ .../features}/initialize/variables.tf | 0 .../features}/initialize/versions.tf | 0 .../terraform/modules/cloud-nat/README.md | 0 .../terraform/modules/cloud-nat/main.tf | 0 .../terraform/modules/cloud-nat/outputs.tf | 0 .../terraform/modules/cloud-nat/variables.tf | 0 .../terraform/modules/cloud-nat/versions.tf | 0 .../terraform/modules/cluster/gke.tf | 0 .../terraform/modules/cluster/outputs.tf | 0 .../terraform/modules/cluster/variables.tf | 0 .../terraform/modules/cluster/versions.tf | 0 .../terraform/modules/network/README.md | 0 .../terraform/modules/network/outputs.tf | 0 .../terraform/modules/network/variables.tf | 0 .../terraform/modules/network/versions.tf | 0 .../terraform/modules/network/vpc.tf | 0 .../terraform/modules/node-pools/nodepools.tf | 0 .../terraform/modules/node-pools/variables.tf | 0 .../terraform/modules/node-pools/versions.tf | 0 .../modules/vm-reservations/outputs.tf | 0 .../modules/vm-reservations/reservations.tf | 0 .../modules/vm-reservations/variables.tf | 0 .../modules/vm-reservations/versions.tf | 0 ml-platform/terraform/README.md | 112 ------ 74 files changed, 554 insertions(+), 112 deletions(-) rename {ml-platform => best-practices/ml-platform}/README.md (99%) rename {ml-platform => best-practices/ml-platform}/docs/images/configsync.png (100%) rename {ml-platform => best-practices/ml-platform}/docs/images/ray-dataprocessing-workflow.png (100%) create mode 100644 best-practices/ml-platform/examples/platform/sandbox/README.md rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/backend.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/main.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/mlp.auto.tfvars (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/outputs.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/create_cluster_yamls.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/create_git_cred.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/create_namespace.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/install_kuberay_operator.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/install_ray_cluster.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/scripts/manage_ray_ns.sh (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/manifests/apps/.gitkeep (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/manifests/clusters/.gitkeep (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/cluster.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/config-selector.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kuberay/values.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/kustomization.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/selector.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/team/kustomization.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/team/namespace.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/team/network-policy.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/team/rbac.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_cluster_template/team/reposync.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_namespace_template/app/kustomization.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/templates/acm-template/templates/_namespace_template/app/values.yaml (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/variables.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/examples/platform/sandbox}/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/CONVERSION.md (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/README.md (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/job.yaml (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/src/Dockerfile (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/src/preprocessing.py (100%) rename {ml-platform => best-practices/ml-platform}/examples/ray-dataprocessing/src/requirements.txt (100%) rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/backend.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/backend.tf.bucket (100%) rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/initialize.auto.tfvars (100%) rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/main.tf (100%) create mode 100644 best-practices/ml-platform/terraform/features/initialize/state/default.tfstate create mode 100644 best-practices/ml-platform/terraform/features/initialize/state/default.tfstate.backup rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/variables.tf (100%) rename {ml-platform/terraform => best-practices/ml-platform/terraform/features}/initialize/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cloud-nat/README.md (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cloud-nat/main.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cloud-nat/outputs.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cloud-nat/variables.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cloud-nat/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cluster/gke.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cluster/outputs.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cluster/variables.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/cluster/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/network/README.md (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/network/outputs.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/network/variables.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/network/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/network/vpc.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/node-pools/nodepools.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/node-pools/variables.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/node-pools/versions.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/vm-reservations/outputs.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/vm-reservations/reservations.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/vm-reservations/variables.tf (100%) rename {ml-platform => best-practices/ml-platform}/terraform/modules/vm-reservations/versions.tf (100%) delete mode 100644 ml-platform/terraform/README.md diff --git a/ml-platform/README.md b/best-practices/ml-platform/README.md similarity index 99% rename from ml-platform/README.md rename to best-practices/ml-platform/README.md index d6a1058fb..b5bc89c9c 100644 --- a/ml-platform/README.md +++ b/best-practices/ml-platform/README.md @@ -53,6 +53,8 @@ This reference architecture demonstrates how to build a GKE platform that facili ## Deploy a single environment reference architecture +README + This is the quick-start deployment guide. It can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts. ### Requirements diff --git a/ml-platform/docs/images/configsync.png b/best-practices/ml-platform/docs/images/configsync.png similarity index 100% rename from ml-platform/docs/images/configsync.png rename to best-practices/ml-platform/docs/images/configsync.png diff --git a/ml-platform/docs/images/ray-dataprocessing-workflow.png b/best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png similarity index 100% rename from ml-platform/docs/images/ray-dataprocessing-workflow.png rename to best-practices/ml-platform/docs/images/ray-dataprocessing-workflow.png diff --git a/best-practices/ml-platform/examples/platform/sandbox/README.md b/best-practices/ml-platform/examples/platform/sandbox/README.md new file mode 100644 index 000000000..e4cdb3e6e --- /dev/null +++ b/best-practices/ml-platform/examples/platform/sandbox/README.md @@ -0,0 +1,334 @@ +# Machine learning platform (MLP) on GKE reference architecture: Sandbox + +This quick-start deployment guide can be used to set up an environment to familiarize yourself with the architecture and get an understanding of the concepts. + +### Requirements + +In this guide you can choose to bring your project (BYOP) or have Terraform create a new project for you. The requirements are difference based on the option that you choose. + +#### Bring your own project (BYOP) + +- Project ID of a new Google Cloud Project, preferably with no APIs enabled +- `roles/owner` IAM permissions on the project +- GitHub Personal Access Token, steps to create the token are provided below + +#### Terraform managed project + +- Billing account ID +- Organization or folder ID +- `roles/billing.user` IAM permissions on the billing account specified +- `roles/resourcemanager.projectCreator` IAM permissions on the organization or folder specified +- GitHub Personal Access Token, steps to create the token are provided below + +### Pull the source code + +- Clone the repository and change directory to the guide directory + + ``` + git clone https://github.com/GoogleCloudPlatform/ai-on-gke + cd ai-on-gke/ml-platform + ``` + +- Set environment variables + + ``` + export MLP_BASE_DIR=$(pwd) && \ + echo "export MLP_BASE_DIR=${MLP_BASE_DIR}" >> ${HOME}/.bashrc + ``` + +### GitHub Configuration + +- Create a [Personal Access Token][personal-access-token] in [GitHub][github]: + + Note: It is recommended to use a [machine user account][machine-user-account] for this but you can use a personal user account just to try this reference architecture. + + **Fine-grained personal access token** + + - Go to https://github.com/settings/tokens and login using your credentials + - Click "Generate new token" >> "Generate new token (Beta)". + - Enter a Token name. + - Select the expiration. + - Select the Resource owner. + - Select All repositories + - Set the following Permissions: + - Repository permissions + - Administration: Read and write + - Content: Read and write + - Click "Generate token" + + **Personal access tokens (classic)** + + - Go to https://github.com/settings/tokens and login using your credentials + - Click "Generate new token" >> "Generate new token (classic)". + - You will be directed to a screen to created the new token. Provide the note and expiration. + - Choose the following two access: + - [x] repo - Full control of private repositories + - [x] delete_repo - Delete repositories + - Click "Generate token" + +- Store the token in a secure file. + + ``` + # Create a secure directory + mkdir -p ${HOME}/secrets/ + chmod go-rwx ${HOME}/secrets + + # Create a secure file + touch ${HOME}/secrets/mlp-github-token + chmod go-rwx ${HOME}/secrets/mlp-github-token + + # Put the token in the secure file using your preferred editor + nano ${HOME}/secrets/mlp-github-token + ``` + +- Set the GitHub environment variables in Cloud Shell + + Replace the following values: + + - `` is the GitHub organization or user namespace to use for the repositories + - `` is the GitHub account to use for authentication + - `` is the email address to use for commit + + ``` + export MLP_GITHUB_ORG="" + export MLP_GITHUB_USER="" + export MLP_GITHUB_EMAIL="" + ``` + +- Set the configuration variables + + ``` + sed -i "s/YOUR_GITHUB_EMAIL/${MLP_GITHUB_EMAIL}/g" ${MLP_BASE_DIR}/terraform/mlp.auto.tfvars + sed -i "s/YOUR_GITHUB_ORG/${MLP_GITHUB_ORG}/g" ${MLP_BASE_DIR}/terraform/mlp.auto.tfvars + sed -i "s/YOUR_GITHUB_USER/${MLP_GITHUB_USER}/g" ${MLP_BASE_DIR}/terraform/mlp.auto.tfvars + ``` + +### Project Configuration + +You only need to complete the section for the option that you have selected. + +#### Bring your own project (BYOP) + +- Set the project environment variables in Cloud Shell + + Replace the following values + + - `` is the ID of your existing Google Cloud project + + ``` + export MLP_PROJECT_ID="" + export MLP_STATE_BUCKET="${MLP_PROJECT_ID}-tf-state" + ``` + +- Set the default `gcloud` project + + ``` + gcloud config set project ${MLP_PROJECT_ID} + ``` + +- Authorize `gcloud` + + ``` + gcloud auth login --activate --no-launch-browser --quiet --update-adc + ``` + +- Create a Cloud Storage bucket to store the Terraform state + + ``` + gcloud storage buckets create gs://${MLP_STATE_BUCKET} --project ${MLP_PROJECT_ID} + ``` + +- Set the configuration variables + + ``` + sed -i "s/YOUR_STATE_BUCKET/${MLP_STATE_BUCKET}/g" ${MLP_BASE_DIR}/terraform/backend.tf + sed -i "s/YOUR_PROJECT_ID/${MLP_PROJECT_ID}/g" ${MLP_BASE_DIR}/terraform/mlp.auto.tfvars + ``` + +#### Terraform managed project + +- Set the configuration variables + + ``` + nano ${MLP_BASE_DIR}/terraform/initialize/initialize.auto.tfvars + ``` + + ``` + project = { + billing_account_id = "XXXXXX-XXXXXX-XXXXXX" + folder_id = "############" + name = "mlp" + org_id = "############" + } + ``` + + > `project.billing_account_id` the billing account ID + > + > Enter either `project.folder_id` **OR** `project.org_id` + > `project.folder_id` the folder ID + > `project.org_id` the organization ID + +- Authorize `gcloud` + + ``` + gcloud auth login --activate --no-launch-browser --quiet --update-adc + ``` + +- Create a new project + + ``` + cd ${MLP_BASE_DIR}/terraform/initialize + terraform init && \ + terraform plan -input=false -out=tfplan && \ + terraform apply -input=false tfplan && \ + rm tfplan && \ + terraform init -force-copy -migrate-state && \ + rm -rf state + ``` + +### Run Terraform + +- Create the resources + + ``` + cd ${MLP_BASE_DIR}/terraform && \ + terraform init && \ + terraform plan -input=false -var github_token="$(tr --delete '\n' < ${HOME}/secrets/mlp-github-token)" -out=tfplan && \ + terraform apply -input=false tfplan + rm tfplan + ``` + +### Review the resources + +#### GKE clusters and ConfigSync + +- Go to Google Cloud Console, click on the navigation menu and click on Kubernetes Engine > Clusters. You should see one cluster. + +- Go to Google Cloud Console, click on the navigation menu and click on Kubernetes Engine > Config. If you haven't enabled GKE Enterprise in the project earlier, Click `LEARN AND ENABLE` button and then `ENABLE GKE ENTERPRISE`. You should see a RootSync and RepoSync object. + ![configsync](docs/images/configsync.png) + +#### Software installed via RepoSync and RootSync + +Open Cloud Shell to execute the following commands: + +- Store your GKE cluster name in env variable: + + `export GKE_CLUSTER=` + +- Get cluster credentials: + + ``` + gcloud container fleet memberships get-credentials ${GKE_CLUSTER} + ``` + +- Fetch KubeRay operator CRDs + + ``` + kubectl get crd | grep ray + ``` + + The output will be similar to the following: + + ``` + rayclusters.ray.io 2024-02-12T21:19:06Z + rayjobs.ray.io 2024-02-12T21:19:09Z + rayservices.ray.io 2024-02-12T21:19:12Z + ``` + +- Fetch KubeRay operator pod + + ``` + kubectl get pods + ``` + + The output will be similar to the following: + + ``` + NAME READY STATUS RESTARTS AGE + kuberay-operator-56b8d98766-2nvht 1/1 Running 0 6m26s + ``` + +- Check the namespace `ml-team` created: + + ``` + kubectl get ns | grep ml-team + ``` + +- Check the RepoSync object created `ml-team` namespace: + ``` + kubectl get reposync -n ml-team + ``` +- Check the `raycluster` in `ml-team` namespace + + ``` + kubectl get raycluster -n ml-team + ``` + + The output will be similar to the following: + + ``` + NAME DESIRED WORKERS AVAILABLE WORKERS STATUS AGE + ray-cluster-kuberay 1 1 ready 29m + ``` + +- Check the head and worker pods of kuberay in `ml-team` namespace + ``` + kubectl get pods -n ml-team + ``` + The output will be similar to the following: + ``` + NAME READY STATUS RESTARTS AGE + ray-cluster-kuberay-head-sp6dg 2/2 Running 0 3m21s + ray-cluster-kuberay-worker-workergroup-rzpjw 2/2 Running 0 3m21s + ``` + +### Cleanup + +- Destroy the resources + + ``` + cd ${MLP_BASE_DIR}/terraform && \ + terraform init && \ + terraform destroy -auto-approve -var github_token="$(tr --delete '\n' < ${HOME}/secrets/mlp-github-token)" && \ + rm -rf .terraform .terraform.lock.hcl + ``` + +#### Project + +You only need to complete the section for the option that you have selected. + +##### Bring your own project (BYOP) + +- Delete the project + + ``` + gcloud projects delete ${MLP_PROJECT_ID} + ``` + +#### Terraform managed project + +- Destroy the project + + ``` + cd ${MLP_BASE_DIR}/terraform/initialize && \ + TERRAFORM_BUCKET_NAME=$(grep bucket backend.tf | awk -F"=" '{print $2}' | xargs) && \ + cp backend.tf.local backend.tf && \ + terraform init -force-copy -lock=false -migrate-state && \ + gsutil -m rm -rf gs://${TERRAFORM_BUCKET_NAME}/* && \ + terraform init && \ + terraform destroy -auto-approve && \ + rm -rf .terraform .terraform.lock.hcl + ``` + +[gitops]: https://about.gitlab.com/topics/gitops/ +[repo-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[root-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields +[config-sync]: https://cloud.google.com/anthos-config-management/docs/config-sync-overview +[cloud-deploy]: https://cloud.google.com/deploy?hl=en +[terraform]: https://www.terraform.io/ +[gke]: https://cloud.google.com/kubernetes-engine?hl=en +[git]: https://git-scm.com/ +[github]: https://github.com/ +[gcp-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[personal-access-token]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +[machine-user-account]: https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts diff --git a/ml-platform/terraform/backend.tf b/best-practices/ml-platform/examples/platform/sandbox/backend.tf similarity index 100% rename from ml-platform/terraform/backend.tf rename to best-practices/ml-platform/examples/platform/sandbox/backend.tf diff --git a/ml-platform/terraform/main.tf b/best-practices/ml-platform/examples/platform/sandbox/main.tf similarity index 100% rename from ml-platform/terraform/main.tf rename to best-practices/ml-platform/examples/platform/sandbox/main.tf diff --git a/ml-platform/terraform/mlp.auto.tfvars b/best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars similarity index 100% rename from ml-platform/terraform/mlp.auto.tfvars rename to best-practices/ml-platform/examples/platform/sandbox/mlp.auto.tfvars diff --git a/ml-platform/terraform/outputs.tf b/best-practices/ml-platform/examples/platform/sandbox/outputs.tf similarity index 100% rename from ml-platform/terraform/outputs.tf rename to best-practices/ml-platform/examples/platform/sandbox/outputs.tf diff --git a/ml-platform/terraform/scripts/create_cluster_yamls.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh similarity index 100% rename from ml-platform/terraform/scripts/create_cluster_yamls.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/create_cluster_yamls.sh diff --git a/ml-platform/terraform/scripts/create_git_cred.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh similarity index 100% rename from ml-platform/terraform/scripts/create_git_cred.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/create_git_cred.sh diff --git a/ml-platform/terraform/scripts/create_namespace.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh similarity index 100% rename from ml-platform/terraform/scripts/create_namespace.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/create_namespace.sh diff --git a/ml-platform/terraform/scripts/install_kuberay_operator.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh similarity index 100% rename from ml-platform/terraform/scripts/install_kuberay_operator.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/install_kuberay_operator.sh diff --git a/ml-platform/terraform/scripts/install_ray_cluster.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh similarity index 100% rename from ml-platform/terraform/scripts/install_ray_cluster.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/install_ray_cluster.sh diff --git a/ml-platform/terraform/scripts/manage_ray_ns.sh b/best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh similarity index 100% rename from ml-platform/terraform/scripts/manage_ray_ns.sh rename to best-practices/ml-platform/examples/platform/sandbox/scripts/manage_ray_ns.sh diff --git a/ml-platform/terraform/templates/acm-template/manifests/apps/.gitkeep b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/apps/.gitkeep similarity index 100% rename from ml-platform/terraform/templates/acm-template/manifests/apps/.gitkeep rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/apps/.gitkeep diff --git a/ml-platform/terraform/templates/acm-template/manifests/clusters/.gitkeep b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/clusters/.gitkeep similarity index 100% rename from ml-platform/terraform/templates/acm-template/manifests/clusters/.gitkeep rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/manifests/clusters/.gitkeep diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/cluster.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/cluster.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/cluster.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/config-selector.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/config-selector.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/config-selector.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/kustomization.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayclusters.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayjobs.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rayservices.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/rbac.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/values.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kuberay/values.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kuberay/values.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/kustomization.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/kustomization.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/selector.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/selector.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/selector.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/kustomization.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/kustomization.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/namespace.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/namespace.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/namespace.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/network-policy.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/network-policy.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/network-policy.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/rbac.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/rbac.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/rbac.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/reposync.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_cluster_template/team/reposync.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_cluster_template/team/reposync.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/fluentd_config.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/kustomization.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/kustomization.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/kustomization.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_head.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/serviceaccount_ray_worker.yaml diff --git a/ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/values.yaml b/best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml similarity index 100% rename from ml-platform/terraform/templates/acm-template/templates/_namespace_template/app/values.yaml rename to best-practices/ml-platform/examples/platform/sandbox/templates/acm-template/templates/_namespace_template/app/values.yaml diff --git a/ml-platform/terraform/variables.tf b/best-practices/ml-platform/examples/platform/sandbox/variables.tf similarity index 100% rename from ml-platform/terraform/variables.tf rename to best-practices/ml-platform/examples/platform/sandbox/variables.tf diff --git a/ml-platform/terraform/versions.tf b/best-practices/ml-platform/examples/platform/sandbox/versions.tf similarity index 100% rename from ml-platform/terraform/versions.tf rename to best-practices/ml-platform/examples/platform/sandbox/versions.tf diff --git a/ml-platform/examples/ray-dataprocessing/CONVERSION.md b/best-practices/ml-platform/examples/ray-dataprocessing/CONVERSION.md similarity index 100% rename from ml-platform/examples/ray-dataprocessing/CONVERSION.md rename to best-practices/ml-platform/examples/ray-dataprocessing/CONVERSION.md diff --git a/ml-platform/examples/ray-dataprocessing/README.md b/best-practices/ml-platform/examples/ray-dataprocessing/README.md similarity index 100% rename from ml-platform/examples/ray-dataprocessing/README.md rename to best-practices/ml-platform/examples/ray-dataprocessing/README.md diff --git a/ml-platform/examples/ray-dataprocessing/job.yaml b/best-practices/ml-platform/examples/ray-dataprocessing/job.yaml similarity index 100% rename from ml-platform/examples/ray-dataprocessing/job.yaml rename to best-practices/ml-platform/examples/ray-dataprocessing/job.yaml diff --git a/ml-platform/examples/ray-dataprocessing/src/Dockerfile b/best-practices/ml-platform/examples/ray-dataprocessing/src/Dockerfile similarity index 100% rename from ml-platform/examples/ray-dataprocessing/src/Dockerfile rename to best-practices/ml-platform/examples/ray-dataprocessing/src/Dockerfile diff --git a/ml-platform/examples/ray-dataprocessing/src/preprocessing.py b/best-practices/ml-platform/examples/ray-dataprocessing/src/preprocessing.py similarity index 100% rename from ml-platform/examples/ray-dataprocessing/src/preprocessing.py rename to best-practices/ml-platform/examples/ray-dataprocessing/src/preprocessing.py diff --git a/ml-platform/examples/ray-dataprocessing/src/requirements.txt b/best-practices/ml-platform/examples/ray-dataprocessing/src/requirements.txt similarity index 100% rename from ml-platform/examples/ray-dataprocessing/src/requirements.txt rename to best-practices/ml-platform/examples/ray-dataprocessing/src/requirements.txt diff --git a/ml-platform/terraform/initialize/backend.tf b/best-practices/ml-platform/terraform/features/initialize/backend.tf similarity index 100% rename from ml-platform/terraform/initialize/backend.tf rename to best-practices/ml-platform/terraform/features/initialize/backend.tf diff --git a/ml-platform/terraform/initialize/backend.tf.bucket b/best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket similarity index 100% rename from ml-platform/terraform/initialize/backend.tf.bucket rename to best-practices/ml-platform/terraform/features/initialize/backend.tf.bucket diff --git a/ml-platform/terraform/initialize/initialize.auto.tfvars b/best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars similarity index 100% rename from ml-platform/terraform/initialize/initialize.auto.tfvars rename to best-practices/ml-platform/terraform/features/initialize/initialize.auto.tfvars diff --git a/ml-platform/terraform/initialize/main.tf b/best-practices/ml-platform/terraform/features/initialize/main.tf similarity index 100% rename from ml-platform/terraform/initialize/main.tf rename to best-practices/ml-platform/terraform/features/initialize/main.tf diff --git a/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate b/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate new file mode 100644 index 000000000..60e4e4772 --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate @@ -0,0 +1,21 @@ +{ + "version": 4, + "terraform_version": "1.7.1", + "serial": 46, + "lineage": "048c221a-5b39-cc81-5abf-befd45bd76c5", + "outputs": {}, + "resources": [], + "check_results": [ + { + "object_kind": "var", + "config_addr": "var.project", + "status": "unknown", + "objects": [ + { + "object_addr": "var.project", + "status": "unknown" + } + ] + } + ] +} diff --git a/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate.backup b/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate.backup new file mode 100644 index 000000000..c7ebde5f7 --- /dev/null +++ b/best-practices/ml-platform/terraform/features/initialize/state/default.tfstate.backup @@ -0,0 +1,197 @@ +{ + "version": 4, + "terraform_version": "1.7.1", + "serial": 39, + "lineage": "048c221a-5b39-cc81-5abf-befd45bd76c5", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "google_project", + "name": "environment", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "auto_create_network": true, + "billing_account": "01EF67-93C321-CB8BDE", + "effective_labels": {}, + "folder_id": "", + "id": "projects/mlp-dev-n5w1z3m2u4tmsjksn8q1hg", + "labels": null, + "name": "mlp-dev", + "number": "882788124079", + "org_id": "886449255148", + "project_id": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg", + "skip_delete": null, + "terraform_labels": {}, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJyZWFkIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "random_string.project_id_suffix" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_storage_bucket", + "name": "mlp", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "autoclass": [], + "cors": [], + "custom_placement_config": [], + "default_event_based_hold": false, + "effective_labels": {}, + "enable_object_retention": false, + "encryption": [], + "force_destroy": false, + "id": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg-mlp", + "labels": null, + "lifecycle_rule": [], + "location": "US-CENTRAL1", + "logging": [], + "name": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg-mlp", + "project": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg", + "public_access_prevention": "inherited", + "requester_pays": false, + "retention_policy": [], + "rpo": null, + "self_link": "https://www.googleapis.com/storage/v1/b/mlp-dev-n5w1z3m2u4tmsjksn8q1hg-mlp", + "storage_class": "STANDARD", + "terraform_labels": {}, + "timeouts": null, + "uniform_bucket_level_access": true, + "url": "gs://mlp-dev-n5w1z3m2u4tmsjksn8q1hg-mlp", + "versioning": [ + { + "enabled": true + } + ], + "website": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsInJlYWQiOjI0MDAwMDAwMDAwMCwidXBkYXRlIjoyNDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "google_project.environment", + "random_string.project_id_suffix" + ] + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "write_environment_name", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "7387891238569079315", + "triggers": { + "md5": "dev", + "tfvars_file": "../mlp.auto.tfvars" + } + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "write_project_id", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1485193428210134212", + "triggers": { + "md5": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg", + "tfvars_file": "../mlp.auto.tfvars" + } + }, + "sensitive_attributes": [], + "dependencies": [ + "google_project.environment", + "random_string.project_id_suffix" + ] + } + ] + }, + { + "mode": "managed", + "type": "null_resource", + "name": "write_storage_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "7155781797202565811", + "triggers": { + "backend_file": "../backend.tf", + "md5": "mlp-dev-n5w1z3m2u4tmsjksn8q1hg-mlp" + } + }, + "sensitive_attributes": [], + "dependencies": [ + "google_project.environment", + "google_storage_bucket.mlp", + "random_string.project_id_suffix" + ] + } + ] + }, + { + "mode": "managed", + "type": "random_string", + "name": "project_id_suffix", + "provider": "provider[\"registry.terraform.io/hashicorp/random\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "id": "n5w1z3m2u4tmsjksn8q1hg", + "keepers": null, + "length": 22, + "lower": true, + "min_lower": 0, + "min_numeric": 0, + "min_special": 0, + "min_upper": 0, + "number": true, + "numeric": true, + "override_special": null, + "result": "n5w1z3m2u4tmsjksn8q1hg", + "special": false, + "upper": false + }, + "sensitive_attributes": [] + } + ] + } + ], + "check_results": [ + { + "object_kind": "var", + "config_addr": "var.project", + "status": "pass", + "objects": [ + { + "object_addr": "var.project", + "status": "pass" + } + ] + } + ] +} diff --git a/ml-platform/terraform/initialize/variables.tf b/best-practices/ml-platform/terraform/features/initialize/variables.tf similarity index 100% rename from ml-platform/terraform/initialize/variables.tf rename to best-practices/ml-platform/terraform/features/initialize/variables.tf diff --git a/ml-platform/terraform/initialize/versions.tf b/best-practices/ml-platform/terraform/features/initialize/versions.tf similarity index 100% rename from ml-platform/terraform/initialize/versions.tf rename to best-practices/ml-platform/terraform/features/initialize/versions.tf diff --git a/ml-platform/terraform/modules/cloud-nat/README.md b/best-practices/ml-platform/terraform/modules/cloud-nat/README.md similarity index 100% rename from ml-platform/terraform/modules/cloud-nat/README.md rename to best-practices/ml-platform/terraform/modules/cloud-nat/README.md diff --git a/ml-platform/terraform/modules/cloud-nat/main.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/main.tf similarity index 100% rename from ml-platform/terraform/modules/cloud-nat/main.tf rename to best-practices/ml-platform/terraform/modules/cloud-nat/main.tf diff --git a/ml-platform/terraform/modules/cloud-nat/outputs.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf similarity index 100% rename from ml-platform/terraform/modules/cloud-nat/outputs.tf rename to best-practices/ml-platform/terraform/modules/cloud-nat/outputs.tf diff --git a/ml-platform/terraform/modules/cloud-nat/variables.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf similarity index 100% rename from ml-platform/terraform/modules/cloud-nat/variables.tf rename to best-practices/ml-platform/terraform/modules/cloud-nat/variables.tf diff --git a/ml-platform/terraform/modules/cloud-nat/versions.tf b/best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf similarity index 100% rename from ml-platform/terraform/modules/cloud-nat/versions.tf rename to best-practices/ml-platform/terraform/modules/cloud-nat/versions.tf diff --git a/ml-platform/terraform/modules/cluster/gke.tf b/best-practices/ml-platform/terraform/modules/cluster/gke.tf similarity index 100% rename from ml-platform/terraform/modules/cluster/gke.tf rename to best-practices/ml-platform/terraform/modules/cluster/gke.tf diff --git a/ml-platform/terraform/modules/cluster/outputs.tf b/best-practices/ml-platform/terraform/modules/cluster/outputs.tf similarity index 100% rename from ml-platform/terraform/modules/cluster/outputs.tf rename to best-practices/ml-platform/terraform/modules/cluster/outputs.tf diff --git a/ml-platform/terraform/modules/cluster/variables.tf b/best-practices/ml-platform/terraform/modules/cluster/variables.tf similarity index 100% rename from ml-platform/terraform/modules/cluster/variables.tf rename to best-practices/ml-platform/terraform/modules/cluster/variables.tf diff --git a/ml-platform/terraform/modules/cluster/versions.tf b/best-practices/ml-platform/terraform/modules/cluster/versions.tf similarity index 100% rename from ml-platform/terraform/modules/cluster/versions.tf rename to best-practices/ml-platform/terraform/modules/cluster/versions.tf diff --git a/ml-platform/terraform/modules/network/README.md b/best-practices/ml-platform/terraform/modules/network/README.md similarity index 100% rename from ml-platform/terraform/modules/network/README.md rename to best-practices/ml-platform/terraform/modules/network/README.md diff --git a/ml-platform/terraform/modules/network/outputs.tf b/best-practices/ml-platform/terraform/modules/network/outputs.tf similarity index 100% rename from ml-platform/terraform/modules/network/outputs.tf rename to best-practices/ml-platform/terraform/modules/network/outputs.tf diff --git a/ml-platform/terraform/modules/network/variables.tf b/best-practices/ml-platform/terraform/modules/network/variables.tf similarity index 100% rename from ml-platform/terraform/modules/network/variables.tf rename to best-practices/ml-platform/terraform/modules/network/variables.tf diff --git a/ml-platform/terraform/modules/network/versions.tf b/best-practices/ml-platform/terraform/modules/network/versions.tf similarity index 100% rename from ml-platform/terraform/modules/network/versions.tf rename to best-practices/ml-platform/terraform/modules/network/versions.tf diff --git a/ml-platform/terraform/modules/network/vpc.tf b/best-practices/ml-platform/terraform/modules/network/vpc.tf similarity index 100% rename from ml-platform/terraform/modules/network/vpc.tf rename to best-practices/ml-platform/terraform/modules/network/vpc.tf diff --git a/ml-platform/terraform/modules/node-pools/nodepools.tf b/best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf similarity index 100% rename from ml-platform/terraform/modules/node-pools/nodepools.tf rename to best-practices/ml-platform/terraform/modules/node-pools/nodepools.tf diff --git a/ml-platform/terraform/modules/node-pools/variables.tf b/best-practices/ml-platform/terraform/modules/node-pools/variables.tf similarity index 100% rename from ml-platform/terraform/modules/node-pools/variables.tf rename to best-practices/ml-platform/terraform/modules/node-pools/variables.tf diff --git a/ml-platform/terraform/modules/node-pools/versions.tf b/best-practices/ml-platform/terraform/modules/node-pools/versions.tf similarity index 100% rename from ml-platform/terraform/modules/node-pools/versions.tf rename to best-practices/ml-platform/terraform/modules/node-pools/versions.tf diff --git a/ml-platform/terraform/modules/vm-reservations/outputs.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf similarity index 100% rename from ml-platform/terraform/modules/vm-reservations/outputs.tf rename to best-practices/ml-platform/terraform/modules/vm-reservations/outputs.tf diff --git a/ml-platform/terraform/modules/vm-reservations/reservations.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf similarity index 100% rename from ml-platform/terraform/modules/vm-reservations/reservations.tf rename to best-practices/ml-platform/terraform/modules/vm-reservations/reservations.tf diff --git a/ml-platform/terraform/modules/vm-reservations/variables.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf similarity index 100% rename from ml-platform/terraform/modules/vm-reservations/variables.tf rename to best-practices/ml-platform/terraform/modules/vm-reservations/variables.tf diff --git a/ml-platform/terraform/modules/vm-reservations/versions.tf b/best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf similarity index 100% rename from ml-platform/terraform/modules/vm-reservations/versions.tf rename to best-practices/ml-platform/terraform/modules/vm-reservations/versions.tf diff --git a/ml-platform/terraform/README.md b/ml-platform/terraform/README.md deleted file mode 100644 index 1c765d049..000000000 --- a/ml-platform/terraform/README.md +++ /dev/null @@ -1,112 +0,0 @@ -## Requirements - -| Name | Version | -|------|---------| -| [github](#requirement\_github) | 6.0.1 | -| [google](#requirement\_google) | 5.19.0 | -| [google-beta](#requirement\_google-beta) | 5.19.0 | -| [null](#requirement\_null) | 3.2.2 | - -## Providers - -| Name | Version | -|------|---------| -| [github](#provider\_github) | 6.0.1 | -| [google](#provider\_google) | 5.19.0 | -| [google-beta](#provider\_google-beta) | 5.19.0 | -| [null](#provider\_null) | 3.2.2 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [cloud-nat](#module\_cloud-nat) | ./modules/cloud-nat | n/a | -| [create-vpc](#module\_create-vpc) | ./modules/network | n/a | -| [gcp-project](#module\_gcp-project) | ./modules/projects | n/a | -| [gke](#module\_gke) | ./modules/cluster | n/a | -| [node\_pool-ondemand](#module\_node\_pool-ondemand) | ./modules/node-pools | n/a | -| [node\_pool-reserved](#module\_node\_pool-reserved) | ./modules/node-pools | n/a | -| [node\_pool-spot](#module\_node\_pool-spot) | ./modules/node-pools | n/a | -| [reservation](#module\_reservation) | ./modules/vm-reservations | n/a | - -## Resources - -| Name | Type | -|------|------| -| [github_branch.branch](https://registry.terraform.io/providers/integrations/github/6.0.1/docs/resources/branch) | resource | -| [github_branch_default.default_branch](https://registry.terraform.io/providers/integrations/github/6.0.1/docs/resources/branch_default) | resource | -| [github_branch_protection_v3.branch_protection](https://registry.terraform.io/providers/integrations/github/6.0.1/docs/resources/branch_protection_v3) | resource | -| [github_repository.acm_repo](https://registry.terraform.io/providers/integrations/github/6.0.1/docs/resources/repository) | resource | -| [google-beta_google_gke_hub_feature.configmanagement_acm_feature](https://registry.terraform.io/providers/hashicorp/google-beta/5.19.0/docs/resources/google_gke_hub_feature) | resource | -| [google-beta_google_gke_hub_feature_membership.feature_member](https://registry.terraform.io/providers/hashicorp/google-beta/5.19.0/docs/resources/google_gke_hub_feature_membership) | resource | -| [google-beta_google_gke_hub_membership.membership](https://registry.terraform.io/providers/hashicorp/google-beta/5.19.0/docs/resources/google_gke_hub_membership) | resource | -| [google_project_service.project_services-an](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-anc](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-com](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-con](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-cr](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-gate](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-gkecon](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-gkeh](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [google_project_service.project_services-iam](https://registry.terraform.io/providers/hashicorp/google/5.19.0/docs/resources/project_service) | resource | -| [null_resource.create_git_cred_cms](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | -| [null_resource.create_git_cred_ns](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | -| [null_resource.create_namespace](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | -| [null_resource.install_kuberay_operator](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | -| [null_resource.install_ray_cluster](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | -| [null_resource.manage_ray_ns](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [billing\_account](#input\_billing\_account) | GCP billing account | `string` | `null` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the GKE cluster | `string` | `"gke-ml"` | no | -| [configsync\_repo\_name](#input\_configsync\_repo\_name) | Name of the GitHub repo that will be synced to the cluster with Config sync. | `string` | `"config-sync-repo"` | no | -| [create\_namespace](#input\_create\_namespace) | Setup a namespace to demo. | `number` | `1` | no | -| [create\_projects](#input\_create\_projects) | Flag to create GCP projects | `number` | `0` | no | -| [env](#input\_env) | List of environments | `set(string)` | 
[
  "dev"
]
| no | -| [folder\_id](#input\_folder\_id) | Folder Id where the GCP projects will be created | `string` | `null` | no | -| [github\_email](#input\_github\_email) | GitHub user email. | `string` | n/a | yes | -| [github\_org](#input\_github\_org) | GitHub org. | `string` | n/a | yes | -| [github\_token](#input\_github\_token) | GitHub token. It is a token with write permissions as it will create a repo in the GitHub org. | `string` | n/a | yes | -| [github\_user](#input\_github\_user) | GitHub user name. | `string` | n/a | yes | -| [install\_kuberay](#input\_install\_kuberay) | Flag to install kuberay operator. | `number` | `1` | no | -| [install\_ray\_in\_ns](#input\_install\_ray\_in\_ns) | Flag to install ray cluster in the namespace created with the demo. | `number` | `1` | no | -| [namespace](#input\_namespace) | Name of the namespace to demo. | `string` | `"ml-team"` | no | -| [network\_name](#input\_network\_name) | VPC network where GKE cluster will be created | `string` | `"ml-vpc"` | no | -| [ondemand\_taints](#input\_ondemand\_taints) | Taints to be applied to the on-demand node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "ondemand",
    "value": true
  }
]
| no | -| [org\_id](#input\_org\_id) | The GCP orig id | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The GCP project where the resources will be created. It is a map with environments as keys and project\_ids s values | `map` | n/a | yes | -| [project\_name](#input\_project\_name) | GCP project name | `string` | `null` | no | -| [reserved\_taints](#input\_reserved\_taints) | Taints to be applied to the reserved node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "reserved",
    "value": true
  }
]
| no | -| [routing\_mode](#input\_routing\_mode) | VPC routing mode. | `string` | `"GLOBAL"` | no | -| [secret\_for\_rootsync](#input\_secret\_for\_rootsync) | Create git-cred in config-management-system namespace. | `number` | `1` | no | -| [spot\_taints](#input\_spot\_taints) | Taints to be applied to the spot node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "spot",
    "value": true
  }
]
| no | -| [subnet\_01\_description](#input\_subnet\_01\_description) | Description of the first subnet. | `string` | `"subnet 01"` | no | -| [subnet\_01\_ip](#input\_subnet\_01\_ip) | CIDR of the first subnet. | `string` | `"10.40.0.0/22"` | no | -| [subnet\_01\_name](#input\_subnet\_01\_name) | Name of the first subnet in the VPC network. | `string` | `"ml-vpc-subnet-01"` | no | -| [subnet\_01\_region](#input\_subnet\_01\_region) | Region of the first subnet. | `string` | `"us-central1"` | no | -| [subnet\_02\_description](#input\_subnet\_02\_description) | Description of the second subnet. | `string` | `"subnet 02"` | no | -| [subnet\_02\_ip](#input\_subnet\_02\_ip) | CIDR of the second subnet. | `string` | `"10.12.0.0/22"` | no | -| [subnet\_02\_name](#input\_subnet\_02\_name) | Name of the second subnet in the VPC network. | `string` | `"gke-vpc-subnet-02"` | no | -| [subnet\_02\_region](#input\_subnet\_02\_region) | Region of the second subnet. | `string` | `"us-west2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [project\_ids](#output\_project\_ids) | n/a | - -[gitops]: https://about.gitlab.com/topics/gitops/ -[repo-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields -[root-sync]: https://cloud.google.com/anthos-config-management/docs/reference/rootsync-reposync-fields -[config-sync]: https://cloud.google.com/anthos-config-management/docs/config-sync-overview -[cloud-deploy]: https://cloud.google.com/deploy?hl=en -[terraform]: https://www.terraform.io/ -[gke]: https://cloud.google.com/kubernetes-engine?hl=en -[git]: https://git-scm.com/ -[github]: https://github.com/ -[gcp-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects -[personal-access-token]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens -[machine-user-account]: https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts \ No newline at end of file