From 8585e19868bf563902fd28e19456733bce2440a1 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Fri, 12 Oct 2018 11:08:22 +0100 Subject: [PATCH 01/40] Add configs for deploying Trillian onto Kubernetes via Google Cloud Marketplace Configs adapted from those found in: https://github.com/google/trillian/tree/v1.2.1/examples/deployment/kubernetes --- k8s/trillian/Makefile | 98 +++++++++++++++ k8s/trillian/README.md | 180 +++++++++++++++++++++++++++ k8s/trillian/deployer/Dockerfile | 13 ++ k8s/trillian/manifest/app.yaml | 52 ++++++++ k8s/trillian/manifest/config.yaml | 15 +++ k8s/trillian/manifest/etcd.yaml | 55 ++++++++ k8s/trillian/manifest/logserver.yaml | 108 ++++++++++++++++ k8s/trillian/manifest/logsigner.yaml | 102 +++++++++++++++ k8s/trillian/manifest/mysql.yaml | 55 ++++++++ k8s/trillian/schema.yaml | 133 ++++++++++++++++++++ 10 files changed, 811 insertions(+) create mode 100644 k8s/trillian/Makefile create mode 100644 k8s/trillian/README.md create mode 100644 k8s/trillian/deployer/Dockerfile create mode 100644 k8s/trillian/manifest/app.yaml create mode 100644 k8s/trillian/manifest/config.yaml create mode 100644 k8s/trillian/manifest/etcd.yaml create mode 100644 k8s/trillian/manifest/logserver.yaml create mode 100644 k8s/trillian/manifest/logsigner.yaml create mode 100644 k8s/trillian/manifest/mysql.yaml create mode 100644 k8s/trillian/schema.yaml diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile new file mode 100644 index 0000000000..45b4820932 --- /dev/null +++ b/k8s/trillian/Makefile @@ -0,0 +1,98 @@ +include ../app.Makefile +include ../crd.Makefile +include ../gcloud.Makefile +include ../var.Makefile + +TAG ?= latest +APP_DEPLOYER_IMAGE ?= $(REGISTRY)/trillian/deployer:$(TAG) +NAME ?= trillian-1 + +ifdef IMAGE_TRILLIAN_LOGSERVER + IMAGE_TRILLIAN_LOGSERVER_FIELD = , "IMAGE_TRILLIAN_LOGSERVER": "$(IMAGE_TRILLIAN_LOGSERVER)" +endif + +ifdef IMAGE_TRILLIAN_LOGSIGNER + IMAGE_TRILLIAN_LOGSIGNER_FIELD = , "IMAGE_TRILLIAN_LOGSIGNER": "$(IMAGE_TRILLIAN_LOGSIGNER)" +endif + +ifdef IMAGE_MYSQL + IMAGE_MYSQL_FIELD = , "IMAGE_MYSQL": "$(IMAGE_MYSQL)" +endif + +ifdef IMAGE_ETCD_OPERATOR + IMAGE_ETCD_OPERATOR_FIELD = , "IMAGE_ETCD_OPERATOR": "$(IMAGE_ETCD_OPERATOR)" +endif + +ifdef IMAGE_PROMETHEUS_TO_SD + IMAGE_PROMETHEUS_TO_SD_FIELD = , "IMAGE_PROMETHEUS_TO_SD": "$(IMAGE_PROMETHEUS_TO_SD)" +endif + +APP_PARAMETERS ?= { \ + "NAME": "$(NAME)", \ + "NAMESPACE": "$(NAMESPACE)" \ + $(IMAGE_TRILLIAN_LOGSERVER_FIELD) \ + $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ + $(IMAGE_MYSQL_FIELD) \ + $(IMAGE_ETCD_OPERATOR) \ + $(IMAGE_PROMETHEUS_TO_SD_FIELD) \ +} +APP_TEST_PARAMETERS ?= {} + + +app/build:: .build/trillian/db_server \ + .build/trillian/deployer \ + .build/trillian/log_server \ + .build/trillian/log_signer + + +.build/trillian: | .build + mkdir -p "$@" + + +.build/trillian/db_server: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull gcr.io/trillian-opensource-ci/db_server:$(TAG) + docker tag gcr.io/trillian-opensource-ci/db_server:$(TAG) \ + "$(REGISTRY)/trillian/db_server:$(TAG)" + docker push "$(REGISTRY)/trillian/db_server:$(TAG)" + @touch "$@" + + +.build/trillian/deployer: deployer/* \ + manifest/* \ + schema.yaml \ + .build/var/APP_DEPLOYER_IMAGE \ + .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker build \ + --build-arg REGISTRY="$(REGISTRY)/trillian" \ + --build-arg TAG="$(TAG)" \ + --build-arg MARKETPLACE_TOOLS_TAG="$(MARKETPLACE_TOOLS_TAG)" \ + --tag "$(APP_DEPLOYER_IMAGE)" \ + -f deployer/Dockerfile \ + . + docker push "$(APP_DEPLOYER_IMAGE)" + @touch "$@" + + +.build/trillian/log_server: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull gcr.io/trillian-opensource-ci/log_server:$(TAG) + docker tag gcr.io/trillian-opensource-ci/log_server:$(TAG) \ + "$(REGISTRY)/trillian/log_server:$(TAG)" + docker push "$(REGISTRY)/trillian/log_server:$(TAG)" + @touch "$@" + + +.build/trillian/log_signer: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull gcr.io/trillian-opensource-ci/log_signer:$(TAG) + docker tag gcr.io/trillian-opensource-ci/log_signer:$(TAG) \ + "$(REGISTRY)/trillian/log_signer:$(TAG)" + docker push "$(REGISTRY)/trillian/log_signer:$(TAG)" + @touch "$@" + diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md new file mode 100644 index 0000000000..fc0b440002 --- /dev/null +++ b/k8s/trillian/README.md @@ -0,0 +1,180 @@ +# Overview + +Trillian makes it easy to build cryptographically verifiable solutions enabling +developers to minimize the trust users must place in the solutions they build. +This is accomplished by applying the concept of cryptographic transparency to +the storage and distribution of data. + +When properly implemented this moves solutions from a model of ”trust us” to a +model of ”you can verify we do what we say”. + +Architecturally this is accomplished by providing a relatively simple +abstraction that can be used to store a ledger of values or a set of keys and +associated values and have the data be added to an append only ordered list +known as a [merkle tree](https://en.wikipedia.org/wiki/Merkle_tree). This +abstraction takes care of the management of the cryptographic aspects necessary +to deliver on this transparency principal. + +Behind this abstraction is a database abstraction that enables storing these +values into Cloud Spanner, My SQL and other databases. + +This approach allows solutions to scale to 10s of billions of entries and over +2k transactions a second without struggling complex engineering tasks. + +This platform is the basis of +[Certificate Transparency](https://en.wikipedia.org/wiki/Certificate_Transparency), +[Key Transparency](https://security.googleblog.com/2017/01/security-through-transparency.html), +[Verifiable Data Audit](https://www.wired.com/2017/03/google-deepminds-untrendy-blockchain-play-make-actually-useful/) +and other solutions. + +[Learn more](https://github.com/trillian/). + +## About Google Click to Deploy + +Popular open stacks on Kubernetes packaged by Google. + +## Design + +![Architecture diagram](https://github.com/google/certificate-transparency-go/blob/c0b58057e5831c0fe4c19a193c273b50704fd82a/trillian/docs/images/DeploymentFull.png) + +* **Personality** - translates between Trillian's API and the API you want to + expose to users. This is not included in the deployment - you must implement + and add this yourself. + +* **Log Server** - exposes a gRPC API for interacting with the ledgers + (referred to as "logs" by Trillian). + +* **Log Signer** - adds new queued entries to logs. This is a background + process. + +* **Etcd Cluster** - used to handle rate-limiting/quota and master election. + +* **Prometheus** - used for monitoring Trillian's health. This Trillian + installer will automatically forward monitoring data to StackDriver, but you + can alternatively + [install Prometheus and Grafana](https://pantheon.corp.google.com/marketplace/details/google/prometheus) + and use those instead. + +## Configuration + +* The number of Trillian log server and signer replicas is specified by the + user before installation, and can later be scaled as required. Most + configuration options are stored in a ConfigMap. + +# Installation + +## Quick install with Google Cloud Marketplace + +Get up and running with a few clicks! Install Trillian to a Google Kubernetes +Engine cluster using Google Cloud Marketplace. Follow the +[on-screen instructions](https://console.cloud.google.com/marketplace/details/google/trillian). + +## Command line instructions + +### Prerequisites + +#### Set up command-line tools + +You'll need the following tools in your development environment: - +[gcloud](https://cloud.google.com/sdk/gcloud/) - +[kubectl](https://kubernetes.io/docs/reference/kubectl/overview/) - +[docker](https://docs.docker.com/install/) - +[git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +[make](https://www.gnu.org/software/make/) + +Configure `gcloud` as a Docker credential helper: + +```shell +gcloud auth configure-docker +``` + +#### Create a Google Kubernetes Engine cluster + +Create a new cluster from the command line: + +```shell +export CLUSTER=trillian-cluster +export ZONE=us-west1-a + +gcloud container clusters create "$CLUSTER" --zone "$ZONE" +``` + +Configure `kubectl` to connect to the new cluster. + +```shell +gcloud container clusters get-credentials "$CLUSTER" --zone "$ZONE" +``` + +#### Clone this repo + +Clone this repo and the associated tools repo: + +```shell +git clone --recursive https://github.com/GoogleCloudPlatform/click-to-deploy.git +``` + +#### Install the Application resource definition + +An Application resource is a collection of individual Kubernetes components, +such as Services, Deployments, and so on, that you can manage as a group. + +To set up your cluster to understand Application resources, run the following +command: + +```shell +make -C click-to-deploy/k8s/trillian crd/install +``` + +You need to run this command once. + +The Application resource is defined by the +[Kubernetes SIG-apps](https://github.com/kubernetes/community/tree/master/sig-apps) +community. The source code can be found on +[github.com/kubernetes-sigs/application](https://github.com/kubernetes-sigs/application). + +### Install the Application + +#### Make sure you are a Cluster Admin + +Installing Trillian creates a custom cluster role, in order to provide necessary +privileges to Etcd. You must be a Cluster Admin in order for the installer to do +this on your behalf. To assign the Cluster Admin role to your user account, run +the following command: + +```shell +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user $(gcloud config get-value account) +``` + +Next, choose the instance name, namespace and Trillian release tag to use, then +run the installer. + +```shell +export NAME=trillian-1 +export NAMESPACE=default +export TAG=1.2.1 + +make -C click-to-deploy/k8s/trillian app/install +``` + +#### View the app in the Google Cloud Console + +To get the Console URL for your app, run the following command: + +```shell +echo "https://console.cloud.google.com/kubernetes/application/${ZONE}/${CLUSTER}/${NAMESPACE}/${NAME}" +``` + +To view your app, open the URL in your browser. + +## Forward Trillian gRPC port in local environment + +Use local port forwarding to access Trillian's gRPC API from your machine. In a +terminal, run the following command: + +```shell +kubectl port-forward --namespace ${NAMESPACE} ${NAME}-trillian-logserver 8090 +``` + +You can access the Trillian API at `localhost:8090`. diff --git a/k8s/trillian/deployer/Dockerfile b/k8s/trillian/deployer/Dockerfile new file mode 100644 index 0000000000..15c3f074a9 --- /dev/null +++ b/k8s/trillian/deployer/Dockerfile @@ -0,0 +1,13 @@ +ARG MARKETPLACE_TOOLS_TAG +FROM gcr.io/cloud-marketplace-tools/k8s/deployer_envsubst:$MARKETPLACE_TOOLS_TAG + +ADD manifest/* /data/manifest/ +ADD schema.yaml /data/ + +# Provide registry prefix and tag for default values for images. +ARG REGISTRY +ARG TAG +RUN cat /data/schema.yaml \ + | env -i "REGISTRY=$REGISTRY" "TAG=$TAG" envsubst \ + > /data/schema.yaml.new \ + && mv /data/schema.yaml.new /data/schema.yaml diff --git a/k8s/trillian/manifest/app.yaml b/k8s/trillian/manifest/app.yaml new file mode 100644 index 0000000000..059626f96f --- /dev/null +++ b/k8s/trillian/manifest/app.yaml @@ -0,0 +1,52 @@ +apiVersion: app.k8s.io/v1beta1 +kind: Application +metadata: + name: $NAME + annotations: + kubernetes-engine.cloud.google.com/icon: >- + data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAATvklEQVR4Ae3dBXQUV9/H8Y17gsQVTSjuPMUpTnFtcXd3Dd5QV+AlVKi7u7vjVi/uIbiFJP/3e897s2fPhITdySa7Dy+/cz6VBJLsnZ2ZO/8rsbhpupaqXmp3QvekU1HNo/7l/wfBTG4PCAj4JDExcZefn9+z/H8srpObaZLQM/Hi4NxhMliGyYDLg6Xy2JRsi6dluMWx3NmoUaOse+65R1avXi3jx4+XpKSkvXy8FArKzYRWCf2s1/6+qvGtBmYNkaByQVssDoTG/uf+++9XjW81evRo4VOLUFBuJjQldPOgnKG68aHFdoj93eJA6tWrd0I3vNV9990nwcHB71kKzs14eHmkdfq5s254oMu2bhKcFLzB4kC47m80HoApU6YI94Q0S6G5GY+ydctua/VWa+l/bqDc9l5biWkdc4KPV4Pd8fT0HMU94JJ616vGX7VqlVSqVGmb5f9JvNEFg03e9CK9Q73n+UX4/ch/v4XyMJP6cXFxB8qVK7fH39//Hf4/Bjd8aqfUaXu0bb/50mvCaqlYo8VZPjYUZjICT8Bs2uBLmE0NdeAqVKjwZ3x8/Cb+vzPcO9FJNX5P3bBX0l47o8jKVzKlVpPeF/mUPxxNHWyB2SzAKphJIg1/Ii0tTV3CRHVnGzRokOXj4zPA4sZJbt1nbq5ufKsuI+4RPtcIjsYH5+EHM3kX3eFwQkJCVk6fPl2MN3LOhO0WN05KwzZD8x2A/jOeFj43CmayEfVhJsfNXvdDQ0PfePDBB/MdgNq1a2dY3DkJler9PePRzdbGX/7icUmq2vgcn4qCmaRjDBxNeeyH2dw/bty4fM8RnAE7LW6ehIiYSj+06Tc/t1m3qTmVa7U5ycdmwWzGYB0czZ14BWaTxLPE0YkTJ6rGF3UvuPXWW88Zb8TunPYoh+bYh1CYSQNshKN5CDNgNhPwdmBg4GPqoU53YxPwX5nHkA4z8cN5+MCR/IimMJMKOIaKuCESjH/RBmayFXVgb3xxDgFwNB74ChNxQ6U19iC4wC5UnTaxKU26htgctF7YgP0YAXvTEPvwBLoh0HBwCsskfAkP3HBZhzXIlz6T06cseebA1Qmrvj7o4+P/uYeHRxYfFi8vL9WVfBWVYE2LjiPj+4xfPbRpl0ktLPkTi/V8jQz+Lfz7sn+4/2cdf7z9CKXuQ02eaVHQpakSjqI8bsiEYC9awjbhfSas+V11W+969ZSUT655ku7eQ7r/7wGVCLTEBKwYOPPZg+rPz1v/x9VyKY1ViXk2+qAeykDFE03KlCmzpkLXimfzqqs1FtXaqL4nbOOJbzAWbpWeZcuW3VylSpXDERER6sbW2Am9o78RhGCkUrk8HxFXOafvpDWXeZATPrYQYRiIN3AcksfL20cWPfmv9VmjTot+6uNGu3AXGsEjODF4dMOHG0nzV2/LDogOzFHfU3+fIKhMxadwRqIQjiKnA33fK48++qjqC8uKFSukTp06GbpxipIn8DGXBlVmzuHAqopnEv4DwSVkQWwcwed4FLNvH7zi4F2vnpYpD/woEVGVH9A1n1ewEachNg7jEATDUSE8PFyVKXL5GY7pmtFRJKIoKRsbG7uTwaDs1q1bZ/M91M+SAnNJSEj4VdfTrZYtWybe3t7PWIqWMGTwLlQNWhW22YYcXMZb6I8yMGZIqfAEzgZf1ailYRtP3IpV+A2iZRp6RrX4GT7kIKjGH4kipXTp0p8uXLjQ2laLFy+WypUrH+RTXnA85cuX36++kBH19W8sRU8ErpUqaIgQFJZhEC0UhaUpXsGSgkroKGqadOrUKdfYVh06dBDdm3M8QUFB3xgHtqdOnSqlSpW6x+L6jIRoQXB14jp27JhjPAADBw7Ulz1zacIpdGbu3Lmi7gMzZ84Uzgp1us6FN1yZMRDNHy4PldJM28Z/7LHHpFq1aheKOhJXIzIyci3UTTNdd/Pex6+oBldlPETzgStTAZ/h2erVq1+aMWOGDBo0SBo3bqw6ElNQLBmKY5gHL5R0JkE0T7giXpiODMyGFyphIXohstive/gAv7jgbJgK0VyRGvgFX6ISXJphOIa5JXg2zIAgGyUZXyzDcYxyp5pRvM3ZUBXFndkQXEFJ5VbsxpvuPNH3aW7W5/TlyVnxR0MMx4P4DOcguIjijidW4zh6wW0TSuNn8nT5kRNf+HAcgRTgHEoi3yETIXDbLEUOapht8PDwxLw+cytsgRTiKr5DSaQmcrEQbpkI3v0XKEC9bWYUqtPA5S+MWvrhoakP/Hixxq09/jE2NvZgHSahFSJRomGqygec3Wd0Lctt4snih8XUPDYxayzbzGBG8y6TZjB9xVpeVlXOhOQGeQ1/CjPgC1enGnIZqH+oadOmpS1OTCCm8g5e5+gcnGEk79G7SZMm6t1RFw6l36T054yTueq27K9O90dQFm6T5s2bH1ATtpYvX/43r9cpPaFIRp/+7NOnj8yaNUv69++fGxUV9bm9p1nfvn3bPPDAA1mqVkSdKO9d+xUawq606jl7lW3jr3z5pJSv2kT41Ey4Q0pjMnZQcLPWebp3797O0YdHxj6+xVvWB1dfX991aumObRFJFd+o+T9t94Tc6OhvKVGf0eXeLAh6wJ5Q0g/4pFG74bLkuUPCPUD+036k7bV/DFyddIhCzf8QFc6r7du332exP6WYW7RVvcHVG5WzRw1snWW8u6VFTb1WjW5Eo/5u7+VLDXqrgp06FsjFBQcKZGkQJTCkDCNU3hv01xAtBwNgm4oYgqHaIESjuNIDgiPwoDHX8Zqv2DvthUld80eOHJmvjRk926bu7N8bJ6E+9NBD6pP/6EvJveiL8tf54RpjJAQvwZ7E4BJEm2kstGlX0VP7GLkQgyw8j/+gOLIPglpoBkEne85wJgW88/DDD+c7AFROT1nIne3atcvhmmb9BHf4K7pBW2MuXsM+nMCHWIYuiMYGumbq4554B4IBDkwfFO0T2GYBxEYOxE5PFEON5hEIlsCL13xWz/izTSiaYiLW41ecw0eTJk3K9yZXK3csOhM4Gse5w59nAOFv7gvzCrph43Yswfs4pssOryMAF5GNMnb2vC5CtKYw5m6ISelOPghtIfhGr0F7Ww/kp+IN/IMz+BaPYSQaIADVb7nlljOpqanWG3izZs1yjFNgPDVHk4hwNITgF9iTrhDtu0JKEUchtjw8PKVKvQ7S5PZxiiQmNyzoINwPZyUCgpNQmYhPsBQ9UeE6B7wpD6mv80bfTy/oLw7gLIuTcwcEz8OerIdo03CtdIbYSq7dWuas3SnG54YJd38lEbGV85cqnFsUPA5BFNwu8yBYCXuyB6Il41r5CJKHxX+y4qUTutHzm5f+u5SNLm88CEvhrHwFQUu4V7xDvF8OiA4QO+fXeOEqBFnwhDEpyLW97Mx8bKtu7IINmPmsGCdlwRvOyHoIhsB90mhN42H9TvfP7ZfRXxL7lFOn6c/YhG3YhT/wL/bjMDIg2uWCLmnGdz8NfF0rXs6QoNBw40FIKmRUbZb+9zRMxWRMxHiMxRiMwghmux2cPHmyUDX4mdlv7lOQ6/RLl0/yJr42e76Fo72VU/YcgEZth6kGtktMuerG71EO10oWxB7cQCVviqbSi1jcJU2eb7ZS7WiiDkDVGVWP6G7pS3gWTyEdq/Ew7kMaliMV4110AIIh9uJpVvK2vmGyWna3bt3c6z4QdVvMj2XqlhVPf08azinpCckTlVjVrsZf9NQe8fENMDZgAoxJhmjb4A0f+MIP/gi0mcUdwl4T76mphg0bNnzJLceCIXDWAQjFeUieUUvfv+4BaD9gibHxt+FaaQnRPoI9eR6CPnC7rIBgAYgTex1aWHiczF6zvcDGH5H6tvHdX1ivbDJEWw978j0E9eB26QfBi3BW6kKMB6Hn2Edk2fNHrQ2/8Kl/pePApeLt429s/IxCKpafQLQ7YU8OQ1AaTk0PZkJ/zTVuMztK3W/HsKIHKqAXVqEDqkKwHc5MGsTILyBEIuOrSHRiNRrer6Cb53sF1edxBYJsOxs0GLk2vTYfOCW9KAxlcWe3Tj+nSqeOdDxUfFEbw/AwvsFp/IvX1TQUvK1vYlc0fzgz90FMWgBjUiHa5w6OCXwIlXdwBm9jPtogrJANpPqqkUaKcnv590//N7GAqAEZSqOq8a1GjBiR94224Dy24ilMQQvDN/ofRnYy1ffA1xB0gbOzDBch13EextL1ZMPNPROidYA9eQaCsfDUJfiX0QNp+AJnsBtPYRzqwhvJLPW6mPccsXLlSrXph/o5G1kYGrOZ0w6os4FLkuoZ1LFjNkJnCBpjCgQbUBwpg1nYAzHYiOEIwBDDoE0uhsEL70G0H2FPvJGJXMSiMQTdYBtPVMcIpGMbzsbExGy59957dfsCS5cuVcO+r6jN7X43HoCxY8eqB49Ui30JUEOSrCdbo0vTgovwKeZp4dE2omDMREgeD2+PbP/ogC2GAZ7bYE96Q/AzLGr4ldd8yc4hyQDurTuNbayHJDdbmMvTn83trnCErAvL1CwJR8q41Li3MlvgMsObulroNn3leZBqM6tL9997yh2nBkjlkcni6eMpetjT3uyE4AtEkgy9gNCuhIWFvWVc6qVqSswvesGi05FG34g/OC2edKR6SMNXZSXlVfVFe/ToIfo6+BCS4PIk9Sv/nd6F1yqhZ9LXFvvjYdtlbdu2bY4a32X6zo8W+1OratWqh+fPn29d6sXGsoed0kY1a9aMZC8ddR+RLl26ZOrumrukTtXpVY9YG19r+lyzJ8zswIgX1d6j6rUyvHjUwSFPXxa8v8Ro2M9cvh516kBR586dKycnJ79Lb0hPTXR54rABuZHNosT2DOh/flDOLbOrjzW7BoJuZA47aR0dTSxulkjr5FzXJB6DsAEXIArvUYnrlpDdZXvXrHZfd9xX7+766y0mQ//9TbqfF/VN3y2zHDkluF5sK7KQDYFRDp5BYkBCgOo+ehZ1Yi5WwG0TxllwSm9BUBI5BSnAJ6gNp4TX9L56bfoh1G0TgE/4QU+X0BqqSxBk4wDexGTUgDPTidd0Vs+Hcts0wR94BqVLaI2uaLNRnPHHPTiKO+FWCcSD2I/bS/JyB9HGoyRSHzvwFmLg8jTHX3jCBdfGOIg2yEXrhIei2OOB8uiLUKgE4VHsQTu4IlUgWg+UdGpjCz5CIool3dQUarUPDithhMJdpq6h/I21Ll6y2QCitYEr4o2FOIGxqEpt7TW10RUV0DeKOoMugIGZk7ZjBSxFUtuvZOrBBFenFURr5AaL9z6kvY6zo5h1q2O2u79UlIXdd/Tr1y9fCZWlOTluUm7oCtGqwaVhE6u71Wiicb8gKqfbTF9+KK7lOwBMCBMn9XYqwpgE/IINiEdh6Q9R7LgGJ2M15hc8Db5ooQTz0SOPPJKvvaiCHjfdz65Ro8Yx25EcNTOM69sOJyx86KUWdvB0+YEeQcrLIsMo1i+Yh5Tr7JhVxvizowpm4BfDPNRI20sHP8f3epy7qEnTlVLjdvfbi9TvpcH39+zZU1jGJFQ8j+kGK0oicBgP6kf7XFXk0u/ifyH4GJkQGwfxDZ7BMoZSt6qdqSZMmCCMOKXrgfYN2IRLEBsn8A8ES1BZrdvi3zn6Z9jnhAcsb7VDIishrcO6zKC77IyrhQeaozW8UNS8ipVQCUOaGtKsW7durrqGMkPjjE0P4zY8iv0QWzS89Z2m3hyGz2fjNzyB9vBGE/XnGG7N5U2VqzduXYow1MMRhBf1Zszo13u8If5ies/n+nu7VfphO3xhmzhGic6qxlSjRlxP1S69DxpmoMWiCQZg4eDBgw/nHYCWLVt+q6/vQ1AX/lDxRENmfjzGzr8ZeX+exf1/XWMF/iq8gBs2kTiCusgX3pnLGGU6zurzz/QWwlf15t2H9FnjD2s4veNHjRr16JAhQ1K50RkPaBs140BPlxHOsKv891esVPyb2Qin+DtDC6j3/IEuuCHzhh3LhTwMtZ478DouOnjj/wyZSEd3BNucEd4oKE2xH2G4odIfW+FTQr+MbTbudcFv+3DLROMoasFM5uIeE2t6P4WZBGMPWuG/Mj5cbx9ir4kv9BbH7yAVZvMa+pro6h53whb7kRiG3vivSG1mDRzLWxGu+ukVK1bM0E+iZrMPFeBoDhdxVG4BD6LnunbtKhQi1Tye3brk4b7h4eaLBQsWqMYH9Ew7eiTPFuGhLQNm8gE6wUy8U1JS9qgJWLaFSD6WobvQ7hl+wH0FzIH8zuxYLD6GmdyF+TCT7gMGDMj3Otq0aSNufTniKTDf1jdLliwRauNmz4BUrISZ9MUrMJN4Jpvl24qelZGiq7Bumy48IFkn+qpTmOvohSLs4fMOusNMkvEHTIUn5xO2E2rVa+KNtEeXY9w6Pfgt1jtU4Y57wiaz12GdI4iHmXjiLIJgJq14HYdVIVJdeng9h/XAzw2fQLqx9/Liv+be8XsRqpTR+FLt9cmlcZXJ3wPpjfZoBw/c+KHhP2fTQGvPg60hc1gsstDRL0O38cCYMWNE3ZOmTZumusP79QLsmylk+t9QhkFzjTc/ni3+dvC3YS+1KVdbr+GcDW9abqbgULVcbf01UDY4C847smiEd/8P1+oOs6/pXkuhuZnRPHXaNppaUaLOgJOOFPL8/Pw+UL90zvbrqEuR2rvTUmhuxlctGmQBhLUby5KgXF2QcyTt2M/H2h1WWrRokaVXNd7MdVKGXRvX0+37l27sxiLsnNuNAfFdtWrVOsW//+KsGGlxw/wvK87UoQAXTwYAAAAASUVORK5CYII= + marketplace.cloud.google.com/deploy-info: '{"partner_id": "click-to-deploy-containers", "product_id": "trillian", "partner_name": "Google Click to Deploy"}' + labels: + app.kubernetes.io/name: "$NAME" +spec: + descriptor: + type: "trillian" + version: "1.2.1" + description: >- + Trillian implements a Merkle tree whose contents are served from a data + storage layer, to allow scalability to extremely large trees. + maintainers: + - name: Google + url: "https://github.com/google/trillian/" + keywords: + - "blockchain" + - "google" + - "merkle" + - "transparency" + - "tree" + - "trillian" + - "verifiable" + - "append-only" + links: + - description: GitHub + url: "https://github.com/google/trillian/" + - description: Support + url: "https://github.com/google/trillian#support" + info: + - name: Trillian log server IP address + type: Reference + valueFrom: + serviceRef: + name: $NAME-logserver-service + selector: + matchLabels: + app.kubernetes.io/name: "$NAME" + componentKinds: + - group: apps/v1 + kind: Deployment + - group: v1 + kind: ConfigMap + - group: v1 + kind: Secret + - group: v1 + kind: Service diff --git a/k8s/trillian/manifest/config.yaml b/k8s/trillian/manifest/config.yaml new file mode 100644 index 0000000000..c949cee275 --- /dev/null +++ b/k8s/trillian/manifest/config.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: $NAME-config + labels: + app.kubernetes.io/name: "$NAME" + data: + STORAGE_SYSTEM: mysql + SIGNER_BATCH_SIZE: "500" + SIGNER_INTERVAL: "20ms" + SIGNER_NUM_SEQUENCERS: "10" + SIGNER_RESIGN_ODDS: "1000" diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml new file mode 100644 index 0000000000..4e7752f620 --- /dev/null +++ b/k8s/trillian/manifest/etcd.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: Service + metadata: + name: $NAME-etcd-service + labels: &EtcdLabels + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd + spec: + type: NodePort + selector: *EtcdLabels + ports: + - protocol: TCP + port: 2379 + nodePort: 32379 +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: $NAME-etcd-operator + labels: *EtcdLabels + spec: + replicas: 1 + selector: + matchLabels: *EtcdLabels + template: + metadata: + labels: *EtcdLabels + spec: + serviceAccountName: $ETCD_SERVICE_ACCOUNT + containers: + - name: trillian-etcd-operator + image: $IMAGE_ETCD_OPERATOR + command: ["etcd-operator", "-cluster-wide"] + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +- apiVersion: "etcd.database.coreos.com/v1beta2" + kind: "EtcdCluster" + metadata: + name: "$NAME-etcd-cluster" + labels: *EtcdLabels + annotations: + etcd.database.coreos.com/scope: clusterwide + spec: + serviceAccountName: $ETCD_SERVICE_ACCOUNT + size: $ETCD_CLUSTER_SIZE + version: $ETCD_VERSION diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml new file mode 100644 index 0000000000..c5ed2f645e --- /dev/null +++ b/k8s/trillian/manifest/logserver.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: Service + metadata: + name: $NAME-logserver-service + labels: &LogServerLabels + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logserver + annotations: + cloud.google.com/load-balancer-type: "Internal" + spec: + ports: + - name: grpclb + port: 8090 + - name: metrics + port: 8091 + selector: *LogServerLabels + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: $NAME-logserver-deployment + labels: *LogServerLabels + spec: + replicas: $TRILLIAN_LOGSERVER_REPLICAS + selector: + matchLabels: *LogServerLabels + template: + metadata: + labels: *LogServerLabels + spec: + volumes: + - name: google-cloud-key + secret: + secretName: trillian-key + restartPolicy: Always + containers: + - name: trillian-logserver + args: [ + "$(STORAGE_FLAG)", + "--storage_system=$(STORAGE_SYSTEM)", + "--quota_system=etcd", + "--etcd_servers=$NAME-etcd-cluster-client:2379", + "--etcd_http_service=$NAME-logserver-http", + "--rpc_endpoint=0.0.0.0:8090", + "--http_endpoint=0.0.0.0:8091", + "--tracing", + "--alsologtostderr", + ] + envFrom: + - configMapRef: + name: ${NAME}-config + - secretRef: + name: ${NAME}-secrets + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/var/secrets/google/key.json" + image: $IMAGE_TRILLIAN_LOGSERVER + imagePullPolicy: Always + resources: + limits: + cpu: "1.0" + requests: + cpu: "0.4" + livenessProbe: + httpGet: + path: /healthz + port: 8091 + failureThreshold: 3 + periodSeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8090 + name: rpc + - containerPort: 8091 + name: metrics + volumeMounts: + - name: google-cloud-key + mountPath: /var/secrets/google + - name: prometheus-to-sd + image: $IMAGE_PROMETHEUS_TO_SD + ports: + - name: profiler + containerPort: 6060 + command: + - /monitor + - --stackdriver-prefix=custom.googleapis.com + - --source=logserver:http://localhost:8091/metrics + - --pod-id=$(POD_NAME) + - --namespace-id=$(POD_NAMESPACE) + - --scrape-interval=5s + - --export-interval=60s + resources: + limits: + cpu: 20m + requests: + cpu: 20m + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml new file mode 100644 index 0000000000..175b0d170b --- /dev/null +++ b/k8s/trillian/manifest/logsigner.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: Service + metadata: + name: $NAME-logsigner-service + labels: &LogSignerLabels + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logsigner + annotations: + cloud.google.com/load-balancer-type: "Internal" + spec: + clusterIP: None + ports: + - name: metrics + port: 8092 + targetPort: 8091 + selector: *LogSignerLabels +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: $NAME-logsigner-deployment + labels: *LogSignerLabels + spec: + replicas: $TRILLIAN_LOGSIGNER_REPLICAS + selector: + matchLabels: *LogSignerLabels + template: + metadata: + labels: *LogSignerLabels + spec: + volumes: + - name: google-cloud-key + secret: + secretName: trillian-key + restartPolicy: Always + containers: + - name: trillian-logsigner + args: [ + "$(STORAGE_FLAG)", + "--storage_system=$(STORAGE_SYSTEM)", + "--etcd_servers=$NAME-etcd-cluster-client:2379", + "--quota_system=etcd", + "--etcd_http_service=$NAME-logsigner-http", + "--http_endpoint=0.0.0.0:8091", + "--sequencer_guard_window=1s", + "--sequencer_interval=$(SIGNER_INTERVAL)", + "--num_sequencers=$(SIGNER_NUM_SEQUENCERS)", + "--batch_size=$(SIGNER_BATCH_SIZE)", + "--resign_odds=$(SIGNER_RESIGN_ODDS)", + "--alsologtostderr", + ] + envFrom: + - configMapRef: + name: ${NAME}-config + - secretRef: + name: ${NAME}-secrets + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/var/secrets/google/key.json" + image: $IMAGE_TRILLIAN_LOGSIGNER + imagePullPolicy: Always + resources: + limits: + cpu: "1" + requests: + cpu: "1" + livenessProbe: + httpGet: + path: /healthz + port: 8091 + failureThreshold: 3 + periodSeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8091 + volumeMounts: + - name: google-cloud-key + mountPath: /var/secrets/google + - name: prometheus-to-sd + image: $IMAGE_PROMETHEUS_TO_SD + ports: + - name: profiler + containerPort: 6060 + command: + - /monitor + - --stackdriver-prefix=custom.googleapis.com + - --source=logsigner:http://localhost:8091/metrics + - --pod-id=$(POD_NAME) + - --namespace-id=$(POD_NAMESPACE) + - --scrape-interval=5s + - --export-interval=60s + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/k8s/trillian/manifest/mysql.yaml b/k8s/trillian/manifest/mysql.yaml new file mode 100644 index 0000000000..c71ffd6173 --- /dev/null +++ b/k8s/trillian/manifest/mysql.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: List +items: +- apiVersion: v1 + kind: Secret + metadata: + name: $NAME-secrets + labels: &MySQLLabels + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql + type: Opaque + stringData: + STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp(${NAME}-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE +- apiVersion: v1 + kind: Secret + metadata: + name: $NAME-mysql-secrets + labels: *MySQLLabels + type: Opaque + stringData: + MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD}" + MYSQL_USER: "${MYSQL_TRILLIAN_USER}" + MYSQL_PASSWORD: "${MYSQL_TRILLIAN_PASSWORD}" + MYSQL_DATABASE: "${MYSQL_TRILLIAN_DATABASE}" +- apiVersion: v1 + kind: Service + metadata: + name: $NAME-mysql-service + labels: *MySQLLabels + spec: + clusterIP: None + ports: + - name: mysql + port: 3306 + selector: *MySQLLabels +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: $NAME-mysql + labels: *MySQLLabels + spec: + replicas: 1 + selector: + matchLabels: *MySQLLabels + template: + metadata: + labels: *MySQLLabels + spec: + containers: + - image: $IMAGE_MYSQL + name: mysql + envFrom: + - secretRef: + name: $NAME-mysql-secrets + restartPolicy: Always diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml new file mode 100644 index 0000000000..af44973542 --- /dev/null +++ b/k8s/trillian/schema.yaml @@ -0,0 +1,133 @@ +application_api_version: v1beta1 +properties: + NAME: + type: string + x-google-marketplace: + type: NAME + NAMESPACE: + type: string + x-google-marketplace: + type: NAMESPACE + # TODO(RJPercival): One image must be the "primary" image and have the same + # name as the product (i.e. "trillian"). The remaining images must have names + # nested under that, e.g. "trillian/signer". + IMAGE_TRILLIAN_LOGSERVER: + type: string + default: $REGISTRY/log_server:$TAG + x-google-marketplace: + type: IMAGE + IMAGE_TRILLIAN_LOGSIGNER: + type: string + default: $REGISTRY/log_signer:$TAG + x-google-marketplace: + type: IMAGE + TRILLIAN_LOGSERVER_REPLICAS: + type: integer + title: Trillian replicas + description: The number of Trillian log servers to run + default: 4 + minimum: 1 + TRILLIAN_LOGSIGNER_REPLICAS: + type: integer + title: Trillian replicas + description: The number of Trillian log signers to run + default: 2 + minimum: 1 + IMAGE_ETCD_OPERATOR: + type: string + default: quay.io/coreos/etcd-operator:v0.9.1 + x-google-marketplace: + type: IMAGE + ETCD_CLUSTER_SIZE: + type: integer + title: Etcd cluster size + default: 3 + minimum: 3 + ETCD_VERSION: + type: string + title: Etcd version + default: 3.2.13 + ETCD_SERVICE_ACCOUNT: + type: string + title: Etcd Service Account + description: Service account with privileges required by etcd-operator + x-google-marketplace: + type: SERVICE_ACCOUNT + serviceAccount: + roles: + - type: ClusterRole + rulesType: CUSTOM + rules: + - apiGroups: + - etcd.database.coreos.com + resources: + - etcdclusters + - etcdbackups + - etcdrestores + verbs: + - "*" + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + verbs: + - "*" + - apiGroups: + - apps + resources: + - deployments + verbs: + - "*" + IMAGE_MYSQL: + type: string + default: $REGISTRY/db_server:$TAG + x-google-marketplace: + type: IMAGE + MYSQL_ROOT_PASSWORD: + type: string + title: MySQL root password, used to login as root and create the Trillian database + x-google-marketplace: + type: GENERATED_PASSWORD + generatedPassword: + length: 12 + MYSQL_TRILLIAN_USER: + type: string + title: MySQL username for Trillian to use + default: trillian + MYSQL_TRILLIAN_PASSWORD: + type: string + title: MySQL password for Trillian to use + x-google-marketplace: + type: GENERATED_PASSWORD + generatedPassword: + length: 12 + MYSQL_TRILLIAN_DATABASE: + type: string + title: MySQL database for Trillian to create and use + default: trillian +required: +- NAME +- NAMESPACE +- IMAGE_TRILLIAN_LOGSERVER +- IMAGE_TRILLIAN_LOGSIGNER +- TRILLIAN_LOGSERVER_REPLICAS +- TRILLIAN_LOGSIGNER_REPLICAS +- IMAGE_ETCD_OPERATOR +- ETCD_CLUSTER_SIZE +- ETCD_VERSION +- ETCD_SERVICE_ACCOUNT +- IMAGE_MYSQL +- MYSQL_ROOT_PASSWORD +- MYSQL_TRILLIAN_USER +- MYSQL_TRILLIAN_PASSWORD +- MYSQL_TRILLIAN_DATABASE From d877eff5cb8e89f5ef574fffeeb62e0953e65b5d Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 26 Nov 2018 14:05:51 +0000 Subject: [PATCH 02/40] remove "imagePullPolicy: Always" No need to pull an image if it is already cached. This is only helpful if the tag being used is "latest", but we shouldn't recommend that. --- k8s/trillian/manifest/logserver.yaml | 1 - k8s/trillian/manifest/logsigner.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml index c5ed2f645e..03daa3d5b4 100644 --- a/k8s/trillian/manifest/logserver.yaml +++ b/k8s/trillian/manifest/logserver.yaml @@ -58,7 +58,6 @@ items: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/var/secrets/google/key.json" image: $IMAGE_TRILLIAN_LOGSERVER - imagePullPolicy: Always resources: limits: cpu: "1.0" diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index 175b0d170b..cd83a83c93 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -60,7 +60,6 @@ items: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/var/secrets/google/key.json" image: $IMAGE_TRILLIAN_LOGSIGNER - imagePullPolicy: Always resources: limits: cpu: "1" From f6812dc4c646525a41a39431814f40b27d6cb6c6 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Thu, 29 Nov 2018 17:42:47 +0000 Subject: [PATCH 03/40] Add --enable-autoscaling flag to cluster creation command Should ensure that the cluster grows to accommodate Trillian regardless of the number of replicas chosen (within reason). --- k8s/trillian/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index fc0b440002..9fe769ccde 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -96,7 +96,7 @@ Create a new cluster from the command line: export CLUSTER=trillian-cluster export ZONE=us-west1-a -gcloud container clusters create "$CLUSTER" --zone "$ZONE" +gcloud container clusters create "$CLUSTER" --zone "$ZONE" --enable-autoscaling ``` Configure `kubectl` to connect to the new cluster. From 317e12b2652102829668ae1b9a9386e8eeca7ee8 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Thu, 29 Nov 2018 17:46:18 +0000 Subject: [PATCH 04/40] Remove prometheus-to-sd Eliminates the need to communicate a GCP service account's private key to the logserver and logsigner containers. Metrics can still be obtained by setting up a Prometheus server and configuring it to monitor the /metrics endpoints. --- k8s/trillian/Makefile | 5 ---- k8s/trillian/README.md | 6 ----- k8s/trillian/manifest/logserver.yaml | 38 +--------------------------- k8s/trillian/manifest/logsigner.yaml | 33 +----------------------- 4 files changed, 2 insertions(+), 80 deletions(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index 45b4820932..fc48375122 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -23,10 +23,6 @@ ifdef IMAGE_ETCD_OPERATOR IMAGE_ETCD_OPERATOR_FIELD = , "IMAGE_ETCD_OPERATOR": "$(IMAGE_ETCD_OPERATOR)" endif -ifdef IMAGE_PROMETHEUS_TO_SD - IMAGE_PROMETHEUS_TO_SD_FIELD = , "IMAGE_PROMETHEUS_TO_SD": "$(IMAGE_PROMETHEUS_TO_SD)" -endif - APP_PARAMETERS ?= { \ "NAME": "$(NAME)", \ "NAMESPACE": "$(NAMESPACE)" \ @@ -34,7 +30,6 @@ APP_PARAMETERS ?= { \ $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ $(IMAGE_MYSQL_FIELD) \ $(IMAGE_ETCD_OPERATOR) \ - $(IMAGE_PROMETHEUS_TO_SD_FIELD) \ } APP_TEST_PARAMETERS ?= {} diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 9fe769ccde..a3e5102436 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -49,12 +49,6 @@ Popular open stacks on Kubernetes packaged by Google. * **Etcd Cluster** - used to handle rate-limiting/quota and master election. -* **Prometheus** - used for monitoring Trillian's health. This Trillian - installer will automatically forward monitoring data to StackDriver, but you - can alternatively - [install Prometheus and Grafana](https://pantheon.corp.google.com/marketplace/details/google/prometheus) - and use those instead. - ## Configuration * The number of Trillian log server and signer replicas is specified by the diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml index 03daa3d5b4..b2dcca1dc0 100644 --- a/k8s/trillian/manifest/logserver.yaml +++ b/k8s/trillian/manifest/logserver.yaml @@ -31,10 +31,6 @@ items: metadata: labels: *LogServerLabels spec: - volumes: - - name: google-cloud-key - secret: - secretName: trillian-key restartPolicy: Always containers: - name: trillian-logserver @@ -54,9 +50,6 @@ items: name: ${NAME}-config - secretRef: name: ${NAME}-secrets - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/var/secrets/google/key.json" image: $IMAGE_TRILLIAN_LOGSERVER resources: limits: @@ -75,33 +68,4 @@ items: name: rpc - containerPort: 8091 name: metrics - volumeMounts: - - name: google-cloud-key - mountPath: /var/secrets/google - - name: prometheus-to-sd - image: $IMAGE_PROMETHEUS_TO_SD - ports: - - name: profiler - containerPort: 6060 - command: - - /monitor - - --stackdriver-prefix=custom.googleapis.com - - --source=logserver:http://localhost:8091/metrics - - --pod-id=$(POD_NAME) - - --namespace-id=$(POD_NAMESPACE) - - --scrape-interval=5s - - --export-interval=60s - resources: - limits: - cpu: 20m - requests: - cpu: 20m - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index cd83a83c93..be7e741b80 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -30,10 +30,6 @@ items: metadata: labels: *LogSignerLabels spec: - volumes: - - name: google-cloud-key - secret: - secretName: trillian-key restartPolicy: Always containers: - name: trillian-logsigner @@ -56,9 +52,6 @@ items: name: ${NAME}-config - secretRef: name: ${NAME}-secrets - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/var/secrets/google/key.json" image: $IMAGE_TRILLIAN_LOGSIGNER resources: limits: @@ -74,28 +67,4 @@ items: timeoutSeconds: 5 ports: - containerPort: 8091 - volumeMounts: - - name: google-cloud-key - mountPath: /var/secrets/google - - name: prometheus-to-sd - image: $IMAGE_PROMETHEUS_TO_SD - ports: - - name: profiler - containerPort: 6060 - command: - - /monitor - - --stackdriver-prefix=custom.googleapis.com - - --source=logsigner:http://localhost:8091/metrics - - --pod-id=$(POD_NAME) - - --namespace-id=$(POD_NAMESPACE) - - --scrape-interval=5s - - --export-interval=60s - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + From 36bf9af350e8f724972abb7dcf0268e03142888e Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Fri, 30 Nov 2018 23:22:07 +0000 Subject: [PATCH 05/40] Regenerate cloudbuild.yaml --- cloudbuild.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 2174ffdb8e..e49a288480 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -303,6 +303,24 @@ steps: - -j4 - app/verify +- id: Verify trillian + name: gcr.io/cloud-marketplace-tools/k8s/dev:local + waitFor: + - Copy kubectl Credentials + - Copy gcloud Credentials + - Pull Dev Image + env: + - 'KUBE_CONFIG=/workspace/.kube' + - 'GCLOUD_CONFIG=/workspace/.config/gcloud' + # Use local Docker network named cloudbuild as described here: + # https://cloud.google.com/cloud-build/docs/overview#build_configuration_and_build_steps + - 'EXTRA_DOCKER_PARAMS=--net cloudbuild' + dir: k8s/trillian + args: + - make + - -j4 + - app/verify + - id: Verify wordpress name: gcr.io/cloud-marketplace-tools/k8s/dev:local waitFor: From e9c329f15fd09d8c0f8cbcab9e8a9b03679fa033 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 3 Dec 2018 12:05:52 +0000 Subject: [PATCH 06/40] Add machine type and min/max nodes flags to cluster creation command A machine with more CPUs than default is required to meet the requirements of some pods. A minimum of 3 nodes is required. --- k8s/trillian/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index a3e5102436..90d407a729 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -90,7 +90,9 @@ Create a new cluster from the command line: export CLUSTER=trillian-cluster export ZONE=us-west1-a -gcloud container clusters create "$CLUSTER" --zone "$ZONE" --enable-autoscaling +gcloud container clusters create "$CLUSTER" \ + --zone "$ZONE" --machine-type=n1-standard-2 \ + --enable-autoscaling --min-nodes=3 --max-nodes=10 ``` Configure `kubectl` to connect to the new cluster. From 02323e59f79b3022ab13c4f0c0c6913d36f0ae6e Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 11:50:51 +0000 Subject: [PATCH 07/40] Correct name of IMAGE_ETCD_OPERATOR_FIELD in APP_PARAMETERS in Makefile --- k8s/trillian/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index fc48375122..10b1bba9f0 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -29,7 +29,7 @@ APP_PARAMETERS ?= { \ $(IMAGE_TRILLIAN_LOGSERVER_FIELD) \ $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ $(IMAGE_MYSQL_FIELD) \ - $(IMAGE_ETCD_OPERATOR) \ + $(IMAGE_ETCD_OPERATOR_FIELD) \ } APP_TEST_PARAMETERS ?= {} @@ -90,4 +90,3 @@ app/build:: .build/trillian/db_server \ "$(REGISTRY)/trillian/log_signer:$(TAG)" docker push "$(REGISTRY)/trillian/log_signer:$(TAG)" @touch "$@" - From 3565b13e582926f91a901583fef586f9a7d225da Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 11:51:19 +0000 Subject: [PATCH 08/40] Add missing EtcdCluster entry to app CRD componentKinds --- k8s/trillian/manifest/app.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/trillian/manifest/app.yaml b/k8s/trillian/manifest/app.yaml index 059626f96f..d8aac3634c 100644 --- a/k8s/trillian/manifest/app.yaml +++ b/k8s/trillian/manifest/app.yaml @@ -44,6 +44,8 @@ spec: componentKinds: - group: apps/v1 kind: Deployment + - group: etcd.database.coreos.com/v1beta2 + kind: EtcdCluster - group: v1 kind: ConfigMap - group: v1 From e2b02575302e20da0cbc1b37d38c8e394057f6ff Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 11:51:58 +0000 Subject: [PATCH 09/40] Remove braces from environment variable names Previously, there was a mix of braces and not-braces. This makes it consistent. --- k8s/trillian/manifest/logserver.yaml | 5 ++--- k8s/trillian/manifest/logsigner.yaml | 5 ++--- k8s/trillian/manifest/mysql.yaml | 10 +++++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml index b2dcca1dc0..f2c5a9e9d2 100644 --- a/k8s/trillian/manifest/logserver.yaml +++ b/k8s/trillian/manifest/logserver.yaml @@ -47,9 +47,9 @@ items: ] envFrom: - configMapRef: - name: ${NAME}-config + name: $NAME-config - secretRef: - name: ${NAME}-secrets + name: $NAME-secrets image: $IMAGE_TRILLIAN_LOGSERVER resources: limits: @@ -68,4 +68,3 @@ items: name: rpc - containerPort: 8091 name: metrics - diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index be7e741b80..1b3d652d70 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -49,9 +49,9 @@ items: ] envFrom: - configMapRef: - name: ${NAME}-config + name: $NAME-config - secretRef: - name: ${NAME}-secrets + name: $NAME-secrets image: $IMAGE_TRILLIAN_LOGSIGNER resources: limits: @@ -67,4 +67,3 @@ items: timeoutSeconds: 5 ports: - containerPort: 8091 - diff --git a/k8s/trillian/manifest/mysql.yaml b/k8s/trillian/manifest/mysql.yaml index c71ffd6173..d958aa8279 100644 --- a/k8s/trillian/manifest/mysql.yaml +++ b/k8s/trillian/manifest/mysql.yaml @@ -10,7 +10,7 @@ items: app.kubernetes.io/component: mysql type: Opaque stringData: - STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp(${NAME}-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE + STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp($NAME-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE - apiVersion: v1 kind: Secret metadata: @@ -18,10 +18,10 @@ items: labels: *MySQLLabels type: Opaque stringData: - MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD}" - MYSQL_USER: "${MYSQL_TRILLIAN_USER}" - MYSQL_PASSWORD: "${MYSQL_TRILLIAN_PASSWORD}" - MYSQL_DATABASE: "${MYSQL_TRILLIAN_DATABASE}" + MYSQL_ROOT_PASSWORD: "$MYSQL_ROOT_PASSWORD" + MYSQL_USER: "$MYSQL_TRILLIAN_USER" + MYSQL_PASSWORD: "$MYSQL_TRILLIAN_PASSWORD" + MYSQL_DATABASE: "$MYSQL_TRILLIAN_DATABASE" - apiVersion: v1 kind: Service metadata: From a2d1083be3692772989b71a8330db3ec7659f11d Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 13:51:22 +0000 Subject: [PATCH 10/40] Makefile: Use v1.2.1 of Trillian instead of latest version This can be overridden using the TAG parameter. --- k8s/trillian/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index 10b1bba9f0..b8a8bf10fa 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -3,7 +3,7 @@ include ../crd.Makefile include ../gcloud.Makefile include ../var.Makefile -TAG ?= latest +TAG ?= 1.2.1 APP_DEPLOYER_IMAGE ?= $(REGISTRY)/trillian/deployer:$(TAG) NAME ?= trillian-1 From 287b90dd759609d7f305da3729329460a8ad824d Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:37:34 +0000 Subject: [PATCH 11/40] Add/improve descriptions in trillian/schema.yaml --- k8s/trillian/schema.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index af44973542..20c58cf2e7 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -23,14 +23,14 @@ properties: type: IMAGE TRILLIAN_LOGSERVER_REPLICAS: type: integer - title: Trillian replicas - description: The number of Trillian log servers to run + title: Trillian log server replicas + description: The number of Trillian log servers to run. Increasing this provides greater capacity for serving requests. default: 4 minimum: 1 TRILLIAN_LOGSIGNER_REPLICAS: type: integer - title: Trillian replicas - description: The number of Trillian log signers to run + title: Trillian log signer replicas + description: The number of Trillian log signers to run. Increasing this allows for sequencing and signing more logs in parallel. default: 2 minimum: 1 IMAGE_ETCD_OPERATOR: @@ -41,6 +41,7 @@ properties: ETCD_CLUSTER_SIZE: type: integer title: Etcd cluster size + description: Should be an odd number between 3-7 (https://coreos.com/etcd/docs/3.2.13/faq.html#deployment). default: 3 minimum: 3 ETCD_VERSION: From 45775eee34f8dd71ee82892177bfb90c302619f3 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:39:08 +0000 Subject: [PATCH 12/40] Set maximum for etcd cluster size in trillian/schema.yaml The recommended cluster size is 3-7, but the FAQ gives failure tolerance information up to 9, so allow that large a cluster (see https://coreos.com/etcd/docs/3.2.13/faq.html#deployment). --- k8s/trillian/schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 20c58cf2e7..c74c5f183f 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -44,6 +44,7 @@ properties: description: Should be an odd number between 3-7 (https://coreos.com/etcd/docs/3.2.13/faq.html#deployment). default: 3 minimum: 3 + maximum: 9 ETCD_VERSION: type: string title: Etcd version From 8b501cc9d719997c6befd30a7638fb2dcdda7b1c Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:41:53 +0000 Subject: [PATCH 13/40] Add clusterConstraints to trillian/schema.yaml A pool of 3 n1-standard-2 nodes are required to host Trillian with the default number of replicas. --- k8s/trillian/schema.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index c74c5f183f..08af91171d 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -133,3 +133,12 @@ required: - MYSQL_TRILLIAN_USER - MYSQL_TRILLIAN_PASSWORD - MYSQL_TRILLIAN_DATABASE +x-google-marketplace: + clusterConstraints: + resources: + - replicas: 3 + requests: + cpu: 1 + affinity: + simpleNodeAffinity: + type: REQUIRE_ONE_NODE_PER_REPLICA From 2d8e2d6aafdce29e064a8a306c144590cc7cf2a6 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:57:06 +0000 Subject: [PATCH 14/40] Remove -cluster-wide option from etcd-operator The etcd-operator only needs to manage an Etcd cluster for the namespace used by Trillian. --- k8s/trillian/manifest/etcd.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 4e7752f620..5629674d51 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -32,7 +32,7 @@ items: containers: - name: trillian-etcd-operator image: $IMAGE_ETCD_OPERATOR - command: ["etcd-operator", "-cluster-wide"] + command: ["etcd-operator"] env: - name: MY_POD_NAMESPACE valueFrom: @@ -47,8 +47,6 @@ items: metadata: name: "$NAME-etcd-cluster" labels: *EtcdLabels - annotations: - etcd.database.coreos.com/scope: clusterwide spec: serviceAccountName: $ETCD_SERVICE_ACCOUNT size: $ETCD_CLUSTER_SIZE From 8a35c4f5a21231175bf2a051981397e0263f5545 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:57:38 +0000 Subject: [PATCH 15/40] Provide links in trillian/schema.yaml to the Etcd RBAC docs and examples --- k8s/trillian/schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 08af91171d..9d6c7ef15e 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -52,10 +52,11 @@ properties: ETCD_SERVICE_ACCOUNT: type: string title: Etcd Service Account - description: Service account with privileges required by etcd-operator + description: Service account with privileges required by etcd-operator (see https://github.com/coreos/etcd-operator/blob/v0.9.1/doc/user/rbac.md) x-google-marketplace: type: SERVICE_ACCOUNT serviceAccount: + # Role from https://github.com/coreos/etcd-operator/blob/v0.9.1/example/rbac/cluster-role-template.yaml roles: - type: ClusterRole rulesType: CUSTOM From 5a0d2c9e81578c11f0d6dcbfe6e38d9d2202fe3c Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 7 Jan 2019 14:58:36 +0000 Subject: [PATCH 16/40] Add regex for ETCD_VERSION in trillian/schema.yaml Limit the Etcd version to 3.x.x, as anything other than v3 is likely to be incompatible. --- k8s/trillian/schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 9d6c7ef15e..1ad72865c6 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -49,6 +49,7 @@ properties: type: string title: Etcd version default: 3.2.13 + pattern: 3\.\d+\.\d+ ETCD_SERVICE_ACCOUNT: type: string title: Etcd Service Account From e6264a4e1c21047533f5d950d131e5b80c2ceea4 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Wed, 9 Jan 2019 17:20:24 +0000 Subject: [PATCH 17/40] Remove List wrapper around ConfigMap --- k8s/trillian/manifest/config.yaml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/k8s/trillian/manifest/config.yaml b/k8s/trillian/manifest/config.yaml index c949cee275..176c17614a 100644 --- a/k8s/trillian/manifest/config.yaml +++ b/k8s/trillian/manifest/config.yaml @@ -1,15 +1,12 @@ apiVersion: v1 -kind: List -items: -- apiVersion: v1 - kind: ConfigMap - metadata: - name: $NAME-config - labels: - app.kubernetes.io/name: "$NAME" - data: - STORAGE_SYSTEM: mysql - SIGNER_BATCH_SIZE: "500" - SIGNER_INTERVAL: "20ms" - SIGNER_NUM_SEQUENCERS: "10" - SIGNER_RESIGN_ODDS: "1000" +kind: ConfigMap +metadata: + name: $NAME-config + labels: + app.kubernetes.io/name: "$NAME" +data: + STORAGE_SYSTEM: mysql + SIGNER_BATCH_SIZE: "500" + SIGNER_INTERVAL: "20ms" + SIGNER_NUM_SEQUENCERS: "10" + SIGNER_RESIGN_ODDS: "1000" From 03beff37077080391897d3a720a65dd7013965ed Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Wed, 9 Jan 2019 17:27:56 +0000 Subject: [PATCH 18/40] Remove Lists and duplicate labels Marketplace tools do not work properly with List, so removing it. --- k8s/trillian/manifest/etcd.yaml | 109 +++++++++++---------- k8s/trillian/manifest/logserver.yaml | 140 ++++++++++++++------------- k8s/trillian/manifest/logsigner.yaml | 138 +++++++++++++------------- k8s/trillian/manifest/mysql.yaml | 116 ++++++++++++---------- 4 files changed, 268 insertions(+), 235 deletions(-) diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 5629674d51..5ce924a05c 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -1,53 +1,62 @@ apiVersion: v1 -kind: List -items: -- apiVersion: v1 - kind: Service - metadata: - name: $NAME-etcd-service - labels: &EtcdLabels +kind: Service +metadata: + name: $NAME-etcd-service + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd +spec: + type: NodePort + selector: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd + ports: + - protocol: TCP + port: 2379 + nodePort: 32379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $NAME-etcd-operator + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd +spec: + replicas: 1 + selector: + matchLabels: app.kubernetes.io/name: "$NAME" app.kubernetes.io/component: etcd - spec: - type: NodePort - selector: *EtcdLabels - ports: - - protocol: TCP - port: 2379 - nodePort: 32379 -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: $NAME-etcd-operator - labels: *EtcdLabels - spec: - replicas: 1 - selector: - matchLabels: *EtcdLabels - template: - metadata: - labels: *EtcdLabels - spec: - serviceAccountName: $ETCD_SERVICE_ACCOUNT - containers: - - name: trillian-etcd-operator - image: $IMAGE_ETCD_OPERATOR - command: ["etcd-operator"] - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name -- apiVersion: "etcd.database.coreos.com/v1beta2" - kind: "EtcdCluster" - metadata: - name: "$NAME-etcd-cluster" - labels: *EtcdLabels - spec: - serviceAccountName: $ETCD_SERVICE_ACCOUNT - size: $ETCD_CLUSTER_SIZE - version: $ETCD_VERSION + template: + metadata: + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd + spec: + serviceAccountName: $ETCD_SERVICE_ACCOUNT + containers: + - name: trillian-etcd-operator + image: $IMAGE_ETCD_OPERATOR + command: ["etcd-operator"] + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +--- +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdCluster" +metadata: + name: "$NAME-etcd-cluster" + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd +spec: + serviceAccountName: $ETCD_SERVICE_ACCOUNT + size: $ETCD_CLUSTER_SIZE + version: $ETCD_VERSION diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml index f2c5a9e9d2..9d9f65aaf8 100644 --- a/k8s/trillian/manifest/logserver.yaml +++ b/k8s/trillian/manifest/logserver.yaml @@ -1,70 +1,76 @@ apiVersion: v1 -kind: List -items: -- apiVersion: v1 - kind: Service - metadata: - name: $NAME-logserver-service - labels: &LogServerLabels +kind: Service +metadata: + name: $NAME-logserver-service + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logserver + annotations: + cloud.google.com/load-balancer-type: "Internal" +spec: + ports: + - name: grpclb + port: 8090 + - name: metrics + port: 8091 + selector: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logserver + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $NAME-logserver-deployment + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logserver +spec: + replicas: $TRILLIAN_LOGSERVER_REPLICAS + selector: + matchLabels: app.kubernetes.io/name: "$NAME" app.kubernetes.io/component: logserver - annotations: - cloud.google.com/load-balancer-type: "Internal" - spec: - ports: - - name: grpclb - port: 8090 - - name: metrics - port: 8091 - selector: *LogServerLabels - type: LoadBalancer -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: $NAME-logserver-deployment - labels: *LogServerLabels - spec: - replicas: $TRILLIAN_LOGSERVER_REPLICAS - selector: - matchLabels: *LogServerLabels - template: - metadata: - labels: *LogServerLabels - spec: - restartPolicy: Always - containers: - - name: trillian-logserver - args: [ - "$(STORAGE_FLAG)", - "--storage_system=$(STORAGE_SYSTEM)", - "--quota_system=etcd", - "--etcd_servers=$NAME-etcd-cluster-client:2379", - "--etcd_http_service=$NAME-logserver-http", - "--rpc_endpoint=0.0.0.0:8090", - "--http_endpoint=0.0.0.0:8091", - "--tracing", - "--alsologtostderr", - ] - envFrom: - - configMapRef: - name: $NAME-config - - secretRef: - name: $NAME-secrets - image: $IMAGE_TRILLIAN_LOGSERVER - resources: - limits: - cpu: "1.0" - requests: - cpu: "0.4" - livenessProbe: - httpGet: - path: /healthz - port: 8091 - failureThreshold: 3 - periodSeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8090 - name: rpc - - containerPort: 8091 - name: metrics + template: + metadata: + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logserver + spec: + restartPolicy: Always + containers: + - name: trillian-logserver + args: [ + "$(STORAGE_FLAG)", + "--storage_system=$(STORAGE_SYSTEM)", + "--quota_system=etcd", + "--etcd_servers=$NAME-etcd-cluster-client:2379", + "--etcd_http_service=$NAME-logserver-http", + "--rpc_endpoint=0.0.0.0:8090", + "--http_endpoint=0.0.0.0:8091", + "--tracing", + "--alsologtostderr", + ] + envFrom: + - configMapRef: + name: $NAME-config + - secretRef: + name: $NAME-secrets + image: $IMAGE_TRILLIAN_LOGSERVER + resources: + limits: + cpu: "1.0" + requests: + cpu: "0.4" + livenessProbe: + httpGet: + path: /healthz + port: 8091 + failureThreshold: 3 + periodSeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8090 + name: rpc + - containerPort: 8091 + name: metrics diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index 1b3d652d70..1925e42c81 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -1,69 +1,75 @@ apiVersion: v1 -kind: List -items: -- apiVersion: v1 - kind: Service - metadata: - name: $NAME-logsigner-service - labels: &LogSignerLabels +kind: Service +metadata: + name: $NAME-logsigner-service + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logsigner + annotations: + cloud.google.com/load-balancer-type: "Internal" +spec: + clusterIP: None + ports: + - name: metrics + port: 8092 + targetPort: 8091 + selector: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logsigner +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $NAME-logsigner-deployment + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logsigner +spec: + replicas: $TRILLIAN_LOGSIGNER_REPLICAS + selector: + matchLabels: app.kubernetes.io/name: "$NAME" app.kubernetes.io/component: logsigner - annotations: - cloud.google.com/load-balancer-type: "Internal" - spec: - clusterIP: None - ports: - - name: metrics - port: 8092 - targetPort: 8091 - selector: *LogSignerLabels -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: $NAME-logsigner-deployment - labels: *LogSignerLabels - spec: - replicas: $TRILLIAN_LOGSIGNER_REPLICAS - selector: - matchLabels: *LogSignerLabels - template: - metadata: - labels: *LogSignerLabels - spec: - restartPolicy: Always - containers: - - name: trillian-logsigner - args: [ - "$(STORAGE_FLAG)", - "--storage_system=$(STORAGE_SYSTEM)", - "--etcd_servers=$NAME-etcd-cluster-client:2379", - "--quota_system=etcd", - "--etcd_http_service=$NAME-logsigner-http", - "--http_endpoint=0.0.0.0:8091", - "--sequencer_guard_window=1s", - "--sequencer_interval=$(SIGNER_INTERVAL)", - "--num_sequencers=$(SIGNER_NUM_SEQUENCERS)", - "--batch_size=$(SIGNER_BATCH_SIZE)", - "--resign_odds=$(SIGNER_RESIGN_ODDS)", - "--alsologtostderr", - ] - envFrom: - - configMapRef: - name: $NAME-config - - secretRef: - name: $NAME-secrets - image: $IMAGE_TRILLIAN_LOGSIGNER - resources: - limits: - cpu: "1" - requests: - cpu: "1" - livenessProbe: - httpGet: - path: /healthz - port: 8091 - failureThreshold: 3 - periodSeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8091 + template: + metadata: + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: logsigner + spec: + restartPolicy: Always + containers: + - name: trillian-logsigner + args: [ + "$(STORAGE_FLAG)", + "--storage_system=$(STORAGE_SYSTEM)", + "--etcd_servers=$NAME-etcd-cluster-client:2379", + "--quota_system=etcd", + "--etcd_http_service=$NAME-logsigner-http", + "--http_endpoint=0.0.0.0:8091", + "--sequencer_guard_window=1s", + "--sequencer_interval=$(SIGNER_INTERVAL)", + "--num_sequencers=$(SIGNER_NUM_SEQUENCERS)", + "--batch_size=$(SIGNER_BATCH_SIZE)", + "--resign_odds=$(SIGNER_RESIGN_ODDS)", + "--alsologtostderr", + ] + envFrom: + - configMapRef: + name: $NAME-config + - secretRef: + name: $NAME-secrets + image: $IMAGE_TRILLIAN_LOGSIGNER + resources: + limits: + cpu: "1" + requests: + cpu: "1" + livenessProbe: + httpGet: + path: /healthz + port: 8091 + failureThreshold: 3 + periodSeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8091 diff --git a/k8s/trillian/manifest/mysql.yaml b/k8s/trillian/manifest/mysql.yaml index d958aa8279..fb9e5af24b 100644 --- a/k8s/trillian/manifest/mysql.yaml +++ b/k8s/trillian/manifest/mysql.yaml @@ -1,55 +1,67 @@ apiVersion: v1 -kind: List -items: -- apiVersion: v1 - kind: Secret - metadata: - name: $NAME-secrets - labels: &MySQLLabels +kind: Secret +metadata: + name: $NAME-secrets + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +type: Opaque +stringData: + STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp($NAME-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE +--- +apiVersion: v1 +kind: Secret +metadata: + name: $NAME-mysql-secrets + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +type: Opaque +stringData: + MYSQL_ROOT_PASSWORD: "$MYSQL_ROOT_PASSWORD" + MYSQL_USER: "$MYSQL_TRILLIAN_USER" + MYSQL_PASSWORD: "$MYSQL_TRILLIAN_PASSWORD" + MYSQL_DATABASE: "$MYSQL_TRILLIAN_DATABASE" +--- +apiVersion: v1 +kind: Service +metadata: + name: $NAME-mysql-service + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +spec: + clusterIP: None + ports: + - name: mysql + port: 3306 + selector: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $NAME-mysql + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +spec: + replicas: 1 + selector: + matchLabels: app.kubernetes.io/name: "$NAME" app.kubernetes.io/component: mysql - type: Opaque - stringData: - STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp($NAME-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE -- apiVersion: v1 - kind: Secret - metadata: - name: $NAME-mysql-secrets - labels: *MySQLLabels - type: Opaque - stringData: - MYSQL_ROOT_PASSWORD: "$MYSQL_ROOT_PASSWORD" - MYSQL_USER: "$MYSQL_TRILLIAN_USER" - MYSQL_PASSWORD: "$MYSQL_TRILLIAN_PASSWORD" - MYSQL_DATABASE: "$MYSQL_TRILLIAN_DATABASE" -- apiVersion: v1 - kind: Service - metadata: - name: $NAME-mysql-service - labels: *MySQLLabels - spec: - clusterIP: None - ports: - - name: mysql - port: 3306 - selector: *MySQLLabels -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: $NAME-mysql - labels: *MySQLLabels - spec: - replicas: 1 - selector: - matchLabels: *MySQLLabels - template: - metadata: - labels: *MySQLLabels - spec: - containers: - - image: $IMAGE_MYSQL - name: mysql - envFrom: - - secretRef: - name: $NAME-mysql-secrets - restartPolicy: Always + template: + metadata: + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql + spec: + containers: + - image: $IMAGE_MYSQL + name: mysql + envFrom: + - secretRef: + name: $NAME-mysql-secrets + restartPolicy: Always From a87a3c0c6524373aba30f6fe5df5e9f942b3d143 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 21 Jan 2019 13:47:31 +0000 Subject: [PATCH 19/40] Mirror etcd-operator image in $REGISTRY --- k8s/trillian/Makefile | 12 ++++++++++++ k8s/trillian/schema.yaml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index b8a8bf10fa..c0f956a2bc 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -5,6 +5,7 @@ include ../var.Makefile TAG ?= 1.2.1 APP_DEPLOYER_IMAGE ?= $(REGISTRY)/trillian/deployer:$(TAG) +ETCD_OPERATOR_IMAGE ?= quay.io/coreos/etcd-operator:v0.9.1 NAME ?= trillian-1 ifdef IMAGE_TRILLIAN_LOGSERVER @@ -36,6 +37,7 @@ APP_TEST_PARAMETERS ?= {} app/build:: .build/trillian/db_server \ .build/trillian/deployer \ + .build/trillian/etcd_operator \ .build/trillian/log_server \ .build/trillian/log_signer @@ -72,6 +74,16 @@ app/build:: .build/trillian/db_server \ @touch "$@" +.build/trillian/etcd_operator: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull "$(ETCD_OPERATOR_IMAGE)" + docker tag "$(ETCD_OPERATOR_IMAGE)" \ + "$(REGISTRY)/trillian/etcd_operator:$(TAG)" + docker push "$(REGISTRY)/trillian/etcd_operator:$(TAG)" + @touch "$@" + + .build/trillian/log_server: .build/var/REGISTRY \ .build/var/TAG \ | .build/trillian diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 1ad72865c6..80a5e5e829 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -35,7 +35,7 @@ properties: minimum: 1 IMAGE_ETCD_OPERATOR: type: string - default: quay.io/coreos/etcd-operator:v0.9.1 + default: $REGISTRY/etcd_operator:$TAG x-google-marketplace: type: IMAGE ETCD_CLUSTER_SIZE: From 2e239cc08240f81138aed22c232c0c8cd5d761d1 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 21 Jan 2019 13:48:05 +0000 Subject: [PATCH 20/40] Add 500GiB of persistent storage for MySQL database --- k8s/trillian/manifest/mysql.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/k8s/trillian/manifest/mysql.yaml b/k8s/trillian/manifest/mysql.yaml index fb9e5af24b..2dc320621b 100644 --- a/k8s/trillian/manifest/mysql.yaml +++ b/k8s/trillian/manifest/mysql.yaml @@ -64,4 +64,26 @@ spec: envFrom: - secretRef: name: $NAME-mysql-secrets + volumeMounts: + - name: $NAME-mysql-storage + mountPath: /var/lib/mysql + subPath: mysql restartPolicy: Always + volumes: + - name: $NAME-mysql-storage + persistentVolumeClaim: + claimName: $NAME-mysql-storage-claim +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: $NAME-mysql-storage-claim + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: mysql +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi From 7b7fbf8633c9513f413bec80dfd0668d0db7b36b Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 21 Jan 2019 13:48:26 +0000 Subject: [PATCH 21/40] Remove --resign_odds flag from logsigner This flag has been removed from the binary. --- k8s/trillian/manifest/logsigner.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index 1925e42c81..c3e8082f95 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -50,7 +50,6 @@ spec: "--sequencer_interval=$(SIGNER_INTERVAL)", "--num_sequencers=$(SIGNER_NUM_SEQUENCERS)", "--batch_size=$(SIGNER_BATCH_SIZE)", - "--resign_odds=$(SIGNER_RESIGN_ODDS)", "--alsologtostderr", ] envFrom: From 547688ecd1e2920e38f600bb2cf13b4c536c5faf Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 28 Jan 2019 15:00:02 +0000 Subject: [PATCH 22/40] Increase default size of Etcd cluster to 5 Makes it more resilient to etcd pods restarting. --- k8s/trillian/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 80a5e5e829..b08e794377 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -42,7 +42,7 @@ properties: type: integer title: Etcd cluster size description: Should be an odd number between 3-7 (https://coreos.com/etcd/docs/3.2.13/faq.html#deployment). - default: 3 + default: 5 minimum: 3 maximum: 9 ETCD_VERSION: From dcc0db6c30f168cb81d7f484909038f52040482d Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Fri, 1 Feb 2019 13:54:14 +0000 Subject: [PATCH 23/40] Do not delete Secrets when uninstalling Trillian This contains things like the MySQL root password, which is still useful after uninstallation because the MySQL database is preserved on a persistent volume that isn't deleted by uninstallation. This makes it possible to re-install Trillian without losing access to the database. --- k8s/trillian/manifest/app.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/k8s/trillian/manifest/app.yaml b/k8s/trillian/manifest/app.yaml index d8aac3634c..364d0428b7 100644 --- a/k8s/trillian/manifest/app.yaml +++ b/k8s/trillian/manifest/app.yaml @@ -48,7 +48,5 @@ spec: kind: EtcdCluster - group: v1 kind: ConfigMap - - group: v1 - kind: Secret - group: v1 kind: Service From d91c6d83246e0cc2f7a33f8526d094733776facd Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Fri, 1 Feb 2019 13:56:45 +0000 Subject: [PATCH 24/40] Remove EtcdCluster from Trillian app.yaml The deployer does not wait for etcd-operator to install the CRD for EtcdCluster before trying to use it, and so fails. Unfortunately, by removing EtcdCluster from componentKinds, ownership information will not be setup on the EtcdCluster and so it won't be uninstalled along with Trillian. https://github.com/GoogleCloudPlatform/marketplace-k8s-app-tools/issues/303 tracks a feature request for supporting Kubernetes operators that install a CRD. --- k8s/trillian/manifest/app.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/k8s/trillian/manifest/app.yaml b/k8s/trillian/manifest/app.yaml index 364d0428b7..691f1bc183 100644 --- a/k8s/trillian/manifest/app.yaml +++ b/k8s/trillian/manifest/app.yaml @@ -44,8 +44,6 @@ spec: componentKinds: - group: apps/v1 kind: Deployment - - group: etcd.database.coreos.com/v1beta2 - kind: EtcdCluster - group: v1 kind: ConfigMap - group: v1 From 9afd70bea5a2aea017f15676c3d9f9f3ef6ea715 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Tue, 19 Feb 2019 16:10:34 +0000 Subject: [PATCH 25/40] Make the Busybox image used by etcd-operator configurable --- k8s/trillian/Makefile | 16 +++++++++++++++- k8s/trillian/manifest/etcd.yaml | 2 ++ k8s/trillian/schema.yaml | 8 ++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index c0f956a2bc..2b60944713 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -5,6 +5,9 @@ include ../var.Makefile TAG ?= 1.2.1 APP_DEPLOYER_IMAGE ?= $(REGISTRY)/trillian/deployer:$(TAG) +# Default Busybox image used by etcd-operator: +# https://github.com/coreos/etcd-operator/blob/16f0e1b3693483061f2a8252c0e361b06d216424/pkg/util/k8sutil/k8sutil.go#L64 +BUSYBOX_IMAGE ?= busybox:1.28.0-glibc ETCD_OPERATOR_IMAGE ?= quay.io/coreos/etcd-operator:v0.9.1 NAME ?= trillian-1 @@ -35,7 +38,8 @@ APP_PARAMETERS ?= { \ APP_TEST_PARAMETERS ?= {} -app/build:: .build/trillian/db_server \ +app/build:: .build/trillian/busybox \ + .build/trillian/db_server \ .build/trillian/deployer \ .build/trillian/etcd_operator \ .build/trillian/log_server \ @@ -46,6 +50,16 @@ app/build:: .build/trillian/db_server \ mkdir -p "$@" +.build/trillian/busybox: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull "$(BUSYBOX_IMAGE)" + docker tag "$(BUSYBOX_IMAGE)" \ + "$(REGISTRY)/trillian/busybox:$(TAG)" + docker push "$(REGISTRY)/trillian/busybox:$(TAG)" + @touch "$@" + + .build/trillian/db_server: .build/var/REGISTRY \ .build/var/TAG \ | .build/trillian diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 5ce924a05c..d2c7c53852 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -60,3 +60,5 @@ spec: serviceAccountName: $ETCD_SERVICE_ACCOUNT size: $ETCD_CLUSTER_SIZE version: $ETCD_VERSION + pod: + busyboxImage: $IMAGE_BUSYBOX diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index b08e794377..bdcfaff7fc 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -38,6 +38,13 @@ properties: default: $REGISTRY/etcd_operator:$TAG x-google-marketplace: type: IMAGE + IMAGE_BUSYBOX: + type: string + title: Docker image to use for Etcd init containers + description: Must be a busybox image. + default: $REGISTRY/busybox:$TAG + x-google-marketplace: + type: IMAGE ETCD_CLUSTER_SIZE: type: integer title: Etcd cluster size @@ -127,6 +134,7 @@ required: - TRILLIAN_LOGSERVER_REPLICAS - TRILLIAN_LOGSIGNER_REPLICAS - IMAGE_ETCD_OPERATOR +- IMAGE_BUSYBOX - ETCD_CLUSTER_SIZE - ETCD_VERSION - ETCD_SERVICE_ACCOUNT From d72653b4d1fb013464e4a0bbd1247301bf47dfc6 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Tue, 19 Feb 2019 16:11:22 +0000 Subject: [PATCH 26/40] Prefer Etcd cluster members to be scheduled on different Kubernetes nodes This minimises the number that are lost when a node goes down, improving cluster resilience. --- k8s/trillian/manifest/etcd.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index d2c7c53852..cf988deb43 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -62,3 +62,15 @@ spec: version: $ETCD_VERSION pod: busyboxImage: $IMAGE_BUSYBOX + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: etcd_cluster + operator: In + values: + - $NAME-etcd-cluster + topologyKey: kubernetes.io/hostname From 7c5ce2500ee9a2ca626ff11017ac0cb2a1179ab7 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Tue, 19 Feb 2019 16:13:13 +0000 Subject: [PATCH 27/40] Make the image used by Etcd cluster members configurable --- k8s/trillian/Makefile | 21 +++++++++++++++++---- k8s/trillian/manifest/etcd.yaml | 1 + k8s/trillian/schema.yaml | 17 ++++++++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index 2b60944713..0f0f560505 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -3,13 +3,15 @@ include ../crd.Makefile include ../gcloud.Makefile include ../var.Makefile +NAME ?= trillian-1 TAG ?= 1.2.1 APP_DEPLOYER_IMAGE ?= $(REGISTRY)/trillian/deployer:$(TAG) # Default Busybox image used by etcd-operator: # https://github.com/coreos/etcd-operator/blob/16f0e1b3693483061f2a8252c0e361b06d216424/pkg/util/k8sutil/k8sutil.go#L64 BUSYBOX_IMAGE ?= busybox:1.28.0-glibc ETCD_OPERATOR_IMAGE ?= quay.io/coreos/etcd-operator:v0.9.1 -NAME ?= trillian-1 +ETCD_REPOSITORY ?= quay.io/coreos/etcd +ETCD_VERSION ?= 3.2.13 ifdef IMAGE_TRILLIAN_LOGSERVER IMAGE_TRILLIAN_LOGSERVER_FIELD = , "IMAGE_TRILLIAN_LOGSERVER": "$(IMAGE_TRILLIAN_LOGSERVER)" @@ -23,8 +25,8 @@ ifdef IMAGE_MYSQL IMAGE_MYSQL_FIELD = , "IMAGE_MYSQL": "$(IMAGE_MYSQL)" endif -ifdef IMAGE_ETCD_OPERATOR - IMAGE_ETCD_OPERATOR_FIELD = , "IMAGE_ETCD_OPERATOR": "$(IMAGE_ETCD_OPERATOR)" +ifdef ETCD_VERSION + ETCD_VERSION_FIELD = , "ETCD_VERSION": "$(ETCD_VERSION)" endif APP_PARAMETERS ?= { \ @@ -33,7 +35,7 @@ APP_PARAMETERS ?= { \ $(IMAGE_TRILLIAN_LOGSERVER_FIELD) \ $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ $(IMAGE_MYSQL_FIELD) \ - $(IMAGE_ETCD_OPERATOR_FIELD) \ + $(ETCD_VERSION_FIELD) \ } APP_TEST_PARAMETERS ?= {} @@ -42,6 +44,7 @@ app/build:: .build/trillian/busybox \ .build/trillian/db_server \ .build/trillian/deployer \ .build/trillian/etcd_operator \ + .build/trillian/etcd \ .build/trillian/log_server \ .build/trillian/log_signer @@ -98,6 +101,16 @@ app/build:: .build/trillian/busybox \ @touch "$@" +.build/trillian/etcd: .build/var/REGISTRY \ + .build/var/TAG \ + | .build/trillian + docker pull "$(ETCD_REPOSITORY):v$(ETCD_VERSION)" + docker tag "$(ETCD_REPOSITORY):v$(ETCD_VERSION)" \ + "$(REGISTRY)/trillian/etcd:v$(ETCD_VERSION)" + docker push "$(REGISTRY)/trillian/etcd:v$(ETCD_VERSION)" + @touch "$@" + + .build/trillian/log_server: .build/var/REGISTRY \ .build/var/TAG \ | .build/trillian diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index cf988deb43..567a0a1aec 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -59,6 +59,7 @@ metadata: spec: serviceAccountName: $ETCD_SERVICE_ACCOUNT size: $ETCD_CLUSTER_SIZE + repository: $ETCD_REPOSITORY version: $ETCD_VERSION pod: busyboxImage: $IMAGE_BUSYBOX diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index bdcfaff7fc..56860a5ff4 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -35,6 +35,8 @@ properties: minimum: 1 IMAGE_ETCD_OPERATOR: type: string + title: Docker image to use for etcd-operator + description: See https://github.com/coreos/etcd-operator. default: $REGISTRY/etcd_operator:$TAG x-google-marketplace: type: IMAGE @@ -45,6 +47,15 @@ properties: default: $REGISTRY/busybox:$TAG x-google-marketplace: type: IMAGE + ETCD_REPOSITORY: + type: string + description: Docker repository where Etcd images are stored + default: $REGISTRY/etcd + ETCD_VERSION: + type: string + description: The version of Etcd to use. A Docker image for this version must exist in the Etcd repository. + default: 3.2.13 + pattern: 3\.\d+\.\d+ ETCD_CLUSTER_SIZE: type: integer title: Etcd cluster size @@ -52,11 +63,6 @@ properties: default: 5 minimum: 3 maximum: 9 - ETCD_VERSION: - type: string - title: Etcd version - default: 3.2.13 - pattern: 3\.\d+\.\d+ ETCD_SERVICE_ACCOUNT: type: string title: Etcd Service Account @@ -136,6 +142,7 @@ required: - IMAGE_ETCD_OPERATOR - IMAGE_BUSYBOX - ETCD_CLUSTER_SIZE +- ETCD_REPOSITORY - ETCD_VERSION - ETCD_SERVICE_ACCOUNT - IMAGE_MYSQL From 3c4e7e6c3662008a4707cd2ae04c10f37a84c4c8 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Tue, 19 Feb 2019 16:13:39 +0000 Subject: [PATCH 28/40] Make the MySQL passwords configurable via Makefile --- k8s/trillian/Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index 0f0f560505..d8541bfbfb 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -25,6 +25,14 @@ ifdef IMAGE_MYSQL IMAGE_MYSQL_FIELD = , "IMAGE_MYSQL": "$(IMAGE_MYSQL)" endif +ifdef MYSQL_ROOT_PASSWORD + MYSQL_ROOT_PASSWORD_FIELD = , "MYSQL_ROOT_PASSWORD": "$(MYSQL_ROOT_PASSWORD)" +endif + +ifdef MYSQL_TRILLIAN_PASSWORD + MYSQL_TRILLIAN_PASSWORD_FIELD = , "MYSQL_TRILLIAN_PASSWORD": "$(MYSQL_TRILLIAN_PASSWORD)" +endif + ifdef ETCD_VERSION ETCD_VERSION_FIELD = , "ETCD_VERSION": "$(ETCD_VERSION)" endif @@ -35,6 +43,8 @@ APP_PARAMETERS ?= { \ $(IMAGE_TRILLIAN_LOGSERVER_FIELD) \ $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ $(IMAGE_MYSQL_FIELD) \ + $(MYSQL_ROOT_PASSWORD_FIELD) \ + $(MYSQL_TRILLIAN_PASSWORD_FIELD) \ $(ETCD_VERSION_FIELD) \ } APP_TEST_PARAMETERS ?= {} From 968c54d0bf275a38788cebbbbabb82e7010160a9 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Thu, 21 Feb 2019 16:14:30 +0000 Subject: [PATCH 29/40] Add README sections on scaling, backup & restore, and uninstallation --- k8s/trillian/README.md | 142 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 90d407a729..e1f6c5dd54 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -174,3 +174,145 @@ kubectl port-forward --namespace ${NAMESPACE} ${NAME}-trillian-logserver 8090 ``` You can access the Trillian API at `localhost:8090`. + +# Scaling the Trillian app + +## Scaling the servers + +By default, the Trillian app is deployed using 4 server replicas. To change the +number of replicas, use the following commands. Change `$LOGSERVER_REPLICAS` to +the number of replicas you require. Increasing the number will increase capacity +for serving requests. + +```shell +LOGSERVER_REPLICAS=4 + +kubectl scale "deployments/$NAME-logserver-deployment" \ + --namespace "$NAMESPACE" --replicas=$LOGSERVER_REPLICAS +``` + +## Scaling the signers + +By default, the Trillian app is deployed using 2 signer replicas. To change the +number of replicas, use the following commands. Change `$LOGSIGNER_REPLICAS` to +the number of replicas you require. Increasing the number will increase the +number of logs that can be signed in parallel. + +```shell +LOGSIGNER_REPLICAS=2 + +kubectl scale "deployments/$NAME-logsigner-deployment" \ + --namespace "$NAMESPACE" --replicas=$LOGSIGNER_REPLICAS +``` + +## Scaling etcd + +Shrinking an etcd cluster is not recommended, but increasing the cluster size is +safe. This can be achieved by the following command. Change `$ETCD_CLUSTER_SIZE` +to the desired cluster size. + +```shell +ECTD_CLUSTER_SIZE=5 + +kubectl patch EtcdCluster "$NAME-etcd-cluster" --type "merge" \ + --patch "{\"spec\":{\"size\":$ETCD_CLUSTER_SIZE}}" +``` + +# Backup and Restore + +Backup and restore is not supported for Trillian. Due to the append-only nature +of logs, backup and restore is a dangerous operation because it could result in +entries being removed from a log. + +# Updating the app + +To update the Trillian app, select a new version from the +[GitHub releases page](https://github.com/google/trillian/releases) and use it +as the value of `TAG` below. Then re-install the Trillian app. For example, to +update to version "v1.2.1": + +```shell +export APP_INSTANCE_NAME=trillian-1 +export NAMESPACE=default +export TAG="v1.2.1" + +make -C click-to-deploy/k8s/trillian app/install +``` + +# Uninstall the Application + +## Using the Google Cloud Platform Console + +1. In the GCP Console, open + [Kubernetes Applications](https://console.cloud.google.com/kubernetes/application). + +1. From the list of applications, click **Trillian**. + +1. On the Application Details page, click **Delete**. + +1. To cleanup the Etcd cluster, run the following command: + + ```shell + export APP_INSTANCE_NAME=trillian-1 + export NAMESPACE=default + + kubectl delete EtcdCluster $APP_INSTANCE_NAME-etcd-cluster --namespace=$NAMESPACE + ``` + +## Using the command line + +### Prepare the environment + +Set your installation name and Kubernetes namespace: + +```shell +export APP_INSTANCE_NAME=trillian-1 +export NAMESPACE=default +``` + +### Delete the resources + +> **NOTE:** We recommend to use a kubectl version that is the same as the +> version of your cluster. Using the same versions of kubectl and the cluster +> helps avoid unforeseen issues. + +To delete the resources, use the expanded manifest file used for the +installation. + +Run `kubectl` on the expanded manifest file: + +```shell +kubectl delete -f ${APP_INSTANCE_NAME}_manifest.yaml --namespace $NAMESPACE +``` + +Otherwise, delete the resources using types and a label: + +```shell +kubectl delete application,deployment,service \ + --namespace $NAMESPACE \ + --selector app.kubernetes.io/name=$APP_INSTANCE_NAME +``` + +### Delete the MySQL persistent volume + +By design, the removal of the Trillian app does not remove the +PersistentVolumeClaim used by the MySQL Deployment. This prevents your +installations from accidentally deleting the database. + +To remove the PersistentVolumeClaim with its attached persistent disk, run the +following `kubectl` command: + +```shell +kubectl delete persistentvolumeclaims \ + --namespace $NAMESPACE \ + --selector app.kubernetes.io/name=$APP_INSTANCE_NAME +``` + +### Delete the GKE cluster + +Optionally, if you don't need the deployed application or the GKE cluster, +delete the cluster using this command: + +``` +gcloud container clusters delete "$CLUSTER" --zone "$ZONE" +``` From 31561ee1daffbd707353637c6f337da69e3ed836 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Thu, 21 Feb 2019 16:33:54 +0000 Subject: [PATCH 30/40] Fix service name in port-forwarding section of README --- k8s/trillian/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index e1f6c5dd54..5473ec23c0 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -170,10 +170,10 @@ Use local port forwarding to access Trillian's gRPC API from your machine. In a terminal, run the following command: ```shell -kubectl port-forward --namespace ${NAMESPACE} ${NAME}-trillian-logserver 8090 +kubectl port-forward --namespace ${NAMESPACE} service/${NAME}-logserver-service 8090 ``` -You can access the Trillian API at `localhost:8090`. +You can then access the Trillian API at `localhost:8090`. # Scaling the Trillian app From fa9d190b9150b5c7c2d8a4b72a7f16a3555ee7c3 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 09:13:24 +0000 Subject: [PATCH 31/40] Use correct tag in Trillian README It was missing a "v" prefix. --- k8s/trillian/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 5473ec23c0..6e70c61844 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -149,7 +149,7 @@ run the installer. ```shell export NAME=trillian-1 export NAMESPACE=default -export TAG=1.2.1 +export TAG=v1.2.1 make -C click-to-deploy/k8s/trillian app/install ``` From ce9315aed161dc497a15e08724dd66a23f349205 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 09:13:47 +0000 Subject: [PATCH 32/40] Remove unnecessary quotation marks --- k8s/trillian/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 6e70c61844..42c9f66b57 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -234,7 +234,7 @@ update to version "v1.2.1": ```shell export APP_INSTANCE_NAME=trillian-1 export NAMESPACE=default -export TAG="v1.2.1" +export TAG=v1.2.1 make -C click-to-deploy/k8s/trillian app/install ``` From 631129781ac7e6cd85b6f803bcd32891b3a9f30d Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 09:49:56 +0000 Subject: [PATCH 33/40] Add labels to Etcd cluster pods --- k8s/trillian/manifest/etcd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 567a0a1aec..1ba286dcfd 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -62,6 +62,9 @@ spec: repository: $ETCD_REPOSITORY version: $ETCD_VERSION pod: + labels: + app.kubernetes.io/name: "$NAME" + app.kubernetes.io/component: etcd busyboxImage: $IMAGE_BUSYBOX affinity: podAntiAffinity: From 45a595aa12a5e19264172240765fa4a91a2137bf Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 09:50:52 +0000 Subject: [PATCH 34/40] Remove documentation referring to "expanded manifest file" The documentation never mentions how to create an expanded manifest file, so it shouldn't rely on it. Just direct users to use `kubectl delete` instead. --- k8s/trillian/README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 42c9f66b57..672eff45ba 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -276,16 +276,7 @@ export NAMESPACE=default > version of your cluster. Using the same versions of kubectl and the cluster > helps avoid unforeseen issues. -To delete the resources, use the expanded manifest file used for the -installation. - -Run `kubectl` on the expanded manifest file: - -```shell -kubectl delete -f ${APP_INSTANCE_NAME}_manifest.yaml --namespace $NAMESPACE -``` - -Otherwise, delete the resources using types and a label: +Delete all resources matching the name you used during installation: ```shell kubectl delete application,deployment,service \ From 086d9cf9d58841520404cd337c02d2cd7c328888 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 10:04:24 +0000 Subject: [PATCH 35/40] Consistently use $APP_INSTANCE_NAME instead of $NAME --- k8s/trillian/Makefile | 2 +- k8s/trillian/README.md | 26 ++++++++++---------- k8s/trillian/manifest/app.yaml | 8 +++---- k8s/trillian/manifest/config.yaml | 4 ++-- k8s/trillian/manifest/etcd.yaml | 2 +- k8s/trillian/manifest/logserver.yaml | 22 ++++++++--------- k8s/trillian/manifest/logsigner.yaml | 22 ++++++++--------- k8s/trillian/manifest/mysql.yaml | 36 ++++++++++++++-------------- k8s/trillian/schema.yaml | 4 ++-- 9 files changed, 63 insertions(+), 63 deletions(-) diff --git a/k8s/trillian/Makefile b/k8s/trillian/Makefile index d8541bfbfb..3f6650f119 100644 --- a/k8s/trillian/Makefile +++ b/k8s/trillian/Makefile @@ -38,7 +38,7 @@ ifdef ETCD_VERSION endif APP_PARAMETERS ?= { \ - "NAME": "$(NAME)", \ + "APP_INSTANCE_NAME": "$(NAME)", \ "NAMESPACE": "$(NAMESPACE)" \ $(IMAGE_TRILLIAN_LOGSERVER_FIELD) \ $(IMAGE_TRILLIAN_LOGSIGNER_FIELD) \ diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 672eff45ba..ea8d5be5c9 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -147,7 +147,7 @@ Next, choose the instance name, namespace and Trillian release tag to use, then run the installer. ```shell -export NAME=trillian-1 +export APP_INSTANCE_NAME=trillian-1 export NAMESPACE=default export TAG=v1.2.1 @@ -159,7 +159,7 @@ make -C click-to-deploy/k8s/trillian app/install To get the Console URL for your app, run the following command: ```shell -echo "https://console.cloud.google.com/kubernetes/application/${ZONE}/${CLUSTER}/${NAMESPACE}/${NAME}" +echo "https://console.cloud.google.com/kubernetes/application/${ZONE}/${CLUSTER}/${NAMESPACE}/${APP_INSTANCE_NAME}" ``` To view your app, open the URL in your browser. @@ -170,7 +170,7 @@ Use local port forwarding to access Trillian's gRPC API from your machine. In a terminal, run the following command: ```shell -kubectl port-forward --namespace ${NAMESPACE} service/${NAME}-logserver-service 8090 +kubectl port-forward --namespace ${NAMESPACE} service/${APP_INSTANCE_NAME}-logserver-service 8090 ``` You can then access the Trillian API at `localhost:8090`. @@ -187,8 +187,8 @@ for serving requests. ```shell LOGSERVER_REPLICAS=4 -kubectl scale "deployments/$NAME-logserver-deployment" \ - --namespace "$NAMESPACE" --replicas=$LOGSERVER_REPLICAS +kubectl scale "deployments/${APP_INSTANCE_NAME}-logserver-deployment" \ + --namespace "${NAMESPACE}" --replicas ${LOGSERVER_REPLICAS} ``` ## Scaling the signers @@ -201,8 +201,8 @@ number of logs that can be signed in parallel. ```shell LOGSIGNER_REPLICAS=2 -kubectl scale "deployments/$NAME-logsigner-deployment" \ - --namespace "$NAMESPACE" --replicas=$LOGSIGNER_REPLICAS +kubectl scale "deployments/${APP_INSTANCE_NAME}-logsigner-deployment" \ + --namespace "$NAMESPACE" --replicas $LOGSIGNER_REPLICAS ``` ## Scaling etcd @@ -214,7 +214,7 @@ to the desired cluster size. ```shell ECTD_CLUSTER_SIZE=5 -kubectl patch EtcdCluster "$NAME-etcd-cluster" --type "merge" \ +kubectl patch EtcdCluster "${APP_INSTANCE_NAME}-etcd-cluster" --type "merge" \ --patch "{\"spec\":{\"size\":$ETCD_CLUSTER_SIZE}}" ``` @@ -256,7 +256,7 @@ make -C click-to-deploy/k8s/trillian app/install export APP_INSTANCE_NAME=trillian-1 export NAMESPACE=default - kubectl delete EtcdCluster $APP_INSTANCE_NAME-etcd-cluster --namespace=$NAMESPACE + kubectl delete EtcdCluster "${APP_INSTANCE_NAME}-etcd-cluster" --namespace "$NAMESPACE" ``` ## Using the command line @@ -280,8 +280,8 @@ Delete all resources matching the name you used during installation: ```shell kubectl delete application,deployment,service \ - --namespace $NAMESPACE \ - --selector app.kubernetes.io/name=$APP_INSTANCE_NAME + --namespace "${NAMESPACE}" \ + --selector "app.kubernetes.io/name=${APP_INSTANCE_NAME}" ``` ### Delete the MySQL persistent volume @@ -295,8 +295,8 @@ following `kubectl` command: ```shell kubectl delete persistentvolumeclaims \ - --namespace $NAMESPACE \ - --selector app.kubernetes.io/name=$APP_INSTANCE_NAME + --namespace "${NAMESPACE}" \ + --selector "app.kubernetes.io/name=${APP_INSTANCE_NAME}" ``` ### Delete the GKE cluster diff --git a/k8s/trillian/manifest/app.yaml b/k8s/trillian/manifest/app.yaml index 691f1bc183..ff2afddb95 100644 --- a/k8s/trillian/manifest/app.yaml +++ b/k8s/trillian/manifest/app.yaml @@ -1,13 +1,13 @@ apiVersion: app.k8s.io/v1beta1 kind: Application metadata: - name: $NAME + name: $APP_INSTANCE_NAME annotations: kubernetes-engine.cloud.google.com/icon: >- data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAATvklEQVR4Ae3dBXQUV9/H8Y17gsQVTSjuPMUpTnFtcXd3Dd5QV+AlVKi7u7vjVi/uIbiFJP/3e897s2fPhITdySa7Dy+/cz6VBJLsnZ2ZO/8rsbhpupaqXmp3QvekU1HNo/7l/wfBTG4PCAj4JDExcZefn9+z/H8srpObaZLQM/Hi4NxhMliGyYDLg6Xy2JRsi6dluMWx3NmoUaOse+65R1avXi3jx4+XpKSkvXy8FArKzYRWCf2s1/6+qvGtBmYNkaByQVssDoTG/uf+++9XjW81evRo4VOLUFBuJjQldPOgnKG68aHFdoj93eJA6tWrd0I3vNV9990nwcHB71kKzs14eHmkdfq5s254oMu2bhKcFLzB4kC47m80HoApU6YI94Q0S6G5GY+ydctua/VWa+l/bqDc9l5biWkdc4KPV4Pd8fT0HMU94JJ616vGX7VqlVSqVGmb5f9JvNEFg03e9CK9Q73n+UX4/ch/v4XyMJP6cXFxB8qVK7fH39//Hf4/Bjd8aqfUaXu0bb/50mvCaqlYo8VZPjYUZjICT8Bs2uBLmE0NdeAqVKjwZ3x8/Cb+vzPcO9FJNX5P3bBX0l47o8jKVzKlVpPeF/mUPxxNHWyB2SzAKphJIg1/Ii0tTV3CRHVnGzRokOXj4zPA4sZJbt1nbq5ufKsuI+4RPtcIjsYH5+EHM3kX3eFwQkJCVk6fPl2MN3LOhO0WN05KwzZD8x2A/jOeFj43CmayEfVhJsfNXvdDQ0PfePDBB/MdgNq1a2dY3DkJler9PePRzdbGX/7icUmq2vgcn4qCmaRjDBxNeeyH2dw/bty4fM8RnAE7LW6ehIiYSj+06Tc/t1m3qTmVa7U5ycdmwWzGYB0czZ14BWaTxLPE0YkTJ6rGF3UvuPXWW88Zb8TunPYoh+bYh1CYSQNshKN5CDNgNhPwdmBg4GPqoU53YxPwX5nHkA4z8cN5+MCR/IimMJMKOIaKuCESjH/RBmayFXVgb3xxDgFwNB74ChNxQ6U19iC4wC5UnTaxKU26htgctF7YgP0YAXvTEPvwBLoh0HBwCsskfAkP3HBZhzXIlz6T06cseebA1Qmrvj7o4+P/uYeHRxYfFi8vL9WVfBWVYE2LjiPj+4xfPbRpl0ktLPkTi/V8jQz+Lfz7sn+4/2cdf7z9CKXuQ02eaVHQpakSjqI8bsiEYC9awjbhfSas+V11W+969ZSUT655ku7eQ7r/7wGVCLTEBKwYOPPZg+rPz1v/x9VyKY1ViXk2+qAeykDFE03KlCmzpkLXimfzqqs1FtXaqL4nbOOJbzAWbpWeZcuW3VylSpXDERER6sbW2Am9o78RhGCkUrk8HxFXOafvpDWXeZATPrYQYRiIN3AcksfL20cWPfmv9VmjTot+6uNGu3AXGsEjODF4dMOHG0nzV2/LDogOzFHfU3+fIKhMxadwRqIQjiKnA33fK48++qjqC8uKFSukTp06GbpxipIn8DGXBlVmzuHAqopnEv4DwSVkQWwcwed4FLNvH7zi4F2vnpYpD/woEVGVH9A1n1ewEachNg7jEATDUSE8PFyVKXL5GY7pmtFRJKIoKRsbG7uTwaDs1q1bZ/M91M+SAnNJSEj4VdfTrZYtWybe3t7PWIqWMGTwLlQNWhW22YYcXMZb6I8yMGZIqfAEzgZf1ailYRtP3IpV+A2iZRp6RrX4GT7kIKjGH4kipXTp0p8uXLjQ2laLFy+WypUrH+RTXnA85cuX36++kBH19W8sRU8ErpUqaIgQFJZhEC0UhaUpXsGSgkroKGqadOrUKdfYVh06dBDdm3M8QUFB3xgHtqdOnSqlSpW6x+L6jIRoQXB14jp27JhjPAADBw7Ulz1zacIpdGbu3Lmi7gMzZ84Uzgp1us6FN1yZMRDNHy4PldJM28Z/7LHHpFq1aheKOhJXIzIyci3UTTNdd/Pex6+oBldlPETzgStTAZ/h2erVq1+aMWOGDBo0SBo3bqw6ElNQLBmKY5gHL5R0JkE0T7giXpiODMyGFyphIXohstive/gAv7jgbJgK0VyRGvgFX6ISXJphOIa5JXg2zIAgGyUZXyzDcYxyp5pRvM3ZUBXFndkQXEFJ5VbsxpvuPNH3aW7W5/TlyVnxR0MMx4P4DOcguIjijidW4zh6wW0TSuNn8nT5kRNf+HAcgRTgHEoi3yETIXDbLEUOapht8PDwxLw+cytsgRTiKr5DSaQmcrEQbpkI3v0XKEC9bWYUqtPA5S+MWvrhoakP/Hixxq09/jE2NvZgHSahFSJRomGqygec3Wd0Lctt4snih8XUPDYxayzbzGBG8y6TZjB9xVpeVlXOhOQGeQ1/CjPgC1enGnIZqH+oadOmpS1OTCCm8g5e5+gcnGEk79G7SZMm6t1RFw6l36T054yTueq27K9O90dQFm6T5s2bH1ATtpYvX/43r9cpPaFIRp/+7NOnj8yaNUv69++fGxUV9bm9p1nfvn3bPPDAA1mqVkSdKO9d+xUawq606jl7lW3jr3z5pJSv2kT41Ey4Q0pjMnZQcLPWebp3797O0YdHxj6+xVvWB1dfX991aumObRFJFd+o+T9t94Tc6OhvKVGf0eXeLAh6wJ5Q0g/4pFG74bLkuUPCPUD+036k7bV/DFyddIhCzf8QFc6r7du332exP6WYW7RVvcHVG5WzRw1snWW8u6VFTb1WjW5Eo/5u7+VLDXqrgp06FsjFBQcKZGkQJTCkDCNU3hv01xAtBwNgm4oYgqHaIESjuNIDgiPwoDHX8Zqv2DvthUld80eOHJmvjRk926bu7N8bJ6E+9NBD6pP/6EvJveiL8tf54RpjJAQvwZ7E4BJEm2kstGlX0VP7GLkQgyw8j/+gOLIPglpoBkEne85wJgW88/DDD+c7AFROT1nIne3atcvhmmb9BHf4K7pBW2MuXsM+nMCHWIYuiMYGumbq4554B4IBDkwfFO0T2GYBxEYOxE5PFEON5hEIlsCL13xWz/izTSiaYiLW41ecw0eTJk3K9yZXK3csOhM4Gse5w59nAOFv7gvzCrph43Yswfs4pssOryMAF5GNMnb2vC5CtKYw5m6ISelOPghtIfhGr0F7Ww/kp+IN/IMz+BaPYSQaIADVb7nlljOpqanWG3izZs1yjFNgPDVHk4hwNITgF9iTrhDtu0JKEUchtjw8PKVKvQ7S5PZxiiQmNyzoINwPZyUCgpNQmYhPsBQ9UeE6B7wpD6mv80bfTy/oLw7gLIuTcwcEz8OerIdo03CtdIbYSq7dWuas3SnG54YJd38lEbGV85cqnFsUPA5BFNwu8yBYCXuyB6Il41r5CJKHxX+y4qUTutHzm5f+u5SNLm88CEvhrHwFQUu4V7xDvF8OiA4QO+fXeOEqBFnwhDEpyLW97Mx8bKtu7IINmPmsGCdlwRvOyHoIhsB90mhN42H9TvfP7ZfRXxL7lFOn6c/YhG3YhT/wL/bjMDIg2uWCLmnGdz8NfF0rXs6QoNBw40FIKmRUbZb+9zRMxWRMxHiMxRiMwghmux2cPHmyUDX4mdlv7lOQ6/RLl0/yJr42e76Fo72VU/YcgEZth6kGtktMuerG71EO10oWxB7cQCVviqbSi1jcJU2eb7ZS7WiiDkDVGVWP6G7pS3gWTyEdq/Ew7kMaliMV4110AIIh9uJpVvK2vmGyWna3bt3c6z4QdVvMj2XqlhVPf08azinpCckTlVjVrsZf9NQe8fENMDZgAoxJhmjb4A0f+MIP/gi0mcUdwl4T76mphg0bNnzJLceCIXDWAQjFeUieUUvfv+4BaD9gibHxt+FaaQnRPoI9eR6CPnC7rIBgAYgTex1aWHiczF6zvcDGH5H6tvHdX1ivbDJEWw978j0E9eB26QfBi3BW6kKMB6Hn2Edk2fNHrQ2/8Kl/pePApeLt429s/IxCKpafQLQ7YU8OQ1AaTk0PZkJ/zTVuMztK3W/HsKIHKqAXVqEDqkKwHc5MGsTILyBEIuOrSHRiNRrer6Cb53sF1edxBYJsOxs0GLk2vTYfOCW9KAxlcWe3Tj+nSqeOdDxUfFEbw/AwvsFp/IvX1TQUvK1vYlc0fzgz90FMWgBjUiHa5w6OCXwIlXdwBm9jPtogrJANpPqqkUaKcnv590//N7GAqAEZSqOq8a1GjBiR94224Dy24ilMQQvDN/ofRnYy1ffA1xB0gbOzDBch13EextL1ZMPNPROidYA9eQaCsfDUJfiX0QNp+AJnsBtPYRzqwhvJLPW6mPccsXLlSrXph/o5G1kYGrOZ0w6os4FLkuoZ1LFjNkJnCBpjCgQbUBwpg1nYAzHYiOEIwBDDoE0uhsEL70G0H2FPvJGJXMSiMQTdYBtPVMcIpGMbzsbExGy59957dfsCS5cuVcO+r6jN7X43HoCxY8eqB49Ui30JUEOSrCdbo0vTgovwKeZp4dE2omDMREgeD2+PbP/ogC2GAZ7bYE96Q/AzLGr4ldd8yc4hyQDurTuNbayHJDdbmMvTn83trnCErAvL1CwJR8q41Li3MlvgMsObulroNn3leZBqM6tL9997yh2nBkjlkcni6eMpetjT3uyE4AtEkgy9gNCuhIWFvWVc6qVqSswvesGi05FG34g/OC2edKR6SMNXZSXlVfVFe/ToIfo6+BCS4PIk9Sv/nd6F1yqhZ9LXFvvjYdtlbdu2bY4a32X6zo8W+1OratWqh+fPn29d6sXGsoed0kY1a9aMZC8ddR+RLl26ZOrumrukTtXpVY9YG19r+lyzJ8zswIgX1d6j6rUyvHjUwSFPXxa8v8Ro2M9cvh516kBR586dKycnJ79Lb0hPTXR54rABuZHNosT2DOh/flDOLbOrjzW7BoJuZA47aR0dTSxulkjr5FzXJB6DsAEXIArvUYnrlpDdZXvXrHZfd9xX7+766y0mQ//9TbqfF/VN3y2zHDkluF5sK7KQDYFRDp5BYkBCgOo+ehZ1Yi5WwG0TxllwSm9BUBI5BSnAJ6gNp4TX9L56bfoh1G0TgE/4QU+X0BqqSxBk4wDexGTUgDPTidd0Vs+Hcts0wR94BqVLaI2uaLNRnPHHPTiKO+FWCcSD2I/bS/JyB9HGoyRSHzvwFmLg8jTHX3jCBdfGOIg2yEXrhIei2OOB8uiLUKgE4VHsQTu4IlUgWg+UdGpjCz5CIool3dQUarUPDithhMJdpq6h/I21Ll6y2QCitYEr4o2FOIGxqEpt7TW10RUV0DeKOoMugIGZk7ZjBSxFUtuvZOrBBFenFURr5AaL9z6kvY6zo5h1q2O2u79UlIXdd/Tr1y9fCZWlOTluUm7oCtGqwaVhE6u71Wiicb8gKqfbTF9+KK7lOwBMCBMn9XYqwpgE/IINiEdh6Q9R7LgGJ2M15hc8Db5ooQTz0SOPPJKvvaiCHjfdz65Ro8Yx25EcNTOM69sOJyx86KUWdvB0+YEeQcrLIsMo1i+Yh5Tr7JhVxvizowpm4BfDPNRI20sHP8f3epy7qEnTlVLjdvfbi9TvpcH39+zZU1jGJFQ8j+kGK0oicBgP6kf7XFXk0u/ifyH4GJkQGwfxDZ7BMoZSt6qdqSZMmCCMOKXrgfYN2IRLEBsn8A8ES1BZrdvi3zn6Z9jnhAcsb7VDIishrcO6zKC77IyrhQeaozW8UNS8ipVQCUOaGtKsW7durrqGMkPjjE0P4zY8iv0QWzS89Z2m3hyGz2fjNzyB9vBGE/XnGG7N5U2VqzduXYow1MMRhBf1Zszo13u8If5ies/n+nu7VfphO3xhmzhGic6qxlSjRlxP1S69DxpmoMWiCQZg4eDBgw/nHYCWLVt+q6/vQ1AX/lDxRENmfjzGzr8ZeX+exf1/XWMF/iq8gBs2kTiCusgX3pnLGGU6zurzz/QWwlf15t2H9FnjD2s4veNHjRr16JAhQ1K50RkPaBs140BPlxHOsKv891esVPyb2Qin+DtDC6j3/IEuuCHzhh3LhTwMtZ478DouOnjj/wyZSEd3BNucEd4oKE2xH2G4odIfW+FTQr+MbTbudcFv+3DLROMoasFM5uIeE2t6P4WZBGMPWuG/Mj5cbx9ir4kv9BbH7yAVZvMa+pro6h53whb7kRiG3vivSG1mDRzLWxGu+ukVK1bM0E+iZrMPFeBoDhdxVG4BD6LnunbtKhQi1Tye3brk4b7h4eaLBQsWqMYH9Ew7eiTPFuGhLQNm8gE6wUy8U1JS9qgJWLaFSD6WobvQ7hl+wH0FzIH8zuxYLD6GmdyF+TCT7gMGDMj3Otq0aSNufTniKTDf1jdLliwRauNmz4BUrISZ9MUrMJN4Jpvl24qelZGiq7Bumy48IFkn+qpTmOvohSLs4fMOusNMkvEHTIUn5xO2E2rVa+KNtEeXY9w6Pfgt1jtU4Y57wiaz12GdI4iHmXjiLIJgJq14HYdVIVJdeng9h/XAzw2fQLqx9/Liv+be8XsRqpTR+FLt9cmlcZXJ3wPpjfZoBw/c+KHhP2fTQGvPg60hc1gsstDRL0O38cCYMWNE3ZOmTZumusP79QLsmylk+t9QhkFzjTc/ni3+dvC3YS+1KVdbr+GcDW9abqbgULVcbf01UDY4C847smiEd/8P1+oOs6/pXkuhuZnRPHXaNppaUaLOgJOOFPL8/Pw+UL90zvbrqEuR2rvTUmhuxlctGmQBhLUby5KgXF2QcyTt2M/H2h1WWrRokaVXNd7MdVKGXRvX0+37l27sxiLsnNuNAfFdtWrVOsW//+KsGGlxw/wvK87UoQAXTwYAAAAASUVORK5CYII= marketplace.cloud.google.com/deploy-info: '{"partner_id": "click-to-deploy-containers", "product_id": "trillian", "partner_name": "Google Click to Deploy"}' labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" spec: descriptor: type: "trillian" @@ -37,10 +37,10 @@ spec: type: Reference valueFrom: serviceRef: - name: $NAME-logserver-service + name: $APP_INSTANCE_NAME-logserver-service selector: matchLabels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" componentKinds: - group: apps/v1 kind: Deployment diff --git a/k8s/trillian/manifest/config.yaml b/k8s/trillian/manifest/config.yaml index 176c17614a..f9a784c867 100644 --- a/k8s/trillian/manifest/config.yaml +++ b/k8s/trillian/manifest/config.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: $NAME-config + name: $APP_INSTANCE_NAME-config labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" data: STORAGE_SYSTEM: mysql SIGNER_BATCH_SIZE: "500" diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 1ba286dcfd..45896a376e 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -76,5 +76,5 @@ spec: - key: etcd_cluster operator: In values: - - $NAME-etcd-cluster + - $APP_INSTANCE_NAME-etcd-cluster topologyKey: kubernetes.io/hostname diff --git a/k8s/trillian/manifest/logserver.yaml b/k8s/trillian/manifest/logserver.yaml index 9d9f65aaf8..087681149d 100644 --- a/k8s/trillian/manifest/logserver.yaml +++ b/k8s/trillian/manifest/logserver.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: $NAME-logserver-service + name: $APP_INSTANCE_NAME-logserver-service labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logserver annotations: cloud.google.com/load-balancer-type: "Internal" @@ -14,27 +14,27 @@ spec: - name: metrics port: 8091 selector: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logserver type: LoadBalancer --- apiVersion: apps/v1 kind: Deployment metadata: - name: $NAME-logserver-deployment + name: $APP_INSTANCE_NAME-logserver-deployment labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logserver spec: replicas: $TRILLIAN_LOGSERVER_REPLICAS selector: matchLabels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logserver template: metadata: labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logserver spec: restartPolicy: Always @@ -44,8 +44,8 @@ spec: "$(STORAGE_FLAG)", "--storage_system=$(STORAGE_SYSTEM)", "--quota_system=etcd", - "--etcd_servers=$NAME-etcd-cluster-client:2379", - "--etcd_http_service=$NAME-logserver-http", + "--etcd_servers=$APP_INSTANCE_NAME-etcd-cluster-client:2379", + "--etcd_http_service=$APP_INSTANCE_NAME-logserver-http", "--rpc_endpoint=0.0.0.0:8090", "--http_endpoint=0.0.0.0:8091", "--tracing", @@ -53,9 +53,9 @@ spec: ] envFrom: - configMapRef: - name: $NAME-config + name: $APP_INSTANCE_NAME-config - secretRef: - name: $NAME-secrets + name: $APP_INSTANCE_NAME-secrets image: $IMAGE_TRILLIAN_LOGSERVER resources: limits: diff --git a/k8s/trillian/manifest/logsigner.yaml b/k8s/trillian/manifest/logsigner.yaml index c3e8082f95..4363755eab 100644 --- a/k8s/trillian/manifest/logsigner.yaml +++ b/k8s/trillian/manifest/logsigner.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: $NAME-logsigner-service + name: $APP_INSTANCE_NAME-logsigner-service labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logsigner annotations: cloud.google.com/load-balancer-type: "Internal" @@ -14,26 +14,26 @@ spec: port: 8092 targetPort: 8091 selector: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logsigner --- apiVersion: apps/v1 kind: Deployment metadata: - name: $NAME-logsigner-deployment + name: $APP_INSTANCE_NAME-logsigner-deployment labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logsigner spec: replicas: $TRILLIAN_LOGSIGNER_REPLICAS selector: matchLabels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logsigner template: metadata: labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: logsigner spec: restartPolicy: Always @@ -42,9 +42,9 @@ spec: args: [ "$(STORAGE_FLAG)", "--storage_system=$(STORAGE_SYSTEM)", - "--etcd_servers=$NAME-etcd-cluster-client:2379", + "--etcd_servers=$APP_INSTANCE_NAME-etcd-cluster-client:2379", "--quota_system=etcd", - "--etcd_http_service=$NAME-logsigner-http", + "--etcd_http_service=$APP_INSTANCE_NAME-logsigner-http", "--http_endpoint=0.0.0.0:8091", "--sequencer_guard_window=1s", "--sequencer_interval=$(SIGNER_INTERVAL)", @@ -54,9 +54,9 @@ spec: ] envFrom: - configMapRef: - name: $NAME-config + name: $APP_INSTANCE_NAME-config - secretRef: - name: $NAME-secrets + name: $APP_INSTANCE_NAME-secrets image: $IMAGE_TRILLIAN_LOGSIGNER resources: limits: diff --git a/k8s/trillian/manifest/mysql.yaml b/k8s/trillian/manifest/mysql.yaml index 2dc320621b..0570d909fe 100644 --- a/k8s/trillian/manifest/mysql.yaml +++ b/k8s/trillian/manifest/mysql.yaml @@ -1,20 +1,20 @@ apiVersion: v1 kind: Secret metadata: - name: $NAME-secrets + name: $APP_INSTANCE_NAME-secrets labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql type: Opaque stringData: - STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp($NAME-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE + STORAGE_FLAG: --mysql_uri=$MYSQL_TRILLIAN_USER:$MYSQL_TRILLIAN_PASSWORD@tcp($APP_INSTANCE_NAME-mysql-service:3306)/$MYSQL_TRILLIAN_DATABASE --- apiVersion: v1 kind: Secret metadata: - name: $NAME-mysql-secrets + name: $APP_INSTANCE_NAME-mysql-secrets labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql type: Opaque stringData: @@ -26,9 +26,9 @@ stringData: apiVersion: v1 kind: Service metadata: - name: $NAME-mysql-service + name: $APP_INSTANCE_NAME-mysql-service labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql spec: clusterIP: None @@ -36,26 +36,26 @@ spec: - name: mysql port: 3306 selector: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql --- apiVersion: apps/v1 kind: Deployment metadata: - name: $NAME-mysql + name: $APP_INSTANCE_NAME-mysql labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql template: metadata: labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql spec: containers: @@ -63,23 +63,23 @@ spec: name: mysql envFrom: - secretRef: - name: $NAME-mysql-secrets + name: $APP_INSTANCE_NAME-mysql-secrets volumeMounts: - - name: $NAME-mysql-storage + - name: $APP_INSTANCE_NAME-mysql-storage mountPath: /var/lib/mysql subPath: mysql restartPolicy: Always volumes: - - name: $NAME-mysql-storage + - name: $APP_INSTANCE_NAME-mysql-storage persistentVolumeClaim: - claimName: $NAME-mysql-storage-claim + claimName: $APP_INSTANCE_NAME-mysql-storage-claim --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: $NAME-mysql-storage-claim + name: $APP_INSTANCE_NAME-mysql-storage-claim labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: mysql spec: accessModes: diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 56860a5ff4..359832a3e0 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -1,6 +1,6 @@ application_api_version: v1beta1 properties: - NAME: + APP_INSTANCE_NAME: type: string x-google-marketplace: type: NAME @@ -133,7 +133,7 @@ properties: title: MySQL database for Trillian to create and use default: trillian required: -- NAME +- APP_INSTANCE_NAME - NAMESPACE - IMAGE_TRILLIAN_LOGSERVER - IMAGE_TRILLIAN_LOGSIGNER From f8a67e06b5458238d1a2b0a8466b5ae7ffc1df90 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 10:04:39 +0000 Subject: [PATCH 36/40] Document the need to create a namespace if not using "default" --- k8s/trillian/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index ea8d5be5c9..f5621348a1 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -144,7 +144,9 @@ kubectl create clusterrolebinding cluster-admin-binding \ ``` Next, choose the instance name, namespace and Trillian release tag to use, then -run the installer. +run the installer. If you are not using the "default" namespace, you may need to +[create the namespace](https://kubernetes.io/docs/tasks/administer-cluster/namespaces/#creating-a-new-namespace) +first. ```shell export APP_INSTANCE_NAME=trillian-1 From be200d37fb0378aae04f0287fd08bb81833a3eaf Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 10:16:36 +0000 Subject: [PATCH 37/40] Require Etcd cluster pods to be scheduled on different nodes Otherwise, a single node being restarted (e.g. during a Kubernetes upgrade) can kill the cluster via loss of quorum. --- k8s/trillian/manifest/etcd.yaml | 25 ++++++++++++------------- k8s/trillian/schema.yaml | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/k8s/trillian/manifest/etcd.yaml b/k8s/trillian/manifest/etcd.yaml index 45896a376e..e6284cd6e3 100644 --- a/k8s/trillian/manifest/etcd.yaml +++ b/k8s/trillian/manifest/etcd.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: $NAME-etcd-service + name: $APP_INSTANCE_NAME-etcd-service labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd spec: type: NodePort selector: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd ports: - protocol: TCP @@ -18,20 +18,20 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: $NAME-etcd-operator + name: $APP_INSTANCE_NAME-etcd-operator labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd template: metadata: labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd spec: serviceAccountName: $ETCD_SERVICE_ACCOUNT @@ -52,9 +52,9 @@ spec: apiVersion: "etcd.database.coreos.com/v1beta2" kind: "EtcdCluster" metadata: - name: "$NAME-etcd-cluster" + name: "$APP_INSTANCE_NAME-etcd-cluster" labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd spec: serviceAccountName: $ETCD_SERVICE_ACCOUNT @@ -63,14 +63,13 @@ spec: version: $ETCD_VERSION pod: labels: - app.kubernetes.io/name: "$NAME" + app.kubernetes.io/name: "$APP_INSTANCE_NAME" app.kubernetes.io/component: etcd busyboxImage: $IMAGE_BUSYBOX affinity: podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: + requiredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: labelSelector: matchExpressions: - key: etcd_cluster diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index 359832a3e0..f290ae183c 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -153,7 +153,7 @@ required: x-google-marketplace: clusterConstraints: resources: - - replicas: 3 + - replicas: 5 requests: cpu: 1 affinity: From 2805e4752995c2198752ab334a6fae2020ee7a65 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 10:49:25 +0000 Subject: [PATCH 38/40] Reduce maximum size of Etcd cluster from 9 to 7 The etcd-operator documentation states that 7 is the maximum: https://github.com/coreos/etcd-operator/blob/16f0e1b3693483061f2a8252c0e361b06d216424/pkg/apis/etcd/v1beta2/cluster.go#L71https://godoc.org/github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2#ClusterSpec --- k8s/trillian/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/trillian/schema.yaml b/k8s/trillian/schema.yaml index f290ae183c..8a4066d4b1 100644 --- a/k8s/trillian/schema.yaml +++ b/k8s/trillian/schema.yaml @@ -62,7 +62,7 @@ properties: description: Should be an odd number between 3-7 (https://coreos.com/etcd/docs/3.2.13/faq.html#deployment). default: 5 minimum: 3 - maximum: 9 + maximum: 7 ETCD_SERVICE_ACCOUNT: type: string title: Etcd Service Account From 0c431dfa33806ee5afacf70394e5cb23cb9607c0 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 11:03:01 +0000 Subject: [PATCH 39/40] Document how to delete Etcd entirely during uninstallation The etcd-operator creates some services on which it isn't possible to set labels, so an extra command is required to delete them. --- k8s/trillian/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index f5621348a1..16fb267319 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -281,9 +281,13 @@ export NAMESPACE=default Delete all resources matching the name you used during installation: ```shell -kubectl delete application,deployment,service \ +kubectl delete application,deployment,service,EtcdCluster \ --namespace "${NAMESPACE}" \ --selector "app.kubernetes.io/name=${APP_INSTANCE_NAME}" + +kubectl delete service \ + --namespace "${NAMESPACE}" \ + --selector "etcd_cluster=${APP_INSTANCE_NAME}-etcd-cluster" ``` ### Delete the MySQL persistent volume From 3ad97fec5745b6952beef01e9e3916a2301c3d20 Mon Sep 17 00:00:00 2001 From: Rob Percival Date: Mon, 25 Feb 2019 11:10:56 +0000 Subject: [PATCH 40/40] Document how to delete secrets --- k8s/trillian/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/k8s/trillian/README.md b/k8s/trillian/README.md index 16fb267319..1b9be3408b 100644 --- a/k8s/trillian/README.md +++ b/k8s/trillian/README.md @@ -305,6 +305,17 @@ kubectl delete persistentvolumeclaims \ --selector "app.kubernetes.io/name=${APP_INSTANCE_NAME}" ``` +### Delete secrets + +Secrets stored in Kubernetes (e.g. database password) are not automatically +deleted. To delete these, run the following `kubectl` command. + +```shell +kubectl delete secret \ + --namespace "${NAMESPACE}" \ + --selector "app.kubernetes.io/name=${APP_INSTANCE_NAME}" +``` + ### Delete the GKE cluster Optionally, if you don't need the deployed application or the GKE cluster,