Implement a new tool to watch for cherrypicks

This requires moving some things around as we now have one binary which
serves 2 very distinct web pages (submit-queue and cherrypick-queue)

Both queues need to look at very different sets of PRs. submit-queue
cares about open PRs. Cherrypick needs to look at open and closed PRs.
While at first glance it might seem easy to just make the submit-queue
ignore closed PRs (it is easy) it would mean that the submit queue would
have to walk all 22000+ issues on every loop, obviously ignoring most of
them.

Instead for the cherry-pick we are able to limit the issues we walk to a
given label (cherrypick-candidate), so even though we are walking all
issues (open or close) that number becomes quite managable. Obviously we
can't just walk the issues with that label for the submit queue...

Thus we end up with 1 binary which is becoming two distinct 'apps' with
distinct configuration. The makefile defaults to deploying and working
with the 'submit-queue' APP, but one can use APP=cherrypick

mungegithub/.gitignore

@@ -1,5 +1 @@
 mungegithub
-
-local.rc.yaml
-local.secret.yaml
-local.service.yaml

mungegithub/Dockerfile-cherrypick

@@ -0,0 +1,25 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM google/debian:wheezy
+MAINTAINER Brendan Burns <[email protected]>
+RUN apt-get update
+RUN apt-get install -y -qq ca-certificates git
+
+EXPOSE 8080
+ENTRYPOINT ["/mungegithub"]
+CMD ["--dry-run", "--token-file=/token"]
+
+ADD cherrypick/www /www
+ADD mungegithub /mungegithub

mungegithub/Dockerfile → mungegithub/Dockerfile-submit-queue

@@ -28,5 +28,5 @@ ADD generated-files.txt /generated-files.txt
 ADD committers.txt /committers.txt
 ADD whitelist.txt /whitelist.txt
 # Submit queue web interface
-ADD www /www
+ADD submit-queue/www /www
 ADD mungegithub /mungegithub

mungegithub/Makefile

@@ -7,7 +7,8 @@ TAG ?= $(DATE)-$(GIT)
 
 DEFAULTREPO := gcr.io/google_containers
 REPO ?= $(DEFAULTREPO)
-CONTAINER := $(REPO)/mungegithub:$(TAG)
+APP ?= submit-queue
+CONTAINER := $(REPO)/$(APP):$(TAG)
 
 KUBECONFIG ?= $(HOME)/.kube/config
 
@@ -22,7 +23,7 @@ mungegithub:
 
 # build the container with the binary
 container: mungegithub
-	docker build -t $(CONTAINER) .
+	docker build -t $(CONTAINER) -f Dockerfile-$(APP) .
 
 # push the container
 push: container
@@ -37,17 +38,17 @@ endif
 # The cluster will need a github oauth token (the secret target makes that easy to create)
 deploy: push rc
 	# Get existing RCs
-	$(eval rcs := $(shell kubectl --kubeconfig=$(KUBECONFIG) get rc --selector='app=mungegithub,readonly=$(READONLY)' --output=template --template='{{range .items}}{{.metadata.name}} {{end}}'))
+	$(eval rcs := $(shell kubectl --kubeconfig=$(KUBECONFIG) get rc --selector='app=$(APP),readonly=$(READONLY)' --output=template --template='{{range .items}}{{.metadata.name}} {{end}}'))
 	# Deploy the new RC
-	kubectl --kubeconfig=$(KUBECONFIG) create -f local.rc.yaml
+	kubectl --kubeconfig=$(KUBECONFIG) create -f $(APP)/local.rc.yaml
 	# Scale the old RCs to 0
 	$(foreach rc,$(rcs),kubectl --kubeconfig=$(KUBECONFIG) scale --replicas=0 rc $(rc);)
 	# Current RCs, you might want to clean some up!
 	kubectl --kubeconfig=$(KUBECONFIG) get rc
 
 # Removes all mungegithub RCs
 cleanrcs:
-	$(eval rcs := $(shell kubectl --kubeconfig=$(KUBECONFIG) get rc --selector='app=mungegithub,readonly=$(READONLY)' --output=template --template='{{range .items}}{{.metadata.name}} {{end}}'))
+	$(eval rcs := $(shell kubectl --kubeconfig=$(KUBECONFIG) get rc --selector='app=$(APP),readonly=$(READONLY)' --output=template --template='{{range .items}}{{.metadata.name}} {{end}}'))
 	$(foreach rc,$(rcs),kubectl --kubeconfig=$(KUBECONFIG) delete rc $(rc);)
 
 # Try to run the binary locally using docker, doesn't need to push or have a running kube cluster.
@@ -58,41 +59,42 @@ local_dryrun: container
 # updates the rc.yaml with current build information and sets it to --dry-run
 rc:
 	# update the rc.yaml with the current date and git hash
-	sed -e 's|[[:digit:]]\{4\}-[[:digit:]]\{2\}-[[:digit:]]\{2\}-[[:xdigit:]]\+|$(TAG)|g' rc.yaml > local.rc.yaml
+	sed -e 's|[[:digit:]]\{4\}-[[:digit:]]\{2\}-[[:digit:]]\{2\}-[[:xdigit:]]\+|$(TAG)|g' $(APP)/rc.yaml > $(APP)/local.rc.yaml
 	# update the rc.yaml with the current repo (if not gcr.io
-	sed -i -e 's|gcr.io/google_containers|$(REPO)|g' local.rc.yaml
+	sed -i -e 's|gcr.io/google_containers|$(REPO)|g' $(APP)/local.rc.yaml
 ifeq ($(READONLY),false)
 	# update the rc.yaml with --dry-run=false
-	sed -i -e 's!^\([[:space:]]\+\)- --dry-run=true!\1- --dry-run=false!g' local.rc.yaml
+	sed -i -e 's!^\([[:space:]]\+\)- --dry-run=true!\1- --dry-run=false!g' $(APP)/local.rc.yaml
 endif
 	# update the rc.yaml with label "readonly: true"
-	sed -i -e 's!^\([[:space:]]\+\)app: mungegithub!\1app: mungegithub\n\1readonly: "$(READONLY)"!g' local.rc.yaml
+	sed -i -e 's!^\([[:space:]]\+\)app: $(APP)!\1app: $(APP)\n\1readonly: "$(READONLY)"!g' $(APP)/local.rc.yaml
 
 # simple transformation of a github oauth secret file to a kubernetes secret
 secret:
 	@echo $(token)
-	sed -e 's|1234567890123456789012345678901234567890123456789012345=|$(token)|' secret.yaml > local.secret.yaml
+	sed -e 's|1234567890123456789012345678901234567890123456789012345=|$(token)|' $(APP)/secret.yaml > $(APP)/local.secret.yaml
 
 clean:
-	rm -f mungegithub local.rc.yaml local.secret.yaml
+	rm -f mungegithub $(APP)/local.rc.yaml $(APP)/local.secret.yaml
 
 help:
 	@echo "ENVIRONMENT VARS:"
 	@echo " REPO=       repository for the docker image being build. Default: $(REPO)"
 	@echo " TOKEN=      file with github oauth token, needed in local_dryrun and secret. Default: $(TOKEN)"
 	@echo " KUBECONFIG= kubeconfig file for deployment. Default: $(KUBECONFIG)"
 	@echo " READONLY=   should the container actually mute github objects or just do everything else. Default: $(READONLY)"
+	@echo " APP=        which application you are trying to deploy. cherrypick or submit-queue. Default: $(APP)"
 	@echo ""
 	@echo "TARGETS:"
 	@echo " all:          runs 'container'"
 	@echo " mungegithub:  builds the binary"
 	@echo " container:    builds the binary and creates a container with the binary"
 	@echo " push:         pushes the container to the registry"
 	@echo " deploy:       launches the container on a kubernetes cluster, it will scale all other RCs to 0, but not delete them"
-	@echo " cleanrcs:     removes all mungegithub RCs deployed READONLY=$(READONLY)"
+	@echo " cleanrcs:     removes all RCs deployed READONLY=$(READONLY)"
 	@echo " local_dryrun: tries to launch the container locally with docker"
-	@echo " rc:           updates rc.yaml and places results in local.rc.yaml"
-	@echo " secret:       updates secret.yaml with TOKEN an creates local.secret.yaml"
+	@echo " rc:           updates $(APP)/rc.yaml and places results in $(APP)/local.rc.yaml"
+	@echo " secret:       updates $(APP)/secret.yaml with TOKEN an creates $(APP)/local.secret.yaml"
 	@echo " clean:        deletes the binary and local files (does not delete old containers)"
 
 

mungegithub/cherrypick/.gitignore

@@ -0,0 +1,3 @@
+local.rc.yaml
+local.secret.yaml
+local.service.yaml

mungegithub/cherrypick/rc.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  labels:
+    app: cherrypick
+    version: "2015-10-26-2a7f8fd"
+  name: cherrypick-2015-10-26-2a7f8fd
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    app: cherrypick
+    version: "2015-10-26-2a7f8fd"
+  template:
+    metadata:
+      name: cherrypick
+      namespace: default
+      labels:
+        app: cherrypick
+        version: "2015-10-26-2a7f8fd"
+    spec:
+      containers:
+      - command:
+        - /mungegithub
+        - --token-file=/etc/secret-volume/token
+        - --pr-mungers=cherrypick-queue,cherrypick-must-have-milestone
+        - --state=all
+        - --labels=cherrypick-candidate
+        - --dry-run=true
+        - --kubernetes-dir=/gitrepo/kubernetes
+        - --period=3m
+        image: gcr.io/google_containers/cherrypick:2015-10-26-2a7f8fd
+        imagePullPolicy: IfNotPresent
+        name: cherrypick
+        ports:
+        - name: status
+          containerPort: 8080
+        resources:
+          limits:
+            cpu: 100m
+        volumeMounts:
+        - mountPath: /etc/secret-volume
+          name: cherrypick-secret-volume
+        - mountPath: /gitrepo
+          name: kubernetes-repo
+      dnsPolicy: ClusterFirst
+      volumes:
+      - name: cherrypick-secret-volume
+        secret:
+          secretName: cherrypick-secret-volume
+      - name: kubernetes-repo
+        gitRepo:
+          repository: "https://github.com/kubernetes/kubernetes.git"
+          revision: "master"

mungegithub/cherrypick/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  token: 1234567890123456789012345678901234567890123456789012345=
+kind: Secret
+metadata:
+  name: cherrypick-secret-volume
+  namespace: default
+type: Opaque

mungegithub/status-service.yaml → mungegithub/cherrypick/service.yaml

@@ -2,18 +2,14 @@ apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app: mungegithub
-  name: submit-queue-status
+    app: cherrypick
+  name: cherrypick-status
   namespace: default
 spec:
-  clusterIP: 10.147.242.20
   ports:
   - name: status
-    nodePort: 31624
     port: 80
-    protocol: TCP
     targetPort: status
   selector:
-    app: mungegithub
-  sessionAffinity: None
+    app: cherrypick
   type: LoadBalancer

mungegithub/cherrypick/www/index.html

@@ -0,0 +1,124 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/angular_material/1.0.5/angular-material.min.css" type="text/css">
+  <link rel="stylesheet" href="md-data-table.min.css" type="text/css">
+  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=RobotoDraft:300,400,500,700,400italic" type="text/css">
+  <link rel="stylesheet" href="style.css" type="text/css">
+  <link rel="shortcut icon" href="https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo.png">
+  <meta name="viewport" content="initial-scale=1">
+
+  <title>Kubernetes Cherrypick Queue</title>
+</head>
+
+<body ng-app="CherrypickModule" ng-controller="CPCntl as cntl">
+  <!-- Indent the whole thing -->
+  <md-content class="md-padding" layout="column">
+    <!-- Display the white border around the indented portion -->
+    <md-content class="md-whiteframe-z2">
+      <md-toolbar layout="row" layout-align="space-around center">
+        <h1 class="md-display-2">Cherrypick Queue Status</h1>
+        <a ng-href="https://k8s.io"><img src="https://raw.githubusercontent.com/kubernetes/kubernetes/master/logo.png" class="titleLogo"></a>
+      </md-toolbar>
+      <md-content class="md-padding">
+        <md-tabs md-selected="cntl.selected" md-dynamic-height md-border-bottom>
+
+          <md-tab label="Queue" md-on-select="cntl.selectTab('queue')">
+            <md-content class="md-padding">
+              <md-toolbar class="md-whiteframe-z2">
+                <h2 class="md-toolbar-tools">PRs proposed to be cherrypicked<span id="queue-len"></span></h2>
+              </md-toolbar>
+                <md-list layout-padding>
+                  <md-list-item class="md-2-line" ng-repeat="pr in cntl.queue track by pr.Number">
+                    <a class="md-avatar" ng-href="https://github.com/kubernetes/kubernetes/pulls/{{pr.Login}}">
+                      <img ng-src="{{pr.AvatarURL}}" alt="{{pr.Login}}">
+                    </a>
+                    <md-content class="md-list-item-text" layout="column">
+                      <h3 class="md-body-1">
+                        <a ng-href="{{pr.URL}}">#{{pr.Number}}: {{pr.Title}}</a>
+                      </h3>
+		      <h4 class="md-body2">
+			{{pr.ExtraInfo}}
+		      </h4>
+                    </md-content>
+                    <md-divider md-inset ng-if="!$last"></md-divider>
+                  </md-list-item>
+                </md-list>
+              </section>
+            </md-content>
+          </md-tab>
+
+          <md-tab label="Info" md-on-select="cntl.selectTab('info')">
+	    <md-content class="md-padding">
+	      <md-tabs md-dynamic-height md-border-bottom>
+
+                <md-tab label="Purpose">
+	          <md-content class="md-padding">
+                    <md-content class="md-whiteframe-z2">
+                      <md-toolbar>
+                        <h2 class="md-toolbar-tools">How PRs get ordered in the queue</h2>
+                      </md-toolbar>
+                    </md-content>
+	            <md-content class="md-padding">
+	              <span id="queue-info"></span>
+	            </md-content>
+	          </md-content>
+	        </md-tab>
+
+                <md-tab label="Bot Stats">
+                  <md-content class="md-padding">
+                    <md-content class="md-whiteframe-z2">
+                      <md-toolbar>
+                        <h2 class="md-toolbar-tools">Statistics About the Bot</h2>
+                      </md-toolbar>
+                    </md-content>
+                    <br>
+                    <h3 class=md-title>Next Run Loop: <span id="next-run-time"></span></h3>
+                    <h3 class=md-title>API Calls Per Second: <span id="api-calls-per-sec"></span></h3>
+                    <h3 class=md-title>Github Rate Limit Count: <span id="github-api-limit-count"></span></h3>
+                    <h3 class=md-title>Github Rate Limit Next Reset: <span id="github-api-limit-reset"></span></h3>
+                    <md-content class="md-whiteframe-z2">
+                      <md-data-table-container>
+                        <md-data-table-toolbar>
+                          <h2 class="md-title">API Calls During Last Loop</h2>
+                        </md-data-table-toolbar>
+                        <table md-data-table class="md-primary">
+                          <thead md-order="cntl.StatOrder">
+                            <tr>
+                              <th order-by="$key" name="Call"></th>
+                              <th order-by="Count" name="Count ({{ cntl.APICount }})" numeric descend-first></th>
+                              <th order-by="CachedCount" name="Cached Count ({{ cntl.CachedAPICount }})" numeric descend-first></th>
+                            </tr>
+                          </thead>
+                          <tbody>
+                            <tr ng-repeat="stat in cntl.botStats | toArray | orderBy: cntl.StatOrder track by stat.$key">
+                              <td>{{stat.$key}}</td>
+                              <td>{{stat.Count}}</td>
+                              <td>{{stat.CachedCount}}</td>
+                            </tr>
+                          </tbody>
+                        </table>
+                      </md-data-table-container>
+                    </md-content>
+                  </md-content>
+                </md-tab>
+	      </md-tabs>
+	    </md-content>
+	  </md-tab>
+
+        </md-tabs>
+      </md-content>
+    </md-content>
+  </md-content>
+  <!-- Angular Material Dependencies -->
+  <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.0/angular.min.js" type="text/javascript"></script>
+  <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.0/angular-animate.min.js" type="text/javascript"></script>
+  <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.0/angular-aria.min.js" type="text/javascript"></script>
+  <script src="https://ajax.googleapis.com/ajax/libs/angular_material/1.0.5/angular-material.min.js" type="text/javascript"></script>
+  <script src="md-data-table.min.js" type="text/javascript"></script>
+  <script src="toArrayFilter.js" type="text/javascript"></script>
+  <script src="script.js" type="text/javascript"></script>
+</body>
+
+</html>