Merge pull request #180 from ncdc/picks-for-0.5.1

WIP Picks for 0.5.1

.travis.yml

@@ -0,0 +1,6 @@
+language: go
+
+go:
+  - 1.9.x
+
+script: make ci

Makefile

@@ -87,6 +87,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs
 		VERSION=$(VERSION) \
 		PKG=$(PKG) \
 		BIN=$(BIN) \
+		OUTPUT_DIR=/output/$(GOOS)/$(GOARCH) \
 		./hack/build.sh'"
 
 TTY := $(shell tty -s && echo "-t")
@@ -181,3 +182,8 @@ container-clean:
 
 bin-clean:
 	rm -rf .go _output
+
+ci:
+	hack/verify-all.sh
+	hack/test.sh $(SRC_DIRS)
+	GOOS=$(GOOS) GOARCH=$(GOARCH) VERSION=$(VERSION) PKG=$(PKG) BIN=$(BIN) ./hack/build.sh

hack/build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -53,7 +53,10 @@ LDFLAGS="-X ${PKG}/pkg/buildinfo.Version=${VERSION}"
 LDFLAGS="${LDFLAGS} -X ${PKG}/pkg/buildinfo.GitSHA=${GIT_SHA}"
 LDFLAGS="${LDFLAGS} -X ${PKG}/pkg/buildinfo.GitTreeState=${GIT_TREE_STATE}"
 
-OUTPUT=/output/${GOOS}/${GOARCH}/${BIN}
+if [[ -z "${OUTPUT_DIR:-}" ]]; then
+  OUTPUT_DIR=.
+fi
+OUTPUT=${OUTPUT_DIR}/${BIN}
 if [[ "${GOOS}" = "windows" ]]; then
   OUTPUT="${OUTPUT}.exe"
 fi

hack/test.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2016 The Kubernetes Authors.
 #

hack/update-generated-clientsets.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARK_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+ARK_ROOT=$(dirname ${BASH_SOURCE})/..
 BIN=${ARK_ROOT}/_output/bin
 mkdir -p ${BIN}
 go build -o ${BIN}/client-gen ./vendor/k8s.io/kubernetes/cmd/libs/go2idl/client-gen

hack/update-generated-docs.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARK_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+ARK_ROOT=$(dirname ${BASH_SOURCE})/..
 BIN=${ARK_ROOT}/_output/bin
 mkdir -p ${BIN}
 

hack/update-generated-informers.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARK_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+ARK_ROOT=$(dirname ${BASH_SOURCE})/..
 BIN=${ARK_ROOT}/_output/bin
 mkdir -p ${BIN}
 

hack/update-generated-listers.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARK_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+ARK_ROOT=$(dirname ${BASH_SOURCE})/..
 BIN=${ARK_ROOT}/_output/bin
 mkdir -p ${BIN}
 

hack/verify-generated-docs.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARK_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
+ARK_ROOT=$(dirname ${BASH_SOURCE})/..
 HACK_DIR=$(dirname "${BASH_SOURCE}")
 DOCS_DIR=${ARK_ROOT}/docs/cli-reference
 TMP_DIR="$(mktemp -d)"

pkg/backup/backup_test.go

@@ -230,7 +230,7 @@ func TestGetNamespaceIncludesExcludes(t *testing.T) {
 var (
 	v1Group = &metav1.APIResourceList{
 		GroupVersion: "v1",
-		APIResources: []metav1.APIResource{configMapsResource, podsResource},
+		APIResources: []metav1.APIResource{configMapsResource, podsResource, namespacesResource},
 	}
 
 	configMapsResource = metav1.APIResource{
@@ -266,6 +266,14 @@ var (
 		Verbs:        metav1.Verbs([]string{"create", "update", "get", "list", "watch", "delete"}),
 	}
 
+	namespacesResource = metav1.APIResource{
+		Name:         "namespaces",
+		SingularName: "namespace",
+		Namespaced:   false,
+		Kind:         "Namespace",
+		Verbs:        metav1.Verbs([]string{"create", "update", "get", "list", "watch", "delete"}),
+	}
+
 	certificatesGroup = &metav1.APIResourceList{
 		GroupVersion: "certificates.k8s.io/v1beta1",
 		APIResources: []metav1.APIResource{certificateSigningRequestsResource},

pkg/backup/item_backupper.go

@@ -103,6 +103,7 @@ type defaultItemBackupper struct {
 }
 
 var podsGroupResource = schema.GroupResource{Group: "", Resource: "pods"}
+var namespacesGroupResource = schema.GroupResource{Group: "", Resource: "namespaces"}
 
 // backupItem backs up an individual item to tarWriter. The item may be excluded based on the
 // namespaces IncludesExcludes list.
@@ -127,7 +128,9 @@ func (ib *defaultItemBackupper) backupItem(logger *logrus.Entry, obj runtime.Uns
 		return nil
 	}
 
-	if namespace == "" && ib.backup.Spec.IncludeClusterResources != nil && !*ib.backup.Spec.IncludeClusterResources {
+	// NOTE: we specifically allow namespaces to be backed up even if IncludeClusterResources is
+	// false.
+	if namespace == "" && groupResource != namespacesGroupResource && ib.backup.Spec.IncludeClusterResources != nil && !*ib.backup.Spec.IncludeClusterResources {
 		log.Info("Excluding item because resource is cluster-scoped and backup.spec.includeClusterResources is false")
 		return nil
 	}

pkg/backup/item_backupper_test.go

@@ -103,6 +103,23 @@ func TestBackupItemSkips(t *testing.T) {
 	}
 }
 
+func TestBackupItemSkipsClusterScopedResourceWhenIncludeClusterResourcesFalse(t *testing.T) {
+	f := false
+	ib := &defaultItemBackupper{
+		backup: &v1.Backup{
+			Spec: v1.BackupSpec{
+				IncludeClusterResources: &f,
+			},
+		},
+		namespaces: collections.NewIncludesExcludes(),
+		resources:  collections.NewIncludesExcludes(),
+	}
+
+	u := unstructuredOrDie(`{"apiVersion":"v1","kind":"Foo","metadata":{"name":"bar"}}`)
+	err := ib.backupItem(arktest.NewLogger(), u, schema.GroupResource{Group: "foo", Resource: "bar"})
+	assert.NoError(t, err)
+}
+
 func TestBackupItemNoSkips(t *testing.T) {
 	tests := []struct {
 		name                                  string

pkg/backup/resource_backupper.go

@@ -25,6 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	kuberrs "k8s.io/apimachinery/pkg/util/errors"
@@ -122,24 +123,27 @@ func (rb *defaultResourceBackupper) backupResource(
 
 	log := rb.log.WithField("groupResource", grString)
 
-	switch {
-	case rb.backup.Spec.IncludeClusterResources == nil:
-		// when IncludeClusterResources == nil (auto), only directly
-		// back up cluster-scoped resources if we're doing a full-cluster
-		// (all namespaces) backup. Note that in the case of a subset of
-		// namespaces being backed up, some related cluster-scoped resources
-		// may still be backed up if triggered by a custom action (e.g. PVC->PV).
-		if !resource.Namespaced && !rb.namespaces.IncludeEverything() {
-			log.Info("Skipping resource because it's cluster-scoped and only specific namespaces are included in the backup")
-			return nil
-		}
-	case *rb.backup.Spec.IncludeClusterResources == false:
-		if !resource.Namespaced {
+	clusterScoped := !resource.Namespaced
+
+	// If the resource we are backing up is NOT namespaces, and it is cluster-scoped, check to see if
+	// we should include it based on the IncludeClusterResources setting.
+	if gr != namespacesGroupResource && clusterScoped {
+		if rb.backup.Spec.IncludeClusterResources == nil {
+			if !rb.namespaces.IncludeEverything() {
+				// when IncludeClusterResources == nil (auto), only directly
+				// back up cluster-scoped resources if we're doing a full-cluster
+				// (all namespaces) backup. Note that in the case of a subset of
+				// namespaces being backed up, some related cluster-scoped resources
+				// may still be backed up if triggered by a custom action (e.g. PVC->PV).
+				// If we're processing namespaces themselves, we will not skip here, they may be
+				// filtered out later.
+				log.Info("Skipping resource because it's cluster-scoped and only specific namespaces are included in the backup")
+				return nil
+			}
+		} else if !*rb.backup.Spec.IncludeClusterResources {
 			log.Info("Skipping resource because it's cluster-scoped")
 			return nil
 		}
-	case *rb.backup.Spec.IncludeClusterResources == true:
-		// include the resource, no action required
 	}
 
 	if !rb.resources.ShouldInclude(grString) {
@@ -173,12 +177,50 @@ func (rb *defaultResourceBackupper) backupResource(
 		rb.discoveryHelper,
 	)
 
-	var namespacesToList []string
-	if resource.Namespaced {
-		namespacesToList = getNamespacesToList(rb.namespaces)
-	} else {
+	namespacesToList := getNamespacesToList(rb.namespaces)
+
+	// Check if we're backing up namespaces, and only certain ones
+	if gr == namespacesGroupResource && namespacesToList[0] != "" {
+		resourceClient, err := rb.dynamicFactory.ClientForGroupVersionResource(gv, resource, "")
+		if err != nil {
+			return err
+		}
+
+		var labelSelector labels.Selector
+		if rb.backup.Spec.LabelSelector != nil {
+			labelSelector, err = metav1.LabelSelectorAsSelector(rb.backup.Spec.LabelSelector)
+			if err != nil {
+				// This should never happen...
+				return errors.Wrap(err, "invalid label selector")
+			}
+		}
+
+		for _, ns := range namespacesToList {
+			unstructured, err := resourceClient.Get(ns, metav1.GetOptions{})
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "error getting namespace"))
+				continue
+			}
+
+			labels := labels.Set(unstructured.GetLabels())
+			if labelSelector != nil && !labelSelector.Matches(labels) {
+				log.WithField("name", unstructured.GetName()).Info("skipping item because it does not match the backup's label selector")
+				continue
+			}
+
+			if err := itemBackupper.backupItem(log, unstructured, gr); err != nil {
+				errs = append(errs, err)
+			}
+		}
+
+		return kuberrs.NewAggregate(errs)
+	}
+
+	// If we get here, we're backing up something other than namespaces
+	if clusterScoped {
 		namespacesToList = []string{""}
 	}
+
 	for _, namespace := range namespacesToList {
 		resourceClient, err := rb.dynamicFactory.ClientForGroupVersionResource(gv, resource, namespace)
 		if err != nil {
@@ -203,6 +245,17 @@ func (rb *defaultResourceBackupper) backupResource(
 				continue
 			}
 
+			metadata, err := meta.Accessor(unstructured)
+			if err != nil {
+				errs = append(errs, errors.Wrapf(err, "unable to get a metadata accessor"))
+				continue
+			}
+
+			if gr == namespacesGroupResource && !rb.namespaces.ShouldInclude(metadata.GetName()) {
+				log.WithField("name", metadata.GetName()).Info("skipping namespace because it is excluded")
+				continue
+			}
+
 			if err := itemBackupper.backupItem(log, unstructured, gr); err != nil {
 				errs = append(errs, err)
 			}