Skip to content

Commit

Permalink
Generate user accounts on the fly in tests
Browse files Browse the repository at this point in the history 
instead of relying them being created in advance

User management is backed up by creating service accounts on the fly,
and use their tokens to create CF API rest clients.

For tests that verify certificate authentication we generate
certificates on the fly as well. However, on EKS signing certificates
with `client auth` usage is not supported, therefore we skip those
tests. As a matter of fact, those tests have already been skipped on
EKS by simply not injecting the certificate related environment
variables.

All of the above will simplify CI setup significantly.

Co-authored-by: Danail Branekov <[email protected]>
Co-authored-by: Georgi Sabev <[email protected]>
  • Loading branch information
@danail-branekov @georgethebeatle
danail-branekov and georgethebeatle committed Aug 15, 2023
1 parent 4c9582d commit 914d839
Show file tree
Hide file tree
Showing 10 changed files with 320 additions and 146 deletions.
2 changes: 1 addition & 1 deletion api/authorization/user_client_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ var _ = Describe("Unprivileged User Client Factory", func() {

When("the cert is not valid on this cluster", func() {
BeforeEach(func() {
authInfo.CertData = helpers.CreateCertificatePEM()
authInfo.CertData = helpers.CreateSelfSignedCertificatePEM()
})

It("creates an unusable client", func() {
Expand Down
20 changes: 5 additions & 15 deletions tests/e2e/authorization_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package e2e_test

import (
"crypto/tls"
"net/http"

"code.cloudfoundry.org/korifi/tests/helpers"
Expand All @@ -13,9 +12,6 @@ import (

var _ = Describe("Authorization", func() {
var (
serviceaccountFactory *helpers.ServiceAccountFactory
svcAcctName string

userName string
userClient *helpers.CorrelatedRestyClient

Expand All @@ -24,18 +20,12 @@ var _ = Describe("Authorization", func() {
)

BeforeEach(func() {
serviceaccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
svcAcctName = uuid.NewString()

userName = "system:serviceaccount:cf:" + svcAcctName
userToken := serviceaccountFactory.CreateServiceAccount(svcAcctName)
userClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
SetAuthToken(userToken).
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
userName = uuid.NewString()
userClient = makeTokenClient(serviceAccountFactory.CreateServiceAccount(userName))
})

AfterEach(func() {
serviceaccountFactory.DeleteServiceAccount(svcAcctName)
serviceAccountFactory.DeleteServiceAccount(userName)
})

Describe("Unauthorized users", func() {
Expand Down Expand Up @@ -77,11 +67,11 @@ var _ = Describe("Authorization", func() {
BeforeEach(func() {
orgName := generateGUID("org")
orgGUID = createOrg(orgName)
createOrgRole("organization_user", userName, orgGUID)
createOrgRole("organization_user", serviceAccountFactory.FullyQualifiedName(userName), orgGUID)

spaceName := generateGUID("space")
spaceGUID = createSpace(spaceName, orgGUID)
createSpaceRole("space_developer", userName, spaceGUID)
createSpaceRole("space_developer", serviceAccountFactory.FullyQualifiedName(userName), spaceGUID)
})

AfterEach(func() {
Expand Down
88 changes: 40 additions & 48 deletions tests/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"archive/zip"
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -38,21 +39,12 @@ import (
var (
correlationId string

adminClient *helpers.CorrelatedRestyClient
privilegedServiceAccountClient *helpers.CorrelatedRestyClient
longCertClient *helpers.CorrelatedRestyClient
serviceAccountFactory *helpers.ServiceAccountFactory
adminServiceAccount string
adminClient *helpers.CorrelatedRestyClient

apiServerRoot string

serviceAccountName string
serviceAccountToken string

certUserName string
certPEM string

longCertUserName string
longCertPEM string

rootNamespace string
appFQDN string
commonTestOrgGUID string
Expand Down Expand Up @@ -307,14 +299,22 @@ func TestE2E(t *testing.T) {
}

type sharedSetupData struct {
CommonOrgName string `json:"commonOrgName"`
CommonOrgGUID string `json:"commonOrgGuid"`
DefaultAppBitsFile string `json:"defaultAppBitsFile"`
MultiProcessAppBitsFile string `json:"multiProcessAppBitsFile"`
CommonOrgName string `json:"commonOrgName"`
CommonOrgGUID string `json:"commonOrgGuid"`
DefaultAppBitsFile string `json:"defaultAppBitsFile"`
MultiProcessAppBitsFile string `json:"multiProcessAppBitsFile"`
AdminServiceAccount string `json:"admin_service_account"`
AdminServiceAccountToken string `json:"admin_service_account_token"`
}

var _ = SynchronizedBeforeSuite(func() []byte {
commonTestSetup()

adminServiceAccount = uuid.NewString()
adminServiceAccountToken := serviceAccountFactory.CreateAdminServiceAccount(adminServiceAccount)

adminClient = makeTokenClient(adminServiceAccountToken)

commonTestOrgName = generateGUID("common-test-org")
commonTestOrgGUID = createOrg(commonTestOrgName)

Expand All @@ -329,15 +329,19 @@ var _ = SynchronizedBeforeSuite(func() []byte {
// The DEFAULT_APP_BITS_PATH and DEFAULT_APP_RESPONSE environment variables are a workaround to allow e2e tests to run
// with a different app in these environments.
// See https://github.com/cloudfoundry/korifi/issues/2355 for refactoring ideas
DefaultAppBitsFile: zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
MultiProcessAppBitsFile: zipAsset("../assets/multi-process"),
DefaultAppBitsFile: zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
MultiProcessAppBitsFile: zipAsset("../assets/multi-process"),
AdminServiceAccount: adminServiceAccount,
AdminServiceAccountToken: adminServiceAccountToken,
}

bs, err := json.Marshal(sharedData)
Expect(err).NotTo(HaveOccurred())

return bs
}, func(bs []byte) {
commonTestSetup()

var sharedSetup sharedSetupData
err := json.Unmarshal(bs, &sharedSetup)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -346,46 +350,44 @@ var _ = SynchronizedBeforeSuite(func() []byte {
commonTestOrgName = sharedSetup.CommonOrgName
defaultAppBitsFile = sharedSetup.DefaultAppBitsFile
multiProcessAppBitsFile = sharedSetup.MultiProcessAppBitsFile
adminServiceAccount = sharedSetup.AdminServiceAccount
adminClient = makeTokenClient(sharedSetup.AdminServiceAccountToken)

eventuallyTimeoutSeconds := 240
customEventuallyTimeoutSeconds := os.Getenv("E2E_EVENTUALLY_TIMEOUT_SECONDS")
if customEventuallyTimeoutSeconds != "" {
eventuallyTimeoutSeconds, err = strconv.Atoi(customEventuallyTimeoutSeconds)
Expect(err).NotTo(HaveOccurred())
}
SetDefaultEventuallyTimeout(time.Duration(eventuallyTimeoutSeconds) * time.Second)
SetDefaultEventuallyTimeout(helpers.EventuallyTimeout())
SetDefaultEventuallyPollingInterval(2 * time.Second)

logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

commonTestSetup()
})

var _ = SynchronizedAfterSuite(func() {
}, func() {
os.RemoveAll(assetsTmpDir)
deleteOrg(commonTestOrgGUID)
serviceAccountFactory.DeleteServiceAccount(adminServiceAccount)
})

var _ = BeforeEach(func() {
correlationId = uuid.NewString()
})

func makeClient(certEnvVar, tokenEnvVar string) *helpers.CorrelatedRestyClient {
func makeCertClientForUserName(userName string, validFor time.Duration) *helpers.CorrelatedRestyClient {
GinkgoHelper()

cert := os.Getenv(certEnvVar)
if cert != "" {
return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(cert).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
if os.Getenv("CLUSTER_TYPE") == "EKS" {
Skip("EKS does not support cert users: https://github.com/aws/containers-roadmap/issues/1604#issuecomment-1072660824")
}

token := os.Getenv(tokenEnvVar)
if token != "" {
return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(token).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
SetAuthScheme("ClientCert").
SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateTrustedCertificatePEM(userName, validFor))).
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}

Fail(fmt.Sprintf("One of %q or %q should have a value, but they are both empty", certEnvVar, tokenEnvVar))
return nil
func makeTokenClient(token string) *helpers.CorrelatedRestyClient {
return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
SetAuthScheme("Bearer").
SetAuthToken(token).
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}

func ensureServerIsUp() {
Expand Down Expand Up @@ -1025,14 +1027,6 @@ func addDestinationForRoute(appGUID, routeGUID string) []string {
func commonTestSetup() {
apiServerRoot = helpers.GetRequiredEnvVar("API_SERVER_ROOT")
rootNamespace = helpers.GetRequiredEnvVar("ROOT_NAMESPACE")
serviceAccountName = fmt.Sprintf("system:serviceaccount:%s:%s", rootNamespace, helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT"))
serviceAccountToken = helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT_TOKEN")

longCertUserName = helpers.GetRequiredEnvVar("E2E_LONGCERT_USER_NAME")
longCertPEM = os.Getenv("E2E_LONGCERT_USER_PEM")

certUserName = helpers.GetRequiredEnvVar("E2E_USER_NAME")
certPEM = os.Getenv("E2E_USER_PEM")

appFQDN = helpers.GetRequiredEnvVar("APP_FQDN")

Expand All @@ -1041,9 +1035,7 @@ func commonTestSetup() {

ensureServerIsUp()

adminClient = makeClient("CF_ADMIN_PEM", "CF_ADMIN_TOKEN")
privilegedServiceAccountClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(serviceAccountToken).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
longCertClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(longCertPEM).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
serviceAccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
}

func zipAsset(src string) string {
Expand Down
11 changes: 6 additions & 5 deletions tests/e2e/orgs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ import (
"fmt"
"net/http"
"sync"
"time"

"github.com/go-resty/resty/v2"
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
Expand Down Expand Up @@ -119,12 +121,11 @@ var _ = Describe("Orgs", func() {
// that gets called by the CLI on each login.
When("The client has a certificate with a long expiry date", func() {
BeforeEach(func() {
if longCertPEM == "" {
Skip("No certificate with a long expiry date provided")
}
restyClient = longCertClient
createOrgRole("organization_manager", longCertUserName, org2GUID)
userName := uuid.NewString()
restyClient = makeCertClientForUserName(userName, 365*24*time.Hour)
createOrgRole("organization_manager", userName, org2GUID)
})

It("returns orgs that the client has a role in and sets an HTTP warning header", func() {
Expect(resp).To(HaveRestyStatusCode(http.StatusOK))
Expect(resp).To(HaveRestyHeaderWithValue("X-Cf-Warnings", HavePrefix("Warning: The client certificate you provided for user authentication expires at")))
Expand Down
36 changes: 22 additions & 14 deletions tests/e2e/whoami_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"crypto/tls"
"encoding/base64"
"net/http"
"time"

"code.cloudfoundry.org/korifi/tests/helpers"
"github.com/go-resty/resty/v2"
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
rbacv1 "k8s.io/api/rbac/v1"
Expand All @@ -19,15 +21,11 @@ type identityResource struct {

var _ = Describe("WhoAmI", func() {
var (
client *resty.Client
client *helpers.CorrelatedRestyClient
httpResp *resty.Response
result identityResource
)

BeforeEach(func() {
client = resty.New().SetBaseURL(apiServerRoot).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
})

JustBeforeEach(func() {
var err error
httpResp, err = client.R().
Expand All @@ -37,13 +35,21 @@ var _ = Describe("WhoAmI", func() {
})

When("authenticating with a Bearer token", func() {
var svcAcctName string

BeforeEach(func() {
client = client.SetAuthToken(serviceAccountToken)
svcAcctName = uuid.NewString()
serviceAccountToken := serviceAccountFactory.CreateServiceAccount(svcAcctName)
client = makeTokenClient(serviceAccountToken)
})

AfterEach(func() {
serviceAccountFactory.DeleteServiceAccount(svcAcctName)
})

It("returns the user identity", func() {
Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
Expect(result.Name).To(Equal(serviceAccountName))
Expect(result.Name).To(Equal(serviceAccountFactory.FullyQualifiedName(svcAcctName)))
Expect(result.Kind).To(Equal(rbacv1.ServiceAccountKind))
})

Expand All @@ -59,16 +65,16 @@ var _ = Describe("WhoAmI", func() {
})

When("authenticating with a client certificate", func() {
var userName string

BeforeEach(func() {
if certPEM == "" {
Skip("No certificate provided.")
}
client = client.SetAuthScheme("ClientCert").SetAuthToken(certPEM)
userName = uuid.NewString()
client = makeCertClientForUserName(userName, time.Hour)
})

It("returns the user identity", func() {
Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
Expect(result.Name).To(Equal(certUserName))
Expect(result.Name).To(Equal(userName))
Expect(result.Kind).To(Equal(rbacv1.UserKind))
})

Expand All @@ -84,8 +90,7 @@ var _ = Describe("WhoAmI", func() {

When("the cert is unauthorized", func() {
BeforeEach(func() {
unauthorisedCertPEM := base64.StdEncoding.EncodeToString(helpers.CreateCertificatePEM())
client = client.SetAuthToken(unauthorisedCertPEM)
client = client.SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateSelfSignedCertificatePEM()))
})

It("returns an unauthorized error", func() {
Expand All @@ -95,6 +100,9 @@ var _ = Describe("WhoAmI", func() {
})

When("no Authorization header is available in the request", func() {
BeforeEach(func() {
client = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
})
It("returns unauthorized error", func() {
Expect(httpResp).To(HaveRestyStatusCode(http.StatusUnauthorized))
})
Expand Down
Loading

0 comments on commit 914d839

Please sign in to comment.