From ae8029c0cb8f7d36305694c2085e66ba320b675b Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 27 Sep 2023 22:55:48 +0000 Subject: [PATCH] Expose CLI for creating tokens This PR exposes the ability for a user at an origin to create tokens for testing. It can be accessed via `pelican origin token create`. There are several claims that all default tokens will posses, namely iat, exp, nbf, iss, and jti. These can be overridden by passing them specifically to the tool as claims. By passing a profile with the --profile flag, the tool will provide information about additional claims that are needed by the profile. NOTE: it only provides information about required values that ARE NOT auto-generated! The lifetime and private-key flags can be used to set the token lifetime (in seconds) and the private signing key, respectively. Next up is to create a CLI for verifying a provided token against a specific profile. While the basic infrastructure was set up for that via `pelican origin token verify`, the function remains unimplemented. --- cmd/origin.go | 38 +++++++ cmd/origin_token.go | 230 +++++++++++++++++++++++++++++++++++++++ cmd/origin_token_test.go | 184 +++++++++++++++++++++++++++++++ go.mod | 2 +- 4 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 cmd/origin_token.go create mode 100644 cmd/origin_token_test.go diff --git a/cmd/origin.go b/cmd/origin.go index 9d707e788..8fdec2988 100644 --- a/cmd/origin.go +++ b/cmd/origin.go @@ -44,6 +44,34 @@ var ( RunE: serveOrigin, SilenceUsage: true, } + + // Expose the token manipulation CLI + originTokenCmd = &cobra.Command{ + Use: "token", + Short: "Manage Pelican origin tokens", + } + + originTokenCreateCmd = &cobra.Command{ + Use: "create", + Short: "Create a Pelican origin token", + Long: `Create a JSON web token (JWT) using the origin's signing keys: + +Pelican origins use JWTs as bearer tokens for authorizing specific requests, +such as reading from or writing to the origin's underlying storage, advertising +to a director, etc. For more information about the makeup of a JWT, see +https://jwt.io/introduction. + +Additional profiles that expand on JWT are supported. They include scitokens2 and +wlcg1. For more information about these profiles, see https://scitokens.org/technical_docs/Claims +and https://github.com/WLCG-AuthZ-WG/common-jwt-profile/blob/master/profile.md, respectively`, + RunE: cliTokenCreate, + } + + originTokenVerifyCmd = &cobra.Command{ + Use: "verify", + Short: "Verify a Pelican origin token", + RunE: verifyToken, + } ) func configOrigin( /*cmd*/ *cobra.Command /*args*/, []string) { @@ -59,4 +87,14 @@ func init() { panic(err) } originServeCmd.Flags().AddFlag(portFlag) + + originCmd.AddCommand(originTokenCmd) + originTokenCmd.AddCommand(originTokenCreateCmd) + originTokenCmd.PersistentFlags().String("profile", "", "Passing a profile ensures the created token adheres to the profile's requirements. Accepted values are scitokens2 and wlcg1") + originTokenCreateCmd.Flags().Int("lifetime", 1200, "The lifetime of the token, in seconds.") + originTokenCreateCmd.Flags().String("private-key", viper.GetString("IssuerKey"), "Filepath designating the location of the private key in PEM format to be used for signing, if different from the origin's default.") + if err := viper.BindPFlag("IssuerKey", originTokenCreateCmd.Flags().Lookup("private-key")); err != nil { + panic(err) + } + originTokenCmd.AddCommand(originTokenVerifyCmd) } diff --git a/cmd/origin_token.go b/cmd/origin_token.go new file mode 100644 index 000000000..b03ab33d0 --- /dev/null +++ b/cmd/origin_token.go @@ -0,0 +1,230 @@ +package main + +import ( + "fmt" + + "github.com/google/uuid" + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jwt" + "net/url" + "regexp" + "time" + + "strconv" + "strings" + + "github.com/pelicanplatform/pelican/config" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// The verifyCreate* funcs only act on the provided claims maps, because they attempt +// to verify certain aspects of the token before it is created for simplicity. To verify +// an actual token object, use the analagous "verifyToken" +func verifyCreateSciTokens2(claimsMap *map[string]string) error { + /* + Don't check for the following claims because ALL base tokens have them: + - iat + - exp + - nbf + - iss + */ + if len(*claimsMap) == 0 { + return errors.New("To create a valid SciToken, the 'aud' and 'scope' claims must be passed, but none were found.") + } + requiredClaims := []string{"aud", "ver", "scope"} + for _, reqClaim := range requiredClaims { + if val, exists := (*claimsMap)[reqClaim]; !exists { + // we can set ver because we know what it should be + if reqClaim == "ver" { + (*claimsMap)["ver"] = "scitokens:2.0" + } else { + // We can't set scope or aud, however + errMsg := "The claim '" + reqClaim + "' is required for the scitokens2 profile, but it could not be found." + return errors.New(errMsg) + } + } else { + // The claim exists. While we're okay setting ver if it's not included, it + // feels wrong to correct an explicitly-provided version that isn't correct, + // so in that event, fail. + if reqClaim == "ver" { + verPattern := `^scitokens:2\.[0-9]+$` + re := regexp.MustCompile(verPattern) + + if !re.MatchString(val) { + errMsg := "The provided version '" + val + "' is not valid. It must match 'scitokens:', where version is of the form 2.x" + return errors.New(errMsg) + } + } + } + } + + return nil +} + +func verifyCreateWLCG1(claimsMap *map[string]string) error { + /* + Don't check for the following claims because ALL base tokens have them: + - iat + - exp + - nbf + - iss + - jti + */ + if len(*claimsMap) == 0 { + return errors.New("To create a valid wlcg, the 'aud' and 'sub' claims must be passed, but none were found.") + } + + requiredClaims := []string{"sub", "wlcg.ver", "aud"} + for _, reqClaim := range requiredClaims { + if val, exists := (*claimsMap)[reqClaim]; !exists { + // we can set wlcg.ver because we know what it should be + if reqClaim == "wlcg.ver" { + (*claimsMap)["wlcg.ver"] = "1.0" + } else { + // We can't set the rest + errMsg := "The claim '" + reqClaim + "' is required for the wlcg1 profile, but it could not be found." + return errors.New(errMsg) + } + } else { + if reqClaim == "wlcg.ver" { + verPattern := `^1\.[0-9]+$` + re := regexp.MustCompile(verPattern) + if !re.MatchString(val) { + errMsg := "The provided version '" + val + "' is not valid. It must be of the form '1.x'" + return errors.New(errMsg) + } + } + } + } + + return nil +} + +func parseClaims(claims []string) (map[string]string, error) { + claimsMap := make(map[string]string) + // We assume each claim has exactly one "=" delimiter + for _, claim := range claims { + parts := strings.Split(claim, "=") + if len(parts) != 2 { + if len(parts) < 2 { + errMsg := "The claim '" + claim + "' is invalid. Did you forget an '='?" + return nil, errors.New(errMsg) + } else { + errMsg := "The claim '" + claim + "' is invalid. Does it contain more than one '='?" + return nil, errors.New(errMsg) + } + } + key := parts[0] + val := parts[1] + + if existingVal, exists := claimsMap[key]; exists { + claimsMap[key] = existingVal + " " + val + } else { + claimsMap[key] = val + } + } + return claimsMap, nil +} + +func createEncodedToken(claimsMap map[string]string, profile string, lifetime int) (string, error) { + var err error + if profile != "" { + if profile == "scitokens2" { + err = verifyCreateSciTokens2(&claimsMap) + if err != nil { + return "", errors.Wrap(err, "Token does not conform to scitokens2 requirements") + } + } else if profile == "wlcg1" { + err = verifyCreateWLCG1(&claimsMap) + if err != nil { + return "", errors.Wrap(err, "Token does not conform to wlcg1 requirements") + } + } else { + errMsg := "The provided profile '" + profile + "' is not recognized. Valid options are 'scitokens2' or 'wlcg1'" + return "", errors.New(errMsg) + } + } + + lifetimeDuration := time.Duration(lifetime) + // Create a jti using uuid4. This will be added to all tokens. + u, err := uuid.NewRandom() + if err != nil { + return "", errors.Wrap(err, "Failed to generate uuid4 for token jti") + } + + issuerUrlStr := viper.GetString("IssuerUrl") + issuerUrl, err := url.Parse(issuerUrlStr) + if err != nil { + return "", errors.Wrap(err, "Failed to parse the configured IssuerUrl") + } + + now := time.Now() + builder := jwt.NewBuilder() + builder.Issuer(issuerUrl.String()). + IssuedAt(now). + Expiration(now.Add(time.Second * lifetimeDuration)). + NotBefore(now). + JwtID(u.String()) + + // Add cli-passed claims after setting up the basic token so that we + // expose a method to override anything we already set. + for key, val := range claimsMap { + builder.Claim(key, val) + } + + tok, err := builder.Build() + if err != nil { + return "", errors.Wrap(err, "Failed to generate token") + } + + // Now that we have a token, it needs signing. Note that GetOriginJWK + // will get the private key passed via the command line because that + // file path has already been bound to IssuerKey + key, err := config.GetOriginJWK() + if err != nil { + return "", errors.Wrap(err, "Failed to load signing keys. Either generate one at the default location by serving an origin, or provide one via the --private-key flag") + } + + // Get/assign the kid, needed for verification by the client + err = jwk.AssignKeyID(*key) + if err != nil { + return "", errors.Wrap(err, "Failed to assign kid to the token") + } + + signed, err := jwt.Sign(tok, jwt.WithKey(jwa.ES512, *key)) + if err != nil { + return "", errors.Wrap(err, "Failed to sign the deletion token") + } + + return string(signed), nil +} + +func cliTokenCreate( /*cmd*/ cmd *cobra.Command /*args*/, args []string) error { + claimsMap, err := parseClaims(args) + if err != nil { + return errors.Wrap(err, "Failed to parse token claims") + } + + // Check if a profile was provided and verify what we need to from the claimsMap + profile := cmd.Flags().Lookup("profile").Value.String() + + lifetime, err := strconv.Atoi(cmd.Flags().Lookup("lifetime").Value.String()) + if err != nil { + return errors.Wrapf(err, "Failed to parse lifetime '%d' as an integer", lifetime) + } + + token, err := createEncodedToken(claimsMap, profile, lifetime) + if err != nil { + return errors.Wrap(err, "Failed to create the token") + } + + fmt.Println(token) + return nil +} + +func verifyToken( /*cmd*/ cmd *cobra.Command /*args*/, args []string) error { + return errors.New("Token verification not yet implemented") +} diff --git a/cmd/origin_token_test.go b/cmd/origin_token_test.go new file mode 100644 index 000000000..0151d36be --- /dev/null +++ b/cmd/origin_token_test.go @@ -0,0 +1,184 @@ +/*************************************************************** + * + * Copyright (C) 2023, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +package main + +import ( + // "net/url" + "os" + "path/filepath" + "testing" + + "github.com/pelicanplatform/pelican/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestVerifyCreateSciTokens2(t *testing.T) { + // Start by feeding it a valid claims map + claimsMap := map[string]string{"aud": "foo", "ver": "scitokens:2.0", "scope": "read:/storage"} + err := verifyCreateSciTokens2(&claimsMap) + assert.NoError(t, err) + + // Fail to give it audience + claimsMap = map[string]string{"ver": "scitokens:2.0", "scope": "read:/storage"} + err = verifyCreateSciTokens2(&claimsMap) + assert.EqualError(t, err, "The claim 'aud' is required for the scitokens2 profile, but it could not be found.") + + // Fail to give it scope + claimsMap = map[string]string{"aud": "foo", "ver": "scitokens:2.0"} + err = verifyCreateSciTokens2(&claimsMap) + assert.EqualError(t, err, "The claim 'scope' is required for the scitokens2 profile, but it could not be found.") + + // Give it bad version + claimsMap = map[string]string{"aud": "foo", "scope": "bar", "ver": "scitokens:2.xxxx"} + err = verifyCreateSciTokens2(&claimsMap) + assert.EqualError(t, err, "The provided version 'scitokens:2.xxxx' is not valid. It must match 'scitokens:', where version is of the form 2.x") + + // Don't give it a version and make sure it gets set correctly + claimsMap = map[string]string{"aud": "foo", "scope": "bar"} + err = verifyCreateSciTokens2(&claimsMap) + assert.NoError(t, err) + assert.Equal(t, claimsMap["ver"], "scitokens:2.0") + + // Give it a non-required claim to make sure it makes it through + claimsMap = map[string]string{"aud": "foo", "scope": "bar", "sub": "origin"} + err = verifyCreateSciTokens2(&claimsMap) + assert.NoError(t, err) + assert.Equal(t, claimsMap["sub"], "origin") +} + +func TestVerifyCreateWLCG1(t *testing.T) { + // Start by feeding it a valid claims map + claimsMap := map[string]string{"sub": "foo", "wlcg.ver": "1.0", "jti": "1234", "aud": "director"} + err := verifyCreateWLCG1(&claimsMap) + assert.NoError(t, err) + + // Fail to give it a sub + claimsMap = map[string]string{"wlcg.ver": "1.0", "jti": "1234", "aud": "director"} + err = verifyCreateWLCG1(&claimsMap) + assert.EqualError(t, err, "The claim 'sub' is required for the wlcg1 profile, but it could not be found.") + + // Fail to give it an aud + claimsMap = map[string]string{"wlcg.ver": "1.0", "jti": "1234", "sub": "foo"} + err = verifyCreateWLCG1(&claimsMap) + assert.EqualError(t, err, "The claim 'aud' is required for the wlcg1 profile, but it could not be found.") + + // Give it bad version + claimsMap = map[string]string{"sub": "foo", "wlcg.ver": "1.xxxx", "jti": "1234", "aud": "director"} + err = verifyCreateWLCG1(&claimsMap) + assert.EqualError(t, err, "The provided version '1.xxxx' is not valid. It must be of the form '1.x'") + + // Don't give it a version and make sure it gets set correctly + claimsMap = map[string]string{"sub": "foo", "jti": "1234", "aud": "director"} + err = verifyCreateWLCG1(&claimsMap) + assert.NoError(t, err) + assert.Equal(t, claimsMap["wlcg.ver"], "1.0") + + // Give it a non-required claim to make sure it makes it through + claimsMap = map[string]string{"sub": "foo", "wlcg.ver": "1.0", "jti": "1234", "aud": "director", "anotherClaim": "bar"} + err = verifyCreateWLCG1(&claimsMap) + assert.NoError(t, err) + assert.Equal(t, claimsMap["anotherClaim"], "bar") +} + +func TestParseClaims(t *testing.T) { + // Give it something valid + claims := []string{"foo=boo", "bar=baz"} + claimsMap, err := parseClaims(claims) + assert.NoError(t, err) + assert.Equal(t, claimsMap["foo"], "boo") + assert.Equal(t, claimsMap["bar"], "baz") + assert.Equal(t, len(claimsMap), 2) + + // Give it something with multiple of the same claim key + claims = []string{"foo=boo", "foo=baz"} + claimsMap, err = parseClaims(claims) + assert.NoError(t, err) + assert.Equal(t, claimsMap["foo"], "boo baz") + assert.Equal(t, len(claimsMap), 1) + + // Give it something without = delimiter + claims = []string{"foo=boo", "barbaz"} + _, err = parseClaims(claims) + assert.EqualError(t, err, "The claim 'barbaz' is invalid. Did you forget an '='?") + + // Give it something with extra = + claims = []string{"foo=boo", "bar==baz"} + _, err = parseClaims(claims) + assert.EqualError(t, err, "The claim 'bar==baz' is invalid. Does it contain more than one '='?") +} + +func TestCreateToken(t *testing.T) { + // For now, the test doesn't actually test for token validity + + // Redirect stdout to a buffer to prevent printing the token during tests + // In theory, we could use this to grab the actual tokens as well. + // TODO: Figure out how to generate consistent tokens + old := os.Stdout + _, w, _ := os.Pipe() + os.Stdout = w + + // Create temp dir for the origin key file + viper.Reset() + tDir := t.TempDir() + kfile := filepath.Join(tDir, "testKey") + viper.Set("IssuerKey", kfile) + + // Generate a private key to use for the test + _, err := config.LoadPublicKey("", kfile) + assert.NoError(t, err) + + // Create a profile-less token + cmd := &cobra.Command{} + cmd.Flags().Int("lifetime", 1200, "Lifetime") + cmd.Flags().String("profile", "", "creation profile") + cmd.Flags().String("private-key", kfile, "private key path") + // Here, we pin various time-related values so we can get a consistent token + testArgs := []string{"scope=foo", "aud=bar", "iat=12345", "exp=12345", "nbf=12345"} + err = cliTokenCreate(cmd, testArgs) + assert.NoError(t, err) + + // Create a scitokens token + cmd = &cobra.Command{} + cmd.Flags().Int("lifetime", 1200, "Lifetime") + cmd.Flags().String("profile", "scitokens2", "creation profile") + testArgs = []string{"aud=foo", "scope=read:/storage", "iat=12345", "exp=12345", "nbf=12345"} + err = cliTokenCreate(cmd, testArgs) + assert.NoError(t, err) + + // Create a wlcg token + cmd = &cobra.Command{} + cmd.Flags().Int("lifetime", 1200, "Lifetime") + cmd.Flags().String("profile", "wlcg1", "creation profile") + testArgs = []string{"sub=foo", "wlcg.ver=1.0", "jti=1234", "aud=director"} + err = cliTokenCreate(cmd, testArgs) + assert.NoError(t, err) + + // Pass an invalid profile + cmd = &cobra.Command{} + cmd.Flags().Int("lifetime", 1200, "Lifetime") + cmd.Flags().String("profile", "foobar", "creation profile") + testArgs = []string{"sub=foo", "wlcg.ver=1.0", "jti=1234", "aud=director"} + err = cliTokenCreate(cmd, testArgs) + assert.EqualError(t, err, "Failed to create the token: The provided profile 'foobar' is not recognized. Valid options are 'scitokens2' or 'wlcg1'") + + w.Close() + os.Stdout = old +} diff --git a/go.mod b/go.mod index e42a0e23e..044abefb0 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/go-kit/kit v0.12.0 github.com/go-kit/log v0.2.1 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/google/uuid v1.3.0 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd github.com/jellydator/ttlcache/v3 v3.0.1 github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c @@ -84,7 +85,6 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect