Skip to content

Commit

Permalink
feat(providers) initial implementation of pulumi provider (#180)
Browse files Browse the repository at this point in the history
Signed-off-by: Andrew Gershman <[email protected]>
  • Loading branch information
agershman authored Nov 25, 2023
1 parent e126a3d commit 88ba797
Show file tree
Hide file tree
Showing 7 changed files with 395 additions and 0 deletions.
27 changes: 27 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ It supports various backends including:
- 1Password Connect
- [Doppler](https://doppler.com/)
- CredHub(Coming soon)
- Pulumi State

- Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets.
- Use `vals exec -f env.yaml -- <COMMAND>` to populate envvars and execute the command.
Expand Down Expand Up @@ -214,6 +215,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/
- [GitLab](#gitlab)
- [1Password Connect](#1password-connect)
- [Doppler](#doppler)
- [Pulumi State](#pulumi-state)

Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes.

Expand Down Expand Up @@ -675,6 +677,31 @@ Examples:
- `ref+doppler://MyProject/development/DB_PASSWORD` fetches the value of secret with name `DB_PASSWORD` for the project named `MyProject` and environment named `development`.
- `ref+doppler://MyProject/development/#DB_PASSWORD` fetches the value of secret with name `DB_PASSWORD` for the project named `MyProject` and environment named `development`.
### Pulumi State
Obtain value in state pulled from Pulumi Cloud REST API:
- `ref+pulumistateapi://RESOURCE_TYPE/RESOURCE_LOGICAL_NAME/ATTRIBUTE_TYPE/ATTRIBUTE_KEY_PATH?project=PROJECT&stack=STACK`
* `RESOURCE_TYPE` is a Pulumi [resource type](https://www.pulumi.com/docs/concepts/resources/names/#types) of the form `<package>:<module>:<type>`, where forward slashes (`/`) are replaced by a double underscore (`__`) and colons (`:`) are replaced by a single underscore (`_`). For example `aws:s3:Bucket` would be encoded as `aws__s3__Bucket` and `kubernetes:storage.k8s.io/v1:StorageClass` would be encoded as `kubernetes_storage.k8s.io__v1_StorageClass`.
* `RESOURCE_LOGICAL_NAME` is the [logical name](https://www.pulumi.com/docs/concepts/resources/names/#logicalname) of the resource in the Pulumi program.
* `ATTRIBUTE_TYPE` is either `outputs` or `inputs`.
* `ATTRIBUTE_KEY_PATH` is a [GJSON](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) expression that selects the desired attribute from the resource's inputs or outputs per the chosen `ATTRIBUTE_TYPE` value. You must encode any characters that would otherwise not comply with URI syntax, for example `#` becomes `%23`.
* `project` is the Pulumi project name.
* `stack` is the Pulumi stack name.
Environment variables:
- `PULUMI_API_ENDPOINT_URL` is the Pulumi API endpoint URL. Defaults to `https://api.pulumi.com`. You may also provide this as the `pulumi_api_endpoint_url` query parameter.
- `PULUMI_ACCESS_TOKEN` is the Pulumi access token to use for authentication.
- `PULUMI_ORGANIZATION` is the Pulumi organization to use for authentication. You may also provide this as an `organization` query parameter.
Examples:
- `ref+pulumistateapi://aws-native_s3_Bucket/my-bucket/outputs/bucketName?project=my-project&stack=my-stack`
- `ref+pulumistateapi://aws-native_s3_Bucket/my-bucket/outputs/tags.%23(key==SomeKey).value?project=my-project&stack=my-stack`
- `ref+pulumistateapi://kubernetes_storage.k8s.io__v1_StorageClass/gp2-encrypted/inputs/metadata.name?project=my-project&stack=my-stack`
## Advanced Usages
### Discriminating config and secrets
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ require (
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/urfave/cli v1.22.14 // indirect
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r
github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
Expand Down Expand Up @@ -183,6 +184,7 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -275,6 +277,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand Down Expand Up @@ -350,6 +353,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
Expand Down Expand Up @@ -380,6 +389,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
Expand Down
185 changes: 185 additions & 0 deletions pkg/providers/pulumi/pulumi.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package pulumi

import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"

"github.com/tidwall/gjson"

"github.com/helmfile/vals/pkg/api"
"github.com/helmfile/vals/pkg/log"
)

const (
defaultPulumiAPIEndpointURL = "https://api.pulumi.com"
)

type provider struct {
log *log.Logger
backend string
pulumiAPIEndpointURL string
pulumiAPIAccessToken string
organization string
project string
stack string
}

type pulumiState struct {
Deployment pulumiDeployment `json:"deployment"`
}

func (r *pulumiState) findResourceByLogicalName(resourceType string, resourceLogicalName string) *pulumiResource {
for _, resource := range r.Deployment.Resources {
if resource.ResourceType == resourceType && strings.HasSuffix(resource.URN, resourceLogicalName) {
return &resource
}
}
return nil
}

type pulumiDeployment struct {
Resources []pulumiResource `json:"resources"`
}

type pulumiResource struct {
URN string `json:"urn"`
Custom bool `json:"custom"`
ID string `json:"id"`
ResourceType string `json:"type"`
Inputs json.RawMessage `json:"inputs"`
Outputs json.RawMessage `json:"outputs"`
}

func (r *pulumiResource) getAttributeValue(resourceAttribute string, resourceAttributePath string) string {
var attributeValue gjson.Result
switch resourceAttribute {
case "inputs":
attributeValue = gjson.GetBytes(r.Inputs, resourceAttributePath)
case "outputs":
attributeValue = gjson.GetBytes(r.Outputs, resourceAttributePath)
}
return attributeValue.String()
}

func New(l *log.Logger, cfg api.StaticConfig, backend string) *provider {
p := &provider{
log: l,
backend: backend,
}

if cfg.Exists("pulumi_api_endpoint_url") {
p.pulumiAPIEndpointURL = cfg.String("pulumi_api_endpoint_url")
} else {
p.pulumiAPIEndpointURL = os.Getenv("PULUMI_API_ENDPOINT_URL")
if p.pulumiAPIEndpointURL == "" {
p.pulumiAPIEndpointURL = defaultPulumiAPIEndpointURL
}
}

p.pulumiAPIAccessToken = os.Getenv("PULUMI_ACCESS_TOKEN")

if cfg.Exists("organization") {
p.organization = cfg.String("organization")
} else {
p.organization = os.Getenv("PULUMI_ORGANIZATION")
}

p.project = cfg.String("project")
p.stack = cfg.String("stack")

p.log.Debugf("pulumi: backend=%q, api_endpoint=%q, organization=%q, project=%q, stack=%q",
p.backend, p.pulumiAPIEndpointURL, p.organization, p.project, p.stack)

return p
}

func (p *provider) GetString(key string) (string, error) {
tokens := strings.Split(key, "/")

// ref+pulumistateapi://RESOURCE_TYPE/RESOURCE_LOGICAL_NAME/ATTRIBUTE_TYPE/ATTRIBUTE_KEY_PATH?project=PROJECT&stack=STACK
if len(tokens) != 4 {
return "", fmt.Errorf("invalid key format. expected key format is RESOURCE_TYPE/RESOURCE_LOGICAL_NAME/ATTRIBUTE_TYPE/ATTRIBUTE_KEY_PATH")
}

resourceType := parsePulumiResourceType(tokens[0])
resourceLogicalName := tokens[1]
resourceAttribute := tokens[2]

// https://github.com/tidwall/gjson/blob/master/SYNTAX.md#gjson-path-syntax
// https://gjson.dev/
resourceAttributePath := tokens[3]

var state *pulumiState
var err error
switch p.backend {
case "pulumistateapi":
state, err = p.getStateFromPulumiAPI()
default:
err = fmt.Errorf("unsupported backend: %s", p.backend)
}
if err != nil {
return "", err
}

resource := state.findResourceByLogicalName(resourceType, resourceLogicalName)
if resource == nil {
return "", fmt.Errorf("resource with logical name not found: %s", resourceLogicalName)
}

attributeValue := resource.getAttributeValue(resourceAttribute, resourceAttributePath)

return attributeValue, nil
}

func (p *provider) GetStringMap(key string) (map[string]interface{}, error) {
return nil, fmt.Errorf("path fragment is not supported for pulumi provider")
}

// double underscore becomes a forward slash
// single underscore becomes a colon
// (e.g. kubernetes_storage.k8s.io__v1_StorageClass -> kubernetes:storage.k8s.io/v1:StorageClass)
func parsePulumiResourceType(str string) string {
return strings.ReplaceAll(strings.ReplaceAll(str, "__", "/"), "_", ":")
}

func (p *provider) getStateFromPulumiAPI() (*pulumiState, error) {
client := &http.Client{}

pulumiApiUrl := fmt.Sprintf("%s/api/stacks/%s/%s/%s/export", p.pulumiAPIEndpointURL, p.organization, p.project, p.stack)
req, err := http.NewRequest(http.MethodGet, pulumiApiUrl, nil)
if err != nil {
return nil, err
}

req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/vnd.pulumi+8")
req.Header.Add("Authorization", fmt.Sprintf("token %s", p.pulumiAPIAccessToken))

response, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() {
_ = response.Body.Close()
}()

responseBody, err := io.ReadAll(response.Body)
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("pulumi api returned a non-200 status code: %d - body: %s",
response.StatusCode, string(responseBody))
}

var state *pulumiState
err = json.Unmarshal(responseBody, &state)
if err != nil {
return nil, err
}
return state, nil
}
Loading

0 comments on commit 88ba797

Please sign in to comment.