Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix provider definition #59

Merged
merged 7 commits into from
Jul 29, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 59 additions & 15 deletions cmd/provider/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"github.com/crossplane/crossplane-runtime/pkg/resource"
"github.com/crossplane/crossplane-runtime/pkg/statemetrics"
upcontroller "github.com/crossplane/upjet/pkg/controller"
"github.com/crossplane/upjet/pkg/terraform"
"gopkg.in/alecthomas/kingpin.v2"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -51,20 +50,37 @@ import (
equinixmetrics "github.com/crossplane-contrib/provider-jet-equinix/internal/metrics"
)

const (
webhookTLSCertDirEnvVar = "WEBHOOK_TLS_CERT_DIR"
tlsServerCertDirEnvVar = "TLS_SERVER_CERTS_DIR"
certsDirEnvVar = "CERTS_DIR"
tlsServerCertDir = "/tls/server"
)

func main() {
var (
app = kingpin.New(filepath.Base(os.Args[0]), "Terraform based Crossplane provider for Equinix").DefaultEnvars()
debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
syncInterval = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration()
leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
pollStateMetricInterval = app.Flag("poll-state-metric", "State metric recording interval").Default("5s").Duration()
maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("10").Int()

app = kingpin.New(filepath.Base(os.Args[0]), "Terraform based Crossplane provider for Equinix").DefaultEnvars()
debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
syncInterval = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration()
leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
terraformVersion = app.Flag("terraform-version", "Terraform version.").Required().Envar("TERRAFORM_VERSION").String()
providerSource = app.Flag("terraform-provider-source", "Terraform provider source.").Required().Envar("TERRAFORM_PROVIDER_SOURCE").String()
displague marked this conversation as resolved.
Show resolved Hide resolved
providerVersion = app.Flag("terraform-provider-version", "Terraform provider version.").Required().Envar("TERRAFORM_PROVIDER_VERSION").String()
pollStateMetricInterval = app.Flag("poll-state-metric", "State metric recording interval").Default("5s").Duration()
maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may checked for drift from the desired state.").Default("10").Int()
namespace = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String()
enableExternalSecretStores = app.Flag("enable-external-secret-stores", "Enable support for ExternalSecretStores.").Default("false").Envar("ENABLE_EXTERNAL_SECRET_STORES").Bool()
essTLSCertsPath = app.Flag("ess-tls-cert-dir", "Path of ESS TLS certificates.").Envar("ESS_TLS_CERTS_DIR").String()
enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for ManagementPolicies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool()

certsDirSet = false
// we record whether the command-line option "--certs-dir" was supplied
// in the registered PreAction for the flag.
certsDir = app.Flag("certs-dir", "The directory that contains the server key and certificate.").Default(tlsServerCertDir).Envar(certsDirEnvVar).PreAction(func(_ *kingpin.ParseContext) error {
certsDirSet = true
return nil
}).String()
)
kingpin.MustParse(app.Parse(os.Args[1:]))

Expand All @@ -84,7 +100,30 @@ func main() {

cfg, err := ctrl.GetConfig()
kingpin.FatalIfError(err, "Cannot get API server rest config")
kingpin.FatalIfError(equinixmetrics.SetupMetrics(), "Cannot setup Linode metrics hook")

// Get the TLS certs directory from the environment variables set by
// Crossplane if they're available.
// In older XP versions we used WEBHOOK_TLS_CERT_DIR, in newer versions
// we use TLS_SERVER_CERTS_DIR. If an explicit certs dir is not supplied
// via the command-line options, then these environment variables are used
// instead.
if !certsDirSet {
// backwards-compatibility concerns
xpCertsDir := os.Getenv(certsDirEnvVar)
if xpCertsDir == "" {
xpCertsDir = os.Getenv(tlsServerCertDirEnvVar)
}
if xpCertsDir == "" {
xpCertsDir = os.Getenv(webhookTLSCertDirEnvVar)
}
// we probably don't need this condition but just to be on the
// safe side, if we are missing any kingpin machinery details...
if xpCertsDir != "" {
*certsDir = xpCertsDir
}
}

kingpin.FatalIfError(equinixmetrics.SetupMetrics(), "Cannot setup Equinix metrics hook")

mgr, err := ctrl.NewManager(ratelimiter.LimitRESTConfig(cfg, *maxReconcileRate), ctrl.Options{
LeaderElection: *leaderElection,
Expand Down Expand Up @@ -113,6 +152,13 @@ func main() {
ctx := context.Background()
provider, err := config.GetProvider(ctx, false)
kingpin.FatalIfError(err, "Cannot initialize the provider configuration")

setupCfg := clients.SetupConfig{
ProviderVersion: providerVersion,
TerraformVersion: terraformVersion,
ProviderSource: providerSource,
TerraformProvider: provider.TerraformProvider,
}
o := upcontroller.Options{
Options: xpcontroller.Options{
Logger: log,
Expand All @@ -122,13 +168,11 @@ func main() {
Features: &feature.Flags{},
MetricOptions: &mo,
},
Provider: provider,
// use the following WorkspaceStoreOption to enable the shared gRPC mode
// terraform.WithProviderRunner(terraform.NewSharedProvider(log, os.Getenv("TERRAFORM_NATIVE_PROVIDER_PATH"), terraform.WithNativeProviderArgs("-debuggable")))
WorkspaceStore: terraform.NewWorkspaceStore(log),
SetupFn: clients.TerraformSetupBuilder(provider.TerraformProvider),
Provider: provider,
SetupFn: clients.TerraformSetupBuilder(setupCfg),
displague marked this conversation as resolved.
Show resolved Hide resolved
PollJitter: pollJitter,
OperationTrackerStore: upcontroller.NewOperationStore(log),
StartWebhooks: *certsDir != "",
}

if *enableManagementPolicies {
Expand Down
7 changes: 7 additions & 0 deletions config/overrides.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ func IdentifierAssignedByEquinix() upconfig.ResourceOption {
}
}

// LongProvision will set the resource to be provisioned asynchronously. Use this for resources with >1m provisions
func LongProvision() upconfig.ResourceOption {
return func(r *upconfig.Resource) {
r.UseAsync = true
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may be a premature optimization. I haven't seen any problem with deploying devices thus far.

}
}

var knownReferencerTFResource = map[string]map[string]string{
"metal": {
"project_id": "equinix_metal_project",
Expand Down
1 change: 1 addition & 0 deletions config/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ func GetProvider(_ context.Context, generationProvider bool) (*upconfig.Provider
KnownReferencers(),
IdentifierAssignedByEquinix(),
SkipOptCompLateInitialization(),
LongProvision(), // TODO: use this only for Device and other long-provisioning resources
),
)

Expand Down
48 changes: 37 additions & 11 deletions internal/clients/equinix.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@ import (
"encoding/json"

"github.com/crossplane/crossplane-runtime/pkg/resource"
"github.com/crossplane/upjet/pkg/terraform"
equinixprovider "github.com/equinix/terraform-provider-equinix/equinix/provider"

"github.com/equinix/terraform-provider-equinix/version"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
terraformsdk "github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/crossplane/upjet/pkg/terraform"

"github.com/crossplane-contrib/provider-jet-equinix/apis/v1alpha1"
)

Expand All @@ -50,12 +53,10 @@ const (
)

type SetupConfig struct {
NativeProviderPath *string
NativeProviderSource *string
NativeProviderVersion *string
TerraformVersion *string
DefaultScheduler terraform.ProviderScheduler
TerraformProvider *schema.Provider
ProviderSource *string
ProviderVersion *string
TerraformVersion *string
TerraformProvider *schema.Provider
}

func prepareTerraformProviderConfiguration(creds map[string]string, pc v1alpha1.ProviderConfiguration) map[string]any {
Expand Down Expand Up @@ -85,9 +86,15 @@ func prepareTerraformProviderConfiguration(creds map[string]string, pc v1alpha1.

// TerraformSetupBuilder builds Terraform a terraform.SetupFn function which
// returns Terraform provider setup configuration
func TerraformSetupBuilder(tfProvider *schema.Provider) terraform.SetupFn {
func TerraformSetupBuilder(setupCfg SetupConfig) terraform.SetupFn {
return func(ctx context.Context, client client.Client, mg resource.Managed) (terraform.Setup, error) {
ps := terraform.Setup{}
ps := terraform.Setup{
Version: *setupCfg.TerraformVersion,
Requirement: terraform.ProviderRequirement{
Source: *setupCfg.ProviderSource,
Version: *setupCfg.ProviderVersion,
},
}

configRef := mg.GetProviderConfigReference()
if configRef == nil {
Expand All @@ -113,6 +120,25 @@ func TerraformSetupBuilder(tfProvider *schema.Provider) terraform.SetupFn {
}

ps.Configuration = prepareTerraformProviderConfiguration(equinixCreds, pc.Spec.Configuration)
return ps, nil
return ps, errors.Wrap(configureNoForkEquinixClient(ctx, &ps, *setupCfg.TerraformProvider), "failed to configure the no-fork equinix client")
}
}

func configureNoForkEquinixClient(ctx context.Context, ps *terraform.Setup, p schema.Provider) error {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm only copying what I see in other providers and I'm not sure why this should be added.

The NoFork concept emerged in crossplane/upjet#294

Both clients employ an in-memory Terraform state cache as these clients no longer construct Terraform workspaces while reconciling their resources. We have an initial evaluation of the memory cost of this cache with 10k MRs and it's in the order of a few hundred megabytes (~200 MB), much smaller than the footprint of the process fork-based clients.

After the introduction of Framework support in Upjet crossplane/upjet#329, NoFork was renamed to TerraformPluginSDK. crossplane/upjet#338

// Please be aware that this implementation relies on the schema.Provider
// parameter `p` being a non-pointer. This is because normally
// the Terraform plugin SDK normally configures the provider
// only once and using a pointer argument here will cause
// race conditions between resources referring to different
// ProviderConfigs.
diag := p.Configure(context.WithoutCancel(ctx), &terraformsdk.ResourceConfig{
Config: ps.Configuration,
})
if diag != nil && diag.HasError() {
return errors.Errorf("failed to configure the provider: %v", diag)
}

fwProvider := equinixprovider.CreateFrameworkProvider(version.ProviderVersion)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

version.ProviderVersion is dev unless overridden at Go compile time. We could define that in the Makefile but it may be more practical to pass TERRAFORM_PROVIDER_VERSION in from main via (SetupConfig)

ps.FrameworkProvider = fwProvider
return nil
}
Loading