Support GITHUB_TOKEN for HTTP Requests to github.com (#912)

* token version * custom detector * config update * config/docs update * debug messages * Update website/docs/cli/configuration/configuration.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> * ATMOS_GITHUB_TOKEN * Update website/docs/cli/configuration/configuration.mdx Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> * Update internal/exec/vendor_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> * Update internal/exec/vendor_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> * Update internal/exec/vendor_utils.go Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> * debug traces --------- Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]>
cloudposse · Jan 11, 2025 · 8d017eb · 8d017eb
1 parent 995e460
commit 8d017eb
Show file tree

Hide file tree

Showing 8 changed files with 140 additions and 10 deletions.
diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go
@@ -105,7 +105,7 @@ func ValidateStacks(atmosConfig schema.AtmosConfiguration) error {
 	} else if u.FileExists(atmosManifestJsonSchemaFileAbsPath) {
 		atmosManifestJsonSchemaFilePath = atmosManifestJsonSchemaFileAbsPath
 	} else if u.IsURL(atmosConfig.Schemas.Atmos.Manifest) {
-		atmosManifestJsonSchemaFilePath, err = downloadSchemaFromURL(atmosConfig.Schemas.Atmos.Manifest)
+		atmosManifestJsonSchemaFilePath, err = downloadSchemaFromURL(atmosConfig)
 		if err != nil {
 			return err
 		}
@@ -372,7 +372,10 @@ func checkComponentStackMap(componentStackMap map[string]map[string][]string) ([
 }
 
 // downloadSchemaFromURL downloads the Atmos JSON Schema file from the provided URL
-func downloadSchemaFromURL(manifestURL string) (string, error) {
+func downloadSchemaFromURL(atmosConfig schema.AtmosConfiguration) (string, error) {
+
+	manifestURL := atmosConfig.Schemas.Atmos.Manifest
+
 	parsedURL, err := url.Parse(manifestURL)
 	if err != nil {
 		return "", fmt.Errorf("invalid URL '%s': %w", manifestURL, err)
@@ -388,6 +391,10 @@ func downloadSchemaFromURL(manifestURL string) (string, error) {
 	atmosManifestJsonSchemaFilePath := filepath.Join(tempDir, fileName)
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
+
+	// Register custom detectors
+	RegisterCustomDetectors(atmosConfig)
+
 	client := &getter.Client{
 		Ctx:  ctx,
 		Dst:  atmosManifestJsonSchemaFilePath,

diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go
@@ -245,7 +245,6 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos
 				name: p.name,
 			}
 		}
-
 		// Create temp directory
 		tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix()))
 		if err != nil {
@@ -269,6 +268,9 @@ func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, atmosConfig schema.Atmos
 		switch p.pkgType {
 		case pkgTypeRemote:
 			// Use go-getter to download remote packages
+			// Register custom detectors
+			RegisterCustomDetectors(atmosConfig)
+
 			client := &getter.Client{
 				Ctx:  ctx,
 				Dst:  tempDir,

diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go
@@ -126,6 +126,9 @@ func installComponent(p *pkgComponentVendor, atmosConfig schema.AtmosConfigurati
 	case pkgTypeRemote:
 		tempDir = filepath.Join(tempDir, sanitizeFileName(p.uri))
 
+		// Register custom detectors
+		RegisterCustomDetectors(atmosConfig)
+
 		client := &getter.Client{
 			Ctx: context.Background(),
 			// Define the destination where the files will be stored. This will create the directory if it doesn't exist
@@ -187,6 +190,10 @@ func installMixin(p *pkgComponentVendor, atmosConfig schema.AtmosConfiguration)
 	defer cancel()
 	switch p.pkgType {
 	case pkgTypeRemote:
+
+		// Register custom detectors
+		RegisterCustomDetectors(atmosConfig)
+
 		client := &getter.Client{
 			Ctx:  ctx,
 			Dst:  filepath.Join(tempDir, p.mixinFilename),

diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/bmatcuk/doublestar/v4"
 	tea "github.com/charmbracelet/bubbletea"
+	"github.com/hashicorp/go-getter"
 	cp "github.com/otiai10/copy"
 	"github.com/samber/lo"
 	"github.com/spf13/cobra"
@@ -668,10 +669,94 @@ func validateURI(uri string) error {
 }
 func isValidScheme(scheme string) bool {
 	validSchemes := map[string]bool{
-		"http":  true,
-		"https": true,
-		"git":   true,
-		"ssh":   true,
+		"http":       true,
+		"https":      true,
+		"git":        true,
+		"ssh":        true,
+		"git::https": true,
 	}
 	return validSchemes[scheme]
 }
+
+// CustomGitHubDetector intercepts GitHub URLs and transforms them
+// into something like git::https://<token>@github.com/... so we can
+// do a git-based clone with a token.
+type CustomGitHubDetector struct {
+	AtmosConfig schema.AtmosConfiguration
+}
+
+// Detect implements the getter.Detector interface for go-getter v1.
+func (d *CustomGitHubDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	if !strings.Contains(src, "://") {
+		src = "https://" + src
+	}
+
+	parsedURL, err := url.Parse(src)
+	if err != nil {
+		u.LogDebug(d.AtmosConfig, fmt.Sprintf("Failed to parse URL %q: %v\n", src, err))
+		return "", false, fmt.Errorf("failed to parse URL %q: %w", src, err)
+	}
+
+	if strings.ToLower(parsedURL.Host) != "github.com" {
+		u.LogDebug(d.AtmosConfig, fmt.Sprintf("Host is %q, not 'github.com', skipping token injection\n", parsedURL.Host))
+		return "", false, nil
+	}
+
+	parts := strings.SplitN(parsedURL.Path, "/", 4)
+	if len(parts) < 3 {
+		u.LogDebug(d.AtmosConfig, fmt.Sprintf("URL path %q doesn't look like /owner/repo\n", parsedURL.Path))
+		return "", false, fmt.Errorf("invalid GitHub URL %q", parsedURL.Path)
+	}
+
+	atmosGitHubToken := os.Getenv("ATMOS_GITHUB_TOKEN")
+	gitHubToken := os.Getenv("GITHUB_TOKEN")
+
+	var usedToken string
+	var tokenSource string
+
+	// 1. If ATMOS_GITHUB_TOKEN is set, always use that
+	if atmosGitHubToken != "" {
+		usedToken = atmosGitHubToken
+		tokenSource = "ATMOS_GITHUB_TOKEN"
+		u.LogDebug(d.AtmosConfig, "ATMOS_GITHUB_TOKEN is set\n")
+	} else {
+		// 2. Otherwise, only inject GITHUB_TOKEN if cfg.Settings.InjectGithubToken == true
+		if d.AtmosConfig.Settings.InjectGithubToken && gitHubToken != "" {
+			usedToken = gitHubToken
+			tokenSource = "GITHUB_TOKEN"
+			u.LogTrace(d.AtmosConfig, "InjectGithubToken=true and GITHUB_TOKEN is set, using it\n")
+		} else {
+			u.LogTrace(d.AtmosConfig, "No ATMOS_GITHUB_TOKEN or GITHUB_TOKEN found\n")
+		}
+	}
+
+	if usedToken != "" {
+		user := parsedURL.User.Username()
+		pass, _ := parsedURL.User.Password()
+		if user == "" && pass == "" {
+			u.LogDebug(d.AtmosConfig, fmt.Sprintf("Injecting token from %s for %s\n", tokenSource, src))
+			parsedURL.User = url.UserPassword("x-access-token", usedToken)
+		} else {
+			u.LogDebug(d.AtmosConfig, "Credentials found, skipping token injection\n")
+		}
+	}
+
+	finalURL := "git::" + parsedURL.String()
+
+	return finalURL, true, nil
+}
+
+// RegisterCustomDetectors prepends the custom detector so it runs before
+// the built-in ones. Any code that calls go-getter should invoke this.
+func RegisterCustomDetectors(atmosConfig schema.AtmosConfiguration) {
+	getter.Detectors = append(
+		[]getter.Detector{
+			&CustomGitHubDetector{AtmosConfig: atmosConfig},
+		},
+		getter.Detectors...,
+	)
+}
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -114,6 +114,7 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
 	// Default configuration values
 	v.SetDefault("components.helmfile.use_eks", true)
 	v.SetDefault("components.terraform.append_user_agent", fmt.Sprintf("Atmos/%s (Cloud Posse; +https://atmos.tools)", version.Version))
+	v.SetDefault("settings.inject_github_token", true)
 
 	// Process config in system folder
 	configFilePath1 := ""

diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go
@@ -50,6 +50,7 @@ type AtmosSettings struct {
 	Terminal          Terminal         `yaml:"terminal,omitempty" json:"terminal,omitempty" mapstructure:"terminal"`
 	Docs              Docs             `yaml:"docs,omitempty" json:"docs,omitempty" mapstructure:"docs"`
 	Markdown          MarkdownSettings `yaml:"markdown,omitempty" json:"markdown,omitempty" mapstructure:"markdown"`
+	InjectGithubToken bool             `yaml:"inject_github_token,omitempty" mapstructure:"inject_github_token"`
 }
 
 type Docs struct {

diff --git a/pkg/utils/github_utils.go b/pkg/utils/github_utils.go
@@ -2,19 +2,39 @@ package utils
 
 import (
 	"context"
+	"os"
 	"time"
 
 	"github.com/google/go-github/v59/github"
+	"golang.org/x/oauth2"
 )
 
+// newGitHubClient creates a new GitHub client. If a token is provided, it returns an authenticated client;
+// otherwise, it returns an unauthenticated client.
+func newGitHubClient(ctx context.Context) *github.Client {
+	githubToken := os.Getenv("GITHUB_TOKEN")
+	if githubToken == "" {
+		return github.NewClient(nil)
+	}
+
+	// Token found, create an authenticated client
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: githubToken},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+
+	return github.NewClient(tc)
+}
+
 // GetLatestGitHubRepoRelease returns the latest release tag for a GitHub repository
 func GetLatestGitHubRepoRelease(owner string, repo string) (string, error) {
 	opt := &github.ListOptions{Page: 1, PerPage: 1}
-	client := github.NewClient(nil)
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
 	defer cancel()
 
+	client := newGitHubClient(ctx)
+
 	releases, _, err := client.Repositories.ListReleases(ctx, owner, repo, opt)
 	if err != nil {
 		return "", err

diff --git a/website/docs/cli/configuration/configuration.mdx b/website/docs/cli/configuration/configuration.mdx
@@ -161,6 +161,7 @@ The `settings` section configures Atmos global settings.
       terminal:
         max_width: 120  # Maximum width for terminal output
         pager: true     # Use pager for long output
+      inject_github_token: true # Adds the GITHUB_TOKEN as a Bearer token for GitHub API requests.
     ```
 </File>
 
@@ -196,6 +197,11 @@ The `settings` section configures Atmos global settings.
     </dl>
   </dd>
 
+  <dt>`settings.inject_github_token`</dt>
+  <dd>
+    Adds the `GITHUB_TOKEN` as a Bearer token for GitHub API requests, enabling authentication for private repositories and increased rate limits. If `ATMOS_GITHUB_TOKEN` is set, it takes precedence, overriding this behavior.
+  </dd>
+
   <dt>`settings.docs` (Deprecated)</dt>
   <dd>
     :::warning Deprecated
@@ -666,11 +672,12 @@ setting `ATMOS_STACKS_BASE_PATH` to a path in `/localhost` to your local develop
 | ATMOS_WORKFLOWS_BASE_PATH                             | workflows.base_path                             | Base path to Atmos workflows                                                                                                                                                                                                 |
 | ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH                    | schemas.jsonschema.base_path                    | Base path to JSON schemas for component validation                                                                                                                                                                           |
 | ATMOS_SCHEMAS_OPA_BASE_PATH                           | schemas.opa.base_path                           | Base path to OPA policies for component validation                                                                                                                                                                           |
-| ATMOS_SCHEMAS_ATMOS_MANIFEST                          | schemas.atmos.manifest                          | Path to JSON Schema to validate Atmos stack manifests. For more details, refer to [Atmos Manifest JSON Schema](/cli/schemas)                                                                                           |
+| ATMOS_SCHEMAS_ATMOS_MANIFEST                          | schemas.atmos.manifest                          | Path to JSON Schema to validate Atmos stack manifests. For more details, refer to [Atmos Manifest JSON Schema](/cli/schemas)                                                                                                 |
 | ATMOS_LOGS_FILE                                       | logs.file                                       | The file to write Atmos logs to. Logs can be written to any file or any standard file descriptor, including `/dev/stdout`, `/dev/stderr` and `/dev/null`). If omitted, `/dev/stdout` will be used                            |
 | ATMOS_LOGS_LEVEL                                      | logs.level                                      | Logs level. Supported log levels are `Trace`, `Debug`, `Info`, `Warning`, `Off`. If the log level is set to `Off`, Atmos will not log any messages (note that this does not prevent other tools like Terraform from logging) |
 | ATMOS_SETTINGS_LIST_MERGE_STRATEGY                    | settings.list_merge_strategy                    | Specifies how lists are merged in Atmos stack manifests. The following strategies are supported: `replace`, `append`, `merge`                                                                                                |
-| ATMOS_VERSION_CHECK_ENABLED                           | version.check.enabled                           | Enable/disable Atmos version checks for updates to the newest release                                                                                                                                                                    |
+| ATMOS_VERSION_CHECK_ENABLED                           | version.check.enabled                           | Enable/disable Atmos version checks for updates to the newest release                                                                                                                                                        |
+| ATMOS_GITHUB_TOKEN                                    | N/A                                             | Bearer token for GitHub API requests, enabling authentication for private repositories and higher rate limits                                                                                                                |
 
 ### Context