From aeb2d8f98f463b311112ea00cd307159bad9d452 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 10 Jan 2025 11:46:18 -0500
Subject: [PATCH 01/29] add majors

---
 instrumentation/packages.go |   4 +
 new_latest_majors.go        | 252 ++++++++++++++++++++++++++++++++++++
 2 files changed, 256 insertions(+)
 create mode 100644 new_latest_majors.go

diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index c3cce149e6..dcda5d0c72 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -739,6 +739,10 @@ var packages = map[Package]PackageInfo{
 	},
 }
 
+func GetPackages() map[Package]PackageInfo {
+	return packages // TODO: change to return a deep copy
+}
+
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name
diff --git a/new_latest_majors.go b/new_latest_majors.go
new file mode 100644
index 0000000000..89cfe226f2
--- /dev/null
+++ b/new_latest_majors.go
@@ -0,0 +1,252 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/DataDog/dd-trace-go/v2/instrumentation"
+	"golang.org/x/mod/semver"
+)
+
+func getLatestMajor(repo string) (string, error) {
+	// Get the latest major version available on GitHub
+	const prefix = "github.com/"
+	const apiBaseURL = "https://api.github.com/repos/%s/tags"
+
+	// Truncate "github.com/" and the version suffix
+	lastSlashIndex := strings.LastIndex(repo, "/")
+	if lastSlashIndex == -1 || !strings.HasPrefix(repo, prefix) {
+		return "", fmt.Errorf("invalid repository format: %s", repo)
+	}
+	trimmedRepo := repo[len(prefix):lastSlashIndex]
+	url := fmt.Sprintf(apiBaseURL, trimmedRepo)
+	fmt.Printf("Base repo: %s\n", trimmedRepo)
+
+	// Fetch tags from GitHub
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to fetch tags: %s", resp.Status)
+	}
+
+	// Parse response
+	var tags []struct {
+		Name string `json:"name"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil {
+		return "", err
+	}
+
+	// Track latest versions by major version
+	latestByMajor := make(map[string]string) // Map major version to the latest full version string
+	for _, tag := range tags {
+		v := semver.Canonical(tag.Name) // Ensure the version is canonical
+		if !semver.IsValid(v) {
+			continue // Skip invalid versions
+		}
+
+		if semver.Prerelease(v) != "" {
+			continue // Skip pre-release versions
+		}
+
+		major := semver.Major(v) // Extract the major version (e.g., "v1")
+		if current, exists := latestByMajor[major]; !exists || semver.Compare(v, current) > 0 {
+			latestByMajor[major] = v
+		}
+	}
+
+	// Determine the largest major version
+	var latestMajor string
+	for major, _ := range latestByMajor {
+		if latestMajor == "" || semver.Compare(major, latestMajor) > 0 {
+			latestMajor = major
+		}
+	}
+
+	if latestMajor != "" {
+		return latestMajor, nil
+	}
+
+	return "", fmt.Errorf("no valid versions found")
+}
+
+// 	// walk through contrib directory
+// 	// match on anything that matches the pattern "contrib/{repository}" (there may be multiple)
+// 	// ex. if repository=go-redis/redis, "contrib/go-redis/redis.v9"
+// 	// inside this repository, look for go.mod
+// 	// if no go.mod, error
+// 	// match on repository in the go.mod in the require() block that contains repository
+// 	// example: if repository=go-redis/redis, should match on line "github.com/redis/go-redis/v9 v9.1.0"
+// 	// if no repository exists that matches this pattern, error
+// 	// return the largest major version associated
+
+// GetLatestMajorContrib finds the latest major version of a repository in the contrib directory.
+// TODO: modify this to only walk through contrib once, and store the repository => major versions
+func GetLatestMajorContrib(repository string) (string, error) {
+	const contribDir = "contrib"
+
+	// Prepare the repository matching pattern -
+	// ex. repository = "redis/go-redis" should match on any repository that contains redis/go-redis
+	repoPattern := fmt.Sprintf(`^%s/%s(/.*)?`, contribDir, strings.ReplaceAll(repository, "/", "/"))
+
+	// Compile regex for matching contrib directory entries
+	repoRegex, err := regexp.Compile(repoPattern)
+	if err != nil {
+		return "", fmt.Errorf("invalid repository pattern: %v", err)
+	}
+
+	var latestVersion string
+
+	// Walk through the contrib directory
+	err = filepath.Walk(contribDir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Check if the path matches the contrib repository pattern
+		if !info.IsDir() || !repoRegex.MatchString(path) {
+			return nil
+		}
+
+		// Look for go.mod file
+
+		goModPath := filepath.Join(path, "go.mod")
+		// fmt.Printf("Looking for go.mod: %s\n", goModPath)
+		if _, err := os.Stat(goModPath); err != nil {
+			return nil // No go.mod file, skip
+		}
+
+		// Log the path where a go.mod file is found
+		fmt.Printf("Found go.mod in: %s\n", goModPath)
+
+		// Parse the go.mod file to find the version
+		version, err := extractVersionFromGoMod(goModPath, repository)
+		if err != nil {
+			return nil
+		}
+
+		// Update the latest version if it's greater
+		if latestVersion == "" || semver.Compare(version, latestVersion) > 0 {
+			latestVersion = version
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("error walking contrib directory: %v", err)
+	}
+
+	if latestVersion == "" {
+		return "", errors.New("no matching contrib repository with a valid version found")
+	}
+
+	return latestVersion, nil
+}
+
+// extractVersionFromGoMod parses the go.mod file and extracts the version of the specified repository.
+func extractVersionFromGoMod(goModPath string, repository string) (string, error) {
+	file, err := os.Open(goModPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open go.mod: %v", err)
+	}
+	defer file.Close()
+
+	repoPattern := fmt.Sprintf(`\b%s\b`, strings.ReplaceAll(repository, "/", "/"))
+	repoRegex, err := regexp.Compile(repoPattern)
+	if err != nil {
+		return "", fmt.Errorf("invalid repository regex pattern: %v", err)
+	}
+
+	var version string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// Match lines in the require block with the repository pattern
+		if repoRegex.MatchString(line) {
+			parts := strings.Fields(line)
+			if len(parts) >= 2 && semver.IsValid(parts[1]) {
+				version = parts[1]
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return "", fmt.Errorf("error reading go.mod: %v", err)
+	}
+
+	if version == "" {
+		return "", errors.New("no valid version found for repository in go.mod")
+	}
+
+	return version, nil
+}
+
+// ValidateRepository checks if the repository string starts with "github.com" and ends with a version suffix.
+// Returns true if the repository is valid, false otherwise.
+func ValidateRepository(repo string) bool {
+	const prefix = "github.com/"
+	if !strings.HasPrefix(repo, prefix) {
+		return false
+	}
+
+	lastSlashIndex := strings.LastIndex(repo, "/")
+	if lastSlashIndex == -1 || !strings.HasPrefix(repo[lastSlashIndex:], "/v") {
+		return false
+	}
+
+	return true
+}
+
+func main() {
+
+	// packageMap := make(map[string]string) // map holding package names and repositories
+	// var modules []ModuleVersion // this should store the module/contrib name, and the version
+	packages := instrumentation.GetPackages()
+	for pkg, info := range packages {
+		fmt.Printf("Package: %s, Traced Package: %s\n", pkg, info.TracedPackage)
+		repository := info.TracedPackage
+
+		// repo starts with github and ends in version suffix
+		// Get the latest major on GH
+		if ValidateRepository(repository) {
+			latest_major_version, err := getLatestMajor(repository)
+			if err != nil {
+				fmt.Printf("Error getting min version for repo %s: %v\n", repository, err)
+				continue
+			}
+			// fmt.Printf("latest major on github: %s", latest_major_version)
+			const prefix = "github.com"
+
+			lastSlashIndex := strings.LastIndex(repository, "/")
+			trimmedRepo := repository[len(prefix)+1 : lastSlashIndex] // TODO: tidy this
+			latest_major_contrib, err := GetLatestMajorContrib(trimmedRepo)
+			if err != nil {
+				fmt.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
+			}
+
+			fmt.Printf("latest major on go.mod: %s", latest_major_contrib)
+
+			if semver.Major(latest_major_contrib) != latest_major_version {
+				fmt.Printf("major on go.mod: %s", semver.Major(latest_major_contrib))
+				fmt.Printf("major on github: %s", latest_major_version)
+				fmt.Printf("repository %s has a new major latest avaialble: %s", trimmedRepo, latest_major_version)
+
+			}
+
+		}
+	}
+}

From cf5ee0b4c6554cd4304cee47922d5dbb405f50af Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 10 Jan 2025 12:35:00 -0500
Subject: [PATCH 02/29] update workflow

---
 .github/workflows/outdated-integrations.yml | 5 ++---
 new_latest_majors.go                        | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index 0e559c8039..2014cb7ecb 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -22,12 +22,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       
-      - run: go get github.com/Masterminds/semver/v3
-
-      - run: go run .github/workflows/apps/latest_major_version.go > latests.txt
+      - run: go run new_latest_majors.go > latests.txt
 
       - run: git diff
 
+      # note: this will only create a PR if there is a new major available
       - name: Create Pull Request
         id: pr
         uses: peter-evans/create-pull-request@v6
diff --git a/new_latest_majors.go b/new_latest_majors.go
index 89cfe226f2..e8e0d2a962 100644
--- a/new_latest_majors.go
+++ b/new_latest_majors.go
@@ -156,7 +156,7 @@ func GetLatestMajorContrib(repository string) (string, error) {
 	return latestVersion, nil
 }
 
-// extractVersionFromGoMod parses the go.mod file and extracts the version of the specified repository.
+// // // extractVersionFromGoMod parses the go.mod file and extracts the version of the specified repository.
 func extractVersionFromGoMod(goModPath string, repository string) (string, error) {
 	file, err := os.Open(goModPath)
 	if err != nil {

From 414b23ad9d9e8a2ed0b19dca3095b88cf181cf55 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 13 Jan 2025 11:36:42 -0500
Subject: [PATCH 03/29] update outdated integrations workflow

---
 .../workflows/apps/outdated_github_majors.go  | 23 ++++++++++++-------
 .github/workflows/outdated-integrations.yml   |  2 +-
 2 files changed, 16 insertions(+), 9 deletions(-)
 rename new_latest_majors.go => .github/workflows/apps/outdated_github_majors.go (91%)

diff --git a/new_latest_majors.go b/.github/workflows/apps/outdated_github_majors.go
similarity index 91%
rename from new_latest_majors.go
rename to .github/workflows/apps/outdated_github_majors.go
index e8e0d2a962..4e2b312739 100644
--- a/new_latest_majors.go
+++ b/.github/workflows/apps/outdated_github_majors.go
@@ -216,6 +216,8 @@ func main() {
 	// packageMap := make(map[string]string) // map holding package names and repositories
 	// var modules []ModuleVersion // this should store the module/contrib name, and the version
 	packages := instrumentation.GetPackages()
+	integrations := make(map[string]struct{})
+
 	for pkg, info := range packages {
 		fmt.Printf("Package: %s, Traced Package: %s\n", pkg, info.TracedPackage)
 		repository := info.TracedPackage
@@ -228,22 +230,27 @@ func main() {
 				fmt.Printf("Error getting min version for repo %s: %v\n", repository, err)
 				continue
 			}
-			// fmt.Printf("latest major on github: %s", latest_major_version)
 			const prefix = "github.com"
 
+			// if we've seen this baseRepo before, continue
+
 			lastSlashIndex := strings.LastIndex(repository, "/")
-			trimmedRepo := repository[len(prefix)+1 : lastSlashIndex] // TODO: tidy this
-			latest_major_contrib, err := GetLatestMajorContrib(trimmedRepo)
+			baseRepo := repository[len(prefix)+1 : lastSlashIndex] // TODO: tidy this
+			if _, exists := integrations[baseRepo]; exists {       // check for duplicates
+				continue
+			}
+
+			integrations[baseRepo] = struct{}{}
+
+			latest_major_contrib, err := GetLatestMajorContrib(baseRepo)
 			if err != nil {
 				fmt.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
 			}
 
-			fmt.Printf("latest major on go.mod: %s", latest_major_contrib)
-
 			if semver.Major(latest_major_contrib) != latest_major_version {
-				fmt.Printf("major on go.mod: %s", semver.Major(latest_major_contrib))
-				fmt.Printf("major on github: %s", latest_major_version)
-				fmt.Printf("repository %s has a new major latest avaialble: %s", trimmedRepo, latest_major_version)
+				fmt.Printf("major on go.mod: %s\n", semver.Major(latest_major_contrib))
+				fmt.Printf("major on github: %s\n", latest_major_version)
+				fmt.Printf("repository %s has a new major latest on github: %s\n", baseRepo, latest_major_version)
 
 			}
 
diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index 2014cb7ecb..364e02f876 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       
-      - run: go run new_latest_majors.go > latests.txt
+      - run: go run .github/workflows/apps/outdated_github_majors.go > latests.txt
 
       - run: git diff
 

From 454160293c2f993b9446c17651140fc9edc6855b Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 13 Jan 2025 13:30:33 -0500
Subject: [PATCH 04/29] cleanup code

---
 .../workflows/apps/outdated_github_majors.go  | 96 ++++++++-----------
 1 file changed, 41 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/apps/outdated_github_majors.go b/.github/workflows/apps/outdated_github_majors.go
index 4e2b312739..b9926765e0 100644
--- a/.github/workflows/apps/outdated_github_majors.go
+++ b/.github/workflows/apps/outdated_github_majors.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bufio"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -10,9 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/DataDog/dd-trace-go/v2/instrumentation"
+	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
 )
 
@@ -20,15 +21,7 @@ func getLatestMajor(repo string) (string, error) {
 	// Get the latest major version available on GitHub
 	const prefix = "github.com/"
 	const apiBaseURL = "https://api.github.com/repos/%s/tags"
-
-	// Truncate "github.com/" and the version suffix
-	lastSlashIndex := strings.LastIndex(repo, "/")
-	if lastSlashIndex == -1 || !strings.HasPrefix(repo, prefix) {
-		return "", fmt.Errorf("invalid repository format: %s", repo)
-	}
-	trimmedRepo := repo[len(prefix):lastSlashIndex]
-	url := fmt.Sprintf(apiBaseURL, trimmedRepo)
-	fmt.Printf("Base repo: %s\n", trimmedRepo)
+	url := fmt.Sprintf(apiBaseURL, repo)
 
 	// Fetch tags from GitHub
 	resp, err := http.Get(url)
@@ -93,12 +86,11 @@ func getLatestMajor(repo string) (string, error) {
 // 	// return the largest major version associated
 
 // GetLatestMajorContrib finds the latest major version of a repository in the contrib directory.
-// TODO: modify this to only walk through contrib once, and store the repository => major versions
 func GetLatestMajorContrib(repository string) (string, error) {
 	const contribDir = "contrib"
 
-	// Prepare the repository matching pattern -
-	// ex. repository = "redis/go-redis" should match on any repository that contains redis/go-redis
+	// Prepare the repository matching pattern
+	// ex. repository = "redis/go-redis" should match on any directory that contains redis/go-redis
 	repoPattern := fmt.Sprintf(`^%s/%s(/.*)?`, contribDir, strings.ReplaceAll(repository, "/", "/"))
 
 	// Compile regex for matching contrib directory entries
@@ -121,16 +113,11 @@ func GetLatestMajorContrib(repository string) (string, error) {
 		}
 
 		// Look for go.mod file
-
 		goModPath := filepath.Join(path, "go.mod")
-		// fmt.Printf("Looking for go.mod: %s\n", goModPath)
 		if _, err := os.Stat(goModPath); err != nil {
 			return nil // No go.mod file, skip
 		}
 
-		// Log the path where a go.mod file is found
-		fmt.Printf("Found go.mod in: %s\n", goModPath)
-
 		// Parse the go.mod file to find the version
 		version, err := extractVersionFromGoMod(goModPath, repository)
 		if err != nil {
@@ -156,43 +143,40 @@ func GetLatestMajorContrib(repository string) (string, error) {
 	return latestVersion, nil
 }
 
-// // // extractVersionFromGoMod parses the go.mod file and extracts the version of the specified repository.
 func extractVersionFromGoMod(goModPath string, repository string) (string, error) {
-	file, err := os.Open(goModPath)
+	data, err := os.ReadFile(goModPath)
 	if err != nil {
-		return "", fmt.Errorf("failed to open go.mod: %v", err)
+		return "", fmt.Errorf("failed to read go.mod: %v", err)
 	}
-	defer file.Close()
+	var versions []string
 
-	repoPattern := fmt.Sprintf(`\b%s\b`, strings.ReplaceAll(repository, "/", "/"))
-	repoRegex, err := regexp.Compile(repoPattern)
+	// parse go.mod
+	modFile, err := modfile.Parse(goModPath, data, nil)
 	if err != nil {
-		return "", fmt.Errorf("invalid repository regex pattern: %v", err)
+		return "", fmt.Errorf("failed to parse go.mod: %v", err)
 	}
 
-	var version string
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		// Match lines in the require block with the repository pattern
-		if repoRegex.MatchString(line) {
-			parts := strings.Fields(line)
-			if len(parts) >= 2 && semver.IsValid(parts[1]) {
-				version = parts[1]
+	for _, req := range modFile.Require {
+		if strings.Contains(req.Mod.Path, repository) { // check if module path contains repository substring
+			if semver.IsValid(req.Mod.Version) {
+				versions = append(versions, req.Mod.Version)
 			}
 		}
-	}
 
-	if err := scanner.Err(); err != nil {
-		return "", fmt.Errorf("error reading go.mod: %v", err)
 	}
 
-	if version == "" {
-		return "", errors.New("no valid version found for repository in go.mod")
+	if len(versions) == 0 {
+		return "", errors.New("no valid versions found for repository in go.mod")
 	}
 
-	return version, nil
+	// Sort versions in descending order
+	// Return the greatest version
+	sort.Slice(versions, func(i, j int) bool {
+		return semver.Compare(versions[i], versions[j]) > 0
+	})
+
+	return versions[0], nil
+
 }
 
 // ValidateRepository checks if the repository string starts with "github.com" and ends with a version suffix.
@@ -213,36 +197,37 @@ func ValidateRepository(repo string) bool {
 
 func main() {
 
-	// packageMap := make(map[string]string) // map holding package names and repositories
-	// var modules []ModuleVersion // this should store the module/contrib name, and the version
 	packages := instrumentation.GetPackages()
 	integrations := make(map[string]struct{})
 
-	for pkg, info := range packages {
-		fmt.Printf("Package: %s, Traced Package: %s\n", pkg, info.TracedPackage)
+	for _, info := range packages {
 		repository := info.TracedPackage
 
 		// repo starts with github and ends in version suffix
-		// Get the latest major on GH
 		if ValidateRepository(repository) {
-			latest_major_version, err := getLatestMajor(repository)
-			if err != nil {
-				fmt.Printf("Error getting min version for repo %s: %v\n", repository, err)
-				continue
-			}
 			const prefix = "github.com"
 
-			// if we've seen this baseRepo before, continue
-
+			// check if we've seen this integration before
 			lastSlashIndex := strings.LastIndex(repository, "/")
-			baseRepo := repository[len(prefix)+1 : lastSlashIndex] // TODO: tidy this
-			if _, exists := integrations[baseRepo]; exists {       // check for duplicates
+			baseRepo := repository[len(prefix)+1 : lastSlashIndex]
+			if _, exists := integrations[baseRepo]; exists {
 				continue
 			}
 
+			fmt.Printf("Base repo: %s\n", baseRepo)
+
+			// Get the latest major from GH
+			latest_major_version, err := getLatestMajor(baseRepo)
+			if err != nil {
+				fmt.Printf("Error getting min version for repo %s: %v\n", repository, err)
+				continue
+			}
 			integrations[baseRepo] = struct{}{}
 
+			// Get the latest major from go.mod
 			latest_major_contrib, err := GetLatestMajorContrib(baseRepo)
+			fmt.Printf("latest major on go.mod: %s\n", latest_major_contrib)
+
 			if err != nil {
 				fmt.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
 			}
@@ -256,4 +241,5 @@ func main() {
 
 		}
 	}
+
 }

From f0aa62ee7edffba4a8a6e41f54adfdf801fe98fe Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Tue, 14 Jan 2025 10:11:47 -0500
Subject: [PATCH 05/29] address nits

---
 .../workflows/apps/outdated_github_majors.go  | 29 +++++++++----------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/apps/outdated_github_majors.go b/.github/workflows/apps/outdated_github_majors.go
index b9926765e0..85778915fb 100644
--- a/.github/workflows/apps/outdated_github_majors.go
+++ b/.github/workflows/apps/outdated_github_majors.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -85,8 +86,8 @@ func getLatestMajor(repo string) (string, error) {
 // 	// if no repository exists that matches this pattern, error
 // 	// return the largest major version associated
 
-// GetLatestMajorContrib finds the latest major version of a repository in the contrib directory.
-func GetLatestMajorContrib(repository string) (string, error) {
+// getLatestMajorContrib finds the latest major version of a repository in the contrib directory.
+func getLatestMajorContrib(repository string) (string, error) {
 	const contribDir = "contrib"
 
 	// Prepare the repository matching pattern
@@ -179,9 +180,9 @@ func extractVersionFromGoMod(goModPath string, repository string) (string, error
 
 }
 
-// ValidateRepository checks if the repository string starts with "github.com" and ends with a version suffix.
+// validateRepository checks if the repository string starts with "github.com" and ends with a version suffix.
 // Returns true if the repository is valid, false otherwise.
-func ValidateRepository(repo string) bool {
+func validateRepository(repo string) bool {
 	const prefix = "github.com/"
 	if !strings.HasPrefix(repo, prefix) {
 		return false
@@ -196,7 +197,7 @@ func ValidateRepository(repo string) bool {
 }
 
 func main() {
-
+	log.SetFlags(0) // disable date and time logging
 	packages := instrumentation.GetPackages()
 	integrations := make(map[string]struct{})
 
@@ -204,7 +205,7 @@ func main() {
 		repository := info.TracedPackage
 
 		// repo starts with github and ends in version suffix
-		if ValidateRepository(repository) {
+		if validateRepository(repository) {
 			const prefix = "github.com"
 
 			// check if we've seen this integration before
@@ -214,29 +215,27 @@ func main() {
 				continue
 			}
 
-			fmt.Printf("Base repo: %s\n", baseRepo)
+			log.Printf("Base repo: %s\n", baseRepo)
 
 			// Get the latest major from GH
 			latest_major_version, err := getLatestMajor(baseRepo)
 			if err != nil {
-				fmt.Printf("Error getting min version for repo %s: %v\n", repository, err)
+				log.Printf("Error getting min version for repo %s: %v\n", repository, err)
 				continue
 			}
 			integrations[baseRepo] = struct{}{}
 
 			// Get the latest major from go.mod
-			latest_major_contrib, err := GetLatestMajorContrib(baseRepo)
-			fmt.Printf("latest major on go.mod: %s\n", latest_major_contrib)
+			latest_major_contrib, err := getLatestMajorContrib(baseRepo)
 
 			if err != nil {
-				fmt.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
+				log.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
 			}
+			log.Printf("major on go.mod: %s\n", semver.Major(latest_major_contrib))
+			log.Printf("major on github: %s\n", latest_major_version)
 
 			if semver.Major(latest_major_contrib) != latest_major_version {
-				fmt.Printf("major on go.mod: %s\n", semver.Major(latest_major_contrib))
-				fmt.Printf("major on github: %s\n", latest_major_version)
-				fmt.Printf("repository %s has a new major latest on github: %s\n", baseRepo, latest_major_version)
-
+				log.Printf("repository %s has a new major latest on github: %s\n", baseRepo, latest_major_version)
 			}
 
 		}

From 6939df738b10deeb31a3be68ff9ec58f69c7737f Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Wed, 15 Jan 2025 09:29:54 -0500
Subject: [PATCH 06/29] add copyright header

---
 .github/workflows/apps/outdated_github_majors.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/apps/outdated_github_majors.go b/.github/workflows/apps/outdated_github_majors.go
index 85778915fb..3a265ef275 100644
--- a/.github/workflows/apps/outdated_github_majors.go
+++ b/.github/workflows/apps/outdated_github_majors.go
@@ -1,3 +1,8 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024 Datadog, Inc.
+
 package main
 
 import (

From 83ad60967c04f4861e536c69122f73412d21a80d Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Wed, 15 Jan 2025 11:26:57 -0500
Subject: [PATCH 07/29] wip

---
 .../workflows/apps/outdated_github_majors.go  | 23 +++++++++++--------
 .github/workflows/outdated-integrations.yml   |  2 +-
 instrumentation/packages.go                   |  4 ----
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/apps/outdated_github_majors.go b/.github/workflows/apps/outdated_github_majors.go
index 3a265ef275..af24b551b5 100644
--- a/.github/workflows/apps/outdated_github_majors.go
+++ b/.github/workflows/apps/outdated_github_majors.go
@@ -206,8 +206,7 @@ func main() {
 	packages := instrumentation.GetPackages()
 	integrations := make(map[string]struct{})
 
-	for _, info := range packages {
-		repository := info.TracedPackage
+	for _, repository := range packages {
 
 		// repo starts with github and ends in version suffix
 		if validateRepository(repository) {
@@ -215,23 +214,23 @@ func main() {
 
 			// check if we've seen this integration before
 			lastSlashIndex := strings.LastIndex(repository, "/")
-			baseRepo := repository[len(prefix)+1 : lastSlashIndex]
-			if _, exists := integrations[baseRepo]; exists {
+			module := repository[len(prefix)+1 : lastSlashIndex]
+			if _, exists := integrations[module]; exists {
 				continue
 			}
 
-			log.Printf("Base repo: %s\n", baseRepo)
+			log.Printf("Module: %s\n", module)
 
 			// Get the latest major from GH
-			latest_major_version, err := getLatestMajor(baseRepo)
+			latest_major_version, err := getLatestMajor(module)
 			if err != nil {
-				log.Printf("Error getting min version for repo %s: %v\n", repository, err)
+				log.Printf("Error getting github major for %s: %v\n", repository, err)
 				continue
 			}
-			integrations[baseRepo] = struct{}{}
+			integrations[module] = struct{}{}
 
 			// Get the latest major from go.mod
-			latest_major_contrib, err := getLatestMajorContrib(baseRepo)
+			latest_major_contrib, err := getLatestMajorContrib(module)
 
 			if err != nil {
 				log.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
@@ -240,7 +239,11 @@ func main() {
 			log.Printf("major on github: %s\n", latest_major_version)
 
 			if semver.Major(latest_major_contrib) != latest_major_version {
-				log.Printf("repository %s has a new major latest on github: %s\n", baseRepo, latest_major_version)
+				// special casing: go-redis/redis renamed to redis/go-redis in v9
+				if module == "go-redis/redis" && latest_major_version == "v9" {
+					continue
+				}
+				log.Printf("repository %s has a new major latest on github: %s\n", module, latest_major_version)
 			}
 
 		}
diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index 364e02f876..b3905aca4a 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       
-      - run: go run .github/workflows/apps/outdated_github_majors.go > latests.txt
+      - run: go run .github/workflows/apps/outdated_github_majors.go > contrib/latests.txt
 
       - run: git diff
 
diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index dcda5d0c72..c3cce149e6 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -739,10 +739,6 @@ var packages = map[Package]PackageInfo{
 	},
 }
 
-func GetPackages() map[Package]PackageInfo {
-	return packages // TODO: change to return a deep copy
-}
-
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name

From 74c7b9ee43fbb277b0d00a8dd27e44d88688dae4 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Tue, 21 Jan 2025 21:58:25 -0500
Subject: [PATCH 08/29] add new checking for new latest majors

---
 .../apps/new_latest_major_versions.go         | 294 ++++++++++++++++++
 1 file changed, 294 insertions(+)
 create mode 100644 .github/workflows/apps/new_latest_major_versions.go

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
new file mode 100644
index 0000000000..77484d88c8
--- /dev/null
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -0,0 +1,294 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024 Datadog, Inc.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"os/exec"
+
+	"github.com/DataDog/dd-trace-go/v2/instrumentation"
+	"golang.org/x/mod/modfile"
+	"golang.org/x/mod/semver"
+)
+
+type ModuleInfo struct {
+	Origin struct {
+		URL string `json:"url"`
+	} `json:"Origin"`
+}
+
+func getGoModVersion(repository string, pkg string) (string, error) {
+	// ex: package: aws/aws-sdk-go-v2
+	// repository: github.com/aws/aws-sdk-go-v2
+	// look for go.mod in contrib/{package}
+	// if it exists, look for repository in the go.mod
+	// parse the version associated with repository
+	// Define the path to the go.mod file within the contrib/{pkg} directory
+	goModPath := fmt.Sprintf("contrib/%s/go.mod", pkg)
+
+	// Read the go.mod file
+	content, err := os.ReadFile(goModPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read go.mod file at %s: %w", goModPath, err)
+	}
+
+	// Parse the go.mod file
+	modFile, err := modfile.Parse(goModPath, content, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse go.mod file at %s: %w", goModPath, err)
+	}
+
+	// Search for the module matching the repository
+	for _, req := range modFile.Require {
+		if strings.HasPrefix(req.Mod.Path, repository) {
+			return req.Mod.Version, nil
+		}
+	}
+
+	// If the repository is not found in the dependencies
+	return "", fmt.Errorf("repository %s not found in go.mod file", repository)
+}
+
+func getModuleOrigin(repository, version string) (string, error) {
+	cmd := exec.Command("go", "list", "-m", "-json", fmt.Sprintf("%s@%s", repository, version))
+	output, err := cmd.Output()
+
+	if err != nil {
+		return "", fmt.Errorf("Failed to execute command: %w", err)
+	}
+
+	var moduleInfo ModuleInfo
+	if err := json.Unmarshal(output, &moduleInfo); err != nil {
+		return "", fmt.Errorf("failed to parse JSON output: %w", err)
+	}
+
+	// Check if the Origin.URL field exists
+	if moduleInfo.Origin.URL != "" {
+		return moduleInfo.Origin.URL, nil
+	}
+
+	// If Origin.URL is not found, return an error
+	return "", fmt.Errorf("Origin.URL not found in JSON for %s@%s", repository, version)
+}
+
+// Run `git ls-remote` and fetch tags
+func getTags(vcsURL string) ([]string, error) {
+	cmd := exec.Command("git", "ls-remote", "--tags", vcsURL)
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewReader(output))
+	tags := []string{}
+	tagRegex := regexp.MustCompile(`refs/tags/(v[\d.]+)$`)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if matches := tagRegex.FindStringSubmatch(line); matches != nil {
+			tags = append(tags, matches[1])
+		}
+	}
+	return tags, nil
+}
+
+// Group tags by major version
+func groupByMajor(tags []string) map[string][]string {
+	majors := make(map[string][]string)
+	for _, tag := range tags {
+		if semver.IsValid(tag) {
+			major := semver.Major(tag)
+			majors[major] = append(majors[major], tag)
+		}
+	}
+	return majors
+}
+
+// Get the latest version for a list of versions
+func getLatestVersion(versions []string) string {
+	sort.Slice(versions, func(i, j int) bool {
+		return semver.Compare(versions[i], versions[j]) < 0
+	})
+	return versions[len(versions)-1]
+}
+
+func fetchGoMod(url string) (bool, string) {
+	cmd := exec.Command("curl", "-v", "-s", "-f", url)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		// If `curl` fails, check if it's because of a 404
+		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 22 { // HTTP 404
+			return false, ""
+		}
+		fmt.Printf("Error running curl: %v\n", err)
+		return false, ""
+	}
+
+	// Parse the output to check for a `module` line
+	scanner := bufio.NewScanner(&stdout)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "module ") {
+			return true, strings.TrimSpace(strings.TrimPrefix(line, "module "))
+		}
+	}
+
+	return false, ""
+}
+
+func truncateVersion(pkg string) string {
+	// Regular expression to match ".v{X}" or "/v{X}" where {X} is a number
+	re := regexp.MustCompile(`(\.v\d+|/v\d+)$`)
+	// Replace the matched pattern with an empty string
+	return re.ReplaceAllString(pkg, "")
+}
+
+// split, ex v5.3.2 => v5
+func truncateMajorVersion(version string) string {
+	// Split the version by "."
+	parts := strings.Split(version, ".")
+	// Return the first part (e.g., "v5")
+	return parts[0]
+}
+
+// Comparator for version strings
+func compareVersions(v1, v2 string) bool {
+	// if v1 > v2 return true
+	re := regexp.MustCompile(`^v(\d+)$`)
+
+	// Extract the numeric part from the first version
+	match1 := re.FindStringSubmatch(v1)
+	match2 := re.FindStringSubmatch(v2)
+
+	if len(match1) < 2 || len(match2) < 2 {
+		panic("Invalid version format") // Ensure valid versions like "v1", "v2", etc.
+	}
+
+	// Convert the numeric part to integers
+	num1, _ := strconv.Atoi(match1[1])
+	num2, _ := strconv.Atoi(match2[1])
+
+	if num1 <= num2 {
+		return false
+	}
+	return true
+}
+
+func main() {
+	log.SetFlags(0) // disable date and time logging
+	// Find latest major
+	github_latests := map[string]string{}  // map module (base name) => latest on github
+	contrib_latests := map[string]string{} // map module (base name) => latest on go.mod
+
+	for pkg, repository := range instrumentation.GetPackages() {
+
+		// Step 1: get the version from the module go.mod
+		fmt.Printf("package: %v\n", pkg)
+		fmt.Printf("repository: %v\n", repository)
+		base := truncateVersion(string(pkg))
+
+		version, err := getGoModVersion(repository, string(pkg))
+		if err != nil {
+			return
+		}
+		fmt.Printf("version: %v\n", version)
+
+		// check if need to update contrib_latests
+		// fmt.Printf("Getting latest contrib major")
+		version_major_contrib := truncateMajorVersion(version)
+		if err != nil {
+			fmt.Println(err)
+		}
+		if latestContribMajor, ok := contrib_latests[base]; ok {
+			if compareVersions(version_major_contrib, latestContribMajor) {
+				contrib_latests[base] = version_major_contrib // TODO: check if this is needed
+			}
+		} else {
+			contrib_latests[base] = version_major_contrib
+		}
+
+		// Step 2:
+		// create the string repository@{version} and run command go list -m -json <repository>@<version>
+		// this should return a JSON
+		// extract Origin[URL] if the JSON contains it, otherwise continue
+
+		origin, err := getModuleOrigin(repository, version)
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			continue
+		} else {
+			fmt.Printf("origin URL: %s\n", origin)
+		}
+
+		// 2. From the VCS url, do `git ls-remote --tags <vcs_url>` to get the tags
+		// output:
+		// 3. Parse the tags, and extract all the majors from them (ex v2, v3, v4)
+		tags, err := getTags(origin)
+		if err != nil {
+			fmt.Println("Error fetching tags:", err)
+			continue
+		}
+
+		majors := groupByMajor(tags)
+
+		// 4. Get the latest version of each major. For each latest version of each major:
+		// curl https://raw.githubusercontent.com/<module>/refs/tags/v4.18.3/go.mod
+		// 5a. If request returns 404, module is not a go module. This means version belongs to the module without any /v at the end.
+		// 5b. If request returns a `go.mod`, parse the modfile and extract the mod name
+
+		// Get the latest version for each major
+		for major, versions := range majors {
+			latest := getLatestVersion(versions)
+			fmt.Printf("Latest version for %s: %s\n", major, latest)
+
+			// Fetch `go.mod`
+			// curl https://raw.githubusercontent.com/<module>/refs/tags/<latest>/go.mod
+			goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
+			isGoModule, modName := fetchGoMod(goModURL)
+			if isGoModule {
+				fmt.Printf("Module name for %s: %s\n", latest, modName)
+			} else {
+				fmt.Printf("Version %s does not belong to a Go module\n", latest)
+				continue
+			}
+			// latest_major := truncateMajorVersion(latest)
+			if latestGithubMajor, ok := github_latests[base]; ok {
+				// fmt.Printf("latest github major:%s", latestGithubMajor)
+				if compareVersions(major, latestGithubMajor) {
+					// if latest > latestGithubMajor
+					github_latests[base] = major
+				}
+			} else {
+				// fmt.Printf("adding to github latests: %s, major %s\n", base, major)
+				github_latests[base] = major
+			}
+		}
+
+	}
+	for base, contribMajor := range contrib_latests {
+		if latestGithubMajor, ok := github_latests[base]; ok {
+			if compareVersions(latestGithubMajor, contribMajor) {
+				fmt.Printf("New latest major on Github: %s", latestGithubMajor)
+			}
+		}
+	}
+
+	// for key, value := range github_latests {
+	// 	fmt.Printf("%s: %s\n", key, value)
+	// }
+}

From b6c3001fb3fd45e22763cd8194cbfe6f3083f8ab Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Tue, 21 Jan 2025 22:00:09 -0500
Subject: [PATCH 09/29] remove old file

---
 .../workflows/apps/outdated_github_majors.go  | 252 ------------------
 1 file changed, 252 deletions(-)
 delete mode 100644 .github/workflows/apps/outdated_github_majors.go

diff --git a/.github/workflows/apps/outdated_github_majors.go b/.github/workflows/apps/outdated_github_majors.go
deleted file mode 100644
index af24b551b5..0000000000
--- a/.github/workflows/apps/outdated_github_majors.go
+++ /dev/null
@@ -1,252 +0,0 @@
-// Unless explicitly stated otherwise all files in this repository are licensed
-// under the Apache License Version 2.0.
-// This product includes software developed at Datadog (https://www.datadoghq.com/).
-// Copyright 2024 Datadog, Inc.
-
-package main
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strings"
-
-	"github.com/DataDog/dd-trace-go/v2/instrumentation"
-	"golang.org/x/mod/modfile"
-	"golang.org/x/mod/semver"
-)
-
-func getLatestMajor(repo string) (string, error) {
-	// Get the latest major version available on GitHub
-	const prefix = "github.com/"
-	const apiBaseURL = "https://api.github.com/repos/%s/tags"
-	url := fmt.Sprintf(apiBaseURL, repo)
-
-	// Fetch tags from GitHub
-	resp, err := http.Get(url)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to fetch tags: %s", resp.Status)
-	}
-
-	// Parse response
-	var tags []struct {
-		Name string `json:"name"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil {
-		return "", err
-	}
-
-	// Track latest versions by major version
-	latestByMajor := make(map[string]string) // Map major version to the latest full version string
-	for _, tag := range tags {
-		v := semver.Canonical(tag.Name) // Ensure the version is canonical
-		if !semver.IsValid(v) {
-			continue // Skip invalid versions
-		}
-
-		if semver.Prerelease(v) != "" {
-			continue // Skip pre-release versions
-		}
-
-		major := semver.Major(v) // Extract the major version (e.g., "v1")
-		if current, exists := latestByMajor[major]; !exists || semver.Compare(v, current) > 0 {
-			latestByMajor[major] = v
-		}
-	}
-
-	// Determine the largest major version
-	var latestMajor string
-	for major, _ := range latestByMajor {
-		if latestMajor == "" || semver.Compare(major, latestMajor) > 0 {
-			latestMajor = major
-		}
-	}
-
-	if latestMajor != "" {
-		return latestMajor, nil
-	}
-
-	return "", fmt.Errorf("no valid versions found")
-}
-
-// 	// walk through contrib directory
-// 	// match on anything that matches the pattern "contrib/{repository}" (there may be multiple)
-// 	// ex. if repository=go-redis/redis, "contrib/go-redis/redis.v9"
-// 	// inside this repository, look for go.mod
-// 	// if no go.mod, error
-// 	// match on repository in the go.mod in the require() block that contains repository
-// 	// example: if repository=go-redis/redis, should match on line "github.com/redis/go-redis/v9 v9.1.0"
-// 	// if no repository exists that matches this pattern, error
-// 	// return the largest major version associated
-
-// getLatestMajorContrib finds the latest major version of a repository in the contrib directory.
-func getLatestMajorContrib(repository string) (string, error) {
-	const contribDir = "contrib"
-
-	// Prepare the repository matching pattern
-	// ex. repository = "redis/go-redis" should match on any directory that contains redis/go-redis
-	repoPattern := fmt.Sprintf(`^%s/%s(/.*)?`, contribDir, strings.ReplaceAll(repository, "/", "/"))
-
-	// Compile regex for matching contrib directory entries
-	repoRegex, err := regexp.Compile(repoPattern)
-	if err != nil {
-		return "", fmt.Errorf("invalid repository pattern: %v", err)
-	}
-
-	var latestVersion string
-
-	// Walk through the contrib directory
-	err = filepath.Walk(contribDir, func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Check if the path matches the contrib repository pattern
-		if !info.IsDir() || !repoRegex.MatchString(path) {
-			return nil
-		}
-
-		// Look for go.mod file
-		goModPath := filepath.Join(path, "go.mod")
-		if _, err := os.Stat(goModPath); err != nil {
-			return nil // No go.mod file, skip
-		}
-
-		// Parse the go.mod file to find the version
-		version, err := extractVersionFromGoMod(goModPath, repository)
-		if err != nil {
-			return nil
-		}
-
-		// Update the latest version if it's greater
-		if latestVersion == "" || semver.Compare(version, latestVersion) > 0 {
-			latestVersion = version
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return "", fmt.Errorf("error walking contrib directory: %v", err)
-	}
-
-	if latestVersion == "" {
-		return "", errors.New("no matching contrib repository with a valid version found")
-	}
-
-	return latestVersion, nil
-}
-
-func extractVersionFromGoMod(goModPath string, repository string) (string, error) {
-	data, err := os.ReadFile(goModPath)
-	if err != nil {
-		return "", fmt.Errorf("failed to read go.mod: %v", err)
-	}
-	var versions []string
-
-	// parse go.mod
-	modFile, err := modfile.Parse(goModPath, data, nil)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse go.mod: %v", err)
-	}
-
-	for _, req := range modFile.Require {
-		if strings.Contains(req.Mod.Path, repository) { // check if module path contains repository substring
-			if semver.IsValid(req.Mod.Version) {
-				versions = append(versions, req.Mod.Version)
-			}
-		}
-
-	}
-
-	if len(versions) == 0 {
-		return "", errors.New("no valid versions found for repository in go.mod")
-	}
-
-	// Sort versions in descending order
-	// Return the greatest version
-	sort.Slice(versions, func(i, j int) bool {
-		return semver.Compare(versions[i], versions[j]) > 0
-	})
-
-	return versions[0], nil
-
-}
-
-// validateRepository checks if the repository string starts with "github.com" and ends with a version suffix.
-// Returns true if the repository is valid, false otherwise.
-func validateRepository(repo string) bool {
-	const prefix = "github.com/"
-	if !strings.HasPrefix(repo, prefix) {
-		return false
-	}
-
-	lastSlashIndex := strings.LastIndex(repo, "/")
-	if lastSlashIndex == -1 || !strings.HasPrefix(repo[lastSlashIndex:], "/v") {
-		return false
-	}
-
-	return true
-}
-
-func main() {
-	log.SetFlags(0) // disable date and time logging
-	packages := instrumentation.GetPackages()
-	integrations := make(map[string]struct{})
-
-	for _, repository := range packages {
-
-		// repo starts with github and ends in version suffix
-		if validateRepository(repository) {
-			const prefix = "github.com"
-
-			// check if we've seen this integration before
-			lastSlashIndex := strings.LastIndex(repository, "/")
-			module := repository[len(prefix)+1 : lastSlashIndex]
-			if _, exists := integrations[module]; exists {
-				continue
-			}
-
-			log.Printf("Module: %s\n", module)
-
-			// Get the latest major from GH
-			latest_major_version, err := getLatestMajor(module)
-			if err != nil {
-				log.Printf("Error getting github major for %s: %v\n", repository, err)
-				continue
-			}
-			integrations[module] = struct{}{}
-
-			// Get the latest major from go.mod
-			latest_major_contrib, err := getLatestMajorContrib(module)
-
-			if err != nil {
-				log.Printf("Error getting latest major from go.mod for %s: %v\n", repository, err)
-			}
-			log.Printf("major on go.mod: %s\n", semver.Major(latest_major_contrib))
-			log.Printf("major on github: %s\n", latest_major_version)
-
-			if semver.Major(latest_major_contrib) != latest_major_version {
-				// special casing: go-redis/redis renamed to redis/go-redis in v9
-				if module == "go-redis/redis" && latest_major_version == "v9" {
-					continue
-				}
-				log.Printf("repository %s has a new major latest on github: %s\n", module, latest_major_version)
-			}
-
-		}
-	}
-
-}

From db61f8790133413c75bfd264755fa34169c83b01 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Tue, 21 Jan 2025 22:00:23 -0500
Subject: [PATCH 10/29] update workflow script

---
 .github/workflows/outdated-integrations.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index b3905aca4a..21812a2503 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       
-      - run: go run .github/workflows/apps/outdated_github_majors.go > contrib/latests.txt
+      - run: go run .github/workflows/apps/new_latest_major_versions.go
 
       - run: git diff
 

From 0b4aac735a4cf710921b116b3db3dc41468291f9 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Wed, 22 Jan 2025 10:32:54 -0500
Subject: [PATCH 11/29] wip

---
 .../apps/new_latest_major_versions.go         | 50 ++++++++++++-------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 77484d88c8..1ccab3d6b0 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -30,6 +30,15 @@ type ModuleInfo struct {
 	} `json:"Origin"`
 }
 
+// stdlibPackages are used to skip in version checking.
+// TODO: can we make this exported and export from gen_supported_versions_doc.go or put it in instrumentation/packages.go?
+var stdLibPackages = map[string]struct{}{
+	"log/slog":     {},
+	"os":           {},
+	"net/http":     {},
+	"database/sql": {},
+}
+
 func getGoModVersion(repository string, pkg string) (string, error) {
 	// ex: package: aws/aws-sdk-go-v2
 	// repository: github.com/aws/aws-sdk-go-v2
@@ -160,9 +169,7 @@ func truncateVersion(pkg string) string {
 
 // split, ex v5.3.2 => v5
 func truncateMajorVersion(version string) string {
-	// Split the version by "."
 	parts := strings.Split(version, ".")
-	// Return the first part (e.g., "v5")
 	return parts[0]
 }
 
@@ -199,28 +206,36 @@ func main() {
 
 		// Step 1: get the version from the module go.mod
 		fmt.Printf("package: %v\n", pkg)
-		fmt.Printf("repository: %v\n", repository)
-		base := truncateVersion(string(pkg))
+		// fmt.Printf("repository: %v\n", repository)
 
+		// if it is part of the standard packages, continue
+		if _, ok := stdLibPackages[repository]; ok {
+			continue
+		}
+
+		base := truncateVersion(string(pkg))
 		version, err := getGoModVersion(repository, string(pkg))
 		if err != nil {
-			return
+			fmt.Printf("%v go.mod not found.", pkg)
+			continue
 		}
 		fmt.Printf("version: %v\n", version)
 
 		// check if need to update contrib_latests
-		// fmt.Printf("Getting latest contrib major")
 		version_major_contrib := truncateMajorVersion(version)
 		if err != nil {
 			fmt.Println(err)
+			continue
 		}
-		if latestContribMajor, ok := contrib_latests[base]; ok {
-			if compareVersions(version_major_contrib, latestContribMajor) {
-				contrib_latests[base] = version_major_contrib // TODO: check if this is needed
-			}
-		} else {
-			contrib_latests[base] = version_major_contrib
-		}
+
+		contrib_latests[base] = version_major_contrib
+		// if latestContribMajor, ok := contrib_latests[base]; ok {
+		// 	if compareVersions(version_major_contrib, latestContribMajor) {
+		// 		contrib_latests[base] = version_major_contrib // TODO: check if this is needed
+		// 	}
+		// } else {
+		// 	contrib_latests[base] = version_major_contrib
+		// }
 
 		// Step 2:
 		// create the string repository@{version} and run command go list -m -json <repository>@<version>
@@ -256,7 +271,7 @@ func main() {
 			latest := getLatestVersion(versions)
 			fmt.Printf("Latest version for %s: %s\n", major, latest)
 
-			// Fetch `go.mod`
+			// Fetch `go.mod` with command
 			// curl https://raw.githubusercontent.com/<module>/refs/tags/<latest>/go.mod
 			goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
 			isGoModule, modName := fetchGoMod(goModURL)
@@ -274,12 +289,14 @@ func main() {
 					github_latests[base] = major
 				}
 			} else {
-				// fmt.Printf("adding to github latests: %s, major %s\n", base, major)
 				github_latests[base] = major
 			}
 		}
 
 	}
+
+	// check if there are any outdated majors
+	// output if there is a new major package we do not support
 	for base, contribMajor := range contrib_latests {
 		if latestGithubMajor, ok := github_latests[base]; ok {
 			if compareVersions(latestGithubMajor, contribMajor) {
@@ -288,7 +305,4 @@ func main() {
 		}
 	}
 
-	// for key, value := range github_latests {
-	// 	fmt.Printf("%s: %s\n", key, value)
-	// }
 }

From 6a80875db8f78e84e6e2766af85515820fa98ac1 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Wed, 22 Jan 2025 10:51:20 -0500
Subject: [PATCH 12/29] wip

---
 .../workflows/apps/new_latest_major_versions.go   | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 1ccab3d6b0..df45ea2592 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -228,14 +228,13 @@ func main() {
 			continue
 		}
 
-		contrib_latests[base] = version_major_contrib
-		// if latestContribMajor, ok := contrib_latests[base]; ok {
-		// 	if compareVersions(version_major_contrib, latestContribMajor) {
-		// 		contrib_latests[base] = version_major_contrib // TODO: check if this is needed
-		// 	}
-		// } else {
-		// 	contrib_latests[base] = version_major_contrib
-		// }
+		if current_latest, ok := contrib_latests[base]; ok {
+			if compareVersions(version_major_contrib, current_latest) {
+				contrib_latests[base] = version_major_contrib
+			}
+		} else {
+			contrib_latests[base] = version_major_contrib
+		}
 
 		// Step 2:
 		// create the string repository@{version} and run command go list -m -json <repository>@<version>

From 287e710898b0344680e24e625e8ec5fa715191b4 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 24 Jan 2025 10:25:36 -0500
Subject: [PATCH 13/29] update workflow

---
 .../apps/new_latest_major_versions.go         | 106 ++++++++----------
 instrumentation/packages.go                   |   7 ++
 2 files changed, 55 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index df45ea2592..a9bc06483d 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -10,11 +10,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"os"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 
 	"os/exec"
@@ -30,15 +31,6 @@ type ModuleInfo struct {
 	} `json:"Origin"`
 }
 
-// stdlibPackages are used to skip in version checking.
-// TODO: can we make this exported and export from gen_supported_versions_doc.go or put it in instrumentation/packages.go?
-var stdLibPackages = map[string]struct{}{
-	"log/slog":     {},
-	"os":           {},
-	"net/http":     {},
-	"database/sql": {},
-}
-
 func getGoModVersion(repository string, pkg string) (string, error) {
 	// ex: package: aws/aws-sdk-go-v2
 	// repository: github.com/aws/aws-sdk-go-v2
@@ -126,32 +118,55 @@ func groupByMajor(tags []string) map[string][]string {
 
 // Get the latest version for a list of versions
 func getLatestVersion(versions []string) string {
-	sort.Slice(versions, func(i, j int) bool {
-		return semver.Compare(versions[i], versions[j]) < 0
+
+	// Filter out pre-release versions and non-valid versions
+	validVersions := make([]string, 0)
+	for _, v := range versions {
+		if semver.IsValid(v) && semver.Prerelease(v) == "" {
+			validVersions = append(validVersions, v)
+		}
+	}
+	sort.Slice(validVersions, func(i, j int) bool {
+		return semver.Compare(validVersions[i], validVersions[j]) < 0
 	})
-	return versions[len(versions)-1]
+	return validVersions[len(validVersions)-1]
 }
 
 func fetchGoMod(url string) (bool, string) {
-	cmd := exec.Command("curl", "-v", "-s", "-f", url)
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
 
-	err := cmd.Run()
+	// Create an HTTP client and make a GET request
+	resp, err := http.Get(url)
 	if err != nil {
-		// If `curl` fails, check if it's because of a 404
-		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 22 { // HTTP 404
-			return false, ""
-		}
-		fmt.Printf("Error running curl: %v\n", err)
+		fmt.Printf("Error making HTTP request: %v\n", err)
 		return false, ""
 	}
+	defer resp.Body.Close()
 
-	// Parse the output to check for a `module` line
-	scanner := bufio.NewScanner(&stdout)
-	for scanner.Scan() {
-		line := scanner.Text()
+	// Check if the HTTP status is 404
+	if resp.StatusCode == http.StatusNotFound {
+		return false, ""
+	}
+
+	// Handle other HTTP errors
+	if resp.StatusCode != http.StatusOK {
+		fmt.Printf("Unexpected HTTP status: %d\n", resp.StatusCode)
+		return false, ""
+	}
+
+	// Read response
+	reader := bufio.NewReader(resp.Body)
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil && err != io.EOF {
+			fmt.Printf("Error reading response body: %v\n", err)
+			return false, ""
+		}
+
+		if err == io.EOF {
+			break
+		}
+
+		line = strings.TrimSpace(line)
 		if strings.HasPrefix(line, "module ") {
 			return true, strings.TrimSpace(strings.TrimPrefix(line, "module "))
 		}
@@ -173,29 +188,6 @@ func truncateMajorVersion(version string) string {
 	return parts[0]
 }
 
-// Comparator for version strings
-func compareVersions(v1, v2 string) bool {
-	// if v1 > v2 return true
-	re := regexp.MustCompile(`^v(\d+)$`)
-
-	// Extract the numeric part from the first version
-	match1 := re.FindStringSubmatch(v1)
-	match2 := re.FindStringSubmatch(v2)
-
-	if len(match1) < 2 || len(match2) < 2 {
-		panic("Invalid version format") // Ensure valid versions like "v1", "v2", etc.
-	}
-
-	// Convert the numeric part to integers
-	num1, _ := strconv.Atoi(match1[1])
-	num2, _ := strconv.Atoi(match2[1])
-
-	if num1 <= num2 {
-		return false
-	}
-	return true
-}
-
 func main() {
 	log.SetFlags(0) // disable date and time logging
 	// Find latest major
@@ -206,10 +198,9 @@ func main() {
 
 		// Step 1: get the version from the module go.mod
 		fmt.Printf("package: %v\n", pkg)
-		// fmt.Printf("repository: %v\n", repository)
 
 		// if it is part of the standard packages, continue
-		if _, ok := stdLibPackages[repository]; ok {
+		if _, ok := instrumentation.StandardPackages[repository]; ok {
 			continue
 		}
 
@@ -229,7 +220,7 @@ func main() {
 		}
 
 		if current_latest, ok := contrib_latests[base]; ok {
-			if compareVersions(version_major_contrib, current_latest) {
+			if semver.Compare(version_major_contrib, current_latest) > 0 {
 				contrib_latests[base] = version_major_contrib
 			}
 		} else {
@@ -262,7 +253,7 @@ func main() {
 
 		// 4. Get the latest version of each major. For each latest version of each major:
 		// curl https://raw.githubusercontent.com/<module>/refs/tags/v4.18.3/go.mod
-		// 5a. If request returns 404, module is not a go module. This means version belongs to the module without any /v at the end.
+		// 5a. If request returns 404, module is not a go module. This means version belongs to the module without /v at the end.
 		// 5b. If request returns a `go.mod`, parse the modfile and extract the mod name
 
 		// Get the latest version for each major
@@ -277,13 +268,12 @@ func main() {
 			if isGoModule {
 				fmt.Printf("Module name for %s: %s\n", latest, modName)
 			} else {
-				fmt.Printf("Version %s does not belong to a Go module\n", latest)
+				// fmt.Printf("Version %s does not belong to a Go module\n", latest)
 				continue
 			}
 			// latest_major := truncateMajorVersion(latest)
 			if latestGithubMajor, ok := github_latests[base]; ok {
-				// fmt.Printf("latest github major:%s", latestGithubMajor)
-				if compareVersions(major, latestGithubMajor) {
+				if semver.Compare(major, latestGithubMajor) > 0 {
 					// if latest > latestGithubMajor
 					github_latests[base] = major
 				}
@@ -298,7 +288,7 @@ func main() {
 	// output if there is a new major package we do not support
 	for base, contribMajor := range contrib_latests {
 		if latestGithubMajor, ok := github_latests[base]; ok {
-			if compareVersions(latestGithubMajor, contribMajor) {
+			if semver.Compare(latestGithubMajor, contribMajor) > 0 {
 				fmt.Printf("New latest major on Github: %s", latestGithubMajor)
 			}
 		}
diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index c3cce149e6..550897fa07 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -739,6 +739,13 @@ var packages = map[Package]PackageInfo{
 	},
 }
 
+var StandardPackages = map[string]struct{}{
+	"log/slog":     {},
+	"os":           {},
+	"net/http":     {},
+	"database/sql": {},
+}
+
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name

From 823f2c2e7e3397f4f64682f47e96fcb28f4a9c0a Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 24 Jan 2025 13:11:51 -0500
Subject: [PATCH 14/29] update workflow files

---
 .../apps/new_latest_major_versions.go         | 38 ++++++++++++-------
 .github/workflows/outdated-integrations.yml   |  2 +
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index a9bc06483d..8ed0726630 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -31,6 +31,11 @@ type ModuleInfo struct {
 	} `json:"Origin"`
 }
 
+type GithubLatests struct {
+	Version string
+	Module  string
+}
+
 func getGoModVersion(repository string, pkg string) (string, error) {
 	// ex: package: aws/aws-sdk-go-v2
 	// repository: github.com/aws/aws-sdk-go-v2
@@ -65,6 +70,7 @@ func getGoModVersion(repository string, pkg string) (string, error) {
 
 func getModuleOrigin(repository, version string) (string, error) {
 	cmd := exec.Command("go", "list", "-m", "-json", fmt.Sprintf("%s@%s", repository, version))
+	cmd.Env = append(os.Environ(), "GOPROXY=direct")
 	output, err := cmd.Output()
 
 	if err != nil {
@@ -191,8 +197,8 @@ func truncateMajorVersion(version string) string {
 func main() {
 	log.SetFlags(0) // disable date and time logging
 	// Find latest major
-	github_latests := map[string]string{}  // map module (base name) => latest on github
-	contrib_latests := map[string]string{} // map module (base name) => latest on go.mod
+	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
+	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
 
 	for pkg, repository := range instrumentation.GetPackages() {
 
@@ -210,10 +216,11 @@ func main() {
 			fmt.Printf("%v go.mod not found.", pkg)
 			continue
 		}
-		fmt.Printf("version: %v\n", version)
 
 		// check if need to update contrib_latests
 		version_major_contrib := truncateMajorVersion(version)
+		fmt.Printf("version: %v\n", version_major_contrib)
+
 		if err != nil {
 			fmt.Println(err)
 			continue
@@ -259,26 +266,24 @@ func main() {
 		// Get the latest version for each major
 		for major, versions := range majors {
 			latest := getLatestVersion(versions)
-			fmt.Printf("Latest version for %s: %s\n", major, latest)
+			// fmt.Printf("Latest version for %s: %s\n", major, latest)
 
-			// Fetch `go.mod` with command
-			// curl https://raw.githubusercontent.com/<module>/refs/tags/<latest>/go.mod
+			// Fetch `go.mod`
 			goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
 			isGoModule, modName := fetchGoMod(goModURL)
 			if isGoModule {
 				fmt.Printf("Module name for %s: %s\n", latest, modName)
 			} else {
-				// fmt.Printf("Version %s does not belong to a Go module\n", latest)
 				continue
 			}
 			// latest_major := truncateMajorVersion(latest)
-			if latestGithubMajor, ok := github_latests[base]; ok {
-				if semver.Compare(major, latestGithubMajor) > 0 {
+			if latestGithub, ok := github_latests[base]; ok {
+				if semver.Compare(major, latestGithub.Version) > 0 {
 					// if latest > latestGithubMajor
-					github_latests[base] = major
+					github_latests[base] = GithubLatests{major, modName}
 				}
 			} else {
-				github_latests[base] = major
+				github_latests[base] = GithubLatests{major, modName}
 			}
 		}
 
@@ -287,9 +292,14 @@ func main() {
 	// check if there are any outdated majors
 	// output if there is a new major package we do not support
 	for base, contribMajor := range contrib_latests {
-		if latestGithubMajor, ok := github_latests[base]; ok {
-			if semver.Compare(latestGithubMajor, contribMajor) > 0 {
-				fmt.Printf("New latest major on Github: %s", latestGithubMajor)
+		if latestGithub, ok := github_latests[base]; ok {
+			if semver.Compare(latestGithub.Version, contribMajor) > 0 {
+				if base == "go-redis/redis" && latestGithub.Version == "v9" {
+					continue // go-redis/redis => redis/go-redis in v9
+				}
+				fmt.Printf("New latest major %s of repository %s on Github at module: %s\n", latestGithub.Version, base, latestGithub.Module)
+				fmt.Printf("latest contrib major: %v\n", contribMajor)
+				fmt.Printf("latest github major: %v\n", latestGithub.Version)
 			}
 		}
 	}
diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index 21812a2503..07693b21c0 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -22,6 +22,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       
+      - run: go clean -modcache
+      
       - run: go run .github/workflows/apps/new_latest_major_versions.go
 
       - run: git diff

From 1978e546498fbc8e637e1b91a598aba6b785adfa Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 10:43:37 -0500
Subject: [PATCH 15/29] wip

---
 instrumentation/packages.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index 550897fa07..f5751ad922 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -445,8 +445,6 @@ var packages = map[Package]PackageInfo{
 			},
 		},
 	},
-	// TODO
-
 	PackageNetHTTP: {
 		TracedPackage: "net/http",
 		EnvVarPrefix:  "HTTP",

From 00829684a712f238a5f2fa146814599a76946d5a Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 10:59:18 -0500
Subject: [PATCH 16/29] rebase off v2-dev

---
 instrumentation/packages.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index f5751ad922..550897fa07 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -445,6 +445,8 @@ var packages = map[Package]PackageInfo{
 			},
 		},
 	},
+	// TODO
+
 	PackageNetHTTP: {
 		TracedPackage: "net/http",
 		EnvVarPrefix:  "HTTP",

From 9453a840185c4795d3f0f0c08fda6bbac57fed35 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 11:01:33 -0500
Subject: [PATCH 17/29] wip

---
 instrumentation/packages.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index 550897fa07..c3cce149e6 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -739,13 +739,6 @@ var packages = map[Package]PackageInfo{
 	},
 }
 
-var StandardPackages = map[string]struct{}{
-	"log/slog":     {},
-	"os":           {},
-	"net/http":     {},
-	"database/sql": {},
-}
-
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name

From 52887bda1b19085649cc733c7457e0f6bc8c826b Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 11:03:28 -0500
Subject: [PATCH 18/29] wip

---
 instrumentation/packages.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index c3cce149e6..550897fa07 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -739,6 +739,13 @@ var packages = map[Package]PackageInfo{
 	},
 }
 
+var StandardPackages = map[string]struct{}{
+	"log/slog":     {},
+	"os":           {},
+	"net/http":     {},
+	"database/sql": {},
+}
+
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name

From 007aaaa602aff1d6eae2133ee9b507b3f86232cc Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 11:48:06 -0500
Subject: [PATCH 19/29] workflow file update

---
 .github/workflows/outdated-integrations.yml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index 07693b21c0..f89264c834 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -25,18 +25,3 @@ jobs:
       - run: go clean -modcache
       
       - run: go run .github/workflows/apps/new_latest_major_versions.go
-
-      - run: git diff
-
-      # note: this will only create a PR if there is a new major available
-      - name: Create Pull Request
-        id: pr
-        uses: peter-evans/create-pull-request@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          branch: "upgrade-latest-major-version"
-          commit-message: "Update latests file"
-          base: main
-          title: "chore: update latest majors"
-          labels: changelog/no-changelog
-          body: "Auto-generated PR from Outdated Integrations workflow to update latests major versions"

From 204a1bfc6725be9574d72665fbc69210b7508fcf Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 27 Jan 2025 12:41:56 -0500
Subject: [PATCH 20/29] update to check latest module version from go.mod

---
 .../apps/new_latest_major_versions.go         | 46 +++++++++++++++----
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 8ed0726630..82b73d751c 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -36,36 +36,66 @@ type GithubLatests struct {
 	Module  string
 }
 
-func getGoModVersion(repository string, pkg string) (string, error) {
+func getGoModVersion(repository string, pkg string) (string, string, error) {
 	// ex: package: aws/aws-sdk-go-v2
 	// repository: github.com/aws/aws-sdk-go-v2
 	// look for go.mod in contrib/{package}
 	// if it exists, look for repository in the go.mod
 	// parse the version associated with repository
 	// Define the path to the go.mod file within the contrib/{pkg} directory
+	repository = truncateVersionSuffix(repository)
+
 	goModPath := fmt.Sprintf("contrib/%s/go.mod", pkg)
 
 	// Read the go.mod file
 	content, err := os.ReadFile(goModPath)
 	if err != nil {
-		return "", fmt.Errorf("failed to read go.mod file at %s: %w", goModPath, err)
+		return "", "", fmt.Errorf("failed to read go.mod file at %s: %w", goModPath, err)
 	}
 
 	// Parse the go.mod file
 	modFile, err := modfile.Parse(goModPath, content, nil)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse go.mod file at %s: %w", goModPath, err)
+		return "", "", fmt.Errorf("failed to parse go.mod file at %s: %w", goModPath, err)
 	}
 
+	// keep track of largest version from go.mod
+	var largestVersion string
+	var largestVersionRepo string
+
 	// Search for the module matching the repository
 	for _, req := range modFile.Require {
 		if strings.HasPrefix(req.Mod.Path, repository) {
-			return req.Mod.Version, nil
+			version := req.Mod.Version
+
+			if !semver.IsValid(version) {
+				return "", "", fmt.Errorf("invalid semantic version %s in go.mod", version)
+			}
+
+			if largestVersion == "" || semver.Compare(version, largestVersion) > 0 {
+				largestVersion = version
+				largestVersionRepo = req.Mod.Path
+			}
 		}
 	}
 
-	// If the repository is not found in the dependencies
-	return "", fmt.Errorf("repository %s not found in go.mod file", repository)
+	if largestVersion == "" {
+		// If the repository is not found in the dependencies
+		return "", "", fmt.Errorf("repository %s not found in go.mod file", repository)
+	}
+	return largestVersionRepo, largestVersion, nil
+
+}
+
+func truncateVersionSuffix(repository string) string {
+	parts := strings.Split(repository, "/")
+	lastPart := parts[len(parts)-1]
+
+	if len(lastPart) > 1 && strings.HasPrefix(lastPart, "v") && semver.IsValid(lastPart) {
+		return strings.Join(parts[:len(parts)-1], "/")
+	}
+
+	return repository
 }
 
 func getModuleOrigin(repository, version string) (string, error) {
@@ -211,7 +241,7 @@ func main() {
 		}
 
 		base := truncateVersion(string(pkg))
-		version, err := getGoModVersion(repository, string(pkg))
+		repo, version, err := getGoModVersion(repository, string(pkg))
 		if err != nil {
 			fmt.Printf("%v go.mod not found.", pkg)
 			continue
@@ -239,7 +269,7 @@ func main() {
 		// this should return a JSON
 		// extract Origin[URL] if the JSON contains it, otherwise continue
 
-		origin, err := getModuleOrigin(repository, version)
+		origin, err := getModuleOrigin(repo, version)
 		if err != nil {
 			fmt.Printf("Error: %v\n", err)
 			continue

From dc655cdb9f840ed452e7ed5cfa59ca86d4771d42 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 18:12:04 -0500
Subject: [PATCH 21/29] change to use IsStdLibPackage variable

---
 .../workflows/apps/new_latest_major_versions.go |  8 +++++---
 instrumentation/packages.go                     | 17 +++++++----------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 82b73d751c..84d447d452 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -173,8 +173,9 @@ func fetchGoMod(url string) (bool, string) {
 	// Create an HTTP client and make a GET request
 	resp, err := http.Get(url)
 	if err != nil {
-		fmt.Printf("Error making HTTP request: %v\n", err)
+		// fmt.Printf("Error making HTTP request: %v\n", err)
 		return false, ""
+		// return false, "", fmt.Errorf("failed to get go.mod from url: %w", err)
 	}
 	defer resp.Body.Close()
 
@@ -230,13 +231,14 @@ func main() {
 	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
 	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
 
-	for pkg, repository := range instrumentation.GetPackages() {
+	for pkg, packageInfo := range instrumentation.GetPackages() {
 
 		// Step 1: get the version from the module go.mod
 		fmt.Printf("package: %v\n", pkg)
+		repository := packageInfo.TracedPackage
 
 		// if it is part of the standard packages, continue
-		if _, ok := instrumentation.StandardPackages[repository]; ok {
+		if packageInfo.IsStdLib {
 			continue
 		}
 
diff --git a/instrumentation/packages.go b/instrumentation/packages.go
index 550897fa07..bf7300c80b 100644
--- a/instrumentation/packages.go
+++ b/instrumentation/packages.go
@@ -93,6 +93,7 @@ type PackageInfo struct {
 	external bool
 
 	TracedPackage string
+	IsStdLib      bool
 	EnvVarPrefix  string
 
 	naming map[Component]componentNames
@@ -218,6 +219,7 @@ var packages = map[Package]PackageInfo{
 	},
 	PackageDatabaseSQL: {
 		TracedPackage: "database/sql",
+		IsStdLib:      true,
 		EnvVarPrefix:  "SQL",
 		naming: map[Component]componentNames{
 			ComponentDefault: {
@@ -449,6 +451,7 @@ var packages = map[Package]PackageInfo{
 
 	PackageNetHTTP: {
 		TracedPackage: "net/http",
+		IsStdLib:      true,
 		EnvVarPrefix:  "HTTP",
 		naming: map[Component]componentNames{
 			ComponentServer: {
@@ -733,19 +736,13 @@ var packages = map[Package]PackageInfo{
 	},
 	PackageLogSlog: {
 		TracedPackage: "log/slog",
+		IsStdLib:      true,
 	},
 	PackageEnvoyProxyGoControlPlane: {
 		TracedPackage: "github.com/envoyproxy/go-control-plane",
 	},
 }
 
-var StandardPackages = map[string]struct{}{
-	"log/slog":     {},
-	"os":           {},
-	"net/http":     {},
-	"database/sql": {},
-}
-
 func staticName(name string) func(OperationContext) string {
 	return func(_ OperationContext) string {
 		return name
@@ -788,10 +785,10 @@ func isAWSMessagingSendOp(awsService, awsOperation string) bool {
 }
 
 // GetPackages returns a map of Package to the corresponding instrumented module.
-func GetPackages() map[Package]string {
-	cp := make(map[Package]string)
+func GetPackages() map[Package]PackageInfo {
+	cp := make(map[Package]PackageInfo)
 	for pkg, info := range packages {
-		cp[pkg] = info.TracedPackage
+		cp[pkg] = info
 	}
 	return cp
 }

From 7850e74be7fcf07b9ae95cbd823e1455d4784ba9 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 19:09:20 -0500
Subject: [PATCH 22/29] update to use go-git and use goroutines

---
 .../apps/new_latest_major_versions.go         | 240 +++++++++++++-----
 1 file changed, 179 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 84d447d452..265f2169b4 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -12,17 +12,23 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"net/http"
 	"os"
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 
 	"os/exec"
 
 	"github.com/DataDog/dd-trace-go/v2/instrumentation"
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
+
+	// Missing dependencies
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/storage/memory"
 )
 
 type ModuleInfo struct {
@@ -36,6 +42,13 @@ type GithubLatests struct {
 	Module  string
 }
 
+type PackageResult struct {
+	Base          string
+	LatestVersion string
+	ModulePath    string
+	Error         error
+}
+
 func getGoModVersion(repository string, pkg string) (string, string, error) {
 	// ex: package: aws/aws-sdk-go-v2
 	// repository: github.com/aws/aws-sdk-go-v2
@@ -87,6 +100,77 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 
 }
 
+func fetchGoModGit(repoURL, tag string, wg *sync.WaitGroup, results chan<- PackageResult) {
+	defer wg.Done()
+
+	log.Printf("fetching %s@%s\n", repoURL, tag)
+	if !strings.HasSuffix(repoURL, ".git") {
+		repoURL = repoURL + ".git"
+	}
+
+	storer := memory.NewStorage()
+	fs := memfs.New()
+
+	repo, err := git.Clone(storer, fs, &git.CloneOptions{
+		URL:           repoURL,
+		Depth:         1,
+		SingleBranch:  true,
+		ReferenceName: plumbing.NewTagReferenceName(tag),
+		Progress:      os.Stdout,
+		NoCheckout:    true,
+	})
+	if err != nil {
+		results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}
+		return
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		results <- PackageResult{Error: fmt.Errorf("failed to get worktree: %w", err)}
+		return
+	}
+
+	err = worktree.Checkout(&git.CheckoutOptions{
+		Force:                     true,
+		Create:                    false,
+		Branch:                    plumbing.NewTagReferenceName(tag),
+		SparseCheckoutDirectories: []string{"go.mod"},
+	})
+	if err != nil {
+		results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}
+		return
+	}
+
+	f, err := fs.Open("go.mod")
+	if err != nil {
+		log.Printf("Failed to open go.mod in repository: %s, tag: %s", repoURL, tag)
+		results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}
+		return
+	}
+	defer f.Close()
+	// defer func() {
+	// 	if cerr := f.Close(); cerr != nil && err == nil {
+	// 		err = cerr
+	// 	}
+	// }()
+
+	b, err := io.ReadAll(f)
+	if err != nil {
+		results <- PackageResult{Error: fmt.Errorf("failed to read content: %w", err)}
+		// return nil, fmt.Errorf("failed to read file content: %w", err)
+		return
+	}
+
+	mf, err := modfile.Parse("go.mod", b, nil)
+	if err != nil {
+		results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}
+		return
+	}
+	fmt.Printf("Success fetching Go mod")
+	results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}
+
+}
+
 func truncateVersionSuffix(repository string) string {
 	parts := strings.Split(repository, "/")
 	lastPart := parts[len(parts)-1]
@@ -168,49 +252,49 @@ func getLatestVersion(versions []string) string {
 	return validVersions[len(validVersions)-1]
 }
 
-func fetchGoMod(url string) (bool, string) {
-
-	// Create an HTTP client and make a GET request
-	resp, err := http.Get(url)
-	if err != nil {
-		// fmt.Printf("Error making HTTP request: %v\n", err)
-		return false, ""
-		// return false, "", fmt.Errorf("failed to get go.mod from url: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// Check if the HTTP status is 404
-	if resp.StatusCode == http.StatusNotFound {
-		return false, ""
-	}
-
-	// Handle other HTTP errors
-	if resp.StatusCode != http.StatusOK {
-		fmt.Printf("Unexpected HTTP status: %d\n", resp.StatusCode)
-		return false, ""
-	}
-
-	// Read response
-	reader := bufio.NewReader(resp.Body)
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil && err != io.EOF {
-			fmt.Printf("Error reading response body: %v\n", err)
-			return false, ""
-		}
-
-		if err == io.EOF {
-			break
-		}
-
-		line = strings.TrimSpace(line)
-		if strings.HasPrefix(line, "module ") {
-			return true, strings.TrimSpace(strings.TrimPrefix(line, "module "))
-		}
-	}
-
-	return false, ""
-}
+// func fetchGoMod(url string) (bool, string) {
+
+// 	// Create an HTTP client and make a GET request
+// 	resp, err := http.Get(url)
+// 	if err != nil {
+// 		// fmt.Printf("Error making HTTP request: %v\n", err)
+// 		return false, ""
+// 		// return false, "", fmt.Errorf("failed to get go.mod from url: %w", err)
+// 	}
+// 	defer resp.Body.Close()
+
+// 	// Check if the HTTP status is 404
+// 	if resp.StatusCode == http.StatusNotFound {
+// 		return false, ""
+// 	}
+
+// 	// Handle other HTTP errors
+// 	if resp.StatusCode != http.StatusOK {
+// 		fmt.Printf("Unexpected HTTP status: %d\n", resp.StatusCode)
+// 		return false, ""
+// 	}
+
+// 	// Read response
+// 	reader := bufio.NewReader(resp.Body)
+// 	for {
+// 		line, err := reader.ReadString('\n')
+// 		if err != nil && err != io.EOF {
+// 			fmt.Printf("Error reading response body: %v\n", err)
+// 			return false, ""
+// 		}
+
+// 		if err == io.EOF {
+// 			break
+// 		}
+
+// 		line = strings.TrimSpace(line)
+// 		if strings.HasPrefix(line, "module ") {
+// 			return true, strings.TrimSpace(strings.TrimPrefix(line, "module "))
+// 		}
+// 	}
+
+// 	return false, ""
+// }
 
 func truncateVersion(pkg string) string {
 	// Regular expression to match ".v{X}" or "/v{X}" where {X} is a number
@@ -231,6 +315,9 @@ func main() {
 	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
 	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
 
+	var wg sync.WaitGroup
+	results := make(chan PackageResult, 10) // Buffered channel to avoid blocking
+
 	for pkg, packageInfo := range instrumentation.GetPackages() {
 
 		// Step 1: get the version from the module go.mod
@@ -296,31 +383,62 @@ func main() {
 		// 5b. If request returns a `go.mod`, parse the modfile and extract the mod name
 
 		// Get the latest version for each major
-		for major, versions := range majors {
+		for _, versions := range majors {
 			latest := getLatestVersion(versions)
+			// modFile, err := fetchGoModGit(origin, latest)
+
+			// Fetch go.mod in a goroutine
+			wg.Add(1)
+			go fetchGoModGit(origin, latest, &wg, results)
+			// if err != nil {
+			// 	continue
+			// }
+
 			// fmt.Printf("Latest version for %s: %s\n", major, latest)
 
-			// Fetch `go.mod`
-			goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
-			isGoModule, modName := fetchGoMod(goModURL)
-			if isGoModule {
-				fmt.Printf("Module name for %s: %s\n", latest, modName)
-			} else {
-				continue
-			}
+			// // Fetch `go.mod`
+			// goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
+			// isGoModule, modName := fetchGoMod(goModURL)
+			// if isGoModule {
+			// 	fmt.Printf("Module name for %s: %s\n", latest, modName)
+			// } else {
+			// 	continue
+			// }
 			// latest_major := truncateMajorVersion(latest)
-			if latestGithub, ok := github_latests[base]; ok {
-				if semver.Compare(major, latestGithub.Version) > 0 {
-					// if latest > latestGithubMajor
-					github_latests[base] = GithubLatests{major, modName}
-				}
-			} else {
-				github_latests[base] = GithubLatests{major, modName}
-			}
+			// if latestGithub, ok := github_latests[base]; ok {
+			// 	if semver.Compare(major, latestGithub.Version) > 0 {
+			// 		// if latest > latestGithubMajor
+			// 		github_latests[base] = GithubLatests{major, modFile.Module.Mod.Path}
+			// 	}
+			// } else {
+			// 	github_latests[base] = GithubLatests{major, modFile.Module.Mod.Path}
+			// }
+
 		}
 
 	}
 
+	// Close the results channel once all goroutines finish
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	// Iterate through results
+	for result := range results {
+		if result.Error != nil {
+			fmt.Println("Error:", result.Error)
+			continue
+		}
+
+		if latestGithub, ok := github_latests[result.Base]; ok {
+			if semver.Compare(result.LatestVersion, latestGithub.Version) > 0 {
+				github_latests[result.Base] = GithubLatests{result.LatestVersion, result.ModulePath}
+			}
+		} else {
+			github_latests[result.Base] = GithubLatests{result.LatestVersion, result.ModulePath}
+		}
+	}
 	// check if there are any outdated majors
 	// output if there is a new major package we do not support
 	for base, contribMajor := range contrib_latests {

From 4bb4a3442e157ca72af0759ae6fc82271f115414 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 19:13:27 -0500
Subject: [PATCH 23/29] add dependencies to workflow file

---
 .github/workflows/apps/new_latest_major_versions.go | 2 +-
 .github/workflows/outdated-integrations.yml         | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 265f2169b4..a333e94878 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -338,7 +338,7 @@ func main() {
 
 		// check if need to update contrib_latests
 		version_major_contrib := truncateMajorVersion(version)
-		fmt.Printf("version: %v\n", version_major_contrib)
+		// fmt.Printf("version: %v\n", version_major_contrib)
 
 		if err != nil {
 			fmt.Println(err)
diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml
index f89264c834..40d86fe23e 100644
--- a/.github/workflows/outdated-integrations.yml
+++ b/.github/workflows/outdated-integrations.yml
@@ -24,4 +24,11 @@ jobs:
       
       - run: go clean -modcache
       
+      - name: Add dependencies
+        run: |
+          go get github.com/go-git/go-billy/v5/memfs
+          go get github.com/go-git/go-git/v5
+          go get github.com/go-git/go-git/v5/plumbing
+          go get github.com/go-git/go-git/v5/storage/memory
+
       - run: go run .github/workflows/apps/new_latest_major_versions.go

From acbdf92e79260c60e2f135961168814b25d98f3d Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 19:42:11 -0500
Subject: [PATCH 24/29] wip

---
 .../apps/new_latest_major_versions.go         | 173 ++++++------------
 1 file changed, 52 insertions(+), 121 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index a333e94878..95eb500e63 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -100,16 +100,15 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 
 }
 
-func fetchGoModGit(repoURL, tag string, wg *sync.WaitGroup, results chan<- PackageResult) {
-	defer wg.Done()
-
-	log.Printf("fetching %s@%s\n", repoURL, tag)
+func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
+	// Ensure WG counter is decreased
+	log.Printf("Fetching go.mod for repo: %s, tag: %s", repoURL, tag)
 	if !strings.HasSuffix(repoURL, ".git") {
 		repoURL = repoURL + ".git"
 	}
 
-	storer := memory.NewStorage()
 	fs := memfs.New()
+	storer := memory.NewStorage()
 
 	repo, err := git.Clone(storer, fs, &git.CloneOptions{
 		URL:           repoURL,
@@ -130,44 +129,53 @@ func fetchGoModGit(repoURL, tag string, wg *sync.WaitGroup, results chan<- Packa
 		return
 	}
 
-	err = worktree.Checkout(&git.CheckoutOptions{
-		Force:                     true,
-		Create:                    false,
-		Branch:                    plumbing.NewTagReferenceName(tag),
-		SparseCheckoutDirectories: []string{"go.mod"},
-	})
-	if err != nil {
-		results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}
-		return
-	}
+	// Parallelize checkout and file reading
+	var wg sync.WaitGroup
+	wg.Add(1)
 
-	f, err := fs.Open("go.mod")
-	if err != nil {
-		log.Printf("Failed to open go.mod in repository: %s, tag: %s", repoURL, tag)
-		results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}
-		return
-	}
-	defer f.Close()
-	// defer func() {
-	// 	if cerr := f.Close(); cerr != nil && err == nil {
-	// 		err = cerr
-	// 	}
-	// }()
-
-	b, err := io.ReadAll(f)
-	if err != nil {
-		results <- PackageResult{Error: fmt.Errorf("failed to read content: %w", err)}
-		// return nil, fmt.Errorf("failed to read file content: %w", err)
-		return
-	}
+	go func() {
+		defer wg.Done()
 
-	mf, err := modfile.Parse("go.mod", b, nil)
-	if err != nil {
-		results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}
-		return
-	}
-	fmt.Printf("Success fetching Go mod")
-	results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}
+		err = worktree.Checkout(&git.CheckoutOptions{
+			Force:                     true,
+			Create:                    false,
+			Branch:                    plumbing.NewTagReferenceName(tag),
+			SparseCheckoutDirectories: []string{"go.mod"},
+		})
+
+		if err != nil {
+			log.Printf("Error checking out file: %v", err)
+			results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}
+			return
+		}
+
+		f, err := fs.Open("go.mod")
+		if err != nil {
+			log.Printf("Failed to open go.mod in repository: %s, tag: %s", repoURL, tag)
+			results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}
+			return
+		}
+		defer f.Close()
+
+		b, err := io.ReadAll(f)
+		if err != nil {
+			log.Printf("Failed to read file content: %v", err)
+			results <- PackageResult{Error: fmt.Errorf("failed to read file content: %w", err)}
+			return
+		}
+
+		mf, err := modfile.Parse("go.mod", b, nil)
+		if err != nil {
+			log.Printf("Failed to parse modfile: %v", err)
+			results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}
+			return
+		}
+
+		results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}
+	}()
+
+	wg.Wait() // Ensure the goroutine finishes before returning
+	log.Printf("Successfully fetched go.mod for %s@%s", repoURL, tag)
 
 }
 
@@ -252,50 +260,6 @@ func getLatestVersion(versions []string) string {
 	return validVersions[len(validVersions)-1]
 }
 
-// func fetchGoMod(url string) (bool, string) {
-
-// 	// Create an HTTP client and make a GET request
-// 	resp, err := http.Get(url)
-// 	if err != nil {
-// 		// fmt.Printf("Error making HTTP request: %v\n", err)
-// 		return false, ""
-// 		// return false, "", fmt.Errorf("failed to get go.mod from url: %w", err)
-// 	}
-// 	defer resp.Body.Close()
-
-// 	// Check if the HTTP status is 404
-// 	if resp.StatusCode == http.StatusNotFound {
-// 		return false, ""
-// 	}
-
-// 	// Handle other HTTP errors
-// 	if resp.StatusCode != http.StatusOK {
-// 		fmt.Printf("Unexpected HTTP status: %d\n", resp.StatusCode)
-// 		return false, ""
-// 	}
-
-// 	// Read response
-// 	reader := bufio.NewReader(resp.Body)
-// 	for {
-// 		line, err := reader.ReadString('\n')
-// 		if err != nil && err != io.EOF {
-// 			fmt.Printf("Error reading response body: %v\n", err)
-// 			return false, ""
-// 		}
-
-// 		if err == io.EOF {
-// 			break
-// 		}
-
-// 		line = strings.TrimSpace(line)
-// 		if strings.HasPrefix(line, "module ") {
-// 			return true, strings.TrimSpace(strings.TrimPrefix(line, "module "))
-// 		}
-// 	}
-
-// 	return false, ""
-// }
-
 func truncateVersion(pkg string) string {
 	// Regular expression to match ".v{X}" or "/v{X}" where {X} is a number
 	re := regexp.MustCompile(`(\.v\d+|/v\d+)$`)
@@ -315,7 +279,7 @@ func main() {
 	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
 	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
 
-	var wg sync.WaitGroup
+	// var wg sync.WaitGroup
 	results := make(chan PackageResult, 10) // Buffered channel to avoid blocking
 
 	for pkg, packageInfo := range instrumentation.GetPackages() {
@@ -338,7 +302,6 @@ func main() {
 
 		// check if need to update contrib_latests
 		version_major_contrib := truncateMajorVersion(version)
-		// fmt.Printf("version: %v\n", version_major_contrib)
 
 		if err != nil {
 			fmt.Println(err)
@@ -385,45 +348,13 @@ func main() {
 		// Get the latest version for each major
 		for _, versions := range majors {
 			latest := getLatestVersion(versions)
-			// modFile, err := fetchGoModGit(origin, latest)
-
-			// Fetch go.mod in a goroutine
-			wg.Add(1)
-			go fetchGoModGit(origin, latest, &wg, results)
-			// if err != nil {
-			// 	continue
-			// }
-
-			// fmt.Printf("Latest version for %s: %s\n", major, latest)
-
-			// // Fetch `go.mod`
-			// goModURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", base, latest)
-			// isGoModule, modName := fetchGoMod(goModURL)
-			// if isGoModule {
-			// 	fmt.Printf("Module name for %s: %s\n", latest, modName)
-			// } else {
-			// 	continue
-			// }
-			// latest_major := truncateMajorVersion(latest)
-			// if latestGithub, ok := github_latests[base]; ok {
-			// 	if semver.Compare(major, latestGithub.Version) > 0 {
-			// 		// if latest > latestGithubMajor
-			// 		github_latests[base] = GithubLatests{major, modFile.Module.Mod.Path}
-			// 	}
-			// } else {
-			// 	github_latests[base] = GithubLatests{major, modFile.Module.Mod.Path}
-			// }
+
+			fetchGoModGit(origin, latest, results)
 
 		}
 
 	}
 
-	// Close the results channel once all goroutines finish
-	go func() {
-		wg.Wait()
-		close(results)
-	}()
-
 	// Iterate through results
 	for result := range results {
 		if result.Error != nil {

From 559a01d7be6760978432d4abf3f1c500a79e691d Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 20:47:26 -0500
Subject: [PATCH 25/29] improve goroutine management

---
 .../apps/new_latest_major_versions.go         | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 95eb500e63..adda4513bf 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -8,6 +8,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -17,6 +18,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	"os/exec"
 
@@ -101,16 +103,15 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 }
 
 func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
-	// Ensure WG counter is decreased
 	log.Printf("Fetching go.mod for repo: %s, tag: %s", repoURL, tag)
-	if !strings.HasSuffix(repoURL, ".git") {
-		repoURL = repoURL + ".git"
-	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
 
 	fs := memfs.New()
 	storer := memory.NewStorage()
 
-	repo, err := git.Clone(storer, fs, &git.CloneOptions{
+	repo, err := git.CloneContext(ctx, storer, fs, &git.CloneOptions{
 		URL:           repoURL,
 		Depth:         1,
 		SingleBranch:  true,
@@ -119,10 +120,19 @@ func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
 		NoCheckout:    true,
 	})
 	if err != nil {
-		results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}
+		select {
+		case results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}:
+		case <-ctx.Done():
+			log.Printf("Context cancelled while sending result")
+		}
 		return
 	}
 
+	// if err != nil {
+	// 	results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}
+	// 	return
+	// }
+
 	worktree, err := repo.Worktree()
 	if err != nil {
 		results <- PackageResult{Error: fmt.Errorf("failed to get worktree: %w", err)}

From 20cc13d25866df30e70669e26d235312dc77b570 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Fri, 31 Jan 2025 20:59:20 -0500
Subject: [PATCH 26/29] update goroutine + add logging

---
 .../apps/new_latest_major_versions.go         | 54 ++++++++++++-------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index adda4513bf..86e3eb59a2 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -101,13 +101,12 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 	return largestVersionRepo, largestVersion, nil
 
 }
-
 func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
 	log.Printf("Fetching go.mod for repo: %s, tag: %s", repoURL, tag)
-
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
+	// Create filesystem without close operation
 	fs := memfs.New()
 	storer := memory.NewStorage()
 
@@ -128,23 +127,22 @@ func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
 		return
 	}
 
-	// if err != nil {
-	// 	results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}
-	// 	return
-	// }
-
 	worktree, err := repo.Worktree()
 	if err != nil {
 		results <- PackageResult{Error: fmt.Errorf("failed to get worktree: %w", err)}
 		return
 	}
 
-	// Parallelize checkout and file reading
 	var wg sync.WaitGroup
 	wg.Add(1)
 
 	go func() {
 		defer wg.Done()
+		defer func() {
+			if r := recover(); r != nil {
+				log.Printf("Recovered from panic: %v", r)
+			}
+		}()
 
 		err = worktree.Checkout(&git.CheckoutOptions{
 			Force:                     true,
@@ -152,41 +150,61 @@ func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
 			Branch:                    plumbing.NewTagReferenceName(tag),
 			SparseCheckoutDirectories: []string{"go.mod"},
 		})
-
 		if err != nil {
 			log.Printf("Error checking out file: %v", err)
-			results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}
+			select {
+			case results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}:
+			case <-ctx.Done():
+				log.Printf("Context cancelled while sending result")
+			}
 			return
 		}
 
 		f, err := fs.Open("go.mod")
 		if err != nil {
-			log.Printf("Failed to open go.mod in repository: %s, tag: %s", repoURL, tag)
-			results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}
+			log.Printf("Failed to open go.mod in repository: %s", repoURL)
+			select {
+			case results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}:
+			case <-ctx.Done():
+				log.Printf("Context cancelled while sending result")
+			}
 			return
 		}
-		defer f.Close()
+		defer f.Close() // Only close the file, not the filesystem
 
 		b, err := io.ReadAll(f)
 		if err != nil {
 			log.Printf("Failed to read file content: %v", err)
-			results <- PackageResult{Error: fmt.Errorf("failed to read file content: %w", err)}
+			select {
+			case results <- PackageResult{Error: fmt.Errorf("failed to read file content: %w", err)}:
+			case <-ctx.Done():
+				log.Printf("Context cancelled while sending result")
+			}
 			return
 		}
 
 		mf, err := modfile.Parse("go.mod", b, nil)
 		if err != nil {
 			log.Printf("Failed to parse modfile: %v", err)
-			results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}
+			select {
+			case results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}:
+			case <-ctx.Done():
+				log.Printf("Context cancelled while sending result")
+			}
 			return
 		}
 
-		results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}
+		select {
+		case results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}:
+		case <-ctx.Done():
+			log.Printf("Context cancelled while sending result")
+		}
 	}()
 
-	wg.Wait() // Ensure the goroutine finishes before returning
+	log.Printf("Waiting for goroutine to complete...")
+	wg.Wait()
+	log.Printf("Goroutine completed")
 	log.Printf("Successfully fetched go.mod for %s@%s", repoURL, tag)
-
 }
 
 func truncateVersionSuffix(repository string) string {

From fa059208243facafbd10b536d3e39a4c7d86e771 Mon Sep 17 00:00:00 2001
From: Rodrigo Arguello <rodrigo.arguello@datadoghq.com>
Date: Mon, 3 Feb 2025 16:43:34 +0100
Subject: [PATCH 27/29] fetch go.mod using http

---
 .../apps/new_latest_major_versions.go         | 144 +++++-------------
 1 file changed, 37 insertions(+), 107 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 86e3eb59a2..8d7aacc79e 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -13,24 +13,17 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"os"
+	"os/exec"
 	"regexp"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
-	"os/exec"
-
 	"github.com/DataDog/dd-trace-go/v2/instrumentation"
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
-
-	// Missing dependencies
-	"github.com/go-git/go-billy/v5/memfs"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/storage/memory"
 )
 
 type ModuleInfo struct {
@@ -101,110 +94,41 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 	return largestVersionRepo, largestVersion, nil
 
 }
-func fetchGoModGit(repoURL, tag string, results chan<- PackageResult) {
-	log.Printf("Fetching go.mod for repo: %s, tag: %s", repoURL, tag)
+
+func fetchGoMod(origin, tag string) (*modfile.File, error) {
+	// https://raw.githubusercontent.com/gin-gonic/gin/refs/tags/v1.7.7/go.mod
+	if !strings.HasPrefix(origin, "https://github.com") {
+		return nil, fmt.Errorf("provider not supported: %s", origin)
+	}
+
+	repoPath := strings.TrimPrefix(origin, "https://github.com/")
+	repoPath = strings.TrimSuffix(repoPath, ".git")
+	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", repoPath, tag)
+
+	log.Printf("fetching %s\n", url)
+
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	// Create filesystem without close operation
-	fs := memfs.New()
-	storer := memory.NewStorage()
-
-	repo, err := git.CloneContext(ctx, storer, fs, &git.CloneOptions{
-		URL:           repoURL,
-		Depth:         1,
-		SingleBranch:  true,
-		ReferenceName: plumbing.NewTagReferenceName(tag),
-		Progress:      os.Stdout,
-		NoCheckout:    true,
-	})
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		select {
-		case results <- PackageResult{Error: fmt.Errorf("failed to clone repo: %w", err)}:
-		case <-ctx.Done():
-			log.Printf("Context cancelled while sending result")
-		}
-		return
+		return nil, err
 	}
 
-	worktree, err := repo.Worktree()
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		results <- PackageResult{Error: fmt.Errorf("failed to get worktree: %w", err)}
-		return
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	go func() {
-		defer wg.Done()
-		defer func() {
-			if r := recover(); r != nil {
-				log.Printf("Recovered from panic: %v", r)
-			}
-		}()
-
-		err = worktree.Checkout(&git.CheckoutOptions{
-			Force:                     true,
-			Create:                    false,
-			Branch:                    plumbing.NewTagReferenceName(tag),
-			SparseCheckoutDirectories: []string{"go.mod"},
-		})
-		if err != nil {
-			log.Printf("Error checking out file: %v", err)
-			select {
-			case results <- PackageResult{Error: fmt.Errorf("failed to checkout file: %w", err)}:
-			case <-ctx.Done():
-				log.Printf("Context cancelled while sending result")
-			}
-			return
-		}
-
-		f, err := fs.Open("go.mod")
-		if err != nil {
-			log.Printf("Failed to open go.mod in repository: %s", repoURL)
-			select {
-			case results <- PackageResult{Error: fmt.Errorf("failed to open file: %w", err)}:
-			case <-ctx.Done():
-				log.Printf("Context cancelled while sending result")
-			}
-			return
-		}
-		defer f.Close() // Only close the file, not the filesystem
-
-		b, err := io.ReadAll(f)
-		if err != nil {
-			log.Printf("Failed to read file content: %v", err)
-			select {
-			case results <- PackageResult{Error: fmt.Errorf("failed to read file content: %w", err)}:
-			case <-ctx.Done():
-				log.Printf("Context cancelled while sending result")
-			}
-			return
-		}
-
-		mf, err := modfile.Parse("go.mod", b, nil)
-		if err != nil {
-			log.Printf("Failed to parse modfile: %v", err)
-			select {
-			case results <- PackageResult{Error: fmt.Errorf("failed to parse modfile: %w", err)}:
-			case <-ctx.Done():
-				log.Printf("Context cancelled while sending result")
-			}
-			return
-		}
-
-		select {
-		case results <- PackageResult{ModulePath: mf.Module.Mod.Path, LatestVersion: tag}:
-		case <-ctx.Done():
-			log.Printf("Context cancelled while sending result")
-		}
-	}()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
 
-	log.Printf("Waiting for goroutine to complete...")
-	wg.Wait()
-	log.Printf("Goroutine completed")
-	log.Printf("Successfully fetched go.mod for %s@%s", repoURL, tag)
+	return modfile.Parse("go.mod", b, nil)
 }
 
 func truncateVersionSuffix(repository string) string {
@@ -302,7 +226,9 @@ func truncateMajorVersion(version string) string {
 }
 
 func main() {
-	log.SetFlags(0) // disable date and time logging
+	// log.SetFlags(0) // disable date and time logging
+	log.Println("starting")
+
 	// Find latest major
 	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
 	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
@@ -377,10 +303,14 @@ func main() {
 		for _, versions := range majors {
 			latest := getLatestVersion(versions)
 
-			fetchGoModGit(origin, latest, results)
-
+			log.Printf("fetching go.mod for %s@%s\n", origin, latest)
+			f, err := fetchGoMod(origin, latest)
+			if err != nil {
+				log.Printf("failed to fetch go.mod for %s@%s: %+v\n", origin, latest, err)
+				continue
+			}
+			log.Printf("go.mod for %s@%s: %s\n", origin, latest, f.Module.Mod.Path)
 		}
-
 	}
 
 	// Iterate through results

From 32ec8ab0731a798fb139073cadf7ff2d2b4cbba3 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Mon, 3 Feb 2025 13:33:50 -0500
Subject: [PATCH 28/29] tidy

---
 .../apps/new_latest_major_versions.go         | 51 +++++++------------
 1 file changed, 18 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index 8d7aacc79e..f19d733a03 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -34,14 +34,7 @@ type ModuleInfo struct {
 
 type GithubLatests struct {
 	Version string
-	Module  string
-}
-
-type PackageResult struct {
-	Base          string
-	LatestVersion string
-	ModulePath    string
-	Error         error
+	Module  string // the base name of the module
 }
 
 func getGoModVersion(repository string, pkg string) (string, string, error) {
@@ -97,15 +90,17 @@ func getGoModVersion(repository string, pkg string) (string, string, error) {
 
 func fetchGoMod(origin, tag string) (*modfile.File, error) {
 	// https://raw.githubusercontent.com/gin-gonic/gin/refs/tags/v1.7.7/go.mod
-	if !strings.HasPrefix(origin, "https://github.com") {
-		return nil, fmt.Errorf("provider not supported: %s", origin)
-	}
 
+	// Process the URL
 	repoPath := strings.TrimPrefix(origin, "https://github.com/")
+	repoPath = strings.TrimPrefix(repoPath, "https://gopkg.in/")
 	repoPath = strings.TrimSuffix(repoPath, ".git")
+	// Remove .vX version suffix if present (e.g., ".v1", ".v2")
+	repoPath = regexp.MustCompile(`\.v\d+$`).ReplaceAllString(repoPath, "")
+
 	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", repoPath, tag)
 
-	log.Printf("fetching %s\n", url)
+	// log.Printf("fetching %s\n", url)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -119,6 +114,8 @@ func fetchGoMod(origin, tag string) (*modfile.File, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
 	}
@@ -226,16 +223,11 @@ func truncateMajorVersion(version string) string {
 }
 
 func main() {
-	// log.SetFlags(0) // disable date and time logging
-	log.Println("starting")
 
 	// Find latest major
 	github_latests := map[string]GithubLatests{} // map module (base name) => latest on github
 	contrib_latests := map[string]string{}       // map module (base name) => latest on go.mod
 
-	// var wg sync.WaitGroup
-	results := make(chan PackageResult, 10) // Buffered channel to avoid blocking
-
 	for pkg, packageInfo := range instrumentation.GetPackages() {
 
 		// Step 1: get the version from the module go.mod
@@ -300,7 +292,7 @@ func main() {
 		// 5b. If request returns a `go.mod`, parse the modfile and extract the mod name
 
 		// Get the latest version for each major
-		for _, versions := range majors {
+		for major, versions := range majors {
 			latest := getLatestVersion(versions)
 
 			log.Printf("fetching go.mod for %s@%s\n", origin, latest)
@@ -310,24 +302,17 @@ func main() {
 				continue
 			}
 			log.Printf("go.mod for %s@%s: %s\n", origin, latest, f.Module.Mod.Path)
-		}
-	}
-
-	// Iterate through results
-	for result := range results {
-		if result.Error != nil {
-			fmt.Println("Error:", result.Error)
-			continue
-		}
-
-		if latestGithub, ok := github_latests[result.Base]; ok {
-			if semver.Compare(result.LatestVersion, latestGithub.Version) > 0 {
-				github_latests[result.Base] = GithubLatests{result.LatestVersion, result.ModulePath}
+			if latestGithub, ok := github_latests[base]; ok {
+				if semver.Compare(major, latestGithub.Version) > 0 {
+					// if latest > latestGithubMajor
+					github_latests[base] = GithubLatests{major, base}
+				}
+			} else {
+				github_latests[base] = GithubLatests{major, base}
 			}
-		} else {
-			github_latests[result.Base] = GithubLatests{result.LatestVersion, result.ModulePath}
 		}
 	}
+
 	// check if there are any outdated majors
 	// output if there is a new major package we do not support
 	for base, contribMajor := range contrib_latests {

From c85378d3656f3a961306ef66ba0e2e27733f1134 Mon Sep 17 00:00:00 2001
From: quinna-h <quinna.halim@datadoghq.com>
Date: Tue, 4 Feb 2025 13:03:35 -0500
Subject: [PATCH 29/29] hardcode k8s.io

---
 .github/workflows/apps/new_latest_major_versions.go | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/apps/new_latest_major_versions.go b/.github/workflows/apps/new_latest_major_versions.go
index f19d733a03..5e8b133953 100644
--- a/.github/workflows/apps/new_latest_major_versions.go
+++ b/.github/workflows/apps/new_latest_major_versions.go
@@ -100,8 +100,6 @@ func fetchGoMod(origin, tag string) (*modfile.File, error) {
 
 	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/refs/tags/%s/go.mod", repoPath, tag)
 
-	// log.Printf("fetching %s\n", url)
-
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -271,16 +269,13 @@ func main() {
 		if err != nil {
 			fmt.Printf("Error: %v\n", err)
 			continue
-		} else {
-			fmt.Printf("origin URL: %s\n", origin)
 		}
-
 		// 2. From the VCS url, do `git ls-remote --tags <vcs_url>` to get the tags
 		// output:
 		// 3. Parse the tags, and extract all the majors from them (ex v2, v3, v4)
 		tags, err := getTags(origin)
 		if err != nil {
-			fmt.Println("Error fetching tags:", err)
+			fmt.Println("Error fetching tags: %v", err)
 			continue
 		}
 
@@ -318,8 +313,10 @@ func main() {
 	for base, contribMajor := range contrib_latests {
 		if latestGithub, ok := github_latests[base]; ok {
 			if semver.Compare(latestGithub.Version, contribMajor) > 0 {
-				if base == "go-redis/redis" && latestGithub.Version == "v9" {
-					continue // go-redis/redis => redis/go-redis in v9
+				if (base == "go-redis/redis" && latestGithub.Version == "v9") || (base == "k8s.io/client-go") {
+					// go-redis/redis => redis/go-redis in v9
+					// for k8s.io we provide a generic http client middleware that can be plugged with any of the versions
+					continue
 				}
 				fmt.Printf("New latest major %s of repository %s on Github at module: %s\n", latestGithub.Version, base, latestGithub.Module)
 				fmt.Printf("latest contrib major: %v\n", contribMajor)