extract downloadFile method

cmd/generate.go

@@ -113,8 +113,13 @@ var generateCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		if err = asset.StoreCLIPluginFiles(connectorPackaging); err != nil {
+			fmt.Println("error downloading the cli plugin files", err)
+			os.Exit(1)
+		}
+
 		if err = asset.ApplyCLIPluginTransform(dataServerURL, connectorPackaging); err != nil {
-			fmt.Println("error applying the cli plugin transform", err)
+			fmt.Println("error applying cli plugin transforms", err)
 			os.Exit(1)
 		}
 	},

internal/asset/cli_plugin.go

@@ -4,6 +4,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/hasura/ddn-assets/internal/ndchub"
 	"golang.org/x/sync/errgroup"
@@ -91,10 +92,12 @@ type BinaryCLIPluginPlatform struct {
 }
 
 func ApplyCLIPluginTransform(dataServerBaseURL *url.URL, connPkgs []ndchub.ConnectorPackaging) error {
+	// TODO: remove after testing
+	return nil
 	var transform errgroup.Group
 	for _, cp := range connPkgs {
 		transform.Go(func() error {
-			connMetadataFilePath := connectorVersionFolderForExtracting(cp.Namespace, cp.Name, cp.Version)
+			connMetadataFilePath := extractedConnectorVersionFolder(cp.Namespace, cp.Name, cp.Version)
 
 			stat, err := os.Stat(connMetadataFilePath)
 			if err != nil {
@@ -141,3 +144,50 @@ func ApplyCLIPluginTransform(dataServerBaseURL *url.URL, connPkgs []ndchub.Conne
 	}
 	return transform.Wait()
 }
+
+func StoreCLIPluginFiles(connPkgs []ndchub.ConnectorPackaging) error {
+	var download errgroup.Group
+	for _, cp := range connPkgs {
+		download.Go(func() error {
+			connMetadataFilePath := filepath.Join(
+				extractedConnectorVersionFolder(cp.Namespace, cp.Name, cp.Version),
+				".hasura-connector", "connector-metadata.yaml",
+			)
+
+			data, err := os.ReadFile(connMetadataFilePath)
+			if err != nil {
+				return err
+			}
+
+			var connMetadata ConnectorMetadataYAML
+			err = yaml.Unmarshal(data, &connMetadata)
+			if err != nil {
+				return err
+			}
+
+			if cliPlugin, ok := connMetadata.CLIPlugin.(*BinaryInlineCLIPluginDefinition); ok {
+				var cliPluginDownload errgroup.Group
+				for _, p := range cliPlugin.Platforms {
+					cliPluginDownload.Go(func() error {
+						downloadUrl, err := url.Parse(p.URI)
+						if err != nil {
+							return err
+						}
+
+						return downloadFile(
+							p.URI,
+							filepath.Join(
+								cliPluginFolder(cp.Namespace, cp.Name, cp.Version),
+								p.Selector, path.Base(downloadUrl.Path),
+							),
+							p.SHA256,
+						)
+					})
+				}
+			}
+
+			return nil
+		})
+	}
+	return download.Wait()
+}

internal/asset/download.go

@@ -25,12 +25,6 @@ func DownloadConnectorTarballs(connPkgs []ndchub.ConnectorPackaging) error {
 			var err error
 			tarballPath := connectorTarballDownloadPath(cp.Namespace, cp.Name, cp.Version)
 
-			sha, _ := getSHAIfFileExists(tarballPath)
-			if sha == cp.Checksum.Value {
-				fmt.Println("checksum matched, so using an existing copy: ", tarballPath)
-				return nil
-			}
-
 			defer func() {
 				if err != nil {
 					fmt.Println("error while creating: ", tarballPath)
@@ -40,28 +34,7 @@ func DownloadConnectorTarballs(connPkgs []ndchub.ConnectorPackaging) error {
 				fmt.Printf("successfully wrote: %s (sha256: %s) \n", tarballPath, sha)
 			}()
 
-			outFile, err := os.Create(tarballPath)
-			if err != nil {
-				return err
-			}
-			defer outFile.Close()
-
-			log.Println("starting download: ", cp.URI)
-			resp, err := http.Get(cp.URI)
-			if err != nil {
-				return err
-			}
-			defer resp.Body.Close()
-
-			if resp.StatusCode != http.StatusOK {
-				return fmt.Errorf("error downloading: status code %d", resp.StatusCode)
-			}
-
-			_, err = io.Copy(outFile, resp.Body)
-			if err != nil {
-				return err
-			}
-			return nil
+			return downloadFile(cp.URI, tarballPath, cp.Checksum.Value)
 		})
 	}
 
@@ -84,3 +57,35 @@ func getSHAIfFileExists(path string) (string, error) {
 	checksum := hash.Sum(nil)
 	return fmt.Sprintf("%x", checksum), nil
 }
+
+func downloadFile(uri, destPath, sha256checksum string) error {
+	sha, _ := getSHAIfFileExists(destPath)
+	if sha == sha256checksum {
+		fmt.Println("checksum matched, so using an existing copy: ", destPath)
+		return nil
+	}
+
+	outFile, err := os.Create(destPath)
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+
+	log.Println("starting download: ", uri)
+	resp, err := http.Get(uri)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("error downloading: status code %d", resp.StatusCode)
+	}
+
+	_, err = io.Copy(outFile, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/asset/extract.go

@@ -23,7 +23,7 @@ func ExtractConnectorTarballs(connPkgs []ndchub.ConnectorPackaging) error {
 			}
 			defer file.Close()
 
-			destFolder := connectorVersionFolderForExtracting(cp.Namespace, cp.Name, cp.Version)
+			destFolder := extractedConnectorVersionFolder(cp.Namespace, cp.Name, cp.Version)
 			err = os.MkdirAll(destFolder, 0777)
 			if err != nil {
 				return fmt.Errorf("error creating folder: %s %w", destFolder, err)

internal/asset/index.go

@@ -26,7 +26,7 @@ func connectorTarballDownloadPath(namespace, name, version string) string {
 	return filepath.Join(connectorVersionFolderForDownload(namespace, name, version), "connector-definition.tar.gz")
 }
 
-func connectorVersionFolderForExtracting(namespace, name, version string) string {
+func extractedConnectorVersionFolder(namespace, name, version string) string {
 	return filepath.Join(ExtractsFolderPath, namespace, name, version)
 }