feat: initialize commands and add tarball downloading logic (#4)

* fix prefix v * avoid aliased connectors * fix connector tarball path to include version v prefix * prints the sha256 of downloaded files * add validation logic for index json file * disable validation for connectors for now * add validation for connector versions * refactor * disable validations for now with a todo statement * fix the missing version prefix problem * fix inconsistent version TODOs * add generate and validate commands * refactor * refactor * refactor
hasura · Oct 24, 2024 · de8aecb · de8aecb
1 parent 505df02
commit de8aecb
Show file tree

Hide file tree

Showing 10 changed files with 546 additions and 248 deletions.
diff --git a/cmd/generate.go b/cmd/generate.go
@@ -0,0 +1,203 @@
+package cmd
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hasura/ddn-assets/internal/asset"
+	"github.com/hasura/ddn-assets/internal/ndchub"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+)
+
+var generateCmd = &cobra.Command{
+	Use:   "generate",
+	Short: "Generate assets",
+	Run: func(cmd *cobra.Command, args []string) {
+		ndcHubGitRepoFilePath := os.Getenv("NDC_HUB_GIT_REPO_FILE_PATH")
+		if ndcHubGitRepoFilePath == "" {
+			fmt.Println("please set a value for NDC_HUB_GIT_REPO_FILE_PATH env var")
+			os.Exit(1)
+			return
+		}
+
+		registryFolder := filepath.Join(ndcHubGitRepoFilePath, "registry")
+		_, err := os.Stat(registryFolder)
+		if err != nil {
+			fmt.Println("error while finding the registry folder", err)
+			os.Exit(1)
+			return
+		}
+		if os.IsNotExist(err) {
+			fmt.Println("registry folder does not exist")
+			os.Exit(1)
+			return
+		}
+
+		var connectors []asset.Connector
+		var connectorPackaging []ndchub.ConnectorPackaging
+		err = filepath.WalkDir(registryFolder, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if filepath.Base(path) == ndchub.MetadataJSON {
+				metadata, err := getConnectorMetadata(path)
+				if err != nil {
+					return err
+				}
+				if metadata != nil {
+					connectors = append(connectors, *metadata)
+				}
+			}
+
+			if filepath.Base(path) == ndchub.ConnectorPackagingJSON {
+				cp, err := ndchub.GetConnectorPackaging(path)
+				if err != nil {
+					return err
+				}
+				if cp != nil {
+					connectorPackaging = append(connectorPackaging, *cp)
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			fmt.Println("error while walking the registry folder", err)
+			os.Exit(1)
+			return
+		}
+
+		connectorVersions := make(map[string][]string)
+		for _, cp := range connectorPackaging {
+			slug := fmt.Sprintf("%s/%s", cp.Namespace, cp.Name)
+			connectorVersions[slug] = append(connectorVersions[slug], cp.Version)
+		}
+
+		err = asset.WriteIndexJSON(&asset.Index{
+			TotalConnectors:   len(connectors),
+			Connectors:        connectors,
+			ConnectorVersions: connectorVersions,
+		})
+		if err != nil {
+			fmt.Println("error writing index.json", err)
+			os.Exit(1)
+			return
+		}
+
+		var connectorTarball errgroup.Group
+		for _, cp := range connectorPackaging {
+			versionFolder := asset.VersionFolder(cp.Namespace, cp.Name, cp.Version)
+			err = os.MkdirAll(versionFolder, 0777)
+			if err != nil {
+				fmt.Println("error creating folder:", versionFolder, err)
+				os.Exit(1)
+			}
+
+			connectorTarball.Go(func() error {
+				var err error
+				tarballPath := filepath.Join(versionFolder, "connector-definition.tar.gz")
+
+				sha, _ := getSHAIfFileExists(tarballPath)
+				if sha == cp.Checksum.Value {
+					fmt.Println("checksum matched, so using an existing copy: ", tarballPath)
+					return nil
+				}
+
+				defer func() {
+					if err != nil {
+						fmt.Println("error while creating: ", tarballPath)
+						return
+					}
+					sha, _ := getSHAIfFileExists(tarballPath)
+					fmt.Printf("successfully wrote: %s (sha256: %s) \n", tarballPath, sha)
+				}()
+
+				outFile, err := os.Create(tarballPath)
+				if err != nil {
+					return err
+				}
+				defer outFile.Close()
+
+				log.Println("starting download: ", cp.URI)
+				resp, err := http.Get(cp.URI)
+				if err != nil {
+					return err
+				}
+				defer resp.Body.Close()
+
+				if resp.StatusCode != http.StatusOK {
+					return fmt.Errorf("error downloading: status code %d", resp.StatusCode)
+				}
+
+				_, err = io.Copy(outFile, resp.Body)
+				if err != nil {
+					return err
+				}
+				return nil
+			})
+		}
+
+		err = connectorTarball.Wait()
+		if err != nil {
+			fmt.Println("error writing connector tarball", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func getSHAIfFileExists(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	hash := sha256.New()
+	_, err = io.Copy(hash, file)
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash.Sum(nil)
+	return fmt.Sprintf("%x", checksum), nil
+}
+
+func getConnectorMetadata(path string) (*asset.Connector, error) {
+	if strings.Contains(path, "aliased_connectors") {
+		// It should be safe to ignore aliased_connectors
+		// as their slug does not in the connector init process
+		return nil, nil
+	}
+
+	metadataContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var metadata struct {
+		Overview struct {
+			Namespace     string `json:"namespace"`
+			LatestVersion string `json:"latest_version"`
+		} `json:"overview"`
+	}
+	err = json.Unmarshal(metadataContent, &metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	return &asset.Connector{
+		Namespace:     metadata.Overview.Namespace,
+		Name:          filepath.Base(filepath.Dir(path)),
+		LatestVersion: metadata.Overview.LatestVersion,
+	}, nil
+}
diff --git a/cmd/root.go b/cmd/root.go
@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "ddn-assets",
+	Short: "Tool for managing Hasura DDN assets",
+	Run: func(cmd *cobra.Command, args []string) {
+		_ = cmd.Help()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(generateCmd)
+	rootCmd.AddCommand(validateCmd)
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
diff --git a/cmd/validate.go b/cmd/validate.go
@@ -0,0 +1,116 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "Validate assets",
+	Run: func(cmd *cobra.Command, args []string) {
+		gqlEndpoint := os.Getenv("HASURA_GRAPHQL_ENDPOINT")
+		if len(gqlEndpoint) == 0 {
+			fmt.Println("please set HASURA_GRAPHQL_ENDPOINT env var")
+			os.Exit(1)
+			return
+		}
+		if !strings.HasSuffix(gqlEndpoint, "/v1/graphql") {
+			gqlEndpoint = gqlEndpoint + "/v1/graphql"
+		}
+
+		gqlAdminSecret := os.Getenv("HASURA_GRAPHQL_ADMIN_SECRET")
+		if len(gqlAdminSecret) == 0 {
+			fmt.Println("please set HASURA_GRAPHQL_ADMIN_SECRET env var")
+			os.Exit(1)
+			return
+		}
+
+		// gqlClient := graphql.NewClient(gqlEndpoint)
+		// connectorsInDB, err := gqldata.GetConnectors(context.Background(), gqlClient, gqlAdminSecret)
+		// if err != nil {
+		// 	fmt.Println("error while getting list of onnectors", err)
+		// 	os.Exit(1)
+		// 	return
+		// }
+		// connectorVersionsInDB, err := gqldata.GetConnectorVersions(context.Background(), gqlClient, gqlAdminSecret)
+		// if err != nil {
+		// 	fmt.Println("error while getting list of connector versions", err)
+		// 	os.Exit(1)
+		// 	return
+		// }
+
+		// // TODO: enable after fixing neo4j and sendgrid
+		// // validate index.json: check for presense of all connectors
+		// hasValidIndexJSON := true
+		// fmt.Printf("total number of connectors: in db = %d, in index.json = %d\n", len(connectorsInDB), index.TotalConnectors)
+		// var unpresentConnectorsInHub []string
+		// for _, dbc := range connectorsInDB {
+		// 	slug := fmt.Sprintf("%s/%s", dbc.Namespace, dbc.Name)
+		// 	if _, ok := index.ConnectorVersions[slug]; !ok {
+		// 		unpresentConnectorsInHub = append(unpresentConnectorsInHub, slug)
+		// 	}
+		// }
+		// if len(unpresentConnectorsInHub) > 0 {
+		// 	fmt.Println("Following connectors are present in DB, but not in ndc-hub:")
+		// 	fmt.Println(strings.Join(unpresentConnectorsInHub, "\n"))
+		// 	hasValidIndexJSON = false
+		// }
+
+		// var unpresentConnectorsInDB []string
+		// for _, hubc := range index.Connectors {
+		// 	foundInDb := false
+		// 	for _, dbc := range connectorsInDB {
+		// 		if dbc.Namespace == hubc.Namespace && dbc.Name == hubc.Name {
+		// 			foundInDb = true
+		// 			break
+		// 		}
+		// 	}
+		// 	if !foundInDb {
+		// 		slug := fmt.Sprintf("%s/%s", hubc.Namespace, hubc.Name)
+		// 		unpresentConnectorsInDB = append(unpresentConnectorsInDB, slug)
+		// 	}
+		// }
+		// if len(unpresentConnectorsInDB) > 0 {
+		// 	fmt.Println("Following connectors are present in ndc-hub, but not in the DB:")
+		// 	fmt.Println(strings.Join(unpresentConnectorsInDB, "\n"))
+		// 	hasValidIndexJSON = false
+		// }
+
+		// if !hasValidIndexJSON {
+		// 	os.Exit(1)
+		// 	return
+		// }
+
+		// TODO: enable after fixing neo4j/neo4j [v0.0.6 v0.0.7 v0.0.10]
+		// validate index.json: check for presence of all connector versions
+		// unfoundConnectorVersions := make(map[string][]string)
+		// for _, dbcv := range connectorVersionsInDB {
+		// 	slug := fmt.Sprintf("%s/%s", dbcv.Namespace, dbcv.Name)
+		// 	foundVersion := false
+		// 	for _, v := range index.ConnectorVersions[slug] {
+		// 		if v == dbcv.Version {
+		// 			foundVersion = true
+		// 			break
+		// 		}
+		// 	}
+		// 	if !foundVersion {
+		// 		unfoundConnectorVersions[slug] = append(unfoundConnectorVersions[slug], dbcv.Version)
+		// 	}
+		// }
+
+		// if len(unfoundConnectorVersions) > 0 {
+		// 	fmt.Println("Following connector versions are found in DB but not in the ndc-hub")
+		// 	count := 1
+		// 	for k, v := range unfoundConnectorVersions {
+		// 		fmt.Printf("%d. %s %+v\n", count, k, v)
+		// 		count++
+		// 	}
+		// 	os.Exit(1)
+		// 	return
+		// }
+	},
+}
diff --git a/go.mod b/go.mod
@@ -2,4 +2,11 @@ module github.com/hasura/ddn-assets
 
 go 1.22.4
 
-require golang.org/x/sync v0.8.0 // indirect
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/machinebox/graphql v0.2.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+)
diff --git a/go.sum b/go.sum
@@ -1,2 +1,16 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
+github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=