add tezos snapshot version on API

cmd/photographer/exec.go

@@ -17,43 +17,48 @@ func createSnapshot(historyMode snapshot.HistoryModeType) {
 		script = script + " --rolling"
 	}
 
-	cmd := exec.Command("sh", "-c", script)
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	if err != nil {
-		log.Fatalf("%v \n", err)
-	}
-
-	log.Printf("snapshot export stdout: \n%s\n", stdout.String())
-	log.Printf("snapshot export stderr: \n%s\n", stderr.String())
+	_, _ = execScript(script)
 }
 
-func getSnapshotNames(historyMode snapshot.HistoryModeType) (string, error) {
+func getSnapshotName(historyMode snapshot.HistoryModeType) (string, error) {
 	log.Println("Getting snapshot names.")
-	var stdout bytes.Buffer
-	cmd := exec.Command("sh", "-c", "mkdir -p /var/run/tezos/snapshots && cd /var/run/tezos/snapshots && /bin/ls -1a")
-	cmd.Stdout = &stdout
-	err := cmd.Run()
-	if err != nil {
-		log.Fatalf("%v \n", err)
-	}
+	script := "mkdir -p /var/run/tezos/snapshots && cd /var/run/tezos/snapshots && /bin/ls -1a"
+	stdout, _ := execScript(script)
+
 	snapshotfilenames := strings.Split(stdout.String(), "\n")
 	log.Printf("All files found: %v \n", snapshotfilenames)
 
-	extension := "full"
-
-	if historyMode == snapshot.ROLLING {
-		extension = "rolling"
-	}
-
 	for _, filename := range snapshotfilenames {
-		if strings.Contains(filename, extension) {
+		if strings.Contains(filename, string(historyMode)) {
 			log.Printf("Snapshot file found is: %q. \n", filename)
 			return filename, nil
 		}
 	}
 
 	return "", fmt.Errorf("Snapshot file not found.")
 }
+
+func getSnapshotHeaderOutput(filepath string) string {
+	script := "/usr/local/bin/octez-node snapshot info --json" + filepath
+	stdout, _ := execScript(script)
+	return stdout.String()
+}
+
+func execScript(script string) (bytes.Buffer, bytes.Buffer) {
+	cmd := exec.Command("sh", "-c", script)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	if err != nil {
+		log.Fatalf("%v \n", err)
+	}
+	if stdout.Len() > 0 {
+		log.Printf("stdout: \n%s\n", stdout.String())
+	}
+	if stderr.Len() > 0 {
+		log.Printf("stderr: \n%s\n", stderr.String())
+	}
+
+	return stdout, stderr
+}

cmd/photographer/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/go-co-op/gocron"
 	"github.com/joho/godotenv"
 	"github.com/marigold-dev/tezos-snapshot/pkg/snapshot"
+	"github.com/marigold-dev/tezos-snapshot/pkg/store"
 	"github.com/marigold-dev/tezos-snapshot/pkg/util"
 	"github.com/samber/lo"
 )
@@ -55,7 +56,7 @@ func task() {
 	}
 	defer client.Close()
 
-	snapshotStorage := util.NewSnapshotStorage(client, bucketName)
+	snapshotStorage := store.NewSnapshotStorage(client, bucketName)
 
 	// Check if today the rolling snapshot already exists
 	execute(ctx, snapshotStorage, snapshot.ROLLING, network)
@@ -68,7 +69,7 @@ func task() {
 	log.Printf("Snapshot job took %s", time.Since(start))
 }
 
-func execute(ctx context.Context, snapshotStorage *util.SnapshotStorage, historyMode snapshot.HistoryModeType, chain string) {
+func execute(ctx context.Context, snapshotStorage *store.SnapshotStorage, historyMode snapshot.HistoryModeType, chain string) {
 	todayItems := snapshotStorage.GetTodaySnapshotsItems(ctx)
 
 	alreadyExist := lo.SomeBy(todayItems, func(item snapshot.SnapshotItem) bool {
@@ -81,9 +82,11 @@ func execute(ctx context.Context, snapshotStorage *util.SnapshotStorage, history
 	}
 
 	createSnapshot(historyMode)
-	snapshotfilename, err := getSnapshotNames(historyMode)
+	snapshotfilename, err := getSnapshotName(historyMode)
+	snapshotHeaderOutput := getSnapshotHeaderOutput(snapshotfilename)
+
 	if err != nil {
 		log.Fatalf("%v \n", err)
 	}
-	snapshotStorage.EphemeralUpload(ctx, snapshotfilename)
+	snapshotStorage.EphemeralUpload(ctx, snapshotfilename, snapshotHeaderOutput)
 }

cmd/schema/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"log"
 
 	"github.com/xeipuuv/gojsonschema"
 )
@@ -16,11 +16,11 @@ func main() {
 	}
 
 	if result.Valid() {
-		fmt.Printf("The document is valid\n")
+		log.Printf("The document is valid\n")
 	} else {
-		fmt.Printf("The document is not valid. see errors :\n")
+		log.Printf("The document is not valid. see errors :\n")
 		for _, desc := range result.Errors() {
-			fmt.Printf("- %s\n", desc)
+			log.Printf("- %s\n", desc)
 		}
 	}
 }

cmd/server/cache.go

@@ -8,7 +8,7 @@ import (
 
 	"cloud.google.com/go/storage"
 	"github.com/marigold-dev/tezos-snapshot/pkg/snapshot"
-	"github.com/marigold-dev/tezos-snapshot/pkg/util"
+	"github.com/marigold-dev/tezos-snapshot/pkg/store"
 	"github.com/patrickmn/go-cache"
 )
 
@@ -26,7 +26,7 @@ func getSnapshotResponseCached(ctx context.Context, goCache *cache.Cache, bucket
 	}
 	defer client.Close()
 
-	snapshotStorage := util.NewSnapshotStorage(client, bucketName)
+	snapshotStorage := store.NewSnapshotStorage(client, bucketName)
 	data := snapshotStorage.GetSnapshotItems(ctx)
 	response := SnapshotResponse{
 		DateGenerated: time.Now().UTC().Format("2006-01-02T15:04:05Z07:00"),

pkg/node/node.go

@@ -0,0 +1,36 @@
+package node
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"time"
+)
+
+func CheckNodesAreReady() {
+	for {
+		r, err := http.Get("http://localhost:8732/version")
+		if err != nil && r.StatusCode != 200 {
+			log.Println("The node is not running. Waiting 5 minutes...")
+			time.Sleep(5 * time.Minute)
+		}
+		defer r.Body.Close()
+		if r.StatusCode == 200 {
+			break
+		}
+	}
+}
+
+func GetTezosVersion() string {
+	reqVersion, err := http.Get("http://localhost:8732/version")
+	if err != nil {
+		log.Fatalf("Unable to get node version. %v \n", err)
+	}
+	defer reqVersion.Body.Close()
+	version, err := io.ReadAll(reqVersion.Body)
+	if err != nil {
+		log.Fatalf("Unable to read node version. %v \n", err)
+	}
+
+	return string(version)
+}

pkg/snapshot/header.go

@@ -0,0 +1,35 @@
+package snapshot
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+type SnapshotHeader struct {
+	Version   int    `json:"version"`
+	ChaiName  string `json:"chain_name"`
+	Mode      string `json:"mode"`
+	BlockHash string `json:"block_hash"`
+	Level     int    `json:"level"`
+	Timestamp string `json:"timestamp"`
+}
+
+func SnapshotHeaderFromJson(snapshotHeaderOutput string) (*SnapshotHeader, error) {
+	var snapshotHeader SnapshotHeader
+	err := json.Unmarshal([]byte(snapshotHeaderOutput), &snapshotHeader)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshotHeader, nil
+}
+
+// Example: TEZOS_MAINNET_2021-01-01_00-00 to mainnet
+func (s *SnapshotHeader) SanitizeChainame() string {
+	parts := strings.Split(s.ChaiName, "_")
+	chainName := strings.ToLower(parts[len(parts)-1])
+	if chainName == "ithacanet" {
+		chainName = "ghostnet"
+	}
+	return chainName
+}

pkg/snapshot/snapshot.go

@@ -14,33 +14,16 @@ type BlockHeaderResponse struct {
 	Signature string `json:"signature"`
 }
 
-type SnapshotItem struct {
-	Filename       string          `json:"filename"`
-	ChainName      string          `json:"chain_name"`
-	BlockTimestamp string          `json:"block_timestamp"`
-	BlockHash      string          `json:"block_hash"`
-	BlockHeight    int             `json:"block_height"`
-	URL            string          `json:"url"`
-	Filesize       string          `json:"filesize"`
-	SHA256         string          `json:"sha256"`
-	ArtifactType   ArtifactType    `json:"artifact_type"`
-	HistoryMode    HistoryModeType `json:"history_mode"`
-	FilesizeBytes  int64           `json:"filesize_bytes"`
-	Date           time.Time       `json:"date"`
-	TezosVersion   TezosVersion    `json:"tezos_version"`
-}
-
 type TezosVersion struct {
 	Implementation string     `json:"implementation"`
 	Version        Version    `json:"version"`
 	CommitInfo     CommitInfo `json:"commit_info"`
 }
 
 type Version struct {
-	Major int `json:"major"`
-	Minor int `json:"minor"`
-	// This could be map[string]int or string
-	AdditionalInfo interface{} `json:"additional_info"`
+	Major          int         `json:"major"`
+	Minor          int         `json:"minor"`
+	AdditionalInfo interface{} `json:"additional_info"` // This could be map[string]int or string
 }
 
 type CommitInfo struct {
@@ -61,25 +44,42 @@ const (
 	ARCHIVE HistoryModeType = "archive"
 )
 
+type SnapshotItem struct {
+	Filename        string          `json:"filename"`
+	ChainName       string          `json:"chain_name"`
+	BlockTimestamp  string          `json:"block_timestamp"`
+	BlockHash       string          `json:"block_hash"`
+	BlockHeight     int             `json:"block_height"`
+	URL             string          `json:"url"`
+	Filesize        string          `json:"filesize"`
+	SHA256          string          `json:"sha256"`
+	ArtifactType    ArtifactType    `json:"artifact_type"`
+	HistoryMode     HistoryModeType `json:"history_mode"`
+	FilesizeBytes   int64           `json:"filesize_bytes"`
+	Date            time.Time       `json:"date"`
+	TezosVersion    TezosVersion    `json:"tezos_version"`
+	SnapshotVersion int             `json:"snapshot_version"`
+}
+
 // NetworkProtocolPriority it's a way to sort like that:
 // 1. Mainnet
 // 2. Ithacanet/Ghostnet
 // 3. Others...
 // 4. Limannet,
 // 5. Mumbainet
-func NetworkProtocolPriority(chain string) int {
+func (s *SnapshotItem) NetworkProtocolPriority() int {
 	// Mainnet then will be the first on the list
-	if chain == "mainnet" {
+	if s.ChainName == "mainnet" {
 		return math.MaxInt
 	}
 
 	// Ithacanet/Ghostnet, then will be the last on the list
-	if chain == "ithacanet" || chain == "ghostnet" {
+	if s.ChainName == "ithacanet" || s.ChainName == "ghostnet" {
 		return math.MaxInt - 1
 	}
 
 	// Others protocol by protocol number
-	network := chain
+	network := s.ChainName
 	network_char := network[0]
 	return int(network_char)
 }

pkg/util/file.go → pkg/store/file.go

@@ -1,4 +1,4 @@
-package util
+package store
 
 import (
 	"log"

pkg/store/fileinfo.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"log"
+	"strconv"
+	"strings"
+
+	"github.com/marigold-dev/tezos-snapshot/pkg/snapshot"
+)
+
+type FileInfo struct {
+	Filename    string
+	ChainName   string
+	HistoryMode snapshot.HistoryModeType
+	BlockHeight int
+	BlockHash   string
+}
+
+func getInfoFromfilename(filename string) *FileInfo {
+	chainName := strings.ToLower(strings.Split(strings.Split(filename, "-")[0], "_")[1])
+
+	if chainName == "ithacanet" {
+		chainName = "ghostnet"
+	}
+
+	historyMode := snapshot.HistoryModeType(snapshot.FULL)
+
+	if strings.Contains(filename, "rolling") {
+		historyMode = snapshot.HistoryModeType(snapshot.ROLLING)
+	}
+
+	splitedByHyphen := strings.Split(filename, "-")
+	blockheight, err := strconv.Atoi(strings.Split(splitedByHyphen[len(splitedByHyphen)-1], ".")[0])
+	if err != nil {
+		log.Fatalf("Unable to parse blockheight. %v \n", err)
+	}
+	blockhash := splitedByHyphen[len(splitedByHyphen)-2]
+
+	return &FileInfo{
+		Filename:    filename,
+		ChainName:   chainName,
+		HistoryMode: historyMode,
+		BlockHeight: blockheight,
+		BlockHash:   blockhash,
+	}
+}