Merge pull request #121 from vsoloviov/oci_object_storage

Store downloaded artifacts in OCI Object Storage

README.md

@@ -176,6 +176,20 @@ n3dr upload -u <new-target-nexus3-user> -n <new-target-nexus3-server-url> \
 -r <new-repo-target-name>
 ```
 
+## Backup to OCI Object Storage
+`n3dr` supports backing up to [OCI Object Storage](https://www.oracle.com/cloud/storage/object-storage.html).
+To enable this option you need to
+- Configure OCI environment and secrets locally: https://docs.cloud.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+- Add following options to `~/.n3dr.yaml`:
+```
+ociBucket: nexus_dev_archives
+```
+
+If you want to remove local copies (after object has been uploaded) add following to `~/.n3dr.yaml`:
+```
+removeLocalFile: true
+```
+
 ## Rationale for N3DR
 
 Although there is a number of equivalent tools:

cli/backup.go

@@ -1,8 +1,12 @@
 package cli
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"github.com/spf13/viper"
+	"io"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -99,24 +103,63 @@ func (n Nexus3) continuationTokenRecursion(t string) ([]string, error) {
 	return append(tokenSlice, token), nil
 }
 
-func createArtifact(d string, f string, content string) error {
+func createArtifact(d string, f string, content string, md5sum string) error {
+	ociBucketname := viper.GetString("ociBucket")
+	Filename := d + "/" + f
+
+	// Check if object exists
+	objectExists := false
+	if ociBucketname != "" {
+		objectExists, err := findObject(ociBucketname, Filename, md5sum)
+
+		if err != nil {
+			return err
+		}
+
+		if objectExists && viper.GetBool("removeLocalFile") {
+			log.Debug("Object " + Filename + " already exist")
+			return nil
+		}
+	}
+
 	log.Debug("Create artifact: '" + d + "/" + f + "'")
 	err := os.MkdirAll(d, os.ModePerm)
 	if err != nil {
 		return err
 	}
 
-	file, err := os.Create(filepath.Join(d, f))
-	if err != nil {
-		return err
+	filename := filepath.Join(d, f)
+
+	md5sumLocal := ""
+	if fileExists(filename) {
+		md5sumLocal, err = HashFileMD5(filename)
+		if err != nil {
+			return err
+		}
 	}
 
-	_, err = file.WriteString(content)
-	defer file.Close()
-	if err != nil {
-		return err
+	if md5sumLocal == md5sum {
+		log.Debug("Skipping as file already exists.")
+	} else {
+		log.Debug("Creating ", filename)
+		file, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+
+		_, err = file.WriteString(content)
+		defer file.Close()
+		if err != nil {
+			return err
+		}
 	}
 
+	if ociBucketname != "" && !objectExists {
+		err := ociBackup(ociBucketname, Filename)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -139,7 +182,9 @@ func (n Nexus3) artifactName(url string) (string, string, error) {
 	return d, f, nil
 }
 
-func (n Nexus3) downloadArtifact(url string) error {
+func (n Nexus3) downloadArtifact(downloadURL interface{}) error {
+	url := fmt.Sprint(downloadURL.(map[string]interface{})["downloadUrl"])
+	md5sum := fmt.Sprint(downloadURL.(map[string]interface{})["md5"])
 	d, f, err := n.artifactName(url)
 	if err != nil {
 		return err
@@ -150,7 +195,7 @@ func (n Nexus3) downloadArtifact(url string) error {
 		return err
 	}
 
-	if err := createArtifact(filepath.Join(downloadDir, n.Repository, d), f, bodyString); err != nil {
+	if err := createArtifact(filepath.Join(downloadDir, n.Repository, d), f, bodyString, md5sum); err != nil {
 		return err
 	}
 	return nil
@@ -177,7 +222,7 @@ func (n Nexus3) downloadURLs() ([]interface{}, error) {
 			json := string(bytes)
 
 			jq := gojsonq.New().JSONString(json)
-			downloadURLsInterface := jq.From("items").Pluck("downloadUrl")
+			downloadURLsInterface := jq.From("items").Only("downloadUrl", "checksum.md5")
 			log.Debug("DownloadURLs: " + fmt.Sprintf("%v", downloadURLsInterface))
 
 			downloadURLsInterfaceArray := downloadURLsInterface.([]interface{})
@@ -202,7 +247,7 @@ func (n Nexus3) StoreArtifactsOnDisk(regex string) error {
 		log.Info("Backing up artifacts '" + n.Repository + "'")
 		bar := pb.StartNew(len(urls))
 		for _, downloadURL := range urls {
-			url := fmt.Sprint(downloadURL)
+			url := fmt.Sprint(downloadURL.(map[string]interface{})["downloadUrl"])
 
 			log.Debug("Only download artifacts that match the regex: '" + regex + "'")
 			r, err := regexp.Compile(regex)
@@ -213,7 +258,7 @@ func (n Nexus3) StoreArtifactsOnDisk(regex string) error {
 				// Exclude download of md5 and sha1 files as these are unavailable
 				// unless the metadata.xml is opened first
 				if !(filepath.Ext(url) == ".md5" || filepath.Ext(url) == ".sha1") {
-					if err := n.downloadArtifact(fmt.Sprint(downloadURL)); err != nil {
+					if err := n.downloadArtifact(downloadURL); err != nil {
 						return err
 					}
 				}
@@ -239,3 +284,29 @@ func (n Nexus3) CreateZip() error {
 	}
 	return nil
 }
+
+// HashFileMD5 returns MD5 checksum of a file
+func HashFileMD5(filePath string) (string, error) {
+	var returnMD5String string
+	file, err := os.Open(filePath)
+	if err != nil {
+		return returnMD5String, err
+	}
+	defer file.Close()
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return returnMD5String, err
+	}
+	hashInBytes := hash.Sum(nil)[:16]
+	returnMD5String = hex.EncodeToString(hashInBytes)
+	return returnMD5String, nil
+
+}
+
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}

cli/backup_test.go

@@ -127,7 +127,7 @@ func TestArtifactNameContainingRepositoryName(t *testing.T) {
 }
 
 func TestCreateArtifact(t *testing.T) {
-	actualErrorFile := createArtifact("testFiles", "file100/file100", "some-content")
+	actualErrorFile := createArtifact("testFiles", "file100/file100", "some-content", "ba1f2511fc30423bdbb183fe33f3dd0f")
 	expectedErrorFile := "open testFiles/file100/file100: no such file or directory"
 
 	if actualErrorFile.Error() != expectedErrorFile {
@@ -136,10 +136,49 @@ func TestCreateArtifact(t *testing.T) {
 }
 
 func TestDownloadArtifact(t *testing.T) {
-	actualError := n.downloadArtifact("http://releasesoftwaremoreoften.com")
+	url := make(map[string]interface{})
+	url["downloadUrl"] = "http://releasesoftwaremoreoften.com"
+	actualError := n.downloadArtifact(url)
 	expectedError := "URL: 'http://releasesoftwaremoreoften.com' does not seem to contain an artifactName"
 
 	if actualError.Error() != expectedError {
 		t.Errorf("Error incorrect. Expected: %v. Actual: %v", expectedError, actualError)
 	}
 }
+
+func TestHashFileMD5(t *testing.T) {
+	file := "file1/file1/1.0.0/file1-1.0.0.jar"
+	_, actualError := HashFileMD5(file)
+	expectedError := "open file1/file1/1.0.0/file1-1.0.0.jar: no such file or directory"
+
+	if actualError.Error() != expectedError {
+		t.Errorf("Error incorrect. Expected: %v. Actual: %v", expectedError, actualError)
+	}
+
+	file = "download/maven-releases/file1/file1/1.0.0/file1-1.0.0.jar"
+	expectedResult := "ad60407c083b4ecc372614b8fcd9f305"
+	result, _ := HashFileMD5(file)
+
+	if result != expectedResult {
+		t.Errorf("Output incorrect. Expected: %v. Actual: %v", expectedResult, result)
+	}
+}
+
+func TestFileExists(t *testing.T) {
+	file := "file1/file1/1.0.0/file1-1.0.0.jar"
+	result := fileExists(file)
+	expectedResult := false
+
+	if result != expectedResult {
+		t.Errorf("Output incorrect. Expected: %v. Actual: %v", expectedResult, result)
+	}
+
+	file = "download/maven-releases/file1/file1/1.0.0/file1-1.0.0.jar"
+	result = fileExists(file)
+	expectedResult = true
+
+	if result != expectedResult {
+		t.Errorf("Output incorrect. Expected: %v. Actual: %v", expectedResult, result)
+	}
+
+}

cli/main_test.go

@@ -124,13 +124,13 @@ func (n Nexus3) submitArtifact(d string, f string) {
 }
 
 func createPOM(d string, f string, number string) {
-	if err := createArtifact(d, f+".pom", "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>file"+number+"</groupId>\n<artifactId>file"+number+"</artifactId>\n<version>1.0.0</version>\n</project>"); err != nil {
+	if err := createArtifact(d, f+".pom", "<project>\n<modelVersion>4.0.0</modelVersion>\n<groupId>file"+number+"</groupId>\n<artifactId>file"+number+"</artifactId>\n<version>1.0.0</version>\n</project>", "ba1f2511fc30423bdbb183fe33f3dd0f"); err != nil {
 		log.Fatal(err)
 	}
 }
 
 func createJAR(d string, f string) {
-	if err := createArtifact(d, f+".jar", "some-content"); err != nil {
+	if err := createArtifact(d, f+".jar", "some-content", "eae7286512c52715673c3878c10d2d55"); err != nil {
 		log.Fatal(err)
 	}
 }