Merge pull request #87 from lbryio/metadata_columns

Fixed bugs and upgraded SQLBoiler

.travis.yml

@@ -52,12 +52,14 @@ before_script:
   - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/ | grep -v /model/ | grep -v /swagger/ | grep -v /migration/)
   - go get golang.org/x/tools/cmd/goimports                     # Used in build script for generated files
   - go get golang.org/x/lint/golint                             # Linter
-  - go get honnef.co/go/tools/cmd/megacheck                     # Badass static analyzer/linter
+  - go get honnef.co/go/tools/cmd/staticcheck                     # Badass static analyzer/linter
   - go get github.com/jgautheron/gocyclo                        # Check against high complexity
   - go get github.com/mdempsky/unconvert                        # Identifies unnecessary type conversions
   - go get github.com/kisielk/errcheck                          # Checks for unhandled errors
   - go get github.com/opennota/check/cmd/varcheck               # Checks for unused vars
   - go get github.com/opennota/check/cmd/structcheck            # Checks for unused fields in structs
+  - go get github.com/jgautheron/goconst/cmd/goconst            # Checks for strings that should be constants
+
 
 
 
@@ -76,12 +78,13 @@ script:
   - structcheck ./...
   # go vet is the official Go static analyzer
   - go vet ./...
-  # forbid code with huge functions
+  -  goconst $(go list ./... | grep -v /vendor/ | grep -v /model | grep -v /swagger/ | grep -v /migration )
+# forbid code with huge functions
   - gocyclo -ignore "_test" -avg -over 19 $GO_FILES
   # checks for unhandled errors
   - errcheck $(go list ./... | grep -v /vendor/ | grep -v /model | grep -v /swagger/ | grep -v /migration )
   # "go vet on steroids" + linter - ignore autogen code
-  - megacheck -simple.exit-non-zero=true $(go list ./... | grep -v /vendor/ | grep -v /model | grep -v /swagger/ | grep -v /migration )
+  - staticcheck $(go list ./... | grep -v /vendor/ | grep -v /model | grep -v /swagger/ | grep -v /migration )
   # check for unnecessary conversions - ignore autogen code
   - unconvert -v $(go list ./... | grep -v /vendor/ | grep -v /model | grep -v /swagger/ | grep -v /migration )
   # one last linter - ignore autogen code

daemon/processing/block.go

@@ -312,7 +312,7 @@ func checkHandleReorg(height uint64, chainPrevHash string) (uint64, error) {
 			return height, errors.Prefix("error getting block@"+strconv.Itoa(int(prevHeight))+": ", err)
 		}
 		//Recursively delete blocks until they match or a reorg of depth 100 == failure of logic.
-		for prevBlock.Hash != chainPrevHash && depth < 100 {
+		for prevBlock.Hash != chainPrevHash && depth < 100 && prevHeight > 0 {
 			// Delete because it needs to be reprocessed due to reorg
 			logrus.Println("block ", prevBlock.Hash, " at height ", prevBlock.Height,
 				" to be removed due to reorg. TX-> ", prevBlock.TransactionHashes)

daemon/processing/claim.go

@@ -5,16 +5,18 @@ import (
 	"encoding/json"
 
 	"github.com/lbryio/chainquery/datastore"
+	"github.com/lbryio/chainquery/global"
 	"github.com/lbryio/chainquery/lbrycrd"
 	"github.com/lbryio/chainquery/model"
+
 	"github.com/lbryio/lbry.go/errors"
 	util "github.com/lbryio/lbry.go/lbrycrd"
 	"github.com/lbryio/lbryschema.go/address/base58"
 	c "github.com/lbryio/lbryschema.go/claim"
 	"github.com/lbryio/types/go"
 
-	"github.com/lbryio/chainquery/global"
 	"github.com/sirupsen/logrus"
+	"github.com/volatiletech/null"
 	"github.com/volatiletech/sqlboiler/boil"
 )
 
@@ -201,6 +203,8 @@ func processUpdateClaim(pbClaim *pb.Claim, claim *model.Claim, value []byte) (*m
 func setPublisherInfo(claim *model.Claim, pbClaim *pb.Claim) {
 	claim.IsCertProcessed = true
 	claim.IsCertValid = false
+	claim.PublisherID = null.NewString("", false)
+	claim.PublisherSig = null.NewString("", false)
 	if pbClaim.GetPublisherSignature() != nil {
 		claim.IsCertProcessed = false
 		publisherClaimID := hex.EncodeToString(pbClaim.GetPublisherSignature().GetCertificateId())
@@ -210,7 +214,8 @@ func setPublisherInfo(claim *model.Claim, pbClaim *pb.Claim) {
 }
 
 func setCertificateInfo(claim *model.Claim, pbClaim *pb.Claim) {
-
+	claim.IsCertProcessed = false
+	claim.Certificate = null.NewString("", false)
 	if pbClaim.GetClaimType() == pb.Claim_certificateType {
 		claim.IsCertProcessed = true
 		certificate := pbClaim.GetCertificate()
@@ -223,16 +228,26 @@ func setCertificateInfo(claim *model.Claim, pbClaim *pb.Claim) {
 }
 
 func setMetaDataInfo(claim *model.Claim, pbClaim *pb.Claim) {
+	resetMetadata(claim)
 	stream := pbClaim.GetStream()
 	if stream != nil {
 		metadata := stream.GetMetadata()
 		if metadata != nil {
 			claim.Title.SetValid(metadata.GetTitle())
-
 			claim.Description.SetValid(metadata.GetDescription())
 			claim.Language.SetValid(metadata.GetLanguage().String())
 			claim.Author.SetValid(metadata.GetAuthor())
 			claim.ThumbnailURL.SetValid(metadata.GetThumbnail())
+			claim.IsNSFW = metadata.GetNsfw()
+			if metadata.License != nil {
+				claim.License.SetValid(metadata.GetLicense())
+			}
+			if metadata.LicenseUrl != nil {
+				claim.LicenseURL.SetValid(metadata.GetLicenseUrl())
+			}
+			if metadata.Preview != nil {
+				claim.Preview.SetValid(metadata.GetPreview())
+			}
 
 			fee := metadata.GetFee()
 			if fee != nil {
@@ -245,7 +260,24 @@ func setMetaDataInfo(claim *model.Claim, pbClaim *pb.Claim) {
 	}
 }
 
+func resetMetadata(claim *model.Claim) {
+	claim.Title = null.NewString("", false)
+	claim.Description = null.NewString("", false)
+	claim.Language = null.NewString("", false)
+	claim.Author = null.NewString("", false)
+	claim.ThumbnailURL = null.NewString("", false)
+	claim.IsNSFW = false
+	claim.FeeCurrency = null.NewString("", false)
+	claim.Fee = 0.0
+	claim.FeeAddress = ""
+	claim.License = null.NewString("", false)
+	claim.LicenseURL = null.NewString("", false)
+	claim.Preview = null.NewString("", false)
+}
+
 func setSourceInfo(claim *model.Claim, pbClaim *pb.Claim) {
+	claim.ContentType = null.NewString("", false)
+	claim.SDHash = null.NewString("", false)
 	stream := pbClaim.GetStream()
 	if stream != nil {
 		source := stream.GetSource()

daemon/upgrademanager/upgrade.go

@@ -13,9 +13,9 @@ import (
 )
 
 const (
-	appVersion  = 10
-	apiVersion  = 10
-	dataVersion = 10
+	appVersion  = 11
+	apiVersion  = 11
+	dataVersion = 11
 )
 
 // RunUpgradesForVersion - Migrations are for structure of the data. Upgrade Manager scripts are for the data itself.
@@ -48,6 +48,7 @@ func RunUpgradesForVersion() {
 		upgradeFrom7(appStatus.AppVersion)
 		upgradeFrom8(appStatus.AppVersion)
 		upgradeFrom9(appStatus.AppVersion)
+		upgradeFrom10(appStatus.AppVersion)
 		////Increment and save
 		//
 		logrus.Debug("Upgrading app status version to App-", appVersion, " Data-", dataVersion, " Api-", apiVersion)
@@ -172,3 +173,10 @@ func upgradeFrom9(version int) {
 		}
 	}
 }
+
+func upgradeFrom10(version int) {
+	if version < 10 {
+		logrus.Info("Re-Processing all claim outputs")
+		reProcessAllClaims()
+	}
+}

db/apiinit.go

@@ -63,12 +63,12 @@ func jsonify(rows *sql.Rows) rowSlice {
 		}
 
 		for i, value := range values {
-			switch value.(type) {
+			switch value := value.(type) {
 			case nil:
 				results[columns[i]] = nil
 
 			case []byte:
-				s := string(value.([]byte))
+				s := string(value)
 				x, err := strconv.Atoi(s)
 
 				if err != nil {

migration/011_add_license.sql

@@ -0,0 +1,15 @@
+-- +migrate Up
+
+-- +migrate StatementBegin
+ALTER TABLE claim
+  ADD COLUMN license NVARCHAR(255),
+  ADD COLUMN license_url NVARCHAR(255),
+  ADD COLUMN preview NVARCHAR(255);
+-- +migrate StatementEnd
+
+-- +migrate StatementBegin
+ALTER TABLE claim
+    ADD INDEX Idx_License (license),
+    ADD INDEX Idx_LicenseURL (license_url),
+    ADD INDEX Idx_Preview (preview);
+-- +migrate StatementEnd