half-baked attempt at indexing title and description after offchain d…

…ownload happens
vocdoni · Aug 28, 2024 · f72f3d4 · f72f3d4
1 parent 43d2089
commit f72f3d4
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 6 deletions.
diff --git a/service/offchaindata.go b/service/offchaindata.go
@@ -32,6 +32,7 @@ func (vs *VocdoniService) OffChainDataHandler() error {
 	}
 	vs.OffChainData = offchaindatahandler.NewOffChainDataHandler(
 		vs.App,
+		vs.Indexer,
 		vs.DataDownloader,
 		vs.CensusDB,
 		vs.Config.SkipPreviousOffchainData,

diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go
@@ -3,13 +3,16 @@ package indexer
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
 	"go.vocdoni.io/proto/build/go/models"
 
+	"go.vocdoni.io/dvote/api"
+	"go.vocdoni.io/dvote/data"
 	"go.vocdoni.io/dvote/log"
 	indexerdb "go.vocdoni.io/dvote/vochain/indexer/db"
 	"go.vocdoni.io/dvote/vochain/indexer/indexertypes"
@@ -315,3 +318,29 @@ func (idx *Indexer) updateProcess(ctx context.Context, queries *indexerdb.Querie
 	}
 	return nil
 }
+
+// updateProcessMetadata synchronize title and description
+// with the information fetched over IPFS.
+func (idx *Indexer) UpdateProcessMetadata(d data.Storage, pid []byte, uri string) error {
+	// Try to retrieve the election metadata
+	if d != nil {
+		stgCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+
+		// TODO: abstract all of this somewhere else and just call from here
+		metadataBytes, err := d.Retrieve(stgCtx, uri, MaxOffchainFileSize)
+		if err != nil {
+			log.Warnf("cannot get metadata from %s: %v", election.MetadataURL, err)
+		} else {
+			// if metadata exists and is not encrypted, add it to the indexed election
+			// if the metadata is not encrypted, unmarshal it, otherwise store it as bytes
+			if !election.ElectionMode.EncryptedMetaData {
+				electionMetadata := api.ElectionMetadata{}
+				if err := json.Unmarshal(metadataBytes, &electionMetadata); err != nil {
+					log.Warnf("cannot unmarshal metadata from %s: %v", election.MetadataURL, err)
+				}
+				election.Metadata = &electionMetadata
+			}
+		}
+	}
+}
diff --git a/vochain/offchaindatahandler/metadata.go b/vochain/offchaindatahandler/metadata.go
@@ -8,12 +8,15 @@ import (
 
 // enqueueMetadata enqueue a election metadata for download.
 // (safe for concurrent use, simply pushes an item to a channel)
-func (d *OffChainDataHandler) enqueueMetadata(uri string) {
-	if !strings.HasPrefix(uri, d.storage.RemoteStorage.URIprefix()) {
-		log.Warnf("metadata URI not valid: %s", uri)
+func (d *OffChainDataHandler) enqueueMetadata(item importItem) {
+	if !strings.HasPrefix(item.uri, d.storage.RemoteStorage.URIprefix()) {
+		log.Warnf("metadata URI not valid: %s", item.uri)
 		return
 	}
-	d.storage.AddToQueue(uri, func(s string, b []byte) {
+	d.storage.AddToQueue(item.uri, func(s string, b []byte) {
 		log.Infof("metadata downloaded successfully from %s (%d bytes)", s, len(b))
+		if item.itemType == itemTypeElectionMetadata && len(item.pid) > 0 {
+			d.indexer.UpdateProcessMetadata(d.storage.RemoteStorage, item.pid, item.uri)
+		}
 	}, true)
 }
diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go
@@ -8,8 +8,10 @@ import (
 	"go.vocdoni.io/dvote/api/censusdb"
 	"go.vocdoni.io/dvote/data/downloader"
 	"go.vocdoni.io/dvote/log"
+	"go.vocdoni.io/dvote/types"
 	"go.vocdoni.io/dvote/util"
 	"go.vocdoni.io/dvote/vochain"
+	"go.vocdoni.io/dvote/vochain/indexer"
 	"go.vocdoni.io/dvote/vochain/state"
 	"go.vocdoni.io/dvote/vochain/transaction/vochaintx"
 	"go.vocdoni.io/proto/build/go/models"
@@ -26,6 +28,7 @@ type importItem struct {
 	itemType   int
 	uri        string
 	censusRoot string
+	pid        types.HexBytes
 }
 
 var itemTypesToString = map[int]string{
@@ -42,6 +45,7 @@ var itemTypesToString = map[int]string{
 // offchain data (usually on IPFS).
 type OffChainDataHandler struct {
 	vochain       *vochain.BaseApplication
+	indexer       *indexer.Indexer
 	census        *censusdb.CensusDB
 	storage       *downloader.Downloader
 	queue         []importItem
@@ -52,11 +56,12 @@ type OffChainDataHandler struct {
 
 // NewOffChainDataHandler creates a new instance of the off chain data downloader daemon.
 // It will subscribe to Vochain events and perform data import.
-func NewOffChainDataHandler(v *vochain.BaseApplication, d *downloader.Downloader,
+func NewOffChainDataHandler(v *vochain.BaseApplication, i *indexer.Indexer, d *downloader.Downloader,
 	c *censusdb.CensusDB, importOnlyNew bool,
 ) *OffChainDataHandler {
 	od := OffChainDataHandler{
 		vochain:       v,
+		indexer:       i,
 		census:        c,
 		storage:       d,
 		importOnlyNew: importOnlyNew,
@@ -87,7 +92,7 @@ func (d *OffChainDataHandler) Commit(_ uint32) error {
 			go d.enqueueOffchainCensus(item.censusRoot, item.uri)
 		case itemTypeElectionMetadata, itemTypeAccountMetadata:
 			log.Infow("importing data", "type", itemTypesToString[item.itemType], "uri", item.uri)
-			go d.enqueueMetadata(item.uri)
+			go d.enqueueMetadata(item)
 		default:
 			log.Errorf("unknown import item %d", item.itemType)
 		}
@@ -107,6 +112,7 @@ func (d *OffChainDataHandler) OnProcess(p *models.Process, _ int32) {
 	if m := p.GetMetadata(); m != "" {
 		log.Debugf("adding election metadata %s to queue", m)
 		d.queue = append(d.queue, importItem{
+			pid:      p.GetProcessId(),
 			uri:      m,
 			itemType: itemTypeElectionMetadata,
 		})