feat: report

go.mod

@@ -7,6 +7,7 @@ toolchain go1.21.3
 require (
 	github.com/bits-and-blooms/bloom/v3 v3.6.0
 	github.com/dgraph-io/badger/v4 v4.2.0
+	github.com/google/uuid v1.5.0
 	github.com/gorilla/handlers v1.5.2
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/nlnwa/gowarc v1.1.2
@@ -39,7 +40,6 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/flatbuffers v23.5.26+incompatible // indirect
-	github.com/google/uuid v1.5.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

index/api.go

@@ -106,3 +106,9 @@ type IdResponse interface {
 	GetValue() string
 	GetError() error
 }
+
+type ReportAPI interface {
+	GenerateReport(context.Context, Request) (*schema.Report, error)
+	ListReport(context.Context) ([]*schema.Report, error)
+	GetReport(context.Context, string) (*schema.Report, error)
+}

internal/tikvidx/api.go

@@ -20,11 +20,14 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/nlnwa/gowarcserver/index"
 	"github.com/nlnwa/gowarcserver/internal/keyvalue"
 	"github.com/nlnwa/gowarcserver/schema"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // iterator mimics tikv's internal iterator interface
@@ -256,3 +259,112 @@ func (db *DB) Delete(ctx context.Context) error {
 
 	return firstErr
 }
+
+func (db *DB) GenerateReport(ctx context.Context, request index.Request) (*schema.Report, error) {
+	taskId, err := uuid.NewV7()
+	if err != nil {
+		return nil, err
+	}
+
+	startTime := time.Now()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	report := &schema.Report{
+		Id: taskId.String(),
+		Meta: &schema.ReportMetadata{
+			StartTime: timestamppb.New(startTime),
+		},
+	}
+	key := keyvalue.KeyWithPrefix(taskId.String(), reportPrefix)
+	value, err := proto.Marshal(report)
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.client.Put(ctx, key, value)
+	if err != nil {
+		return nil, err
+	}
+
+	// // Go routine that actually starts generating report
+	// go func() {
+	// 	ctx, cancel := context.WithCancel(context.Background())
+	// 	defer cancel()
+	// 	db.tasks[taskId.String()] = cancel
+
+	// 		func() {
+	// 		report.Meta.EndTime = timestamppb.New(time.Now())
+
+	// 		key := keyvalue.KeyWithPrefix(taskId.String(), reportPrefix)
+
+	// 	}()
+
+	// }()
+
+	return report, nil
+}
+
+// func (db *DB) Report(ctx context.Context, req index.Request, res chan<- index.ReportResponse) error {
+// 	var it iterator
+// 	var err error
+
+// 	key := keyvalue.SearchKeyWithPrefix(req, cdxPrefix)
+// 	it, err = newIter(ctx, key, db.client, req)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	if it == nil {
+// 		close(res)
+// 		return nil
+// 	}
+
+// 	go func() {
+// 		defer close(res)
+// 		defer it.Close()
+
+// 		count := 0
+
+// 		for it.Valid() {
+// 			reportResponse := func() (reportResponse index.ReportResponse) {
+// 				key := keyvalue.CdxKey(it.Key())
+// 				if !req.DateRange().Contains(key.Unix()) {
+// 					return
+// 				}
+// 				cdx := new(schema.Cdx)
+// 				if err := proto.Unmarshal(it.Value(), cdx); err != nil {
+// 					reportResponse.Error = err
+// 				} else if req.Filter().Eval(cdx) {
+// 					reportResponse.Key = string(key)
+// 					reportResponse.Value = cdx
+// 				}
+// 				return
+// 			}()
+// 			if reportResponse == (index.ReportResponse{}) {
+// 				if err = it.Next(); err != nil {
+// 					res <- index.ReportResponse{Error: err}
+// 					break
+// 				}
+// 				continue
+// 			}
+// 			select {
+// 			case <-ctx.Done():
+// 				res <- index.ReportResponse{Error: ctx.Err()}
+// 				return
+// 			case res <- reportResponse:
+// 				if reportResponse.Error == nil {
+// 					count++
+// 				}
+// 			}
+// 			if req.Limit() > 0 && count >= req.Limit() {
+// 				break
+// 			}
+// 			if err = it.Next(); err != nil {
+// 				res <- index.ReportResponse{Error: err}
+// 				break
+// 			}
+// 		}
+// 	}()
+// 	return nil
+// }

internal/tikvidx/db.go

@@ -34,9 +34,10 @@ import (
 )
 
 var (
-	idPrefix   = "i"
-	filePrefix = "f"
-	cdxPrefix  = "c"
+	idPrefix     = "i"
+	filePrefix   = "f"
+	cdxPrefix    = "c"
+	reportPrefix = "r"
 )
 
 const delimiter = "_"
@@ -46,6 +47,7 @@ type DB struct {
 	batch  chan index.Record
 	done   chan struct{}
 	wg     sync.WaitGroup
+	tasks  map[string]context.CancelFunc
 }
 
 func NewDB(options ...Option) (db *DB, err error) {
@@ -61,6 +63,7 @@ func NewDB(options ...Option) (db *DB, err error) {
 	idPrefix = dbName + delimiter + idPrefix + delimiter
 	filePrefix = dbName + delimiter + filePrefix + delimiter
 	cdxPrefix = dbName + delimiter + cdxPrefix + delimiter
+	reportPrefix = dbName + delimiter + reportPrefix + delimiter
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
@@ -75,6 +78,7 @@ func NewDB(options ...Option) (db *DB, err error) {
 	db = &DB{
 		client: client,
 		done:   done,
+		tasks:  make(map[string]context.CancelFunc),
 	}
 
 	if opts.ReadOnly {