Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query Implementation #9

Merged
merged 8 commits into from
Oct 8, 2024
Merged

Query Implementation #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e2621dc
feat(service): first query implementation
hannahhoward Oct 1, 2024
8573141
refactor(service): cleanup design and add comments
hannahhoward Oct 2, 2024
a404f81
feat(service): implement read through caches
hannahhoward Oct 3, 2024
d6a3bd9
feat: server scaffolding
alanshaw Oct 3, 2024
7ec1692
Merge pull request #10 from storacha/feat/server-scaffolding
hannahhoward Oct 3, 2024
3ffd48f
feat(service): additional tests and refactors
hannahhoward Oct 4, 2024
05faba4
feat(service): fill out remaining components of service
hannahhoward Oct 7, 2024
35d2bc0
Update pkg/metadata/metadata.go
hannahhoward Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 132 additions & 0 deletions cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package main

import (
"fmt"
"os"

logging "github.com/ipfs/go-log/v2"
"github.com/storacha/go-ucanto/did"
ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
"github.com/storacha/go-ucanto/principal/signer"
"github.com/storacha/indexing-service/pkg/server"
"github.com/storacha/indexing-service/pkg/service"
"github.com/urfave/cli/v2"
)

var log = logging.Logger("cmd")

func main() {
logging.SetLogLevel("*", "info")

app := &cli.App{
Name: "indexing-service",
Usage: "Manage running the indexing service.",
Commands: []*cli.Command{
{
Name: "server",
Usage: "HTTP server interface to the indexing service",
Subcommands: []*cli.Command{
{
Name: "start",
Usage: "start an indexing service HTTP server",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "port",
Aliases: []string{"p"},
Value: 9000,
Usage: "port to bind the server to",
},
&cli.StringFlag{
Name: "private-key",
Aliases: []string{"pk"},
Usage: "base64 encoded private key identity for the server",
},
&cli.StringFlag{
Name: "did",
Usage: "DID of the server (only needs to be set if different from what is derived from the private key i.e. a did:web DID)",
},
&cli.StringFlag{
Name: "redis-url",
Aliases: []string{"redis"},
EnvVars: []string{"REDIS_URL"},
Usage: "url for a running redis database",
},
&cli.StringFlag{
Name: "redis-passwd",
Aliases: []string{"rp"},
EnvVars: []string{"REDIS_PASSWD"},
Usage: "passwd for redis",
},
&cli.IntFlag{
Name: "provider-redis-db",
Aliases: []string{"prd"},
Usage: "database number for providers index",
Value: 0,
},
&cli.IntFlag{
Name: "claims-redis-db",
Aliases: []string{"c"},
Usage: "database number for claims",
Value: 1,
},
&cli.IntFlag{
Name: "indexes-redis-db",
Aliases: []string{"c"},
Usage: "database number for indexes cache",
Value: 2,
},
&cli.StringFlag{
Name: "ipni-endpoint",
Aliases: []string{"ipni"},
DefaultText: "Defaults to https://cid.contact",
Value: "https://cid.contact",
Usage: "HTTP endpoint of the IPNI instance used to discover providers.",
},
},
Action: func(cCtx *cli.Context) error {
addr := fmt.Sprintf(":%d", cCtx.Int("port"))
var opts []server.Option
if cCtx.String("private-key") != "" {
id, err := ed25519.Parse(cCtx.String("private-key"))
if err != nil {
return fmt.Errorf("parsing server private key: %w", err)
}
if cCtx.String("did") != "" {
did, err := did.Parse(cCtx.String("did"))
if err != nil {
return fmt.Errorf("parsing server DID: %w", err)
}
id, err = signer.Wrap(id, did)
if err != nil {
return fmt.Errorf("wrapping server DID: %w", err)
}
}
opts = append(opts, server.WithIdentity(id))
}
var sc service.ServiceConfig
sc.RedisURL = cCtx.String("redis-url")
sc.RedisPasswd = cCtx.String("redis-passwd")
sc.ProvidersDB = cCtx.Int("providers-redis-db")
sc.ClaimsDB = cCtx.Int("claims-redis-db")
sc.IndexesDB = cCtx.Int("indexes-redis-db")
sc.IndexerURL = cCtx.String("ipni-endpoint")
indexingService, shutdown, err := service.Construct(sc)
if err != nil {
return err
}
defer func() {
shutdown(cCtx.Context)
}()
opts = append(opts, server.WithService(indexingService))
return server.ListenAndServe(addr, opts...)
},
},
},
},
},
}

if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}
49 changes: 30 additions & 19 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module github.com/storacha-network/indexing-service
module github.com/storacha/indexing-service

go 1.23

Expand All @@ -14,18 +14,28 @@ require (
github.com/multiformats/go-multihash v0.2.3
github.com/multiformats/go-varint v0.0.7
github.com/redis/go-redis/v9 v9.6.1
github.com/storacha-network/go-ucanto v0.1.1-0.20240916072230-3bed7025597b
github.com/storacha/go-ucanto v0.1.1-0.20241003110856-f3261cb2a702
github.com/stretchr/testify v1.9.0
)

require (
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/pion/ice/v2 v2.3.35 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
go.opentelemetry.io/otel/metric v1.30.0 // indirect
)

require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/gammazero/channelqueue v0.2.2 // indirect
github.com/gammazero/deque v0.2.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand All @@ -34,24 +44,24 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-blockservice v0.5.2 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.3 // indirect
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
github.com/ipfs/go-ipld-format v0.6.0 // indirect
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.11.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipfs/go-verifcid v0.0.3 // indirect
github.com/ipld/go-car v0.6.2 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
Expand All @@ -72,16 +82,17 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/whyrusleeping/cbor-gen v0.1.1 // indirect
go.opentelemetry.io/otel v1.13.0 // indirect
go.opentelemetry.io/otel/trace v1.13.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
github.com/urfave/cli/v2 v2.27.4
github.com/whyrusleeping/cbor-gen v0.1.2 // indirect
go.opentelemetry.io/otel v1.30.0 // indirect
go.opentelemetry.io/otel/trace v1.30.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.3.0 // indirect
Expand Down
Loading