diff --git a/commands/helpers.go b/commands/helpers.go index 63e2989..a99fc46 100644 --- a/commands/helpers.go +++ b/commands/helpers.go @@ -13,11 +13,11 @@ import ( func initLogging() { logging := viper.GetString("logging") - if verbose && logging == "" { + if verbose { logging = "*=debug" } - if vverbose && logging == "" { + if vverbose { logging = "*=trace" } diff --git a/commands/run.go b/commands/run.go index 391a58b..73826cd 100644 --- a/commands/run.go +++ b/commands/run.go @@ -14,10 +14,11 @@ import ( "github.com/Alethio/memento/taskmanager" - "github.com/Alethio/memento/core" - "github.com/Alethio/memento/eth/bestblock" "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/Alethio/memento/core" + "github.com/Alethio/memento/eth/bestblock" ) var runCmd = &cobra.Command{ diff --git a/data/data.go b/data/data.go index 52a2177..50093c7 100644 --- a/data/data.go +++ b/data/data.go @@ -2,6 +2,7 @@ package data import ( "database/sql" + "strconv" "github.com/Alethio/memento/metrics" @@ -15,12 +16,23 @@ var log = logrus.WithField("module", "data") type FullBlock struct { Block types.Block - Receipts []types.Receipt + Receipts Receipts Uncles []types.Block storables []Storable } +type Receipts []types.Receipt + +func (a Receipts) Len() int { return len(a) } +func (a Receipts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a Receipts) Less(i, j int) bool { + iIdx, _ := strconv.ParseInt(a[i].TransactionIndex, 0, 64) + jIdx, _ := strconv.ParseInt(a[j].TransactionIndex, 0, 64) + + return iIdx < jIdx +} + // Storable // role: a Storable serves as a means of transforming raw data and inserting it into the database // input: raw Ethereum data + a database transaction diff --git a/go.mod b/go.mod index c15362d..4cde1de 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Alethio/memento require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e // indirect - github.com/alethio/web3-go v0.0.3 + github.com/alethio/web3-go v0.0.6 github.com/davecgh/go-spew v1.1.1 github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect github.com/gin-contrib/cors v1.3.0 @@ -18,7 +18,7 @@ require ( github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/parnurzeal/gorequest v0.2.15 - github.com/pkg/errors v0.8.1 // indirect + github.com/pkg/errors v0.8.1 github.com/pressly/goose v2.6.0+incompatible github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect @@ -26,7 +26,7 @@ require ( github.com/spf13/viper v1.3.2 github.com/stretchr/testify v1.4.0 // indirect github.com/ugorji/go v1.1.7 // indirect - golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect + golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect golang.org/x/text v0.3.2 // indirect gopkg.in/yaml.v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index d971246..98cd695 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/alethio/ethmock v0.0.0-20190607140831-ce4476424f5c/go.mod h1:Lpp9nEVo/nKuq0uf/Le6r1zjaiW4/tu0maG47HBE65A= github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e h1:3g/lGHjagEkqc3L/WiNY1QfpZqMUxtTfZ1uKMu+wMnc= github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e/go.mod h1:Lpp9nEVo/nKuq0uf/Le6r1zjaiW4/tu0maG47HBE65A= -github.com/alethio/web3-go v0.0.3 h1:K7RVdrn6S8MKo0h0s/N8+6P9fKDcOtS/kK9w5dEXLYI= -github.com/alethio/web3-go v0.0.3/go.mod h1:AsfM7UJMW47BDfa7LKPfaaSEkxyxYCsRtCftScgNn+c= +github.com/alethio/web3-go v0.0.6 h1:ZEDJY57OKq9tCmdkxMyy8M11mthX33GmFCzZ8kYuMY0= +github.com/alethio/web3-go v0.0.6/go.mod h1:tnrqWtLdde8ttsGwiiJ6VwmA9WQwI3sFbmyNjMQVT8M= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -49,9 +49,8 @@ github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwK github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -128,14 +127,12 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84 h1:IqXQ59gzdXv58Jmm2xn0tSOR9i6HqroaOFRQ3wR/dJQ= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -153,6 +150,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/scraper/scraper.go b/scraper/scraper.go index 9c11d95..753739c 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -1,9 +1,14 @@ package scraper import ( + "sort" "strconv" + "sync" "time" + "github.com/alethio/web3-go/ethrpc/provider/httprpc" + "github.com/pkg/errors" + "github.com/Alethio/memento/data" "github.com/alethio/web3-go/ethrpc" @@ -24,10 +29,19 @@ type Scraper struct { } func New(config Config) (*Scraper, error) { - c, err := ethrpc.NewWithDefaults(config.NodeURL) + batchLoader, err := httprpc.NewBatchLoader(0, 4*time.Millisecond) if err != nil { - log.Error(err) - return nil, err + return nil, errors.Wrap(err, "could not init batch loader") + } + + provider, err := httprpc.NewWithLoader(config.NodeURL, batchLoader) + if err != nil { + return nil, errors.Wrap(err, "could not init httprpc provider") + } + provider.SetHTTPTimeout(5000 * time.Millisecond) + c, err := ethrpc.New(provider) + if err != nil { + return nil, errors.Wrap(err, "could not init ethrpc") } return &Scraper{ @@ -58,16 +72,35 @@ func (s *Scraper) Exec(block int64) (*data.FullBlock, error) { log.Debug("getting receipts") start = time.Now() + + var wg sync.WaitGroup + var errs []error + var mu sync.Mutex for _, tx := range dataBlock.Transactions { - dataReceipt, err := s.conn.GetTransactionReceipt(tx.Hash) - if err != nil { - log.Error(err) - return nil, err - } + wg.Add(1) + txCopy := tx + + go func() { + defer wg.Done() + + dataReceipt, err := s.conn.GetTransactionReceipt(txCopy.Hash) + if err != nil { + errs = append(errs, err) + return + } - b.Receipts = append(b.Receipts, dataReceipt) + mu.Lock() + b.Receipts = append(b.Receipts, dataReceipt) + mu.Unlock() + }() } + wg.Wait() + sort.Sort(b.Receipts) + log.WithField("duration", time.Since(start)).Debugf("got %d receipts", len(b.Receipts)) + if len(errs) > 0 { + return nil, errs[0] + } if s.config.EnableUncles { log.Debug("getting uncles")