diff --git a/go.mod b/go.mod index f5cfdc50..4120a4b9 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( ) require ( + github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 56657cc6..5cc2e1d3 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,34 @@ +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= +github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= +github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= +github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -35,8 +60,12 @@ github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= @@ -49,11 +78,15 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -65,12 +98,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -78,11 +114,13 @@ golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -100,6 +138,8 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -113,6 +153,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -143,10 +184,12 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/txcache/debugging.go b/txcache/debugging.go new file mode 100644 index 00000000..573ba13e --- /dev/null +++ b/txcache/debugging.go @@ -0,0 +1,249 @@ +package txcache + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "time" + + "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" + "github.com/multiversx/mx-chain-core-go/data/transaction" +) + +var debuggingPeriod = 6 * time.Second +var addressConverter, _ = pubkeyConverter.NewBech32PubkeyConverter(32, "erd") +var debuggingFolder = "txcache_debugging" +var debuggingFolderSizeLimitInBytes = uint64(10_000_000_000) +var numOldestFilesToRemove = 100 + +// MaxNumOfTxsToSelect defines the maximum number of transactions that should be selected from the cache +const MaxNumOfTxsToSelect = 30000 + +// NumTxPerSenderBatchForFillingMiniblock defines the number of transactions to be drawn +// from the transactions pool, for a specific sender, in a single pass. +// Drawing transactions for a miniblock happens in multiple passes, until "MaxItemsInBlock" are drawn. +const NumTxPerSenderBatchForFillingMiniblock = 10 + +// MaxGasBandwidthPerBatchPerSender defines the maximum gas bandwidth that should be selected for a sender per batch from the cache +const MaxGasBandwidthPerBatchPerSender = 5000000 + +type debuggingFileInfo struct { + Name string + Size int64 + ModTime time.Time +} + +type dumpedTransaction struct { + Hash string `json:"hash"` + TxFeeScoreNormalized uint64 `json:"txFeeScoreNormalized"` + + Nonce uint64 `json:"nonce"` + Value string `json:"value"` + Receiver string `json:"receiver"` + Sender string `json:"sender"` + SenderUsername []byte `json:"senderUsername,omitempty"` + ReceiverUsername []byte `json:"receiverUsername,omitempty"` + GasPrice uint64 `json:"gasPrice"` + GasLimit uint64 `json:"gasLimit"` + Data []byte `json:"data,omitempty"` + Signature string `json:"signature,omitempty"` + ChainID string `json:"chainID"` + Version uint32 `json:"version"` + Options uint32 `json:"options,omitempty"` + GuardianAddr string `json:"guardian,omitempty"` + GuardianSignature string `json:"guardianSignature,omitempty"` +} + +type dumpedSender struct { + Address string `json:"address"` + LastComputedScore uint32 `json:"lastComputedScore"` + HasInitialGap bool `json:"hasInitialGap"` + IsInGracePeriod bool `json:"isInGracePeriod"` + TotalBytes uint64 `json:"totalBytes"` + TotalGas uint64 `json:"totalGas"` + TotalFeeScore uint64 `json:"totalFeeScore"` + Txs []string `json:"txs"` +} + +func (cache *TxCache) continuouslyDebug() { + log.Info("TXPOOL starting debugging") + + // Create folder if it doesn't exist: + err := os.MkdirAll(debuggingFolder, 0755) + if err != nil { + log.Error("error creating debugging folder", "error", err) + } + + func() { + for { + cache.saveTransactionsToFile() + + err := limitDebuggingFiles(debuggingFolder) + if err != nil { + log.Error("error limiting debugging files", "error", err) + } + + // // Simulate selection + // cache.SelectTransactionsWithBandwidth(MaxNumOfTxsToSelect, NumTxPerSenderBatchForFillingMiniblock, MaxGasBandwidthPerBatchPerSender) + + time.Sleep(debuggingPeriod) + } + }() +} + +func (cache *TxCache) saveTransactionsToFile() { + timestamp := time.Now().Format("20060102150405") + outfileTxs := path.Join(debuggingFolder, fmt.Sprintf("%s_txcache_%s.json", timestamp, cache.name)) + outfileSenders := path.Join(debuggingFolder, fmt.Sprintf("%s_txcache_senders_%s.json", timestamp, cache.name)) + + numTxs := cache.txByHash.counter.Get() + numSenders := cache.txListBySender.counter.Get() + allTxs := make([]*dumpedTransaction, 0, numTxs) + allSenders := make([]*dumpedSender, 0, numSenders) + + cache.txByHash.forEach(func(txHash []byte, wrappedTx *WrappedTransaction) { + dumpedTx := wrappedTxToDumpedTx(txHash, wrappedTx) + allTxs = append(allTxs, dumpedTx) + }) + + sendersLists := cache.txListBySender.getSnapshotDescending() + + for _, senderList := range sendersLists { + dumpedSender := txListForSenderToDumpedSender(senderList) + allSenders = append(allSenders, dumpedSender) + } + + err := saveJson(outfileTxs, allTxs) + if err != nil { + log.Error("error saving txs to file", "error", err) + } + + err = saveJson(outfileSenders, allSenders) + if err != nil { + log.Error("error saving senders to file", "error", err) + } +} + +func wrappedTxToDumpedTx(txHash []byte, wrappedTx *WrappedTransaction) *dumpedTransaction { + rawTx := wrappedTx.Tx.(*transaction.Transaction) + + receiver, _ := addressConverter.Encode(rawTx.RcvAddr) + sender, _ := addressConverter.Encode(rawTx.SndAddr) + guardian, _ := addressConverter.Encode(rawTx.GuardianAddr) + + value := "" + if rawTx.Value != nil { + value = rawTx.Value.String() + } + + dumpedTx := &dumpedTransaction{ + Hash: hex.EncodeToString(txHash), + TxFeeScoreNormalized: wrappedTx.TxFeeScoreNormalized, + + Nonce: rawTx.Nonce, + Value: value, + Receiver: receiver, + Sender: sender, + SenderUsername: rawTx.SndUserName, + ReceiverUsername: rawTx.RcvUserName, + GasPrice: rawTx.GasPrice, + GasLimit: rawTx.GasLimit, + Data: rawTx.Data, + Signature: hex.EncodeToString(rawTx.Signature), + ChainID: string(rawTx.ChainID), + Version: rawTx.Version, + Options: rawTx.Options, + GuardianAddr: guardian, + GuardianSignature: hex.EncodeToString(rawTx.GuardianSignature), + } + + return dumpedTx +} + +func txListForSenderToDumpedSender(sender *txListForSender) *dumpedSender { + address, _ := addressConverter.Encode([]byte(sender.sender)) + txsHashes := sender.getTxHashes() + + txsHashesStrings := make([]string, len(txsHashes)) + + for i, txHash := range txsHashes { + txsHashesStrings[i] = hex.EncodeToString(txHash) + } + + return &dumpedSender{ + Address: address, + LastComputedScore: sender.getLastComputedScore(), + HasInitialGap: sender.hasInitialGap(), + IsInGracePeriod: sender.isInGracePeriod(), + TotalBytes: sender.totalBytes.GetUint64(), + TotalGas: sender.totalGas.GetUint64(), + TotalFeeScore: sender.totalFeeScore.GetUint64(), + Txs: txsHashesStrings, + } +} + +func saveJson(outfile string, data interface{}) error { + log.Info("saving debugging data", "file", outfile) + + outcomeJSON, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + + err = os.WriteFile(outfile, outcomeJSON, 0644) + if err != nil { + return err + } + + return nil +} + +func limitDebuggingFiles(directory string) error { + var files []debuggingFileInfo + var totalSize uint64 + + err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + files = append(files, debuggingFileInfo{Name: path, Size: info.Size(), ModTime: info.ModTime()}) + totalSize += uint64(info.Size()) + } + + return nil + }) + + if err != nil { + return err + } + + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime.Before(files[j].ModTime) + }) + + // If size is exceeded, remove some old files. + if totalSize < debuggingFolderSizeLimitInBytes { + return nil + } + + if len(files) < numOldestFilesToRemove { + return fmt.Errorf("not enough files to start removal, only %d files found", len(files)) + } + + for i := 0; i < numOldestFilesToRemove; i++ { + log.Info("removing old debugging file", "file", files[i].Name) + + err = os.Remove(files[i].Name) + if err != nil { + return err + } + } + + return nil +} diff --git a/txcache/realworld_test.go b/txcache/realworld_test.go new file mode 100644 index 00000000..57a2047f --- /dev/null +++ b/txcache/realworld_test.go @@ -0,0 +1,100 @@ +package txcache + +import ( + "encoding/json" + "math" + "os" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/hashing/sha256" + "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-storage-go/testscommon/txcachemocks" + "github.com/stretchr/testify/require" +) + +var minPrice = uint64(1000000000) +var minGasLimit = uint64(50000) +var gasProcessingDivisor = uint64(100) + +func TestUsage(t *testing.T) { + _ = logger.SetLogLevel("*:DEBUG") + + config := ConfigSourceMe{ + Name: "test", + NumChunks: 16, + NumBytesPerSenderThreshold: maxNumBytesPerSenderUpperBound, + CountPerSenderThreshold: math.MaxUint32, + } + + txGasHandler := &txcachemocks.TxGasHandlerMock{ + MinimumGasMove: minGasLimit, + MinimumGasPrice: minPrice, + GasProcessingDivisor: gasProcessingDivisor, + } + + cache, err := NewTxCache(config, txGasHandler) + require.Nil(t, err) + require.NotNil(t, cache) + + transactions := loadTransactions(t, "testdata/20240618190000_txcache_0.json") + + for _, tx := range transactions { + ok, added := cache.AddTx(tx) + + require.True(t, ok) + require.True(t, added) + } + + selectedTransactions := cache.SelectTransactionsWithBandwidth(MaxNumOfTxsToSelect, NumTxPerSenderBatchForFillingMiniblock, MaxGasBandwidthPerBatchPerSender) + require.NotNil(t, selectedTransactions) +} + +func loadTransactions(t *testing.T, filePath string) []*WrappedTransaction { + protoMarshalizer := &marshal.GogoProtoMarshalizer{} + hasher := sha256.NewSha256() + + frontendTransactions := make([]*transaction.FrontendTransaction, 0) + transactions := make([]*WrappedTransaction, 0) + + content, err := os.ReadFile(filePath) + require.Nil(t, err) + + err = json.Unmarshal(content, &frontendTransactions) + require.Nil(t, err) + + for _, frontendTx := range frontendTransactions { + senderPubkey, err := addressConverter.Decode(frontendTx.Sender) + require.Nil(t, err) + + receiverPubkey, err := addressConverter.Decode(frontendTx.Receiver) + require.Nil(t, err) + + tx := &transaction.Transaction{ + SndAddr: senderPubkey, + RcvAddr: receiverPubkey, + Nonce: frontendTx.Nonce, + Data: frontendTx.Data, + GasLimit: frontendTx.GasLimit, + GasPrice: frontendTx.GasPrice, + } + + marshalledTx, err := protoMarshalizer.Marshal(tx) + require.Nil(t, err) + + txHash, err := core.CalculateHash(protoMarshalizer, hasher, tx) + require.Nil(t, err) + + wrappedTx := &WrappedTransaction{ + Tx: tx, + TxHash: txHash, + Size: int64(len(marshalledTx)), + } + + transactions = append(transactions, wrappedTx) + } + + return transactions +} diff --git a/txcache/testdata/20240618190000_txcache_0.json b/txcache/testdata/20240618190000_txcache_0.json new file mode 100644 index 00000000..aa646d7a --- /dev/null +++ b/txcache/testdata/20240618190000_txcache_0.json @@ -0,0 +1,156 @@ +[ + { + "hash": "ab3de2f476fb9cc673d511fcd4b615f200eaa1a59ff9513ad9cecc633c81641b", + "txFeeScoreNormalized": 22855088, + "nonce": 35, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1z0nzdnr5swnwnns7huy6q6h8fzetmgcdx5r0qn8a7mw03nz6qwuqkeqcrl", + "gasPrice": 1000000000, + "gasLimit": 130000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "77025f64f6665f9da4bb7298ec79e684c7c938ff8fa91fde7b5cd0a66b1cd28c78a86e7f246d59d767026b76462b31f4786998d682be6e5db3e1070c9cce3c01", + "chainID": "T", + "version": 2 + }, + { + "hash": "1c0440d0e15532850b96334480ab322452334092b0ffcefd0a6fe5c1381d5d18", + "txFeeScoreNormalized": 22855088, + "nonce": 37, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1z0nzdnr5swnwnns7huy6q6h8fzetmgcdx5r0qn8a7mw03nz6qwuqkeqcrl", + "gasPrice": 1000000000, + "gasLimit": 130000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "4de8043c59f25d6a20a9afc3695a28afe6d626555151a133ac98fc19f49f982a3986416cfc1992908c0dbc8b98ee5f6e6fd019d9d010d227851e4644d7eb510f", + "chainID": "T", + "version": 2 + }, + { + "hash": "b7b53b911c1cac97653062ede6c3ffff2459c9ede64de104ab3284e9218444a9", + "txFeeScoreNormalized": 105472208, + "nonce": 32, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "98019decca2f7fe7ba03b2c77d4ca9d934a0aa18d813143cd7b623d9eac87fe6197ade1300bb1278815fb06099ad8b372b0c44e7b566cbdd6b75c6aae1e4310e", + "chainID": "T", + "version": 2 + }, + { + "hash": "cb464bcf8f657587fb72dcdf00913031747c65972da908dc21282131a831be74", + "txFeeScoreNormalized": 22855088, + "nonce": 36, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1z0nzdnr5swnwnns7huy6q6h8fzetmgcdx5r0qn8a7mw03nz6qwuqkeqcrl", + "gasPrice": 1000000000, + "gasLimit": 130000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "3fc97823e135b009d6286bd25d0fa4f61401b806a2a1ec58dcfc9232ecac15a81fa731df34ff99dadf9ddcad4a8db1612d8e6b681df0f919ceb5a9005a774b0a", + "chainID": "T", + "version": 2 + }, + { + "hash": "1b5ba70c0916129df75621a0ae2face45652c97c6ae87f289fba47227d661e6e", + "txFeeScoreNormalized": 105472208, + "nonce": 34, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "674a44813384ac6566825c243a614bfb00a6986dd44e8be143612a03dd2fe04aa010b9b4a778021ad65dfe4b22f7ad5c5b8dc0319c330f141189658702a2c803", + "chainID": "T", + "version": 2 + }, + { + "hash": "7193f13bf9a553fae56b01e94ff7103567515eb5e0476ebc75d8780917f97814", + "txFeeScoreNormalized": 22855088, + "nonce": 34, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1z0nzdnr5swnwnns7huy6q6h8fzetmgcdx5r0qn8a7mw03nz6qwuqkeqcrl", + "gasPrice": 1000000000, + "gasLimit": 130000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "16d6519cd35c5fc4233dc07b1cabafcc3e82cd25a99b8513e5bd62e2e9bba3dd84c9c3de5d534e456d42370fbcac5a6e22342e374c9f80b3b22251feb9047a02", + "chainID": "T", + "version": 2 + }, + { + "hash": "4e776f48c3e26d78dec739325713682d57bb5de0d4921349c58a78508819c589", + "txFeeScoreNormalized": 105472208, + "nonce": 35, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "bfcbe04f03e4fd4b070054e1a80c3634a03a33e4ec7cbf84c8284fa63e4c0ec00520ecc648d6c97f7ced42c0e971220ed31eb9f64b06ea5f48bb471003b26d0d", + "chainID": "T", + "version": 2 + }, + { + "hash": "8a0bb78a40d8e0866a075789d880640523dae04bf17de75ff9a8cc3bf4c321bd", + "txFeeScoreNormalized": 105472208, + "nonce": 36, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "f00bf8d4047fbbbdd391142208b145836ed39e5aa7e9ba4b1c68cffe35bb1e1dcb3fe57c6d80ec7f481f2db8542b8432b3633cd3aff3629e0589aaef779d8006", + "chainID": "T", + "version": 2 + }, + { + "hash": "a709a2bedc9b4189ff1c51d47969af3d4f5aab5bcaf100ff1cfb3f9ba808cbfd", + "txFeeScoreNormalized": 105472208, + "nonce": 31, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "b2ce88ab0fb1364b2ed049acbb08d67c1d7516b3afffd24901e67a7af3eab4b788d7ec692c41d7b326c6977482cd240b12bee4572b6c5e16028baa6bb26ac00f", + "chainID": "T", + "version": 2 + }, + { + "hash": "63c7b085bb4f641cbe2686320563402950b9296e2b5af52f4f61ae2a3d4b205b", + "txFeeScoreNormalized": 105472208, + "nonce": 30, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "3a4cfc309620fbfd1844aed417f92fd0538c3914ba87e3acadc77334479ffd0fe874f1f54c48a5549307bd702e214dac61a230f20e17c1147ea45cc074234b09", + "chainID": "T", + "version": 2 + }, + { + "hash": "3f34755bf1edff74dfe5871b9d3b0b2fb37c608da469cdd861eaab74fa6f8b82", + "txFeeScoreNormalized": 105472208, + "nonce": 33, + "value": "0", + "receiver": "erd1qqqqqqqqqqqqqpgqz045rw74nthgzw2te9lytgah775n3l08396q3wt4qq", + "sender": "erd1wqd499fv90e0d6sgzgxpvn47rma6e90dzf3eeheq3mjkxzusylgqnm52u5", + "gasPrice": 1000000000, + "gasLimit": 600000000, + "data": "Zm9vYmFyQDI3MTA=", + "signature": "1d4c4e6bf318ff18a4d567ef4c9f9990cde8e1380287a862b3862ee066099d849b07ffac19b9ef6a2a49bf014ea6b90a2b5cef392070577a3dcdf181e82dfa04", + "chainID": "T", + "version": 2 + } +] \ No newline at end of file diff --git a/txcache/txCache.go b/txcache/txCache.go index d938b976..f6b29024 100644 --- a/txcache/txCache.go +++ b/txcache/txCache.go @@ -59,6 +59,8 @@ func NewTxCache(config ConfigSourceMe, txGasHandler TxGasHandler) (*TxCache, err } txCache.initSweepable() + go txCache.continuouslyDebug() + return txCache, nil } @@ -120,10 +122,22 @@ func (cache *TxCache) doSelectTransactions(numRequested int, batchSizePerSender snapshotOfSenders := cache.getSendersEligibleForSelection() + log.Debug("TXPOOL_DEBUG senders (below)") + + for i, sender := range snapshotOfSenders { + log.Debug("TXPOOL_DEBUG sender", "i", i, "sender", sender.senderAddress, "score", sender.getLastComputedScore(), "totalGas", sender.totalGas, "totalFeeScore", sender.totalFeeScore) + } + + log.Debug("TXPOOL_DEBUG starting selection loop", "name", cache.name, "numRequested", numRequested, "batchSizePerSender", batchSizePerSender, "bandwidthPerSender", bandwidthPerSender, "numSenders", len(snapshotOfSenders)) + for pass := 0; !resultIsFull; pass++ { + log.Debug("TXPOOL_DEBUG selection LOOP PASS", "pass", pass) + copiedInThisPass := 0 for _, txList := range snapshotOfSenders { + log.Debug("selection loop pass (started) for sender", "sender", txList.senderAddress) + batchSizeWithScoreCoefficient := batchSizePerSender * int(txList.getLastComputedScore()+1) // Reset happens on first pass only isFirstBatch := pass == 0 @@ -137,6 +151,9 @@ func (cache *TxCache) doSelectTransactions(numRequested int, batchSizePerSender resultFillIndex += journal.copied copiedInThisPass += journal.copied resultIsFull = resultFillIndex == numRequested + + log.Debug("selection loop pass (ended) for sender", "copied", journal.copied, "isFirstBatch", isFirstBatch, "resultFillIndex", resultFillIndex, "resultIsFull", resultIsFull) + if resultIsFull { break } diff --git a/txcache/txListBySenderMap.go b/txcache/txListBySenderMap.go index ccda1ce0..421b6bf5 100644 --- a/txcache/txListBySenderMap.go +++ b/txcache/txListBySenderMap.go @@ -87,6 +87,9 @@ func (txMap *txListBySenderMap) addSender(sender string) *txListForSender { func (txMap *txListBySenderMap) notifyScoreChange(txList *txListForSender, scoreParams senderScoreParams) { score := txMap.scoreComputer.computeScore(scoreParams) txList.setLastComputedScore(score) + + log.Info("TXPOOL_DEBUG txListBySenderMap.notifyScoreChange()", "sender", txList.senderAddress, "score", score) + txMap.backingMap.NotifyScoreChange(txList, score) } diff --git a/txcache/txListForSender.go b/txcache/txListForSender.go index a12a91d1..e8b847d9 100644 --- a/txcache/txListForSender.go +++ b/txcache/txListForSender.go @@ -20,6 +20,7 @@ type txListForSender struct { sweepable atomic.Flag copyPreviousNonce uint64 sender string + senderAddress string items *list.List copyBatchIndex *list.Element constraints *senderConstraints @@ -39,9 +40,12 @@ type scoreChangeCallback func(value *txListForSender, scoreParams senderScorePar // newTxListForSender creates a new (sorted) list of transactions func newTxListForSender(sender string, constraints *senderConstraints, onScoreChange scoreChangeCallback) *txListForSender { + senderAddress, _ := addressConverter.Encode([]byte(sender)) + return &txListForSender{ items: list.New(), sender: sender, + senderAddress: senderAddress, constraints: constraints, onScoreChange: onScoreChange, } @@ -102,9 +106,14 @@ func (listForSender *txListForSender) isCapacityExceeded() bool { } func (listForSender *txListForSender) onAddedTransaction(tx *WrappedTransaction, gasHandler TxGasHandler, txFeeHelper feeHelper) { + gas := estimateTxGas(tx) + feeScore := estimateTxFeeScore(tx, gasHandler, txFeeHelper) + listForSender.totalBytes.Add(tx.Size) - listForSender.totalGas.Add(int64(estimateTxGas(tx))) - listForSender.totalFeeScore.Add(int64(estimateTxFeeScore(tx, gasHandler, txFeeHelper))) + listForSender.totalGas.Add(int64(gas)) + listForSender.totalFeeScore.Add(int64(feeScore)) + + log.Info("TXPOOL_DEBUG txListForSender.onAddedTransaction()", "sender", listForSender.sender, "tx", tx.TxHash, "with gas", gas, "and fee score", feeScore) } func (listForSender *txListForSender) triggerScoreChange() { @@ -271,6 +280,8 @@ func (listForSender *txListForSender) selectBatchTo(isFirstBatch bool, destinati break } + log.Debug("txListForSender.selectBatchTo()", "tx", value.TxHash, "nonce", txNonce, "gas", lastTxGasLimit, "copied", copied, "bandwidth", copiedBandwidth) + destination[copied] = value element = element.Next() previousNonce = txNonce