From e3748e3cdfe2d2f5ca775e6370020ff863ad6599 Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Thu, 25 Mar 2021 22:51:36 -0700 Subject: [PATCH 01/40] migrate to go module used the following commands to generate most of the go mod with the same versions as vendored libs: $ for m in $(git submodule | grep -v -e jacobsa/fuse -e genproto | perl -nle 'print "$2\@$1" if $_ =~ m| +(.*) vendor/([^ ]+) |'); do echo $m; go get $m || break ; done $ go mod tidy $ manually added github.com/kahing/fusego replacement from #555 close #555 --- .travis.yml | 2 +- go.mod | 40 ++++++ go.sum | 131 ++++++++++++++++++ .../contrib.go.opencensus.io/exporter/ocagent | 1 - vendor/github.com/Azure/azure-pipeline-go | 1 - vendor/github.com/Azure/azure-sdk-for-go | 1 - vendor/github.com/Azure/azure-storage-blob-go | 1 - vendor/github.com/Azure/go-autorest | 1 - vendor/github.com/aws/aws-sdk-go | 1 - .../census-instrumentation/opencensus-proto | 1 - vendor/github.com/dgrijalva/jwt-go | 1 - vendor/github.com/dimchansky/utfbom | 1 - vendor/github.com/golang/protobuf | 1 - vendor/github.com/google/uuid | 1 - vendor/github.com/grpc-ecosystem/grpc-gateway | 1 - vendor/github.com/hashicorp/golang-lru | 1 - vendor/github.com/jacobsa/fuse | 1 - vendor/github.com/jtolds/gls | 1 - vendor/github.com/kardianos/osext | 1 - vendor/github.com/kr/pretty | 1 - vendor/github.com/kr/text | 1 - vendor/github.com/lsegal/gucumber | 1 - vendor/github.com/mattn/go-ieproxy | 1 - vendor/github.com/mitchellh/go-homedir | 1 - vendor/github.com/satori/go.uuid | 1 - vendor/github.com/sevlyar/go-daemon | 1 - vendor/github.com/shirou/gopsutil | 1 - vendor/github.com/sirupsen/logrus | 1 - vendor/github.com/smartystreets/assertions | 1 - vendor/github.com/smartystreets/goconvey | 1 - vendor/github.com/urfave/cli | 1 - vendor/go.opencensus.io | 1 - vendor/golang.org/x/crypto | 1 - vendor/golang.org/x/net | 1 - vendor/golang.org/x/sync | 1 - vendor/golang.org/x/sys | 1 - vendor/golang.org/x/text | 1 - vendor/google.golang.org/api | 1 - vendor/google.golang.org/genproto | 1 - vendor/google.golang.org/grpc | 1 - vendor/gopkg.in/check.v1 | 1 - vendor/gopkg.in/ini.v1 | 1 - 42 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 go.mod create mode 100644 go.sum delete mode 160000 vendor/contrib.go.opencensus.io/exporter/ocagent delete mode 160000 vendor/github.com/Azure/azure-pipeline-go delete mode 160000 vendor/github.com/Azure/azure-sdk-for-go delete mode 160000 vendor/github.com/Azure/azure-storage-blob-go delete mode 160000 vendor/github.com/Azure/go-autorest delete mode 160000 vendor/github.com/aws/aws-sdk-go delete mode 160000 vendor/github.com/census-instrumentation/opencensus-proto delete mode 160000 vendor/github.com/dgrijalva/jwt-go delete mode 160000 vendor/github.com/dimchansky/utfbom delete mode 160000 vendor/github.com/golang/protobuf delete mode 160000 vendor/github.com/google/uuid delete mode 160000 vendor/github.com/grpc-ecosystem/grpc-gateway delete mode 160000 vendor/github.com/hashicorp/golang-lru delete mode 160000 vendor/github.com/jacobsa/fuse delete mode 160000 vendor/github.com/jtolds/gls delete mode 160000 vendor/github.com/kardianos/osext delete mode 160000 vendor/github.com/kr/pretty delete mode 160000 vendor/github.com/kr/text delete mode 160000 vendor/github.com/lsegal/gucumber delete mode 160000 vendor/github.com/mattn/go-ieproxy delete mode 160000 vendor/github.com/mitchellh/go-homedir delete mode 160000 vendor/github.com/satori/go.uuid delete mode 160000 vendor/github.com/sevlyar/go-daemon delete mode 160000 vendor/github.com/shirou/gopsutil delete mode 160000 vendor/github.com/sirupsen/logrus delete mode 160000 vendor/github.com/smartystreets/assertions delete mode 160000 vendor/github.com/smartystreets/goconvey delete mode 160000 vendor/github.com/urfave/cli delete mode 160000 vendor/go.opencensus.io delete mode 160000 vendor/golang.org/x/crypto delete mode 160000 vendor/golang.org/x/net delete mode 160000 vendor/golang.org/x/sync delete mode 160000 vendor/golang.org/x/sys delete mode 160000 vendor/golang.org/x/text delete mode 160000 vendor/google.golang.org/api delete mode 160000 vendor/google.golang.org/genproto delete mode 160000 vendor/google.golang.org/grpc delete mode 160000 vendor/gopkg.in/check.v1 delete mode 160000 vendor/gopkg.in/ini.v1 diff --git a/.travis.yml b/.travis.yml index e3da16d1..69202383 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - make s3proxy.jar script: travis_retry ./test/run-tests.sh GoofysTest go: - - 1.11.5 + - 1.16 go_import_path: github.com/kahing/goofys matrix: include: diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..98b98ef1 --- /dev/null +++ b/go.mod @@ -0,0 +1,40 @@ +module github.com/kahing/goofys + +go 1.14 + +require ( + github.com/Azure/azure-pipeline-go v0.2.2 + github.com/Azure/azure-sdk-for-go v32.1.0+incompatible + github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 + github.com/Azure/go-autorest/autorest v0.11.18 + github.com/Azure/go-autorest/autorest/adal v0.9.13 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 + github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e + github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f + github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe // indirect + github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f + github.com/jtolds/gls v4.2.0+incompatible // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 + github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect + github.com/mitchellh/go-homedir v1.1.0 + github.com/pkg/errors v0.9.1 // indirect + github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b + github.com/sevlyar/go-daemon v0.1.5 + github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 + github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 + github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 // indirect + github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c // indirect + github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a + golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect + golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 + golang.org/x/text v0.3.2 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/ini.v1 v1.46.0 +) + +replace github.com/jacobsa/fuse => github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..cab425d0 --- /dev/null +++ b/go.sum @@ -0,0 +1,131 @@ +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 h1:8DQB8yl7aLQuP+nuR5e2RO6454OvFlSTXXaNHshc16s= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7/go.mod h1:AkzUsqkrdmNhfP2i54HqINVQopw0CLDnvHpJ88Zz1eI= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e h1:VpbwzFX2LQDqZIqzADiJXE3yvLG2lp/sAV337Z89aqs= +github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= +github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe h1:rcf1P0fm+1l0EjG16p06mYLj9gW9X36KgdHJ/88hS4g= +github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f h1:qMeDFuReF4ZPMVRMfCESfcHD7vQuCFpeiS17Dx4HYxU= +github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jtolds/gls v4.2.0+incompatible h1:f7Me2eb3Wy6xfgSj1uyi0Ho6c4NY1L4KoX7c3auAhOI= +github.com/jtolds/gls v4.2.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc h1:wNmVtupKbdEJcTkMIF2ah/nf3NzHEeNnpTR/1riaahQ= +github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc/go.mod h1:Qk5T6wYqwRWfKHhEKkRsIXwsAtQ0GK8rBJNYf5LVdwo= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 h1:lqGuhK6ejK9x8b6+GPGXm0gzrajIKB7yNKL2fx1oqSU= +github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= +github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= +github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 h1:cPg+LIQqTg4YaxZsUtAus45xKwqpSG5SQrSoOuWKUg4= +github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c h1:RlFdHEfqB3LI6Q8YfB1EQPO2+kED+w9TdVaMwrs6VPo= +github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= +gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/contrib.go.opencensus.io/exporter/ocagent b/vendor/contrib.go.opencensus.io/exporter/ocagent deleted file mode 160000 index a8a6f458..00000000 --- a/vendor/contrib.go.opencensus.io/exporter/ocagent +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a8a6f458bbc1d5042322ad1f9b65eeb0b69be9ea diff --git a/vendor/github.com/Azure/azure-pipeline-go b/vendor/github.com/Azure/azure-pipeline-go deleted file mode 160000 index 232aee85..00000000 --- a/vendor/github.com/Azure/azure-pipeline-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 232aee85e8e3a6223a11c0943f7df2ae0fac00e4 diff --git a/vendor/github.com/Azure/azure-sdk-for-go b/vendor/github.com/Azure/azure-sdk-for-go deleted file mode 160000 index a629ae78..00000000 --- a/vendor/github.com/Azure/azure-sdk-for-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a629ae7873bf2a86184a8bc1b65e65e1ab532f57 diff --git a/vendor/github.com/Azure/azure-storage-blob-go b/vendor/github.com/Azure/azure-storage-blob-go deleted file mode 160000 index 33c102d4..00000000 --- a/vendor/github.com/Azure/azure-storage-blob-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 33c102d4ffd269865af7fce31e927af2752df9a0 diff --git a/vendor/github.com/Azure/go-autorest b/vendor/github.com/Azure/go-autorest deleted file mode 160000 index d9a171ca..00000000 --- a/vendor/github.com/Azure/go-autorest +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d9a171ca366fd4acfaa99107cdc4f91877f3572e diff --git a/vendor/github.com/aws/aws-sdk-go b/vendor/github.com/aws/aws-sdk-go deleted file mode 160000 index 7e7bb328..00000000 --- a/vendor/github.com/aws/aws-sdk-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7e7bb328b06e5d15c0f0b7c935f848a57e289ad8 diff --git a/vendor/github.com/census-instrumentation/opencensus-proto b/vendor/github.com/census-instrumentation/opencensus-proto deleted file mode 160000 index 26aa36c0..00000000 --- a/vendor/github.com/census-instrumentation/opencensus-proto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 26aa36c099c2041b432cf3cc8a26c5fb858d218b diff --git a/vendor/github.com/dgrijalva/jwt-go b/vendor/github.com/dgrijalva/jwt-go deleted file mode 160000 index 5e25c22b..00000000 --- a/vendor/github.com/dgrijalva/jwt-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5e25c22bd5d6de03265bbe5462dcd162f85046f6 diff --git a/vendor/github.com/dimchansky/utfbom b/vendor/github.com/dimchansky/utfbom deleted file mode 160000 index d2133a1c..00000000 --- a/vendor/github.com/dimchansky/utfbom +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d2133a1ce379ef6fa992b0514a77146c60db9d1c diff --git a/vendor/github.com/golang/protobuf b/vendor/github.com/golang/protobuf deleted file mode 160000 index 4c88cc3f..00000000 --- a/vendor/github.com/golang/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4c88cc3f1a34ffade77b79abc53335d1e511f25b diff --git a/vendor/github.com/google/uuid b/vendor/github.com/google/uuid deleted file mode 160000 index c2e93f3a..00000000 --- a/vendor/github.com/google/uuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c2e93f3ae59f2904160ceaab466009f965df46d6 diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway b/vendor/github.com/grpc-ecosystem/grpc-gateway deleted file mode 160000 index fdf06359..00000000 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fdf063599d922ec89a70819e2d5b7b4b5c642b92 diff --git a/vendor/github.com/hashicorp/golang-lru b/vendor/github.com/hashicorp/golang-lru deleted file mode 160000 index 7f827b33..00000000 --- a/vendor/github.com/hashicorp/golang-lru +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7f827b33c0f158ec5dfbba01bb0b14a4541fd81d diff --git a/vendor/github.com/jacobsa/fuse b/vendor/github.com/jacobsa/fuse deleted file mode 160000 index ca77844c..00000000 --- a/vendor/github.com/jacobsa/fuse +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ca77844c7bcca61167af9964208ec3f71ffdc7c8 diff --git a/vendor/github.com/jtolds/gls b/vendor/github.com/jtolds/gls deleted file mode 160000 index 8ddce2a8..00000000 --- a/vendor/github.com/jtolds/gls +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ddce2a84170772b95dd5d576c48d517b22cac63 diff --git a/vendor/github.com/kardianos/osext b/vendor/github.com/kardianos/osext deleted file mode 160000 index 2bc1f35c..00000000 --- a/vendor/github.com/kardianos/osext +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2bc1f35cddc0cc527b4bc3dce8578fc2a6c11384 diff --git a/vendor/github.com/kr/pretty b/vendor/github.com/kr/pretty deleted file mode 160000 index 71e7e499..00000000 --- a/vendor/github.com/kr/pretty +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 71e7e49937503c662b9b636fd6b2c14b1aa818a5 diff --git a/vendor/github.com/kr/text b/vendor/github.com/kr/text deleted file mode 160000 index e2ffdb16..00000000 --- a/vendor/github.com/kr/text +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e2ffdb16a802fe2bb95e2e35ff34f0e53aeef34f diff --git a/vendor/github.com/lsegal/gucumber b/vendor/github.com/lsegal/gucumber deleted file mode 160000 index 44a4d7eb..00000000 --- a/vendor/github.com/lsegal/gucumber +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 44a4d7eb3b14a88cf82b073dfb7e06277afdc549 diff --git a/vendor/github.com/mattn/go-ieproxy b/vendor/github.com/mattn/go-ieproxy deleted file mode 160000 index f9202b1c..00000000 --- a/vendor/github.com/mattn/go-ieproxy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f9202b1cfdeb0c82ddd3dc1e8e9cd94b3c0c1b13 diff --git a/vendor/github.com/mitchellh/go-homedir b/vendor/github.com/mitchellh/go-homedir deleted file mode 160000 index af06845c..00000000 --- a/vendor/github.com/mitchellh/go-homedir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit af06845cf3004701891bf4fdb884bfe4920b3727 diff --git a/vendor/github.com/satori/go.uuid b/vendor/github.com/satori/go.uuid deleted file mode 160000 index b2ce2384..00000000 --- a/vendor/github.com/satori/go.uuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b2ce2384e17bbe0c6d34077efa39dbab3e09123b diff --git a/vendor/github.com/sevlyar/go-daemon b/vendor/github.com/sevlyar/go-daemon deleted file mode 160000 index bf839693..00000000 --- a/vendor/github.com/sevlyar/go-daemon +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bf839693b97cb19a587b25649bbcb59ee16bdb2f diff --git a/vendor/github.com/shirou/gopsutil b/vendor/github.com/shirou/gopsutil deleted file mode 160000 index d80c43f9..00000000 --- a/vendor/github.com/shirou/gopsutil +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d80c43f9c984a48783daf22f4bd9278006ae483a diff --git a/vendor/github.com/sirupsen/logrus b/vendor/github.com/sirupsen/logrus deleted file mode 160000 index de736cf9..00000000 --- a/vendor/github.com/sirupsen/logrus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit de736cf91b921d56253b4010270681d33fdf7cb5 diff --git a/vendor/github.com/smartystreets/assertions b/vendor/github.com/smartystreets/assertions deleted file mode 160000 index 443d8122..00000000 --- a/vendor/github.com/smartystreets/assertions +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 443d812296a84445c202c085f19e18fc238f8250 diff --git a/vendor/github.com/smartystreets/goconvey b/vendor/github.com/smartystreets/goconvey deleted file mode 160000 index 995f5b2e..00000000 --- a/vendor/github.com/smartystreets/goconvey +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 995f5b2e021c69b8b028ba6d0b05c1dd500783db diff --git a/vendor/github.com/urfave/cli b/vendor/github.com/urfave/cli deleted file mode 160000 index 521735b7..00000000 --- a/vendor/github.com/urfave/cli +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 521735b7608a25d771a39d42e2267e061e7e84b8 diff --git a/vendor/go.opencensus.io b/vendor/go.opencensus.io deleted file mode 160000 index b4a14686..00000000 --- a/vendor/go.opencensus.io +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b4a14686f0a98096416fe1b4cb848e384fb2b22b diff --git a/vendor/golang.org/x/crypto b/vendor/golang.org/x/crypto deleted file mode 160000 index 4def268f..00000000 --- a/vendor/golang.org/x/crypto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4def268fd1a49955bfb3dda92fe3db4f924f2285 diff --git a/vendor/golang.org/x/net b/vendor/golang.org/x/net deleted file mode 160000 index ca1201d0..00000000 --- a/vendor/golang.org/x/net +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ca1201d0de80cfde86cb01aea620983605dfe99b diff --git a/vendor/golang.org/x/sync b/vendor/golang.org/x/sync deleted file mode 160000 index 11223019..00000000 --- a/vendor/golang.org/x/sync +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 112230192c580c3556b8cee6403af37a4fc5f28c diff --git a/vendor/golang.org/x/sys b/vendor/golang.org/x/sys deleted file mode 160000 index 51ab0e2d..00000000 --- a/vendor/golang.org/x/sys +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 51ab0e2deafac1f46c46ad59cf0921be2f180c3d diff --git a/vendor/golang.org/x/text b/vendor/golang.org/x/text deleted file mode 160000 index 342b2e1f..00000000 --- a/vendor/golang.org/x/text +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 342b2e1fbaa52c93f31447ad2c6abc048c63e475 diff --git a/vendor/google.golang.org/api b/vendor/google.golang.org/api deleted file mode 160000 index 6f391290..00000000 --- a/vendor/google.golang.org/api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f3912904777a209e099b9dbda3ed7bcb4e25ad7 diff --git a/vendor/google.golang.org/genproto b/vendor/google.golang.org/genproto deleted file mode 160000 index fa694d86..00000000 --- a/vendor/google.golang.org/genproto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fa694d86fc64c7654a660f8908de4e879866748d diff --git a/vendor/google.golang.org/grpc b/vendor/google.golang.org/grpc deleted file mode 160000 index fde0cae1..00000000 --- a/vendor/google.golang.org/grpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fde0cae1c4042d90d749f11d334dd1d122c5c25b diff --git a/vendor/gopkg.in/check.v1 b/vendor/gopkg.in/check.v1 deleted file mode 160000 index 788fd784..00000000 --- a/vendor/gopkg.in/check.v1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 788fd78401277ebd861206a03c884797c6ec5541 diff --git a/vendor/gopkg.in/ini.v1 b/vendor/gopkg.in/ini.v1 deleted file mode 160000 index d4cae42d..00000000 --- a/vendor/gopkg.in/ini.v1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d4cae42d398bc0095297fc3315669590d29166ea From b12cef3a787b50356ca306b7e1947ab91ef0bed7 Mon Sep 17 00:00:00 2001 From: Eric Bressler Date: Thu, 25 Mar 2021 23:19:01 -0700 Subject: [PATCH 02/40] The changes here make it so that there are no longer references being held to the s3 objects allocated during XMLToStruct in the s3 libraries. In my testing, when doing a listObjectsV2 against ~383,000 objects, this change saves about 1/2 of the memory uses. (1gb of use without these changes and ~500mb with them) --- internal/handles.go | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/internal/handles.go b/internal/handles.go index 651faf69..2976be67 100644 --- a/internal/handles.go +++ b/internal/handles.go @@ -105,28 +105,51 @@ func NewInode(fs *Goofys, parent *Inode, name *string) (inode *Inode) { return } +func deepCopyBlobItemOputput(item *BlobItemOutput) BlobItemOutput { + + key := *item.Key + etag := *item.ETag + lastmodified := *item.LastModified + var sc string + if item.StorageClass != nil { + sc = *item.StorageClass + } + + return BlobItemOutput{ + Key: &key, + ETag: &etag, + LastModified: &lastmodified, + Size: item.Size, + StorageClass: &sc, + } +} + func (inode *Inode) SetFromBlobItem(item *BlobItemOutput) { + // copy item so they won't hold back references to the HTTP + // responses and SDK objects. See discussion in + // https://github.com/kahing/goofys/pull/547 + itemcopy := deepCopyBlobItemOputput(item) inode.mu.Lock() defer inode.mu.Unlock() - inode.Attributes.Size = item.Size + inode.Attributes.Size = itemcopy.Size // don't want to point to the attribute because that // can get updated - size := item.Size + size := inode.Attributes.Size inode.KnownSize = &size if item.LastModified != nil { - inode.Attributes.Mtime = *item.LastModified + inode.Attributes.Mtime = *itemcopy.LastModified } else { inode.Attributes.Mtime = inode.fs.rootAttrs.Mtime } if item.ETag != nil { - inode.s3Metadata["etag"] = []byte(*item.ETag) - inode.knownETag = item.ETag + inode.s3Metadata["etag"] = []byte(*itemcopy.ETag) + inode.knownETag = itemcopy.ETag } else { delete(inode.s3Metadata, "etag") } if item.StorageClass != nil { - inode.s3Metadata["storage-class"] = []byte(*item.StorageClass) + inode.s3Metadata["storage-class"] = []byte(*itemcopy.StorageClass) } else { delete(inode.s3Metadata, "storage-class") } From e42018145058392a7595d7bb4bb87ec37a7bf7a2 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Fri, 6 Mar 2020 22:04:48 +0100 Subject: [PATCH 03/40] Add Scaleway to list of compatible non-AWS S3 providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Can confirm it works great 👍 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6bc7b107..cd5e2f49 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ goofys has been tested with the following non-AWS S3 providers: * OpenStack Swift * S3Proxy * Wasabi +* Scaleway Additionally, goofys also works with the following non-S3 object stores: From 8af9f7e35cbbebbd327e586bd44b5534088a4323 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Tue, 17 Mar 2020 09:01:07 +0100 Subject: [PATCH 04/40] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd5e2f49..f55935a7 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,8 @@ goofys has been tested with the following non-AWS S3 providers: * Minio (limited) * OpenStack Swift * S3Proxy -* Wasabi * Scaleway +* Wasabi Additionally, goofys also works with the following non-S3 object stores: From 17fb3d4f56cc34ca26e4e4721704464abf58d319 Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Fri, 26 Mar 2021 15:20:21 -0700 Subject: [PATCH 05/40] ETag could be nil, regression from previous commit --- internal/backend_adlv1.go | 4 ++-- internal/backend_adlv2.go | 12 ++++++------ internal/backend_azblob.go | 15 ++++----------- internal/handles.go | 13 +++++++------ internal/utils.go | 8 ++++++++ 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/internal/backend_adlv1.go b/internal/backend_adlv1.go index 67995b6a..6b695653 100644 --- a/internal/backend_adlv1.go +++ b/internal/backend_adlv1.go @@ -352,8 +352,8 @@ func (b *ADLv1) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { continuationToken = param.StartAfter } - _, prefixes, items, err := b.appendToListResults(nilStr(param.Prefix), - recursive, nilStr(continuationToken), param.MaxKeys, nil, nil) + _, prefixes, items, err := b.appendToListResults(NilStr(param.Prefix), + recursive, NilStr(continuationToken), param.MaxKeys, nil, nil) if err == fuse.ENOENT { err = nil } else if err != nil { diff --git a/internal/backend_adlv2.go b/internal/backend_adlv2.go index 19cb6c1e..baa6d3e3 100644 --- a/internal/backend_adlv2.go +++ b/internal/backend_adlv2.go @@ -384,7 +384,7 @@ func (b *ADLv2) listBlobs(param *ListBlobsInput, maxResults *int32) (adl2PathLis // read errors are transient and should probably be retried more for attempt := 0; attempt < 30; attempt++ { res, err = b.client.List(context.TODO(), param.Delimiter == nil, b.bucket, - nilStr(param.Prefix), nilStr(param.ContinuationToken), maxResults, + NilStr(param.Prefix), NilStr(param.ContinuationToken), maxResults, nil, "", nil, "") err = mapADLv2Error(res.Response.Response, err, false) if err == nil { @@ -479,7 +479,7 @@ func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { items = append(items, BlobItemOutput{ Key: &key, ETag: p.ETag, - LastModified: parseADLv2Time(nilStr(p.LastModified)), + LastModified: parseADLv2Time(NilStr(p.LastModified)), Size: uint64(p.contentLength()), }) } @@ -571,7 +571,7 @@ func (b *ADLv2) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { } res, err := b.client.Read(context.TODO(), b.bucket, param.Key, bytes, - "", nil, nilStr(param.IfMatch), "", "", "", + "", nil, NilStr(param.IfMatch), "", "", "", "", nil, "") if err != nil { return nil, mapADLv2Error(res.Response.Response, err, false) @@ -633,7 +633,7 @@ func (b *ADLv2) toADLProperties(metadata map[string]*string) string { func (b *ADLv2) create(key string, pathType adl2.PathResourceType, contentType *string, metadata map[string]*string, leaseId string) (resp autorest.Response, err error) { resp, err = b.client.Create(context.TODO(), b.bucket, key, - pathType, "", "", "", "", "", "", "", nilStr(contentType), + pathType, "", "", "", "", "", "", "", NilStr(contentType), "", "", "", "", leaseId, "", b.toADLProperties(metadata), "", "", "", "", "", "", "", "", "", "", "", nil, "") if err != nil { @@ -707,7 +707,7 @@ func (b *ADLv2) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { return nil, err } - flush, err := b.flush(param.Key, size, nilStr(param.ContentType), "") + flush, err := b.flush(param.Key, size, NilStr(param.ContentType), "") if err != nil { return nil, err } @@ -760,7 +760,7 @@ func (b *ADLv2) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBl } commitData := &ADLv2MultipartBlobCommitInput{ - ContentType: nilStr(param.ContentType), + ContentType: NilStr(param.ContentType), RenewLeaseStop: make(chan bool, 1), } diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index 7a2e63da..bf0a4d72 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -420,7 +420,7 @@ func nilMetadata(m map[string]*string) map[string]string { metadata := make(map[string]string) for k, v := range m { k = strings.ToLower(k) - metadata[k] = nilStr(v) + metadata[k] = NilStr(v) } return metadata } @@ -470,13 +470,6 @@ func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { }, nil } -func nilStr(v *string) string { - if v == nil { - return "" - } else { - return *v - } -} func nilUint32(v *uint32) uint32 { if v == nil { return 0 @@ -503,7 +496,7 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { var nextMarker *string options := azblob.ListBlobsSegmentOptions{ - Prefix: nilStr(param.Prefix), + Prefix: NilStr(param.Prefix), MaxResults: int32(nilUint32(param.MaxKeys)), Details: azblob.BlobListingDetails{ // blobfuse (following wasb) convention uses @@ -523,7 +516,7 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { azblob.Marker{ param.ContinuationToken, }, - nilStr(param.Delimiter), + NilStr(param.Delimiter), options) if err != nil { return nil, mapAZBError(err) @@ -811,7 +804,7 @@ func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { resp, err := blob.Upload(context.TODO(), body, azblob.BlobHTTPHeaders{ - ContentType: nilStr(param.ContentType), + ContentType: NilStr(param.ContentType), }, nilMetadata(param.Metadata), azblob.BlobAccessConditions{}) if err != nil { diff --git a/internal/handles.go b/internal/handles.go index 2976be67..210f3b67 100644 --- a/internal/handles.go +++ b/internal/handles.go @@ -107,12 +107,13 @@ func NewInode(fs *Goofys, parent *Inode, name *string) (inode *Inode) { func deepCopyBlobItemOputput(item *BlobItemOutput) BlobItemOutput { - key := *item.Key - etag := *item.ETag - lastmodified := *item.LastModified - var sc string - if item.StorageClass != nil { - sc = *item.StorageClass + key := NilStr(item.Key) + etag := NilStr(item.ETag) + sc := NilStr(item.StorageClass) + + var lastmodified time.Time + if item.LastModified != nil { + lastmodified = *item.LastModified } return BlobItemOutput{ diff --git a/internal/utils.go b/internal/utils.go index 08d8ae8a..2a26187e 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -117,6 +117,14 @@ func PTime(v time.Time) *time.Time { return &v } +func NilStr(v *string) string { + if v == nil { + return "" + } else { + return *v + } +} + func xattrEscape(value []byte) (s string) { for _, c := range value { if c == '%' { From ac8dd4bb72b2e3e19866a9977342274d67da0b1b Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Fri, 26 Mar 2021 15:39:17 -0700 Subject: [PATCH 06/40] upgrade s3proxy TestWriteReallyLargeFile seems to be OOM'ing in the JVM although it's not clear when it started happening --- Makefile | 2 +- internal/goofys_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c9ff9272..4d33aee8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ run-test: s3proxy.jar ./test/run-tests.sh s3proxy.jar: - wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.7.0/s3proxy -O s3proxy.jar + wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.8.0/s3proxy -O s3proxy.jar get-deps: s3proxy.jar go get -t ./... diff --git a/internal/goofys_test.go b/internal/goofys_test.go index a51bbeb1..ca3b3e5c 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -1104,6 +1104,9 @@ func (s *GoofysTest) TestWriteLargeFile(t *C) { } func (s *GoofysTest) TestWriteReallyLargeFile(t *C) { + if _, ok := s.cloud.(*S3Backend); ok && s.emulator { + t.Skip("seems to be OOM'ing S3proxy 1.8.0") + } s.testWriteFile(t, "testLargeFile", 512*1024*1024+1, 128*1024) } From a3542b1833a9de0735fcad6b74cc7966c6dba80a Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Fri, 26 Mar 2021 20:29:24 -0700 Subject: [PATCH 07/40] update aws-sdk-go also increased the timeout to one of the tests closes #595 closes #584 --- go.mod | 6 ++---- go.sum | 11 +++++++++++ internal/goofys_test.go | 12 ++++++++---- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 98b98ef1..8b4c9976 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e + github.com/aws/aws-sdk-go v1.38.7 github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe // indirect github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f @@ -30,9 +30,7 @@ require ( github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 // indirect github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c // indirect github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a - golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect - golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 - golang.org/x/text v0.3.2 // indirect + golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/ini.v1 v1.46.0 ) diff --git a/go.sum b/go.sum index cab425d0..a1b720d9 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e h1:VpbwzFX2LQDqZIqzADiJXE3yvLG2lp/sAV337Z89aqs= github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.38.7 h1:uOu2IrTiNhcSNAjBmA21t48lTx5mgGdcFKamDjXMscA= +github.com/aws/aws-sdk-go v1.38.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -56,6 +58,9 @@ github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f h1:qMeDFuReF4ZPMVRMfC github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jtolds/gls v4.2.0+incompatible h1:f7Me2eb3Wy6xfgSj1uyi0Ho6c4NY1L4KoX7c3auAhOI= github.com/jtolds/gls v4.2.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc h1:wNmVtupKbdEJcTkMIF2ah/nf3NzHEeNnpTR/1riaahQ= @@ -106,22 +111,27 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -129,3 +139,4 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/goofys_test.go b/internal/goofys_test.go index ca3b3e5c..8b77f90e 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -437,7 +437,10 @@ func (s *GoofysTest) setupDefaultEnv(t *C, public bool) { s.setupEnv(t, s.env, public) } -func (s *GoofysTest) setUpTestTimeout(t *C) { +func (s *GoofysTest) setUpTestTimeout(t *C, timeout time.Duration) { + if s.timeout != nil { + close(s.timeout) + } s.timeout = make(chan int) debug.SetTraceback("all") started := time.Now() @@ -448,9 +451,9 @@ func (s *GoofysTest) setUpTestTimeout(t *C) { if !ok { return } - case <-time.After(PerTestTimeout): + case <-time.After(timeout): panic(fmt.Sprintf("timeout %v reached. Started %v now %v", - PerTestTimeout, started, time.Now())) + timeout, started, time.Now())) } }() } @@ -458,7 +461,7 @@ func (s *GoofysTest) setUpTestTimeout(t *C) { func (s *GoofysTest) SetUpTest(t *C) { log.Infof("Starting at %v", time.Now()) - s.setUpTestTimeout(t) + s.setUpTestTimeout(t, PerTestTimeout) var bucket string mount := os.Getenv("MOUNT") @@ -1775,6 +1778,7 @@ func (s *GoofysTest) TestBenchLs(t *C) { s.fs.flags.TypeCacheTTL = 1 * time.Minute s.fs.flags.StatCacheTTL = 1 * time.Minute mountPoint := "/tmp/mnt" + s.fs.bucket + s.setUpTestTimeout(t, 20*time.Minute) s.runFuseTest(t, mountPoint, false, "../bench/bench.sh", "cat", mountPoint, "ls") } From 0b22270ba7f1bb2ec8f1bafa2f1f5da1537f5877 Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Fri, 26 Mar 2021 20:45:47 -0700 Subject: [PATCH 08/40] avoid double closing timeout channel --- internal/goofys_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 8b77f90e..3be88ead 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -313,6 +313,7 @@ func (s *GoofysTest) deleteBucket(cloud StorageBackend) error { func (s *GoofysTest) TearDownTest(t *C) { close(s.timeout) + s.timeout = nil for _, cloud := range s.removeBucket { err := s.deleteBucket(cloud) From 0c993271269b539196330a18716a33fbeeebd624 Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Fri, 26 Mar 2021 22:29:20 -0700 Subject: [PATCH 09/40] output done message for each file see if that will workaround travis's 10 minute no output timeout --- bench/bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/bench.sh b/bench/bench.sh index ea030006..5eb0e0e0 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -235,7 +235,7 @@ function create_files_parallel { get_howmany $@ (for i in $(seq 1 $howmany); do - echo $i > file$i & true + (echo $i > file$i && echo "created file$i") & true done wait) } From 697672b38efc714cdb828891ed1be8a6ecd886a7 Mon Sep 17 00:00:00 2001 From: Ben Sussman Date: Wed, 23 Sep 2020 15:32:01 -0400 Subject: [PATCH 10/40] use SetLevel() instead of Level = ? to prevent data race --- api/common/logger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/common/logger.go b/api/common/logger.go index 079f065f..5c321712 100644 --- a/api/common/logger.go +++ b/api/common/logger.go @@ -119,14 +119,14 @@ func GetLogger(name string) *LogHandle { if logger, ok := loggers[name]; ok { if name != "main" && name != "fuse" { - logger.Level = cloudLogLevel + logger.SetLevel(cloudLogLevel) } return logger } else { logger := NewLogger(name) loggers[name] = logger if name != "main" && name != "fuse" { - logger.Level = cloudLogLevel + logger.SetLevel(cloudLogLevel) } return logger } From c6e678cc66390198b28d66144d513f0b93ac5b65 Mon Sep 17 00:00:00 2001 From: Deka Akbar Date: Thu, 1 Apr 2021 01:17:30 +0000 Subject: [PATCH 11/40] Add GCS Backend to Goofys --- .gitmodules | 117 ------- README-gcs.md | 33 ++ api/api.go | 5 + api/common/conf_gcs.go | 35 ++ bench/Dockerfile.gcs | 44 +++ bench/bench.sh | 9 +- bench/run_bench.sh | 12 + go.mod | 46 +-- go.sum | 592 ++++++++++++++++++++++++++++++---- internal/backend.go | 34 +- internal/backend_gcs.go | 464 ++++++++++++++++++++++++++ internal/backend_gcs3.go | 14 +- internal/backend_gcs_test.go | 579 +++++++++++++++++++++++++++++++++ internal/goofys.go | 6 +- internal/goofys_test.go | 30 +- internal/parse_bucket_test.go | 49 +++ internal/utils.go | 59 ++++ test/run-tests.sh | 6 + 18 files changed, 1917 insertions(+), 217 deletions(-) delete mode 100644 .gitmodules create mode 100644 README-gcs.md create mode 100644 api/common/conf_gcs.go create mode 100644 bench/Dockerfile.gcs create mode 100644 internal/backend_gcs.go create mode 100644 internal/backend_gcs_test.go create mode 100644 internal/parse_bucket_test.go diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e7526dbb..00000000 --- a/.gitmodules +++ /dev/null @@ -1,117 +0,0 @@ -[submodule "vendor/github.com/aws/aws-sdk-go"] - path = vendor/github.com/aws/aws-sdk-go - url = https://github.com/aws/aws-sdk-go -[submodule "vendor/github.com/lsegal/gucumber"] - path = vendor/github.com/lsegal/gucumber - url = https://github.com/lsegal/gucumber -[submodule "vendor/github.com/smartystreets/goconvey"] - path = vendor/github.com/smartystreets/goconvey - url = https://github.com/smartystreets/goconvey -[submodule "vendor/github.com/smartystreets/assertions"] - path = vendor/github.com/smartystreets/assertions - url = https://github.com/smartystreets/assertions -[submodule "vendor/github.com/jtolds/gls"] - path = vendor/github.com/jtolds/gls - url = https://github.com/jtolds/gls -[submodule "vendor/github.com/jacobsa/fuse"] - path = vendor/github.com/jacobsa/fuse - url = https://github.com/kahing/fusego -[submodule "vendor/github.com/Azure/go-autorest"] - path = vendor/github.com/Azure/go-autorest - url = https://github.com/Azure/go-autorest/ -[submodule "vendor/github.com/Azure/azure-pipeline-go"] - path = vendor/github.com/Azure/azure-pipeline-go - url = https://github.com/Azure/azure-pipeline-go/ -[submodule "vendor/github.com/Azure/azure-sdk-for-go"] - path = vendor/github.com/Azure/azure-sdk-for-go - url = https://github.com/Azure/azure-sdk-for-go -[submodule "vendor/github.com/Azure/azure-storage-blob-go"] - path = vendor/github.com/Azure/azure-storage-blob-go - url = https://github.com/Azure/azure-storage-blob-go/ -[submodule "vendor/github.com/google/uuid"] - path = vendor/github.com/google/uuid - url = https://github.com/google/uuid -[submodule "vendor/github.com/sirupsen/logrus"] - path = vendor/github.com/sirupsen/logrus - url = https://github.com/sirupsen/logrus -[submodule "vendor/github.com/shirou/gopsutil"] - path = vendor/github.com/shirou/gopsutil - url = https://github.com/shirou/gopsutil/ -[submodule "vendor/github.com/kardianos/osext"] - path = vendor/github.com/kardianos/osext - url = https://github.com/kardianos/osext -[submodule "vendor/github.com/mitchellh/go-homedir"] - path = vendor/github.com/mitchellh/go-homedir - url = https://github.com/mitchellh/go-homedir -[submodule "vendor/github.com/satori/go.uuid"] - path = vendor/github.com/satori/go.uuid - url = https://github.com/satori/go.uuid -[submodule "vendor/github.com/sevlyar/go-daemon"] - path = vendor/github.com/sevlyar/go-daemon - url = https://github.com/sevlyar/go-daemon -[submodule "vendor/github.com/urfave/cli"] - path = vendor/github.com/urfave/cli - url = https://github.com/urfave/cli -[submodule "vendor/contrib.go.opencensus.io/exporter/ocagent"] - path = vendor/contrib.go.opencensus.io/exporter/ocagent - url = https://github.com/census-ecosystem/opencensus-go-exporter-ocagent -[submodule "vendor/github.com/census-instrumentation/opencensus-proto"] - path = vendor/github.com/census-instrumentation/opencensus-proto - url = https://github.com/census-instrumentation/opencensus-proto/ -[submodule "vendor/github.com/mattn/go-ieproxy"] - path = vendor/github.com/mattn/go-ieproxy - url = https://github.com/mattn/go-ieproxy -[submodule "vendor/google.golang.org/grpc"] - path = vendor/google.golang.org/grpc - url = https://github.com/grpc/grpc-go -[submodule "vendor/go.opencensus.io"] - path = vendor/go.opencensus.io - url = https://github.com/census-instrumentation/opencensus-go -[submodule "vendor/github.com/dgrijalva/jwt-go"] - path = vendor/github.com/dgrijalva/jwt-go - url = https://github.com/dgrijalva/jwt-go -[submodule "vendor/github.com/dimchansky/utfbom"] - path = vendor/github.com/dimchansky/utfbom - url = https://github.com/dimchansky/utfbom -[submodule "vendor/github.com/golang/protobuf"] - path = vendor/github.com/golang/protobuf - url = https://github.com/golang/protobuf/ -[submodule "vendor/github.com/grpc-ecosystem/grpc-gateway"] - path = vendor/github.com/grpc-ecosystem/grpc-gateway - url = https://github.com/grpc-ecosystem/grpc-gateway/ -[submodule "vendor/github.com/hashicorp/golang-lru"] - path = vendor/github.com/hashicorp/golang-lru - url = https://github.com/hashicorp/golang-lru/ -[submodule "vendor/golang.org/x/crypto"] - path = vendor/golang.org/x/crypto - url = https://go.googlesource.com/crypto -[submodule "vendor/golang.org/x/net"] - path = vendor/golang.org/x/net - url = https://go.googlesource.com/net -[submodule "vendor/golang.org/x/text"] - path = vendor/golang.org/x/text - url = https://go.googlesource.com/text -[submodule "vendor/golang.org/x/sys"] - path = vendor/golang.org/x/sys - url = https://go.googlesource.com/sys -[submodule "vendor/google.golang.org/genproto"] - path = vendor/google.golang.org/genproto - url = https://github.com/google/go-genproto -[submodule "vendor/google.golang.org/api"] - path = vendor/google.golang.org/api - url = https://code.googlesource.com/google-api-go-client -[submodule "vendor/gopkg.in/ini.v1"] - path = vendor/gopkg.in/ini.v1 - url = https://gopkg.in/ini.v1 -[submodule "vendor/golang.org/x/sync"] - path = vendor/golang.org/x/sync - url = https://go.googlesource.com/sync -[submodule "vendor/gopkg.in/check.v1"] - path = vendor/gopkg.in/check.v1 - url = https://gopkg.in/check.v1 -[submodule "vendor/github.com/kr/pretty"] - path = vendor/github.com/kr/pretty - url = https://github.com/kr/pretty.git -[submodule "vendor/github.com/kr/text"] - path = vendor/github.com/kr/text - url = https://github.com/kr/text.git diff --git a/README-gcs.md b/README-gcs.md new file mode 100644 index 00000000..cf44f812 --- /dev/null +++ b/README-gcs.md @@ -0,0 +1,33 @@ +# Google Cloud Storage (GCS) + + +## Prerequisite + +Service Account credentials or user authentication. Ensure that either the service account or user has the proper permissions to the Bucket / Object under GCS. + +For example, read-only access should be granted the following permissions: +``` +storage.objects.get +storage.objects.list +```` + +### Service Account credentials + +Create a service account credentials (https://cloud.google.com/iam/docs/creating-managing-service-accounts) and generate the JSON credentials file. + +### User Authentication and `gcloud` Default Authentication +User can authenticate to gcloud's default environment by first installing cloud sdk (https://cloud.google.com/sdk/) and running `gcloud auth application-default login` command. + + +## Using Goofys for GCS + +### With service account credentials file +``` +GOOGLE_APPLICATION_CREDENTIALS="/path/to/creds.json" goofys gs://[BUCKET] /path/to/mount +``` + +### With user authentication (`gcloud auth application-default login`) + +``` +goofys gs://[BUCKET] [MOUNT DIRECTORY] +``` \ No newline at end of file diff --git a/api/api.go b/api/api.go index f8ce2e39..f78cf082 100644 --- a/api/api.go +++ b/api/api.go @@ -25,6 +25,7 @@ func Mount( if flags.DebugS3 { SetCloudLogLevel(logrus.DebugLevel) } + // Mount the file system. mountCfg := &fuse.MountConfig{ FSName: bucketName, @@ -106,6 +107,10 @@ func Mount( if spec.Prefix != "" { bucketName += ":" + spec.Prefix } + case "gs": + config := NewGCSConfig() + bucketName = spec.Bucket + flags.Backend = config } } } diff --git a/api/common/conf_gcs.go b/api/common/conf_gcs.go new file mode 100644 index 00000000..01543f96 --- /dev/null +++ b/api/common/conf_gcs.go @@ -0,0 +1,35 @@ +package common + +import ( + "context" + "golang.org/x/oauth2/google" +) + +type GCSConfig struct { + Credentials *google.Credentials + ChunkSize int // determine the size of chunks when uploading a file to GCS +} + +const ( + defaultGCSChunkSize int = 64 * 1024 * 1024 +) + +// NewGCSConfig returns a GCS Config. +func NewGCSConfig() *GCSConfig { + // Credentials will be checked when initializing GCS bucket to create an authenticated / unauthenticated client + // We allow nil credentials for unauthenticated client. + credentials, err := google.FindDefaultCredentials(context.Background()) + if err == nil { + // authenticated config + return &GCSConfig{ + ChunkSize: defaultGCSChunkSize, + Credentials: credentials, + } + } else { + log.Debugf("Initializing an unauthenticated config: %v", err) + // unauthenticated config + return &GCSConfig{ + ChunkSize: defaultGCSChunkSize, + } + } +} diff --git a/bench/Dockerfile.gcs b/bench/Dockerfile.gcs new file mode 100644 index 00000000..8a51c128 --- /dev/null +++ b/bench/Dockerfile.gcs @@ -0,0 +1,44 @@ +FROM golang:1.14 AS goofys-builder + +# install goofys +WORKDIR $GOPATH/src/github.com/kahing/goofys + +COPY . . + +RUN git init +RUN git submodule update --init --recursive +RUN go build + +# install gcsfuse, the binary is added to /go/bin/gcsfuse +RUN go get -u github.com/googlecloudplatform/gcsfuse + +FROM ubuntu:18.04 +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get -y install --no-install-recommends \ + # gcsfuse dependencies \ + fuse \ + # for running goofys benchmark \ + curl python-setuptools python-pip gnuplot-nox imagemagick awscli \ + # finally, clean up to make image smaller \ + && apt-get clean + +# install catfs, required to run goofys with cache +RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ + mv catfs /usr/bin && chmod 0755 /usr/bin/catfs + +# goofys graph generation +RUN pip install numpy + +ENV PATH=$PATH:/root/go/bin + +# copy go binaries +COPY --from=goofys-builder /go/src/github.com/kahing/goofys/goofys /root/go/bin/goofys +COPY --from=goofys-builder /go/bin/gcsfuse /root/go/bin/gcsfuse + +WORKDIR /root/go/src/github.com/kahing/goofys + +# copy bench scripts +COPY bench bench + +ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"] diff --git a/bench/bench.sh b/bench/bench.sh index 5eb0e0e0..d08cb2ef 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -5,6 +5,7 @@ : ${test:=""} : ${CACHE:="false"} : ${CLEANUP:="true"} +: ${MOUNT_LOG:="mount.log"} iter=10 @@ -49,7 +50,7 @@ else BLOBFUSE="false" fi -$cmd >& mount.log & +$cmd >& $MOUNT_LOG & PID=$! function cleanup { @@ -73,7 +74,7 @@ function cleanup { function cleanup_err { err=$? - cat mount.log + cat $MOUNT_LOG if [ $MOUNTED == 1 ]; then popd >&/dev/null || true rmdir $prefix >&/dev/null || true @@ -99,7 +100,7 @@ function wait_for_mount { done if ! grep -q $mnt /proc/mounts; then echo "$mnt not mounted by $cmd" - cat mount.log + cat $MOUNT_LOG exit 1 fi } @@ -144,7 +145,7 @@ function run_test { fusermount -u $mnt sleep 1 fi - $cmd >& mount.log & + $cmd >& $MOUNT_LOG & PID=$! wait_for_mount pushd "$prefix" >/dev/null diff --git a/bench/run_bench.sh b/bench/run_bench.sh index b82e419b..63048586 100755 --- a/bench/run_bench.sh +++ b/bench/run_bench.sh @@ -24,6 +24,7 @@ dir=$(dirname $0) mkdir bench-mnt S3FS_CACHE="-ouse_cache=/tmp/cache" +GCSFUSE_CACHE="" # by default performs caching GOOFYS_CACHE="--cache /tmp/cache -o allow_other" if [ "$CACHE" == "false" ]; then @@ -51,6 +52,12 @@ if [ -f /usr/local/bin/blobfuse ]; then GOOFYS_ENDPOINT="" fi +if [ -f /root/go/bin/gcsfuse ]; then + PROG="gcsfuse" + GOOFYS_BUCKET="gs://${BUCKET}" + GOOFYS_ENDPOINT="" +fi + rm -f $dir/bench.goofys $dir/bench.$PROG $dir/bench.png $dir/bench-cached.png export BUCKET @@ -59,6 +66,7 @@ export ENDPOINT S3FS="s3fs -f -ostat_cache_expire=1 ${S3FS_CACHE} ${S3FS_ENDPOINT} $BUCKET bench-mnt" GOOFYS="goofys -f --stat-cache-ttl 1s --type-cache-ttl 1s ${GOOFYS_CACHE} ${GOOFYS_ENDPOINT} ${GOOFYS_BUCKET} bench-mnt" BLOBFUSE="blobfuse bench-mnt --container-name=$BUCKET --tmp-path=/tmp/cache" +GCSFUSE="gcsfuse --temp-dir /tmp/cache --foreground $GCSFUSE_CACHE $BUCKET bench-mnt" LOCAL="cat" iter=10 @@ -108,6 +116,10 @@ for fs in $PROG goofys; do FS=$BLOBFUSE CREATE_FS=$FS ;; + gcsfuse) + FS=$GCSFUSE + CREATE_FS=$FS + ;; cat) FS=$LOCAL CREATE_FS=$FS diff --git a/go.mod b/go.mod index 8b4c9976..942570e4 100644 --- a/go.mod +++ b/go.mod @@ -3,36 +3,42 @@ module github.com/kahing/goofys go 1.14 require ( - github.com/Azure/azure-pipeline-go v0.2.2 - github.com/Azure/azure-sdk-for-go v32.1.0+incompatible - github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 + cloud.google.com/go/storage v1.14.0 + github.com/Azure/azure-pipeline-go v0.2.3 + github.com/Azure/azure-sdk-for-go v52.6.0+incompatible + github.com/Azure/azure-storage-blob-go v0.13.0 github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/adal v0.9.13 github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/aws/aws-sdk-go v1.38.7 - github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f - github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe // indirect - github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f - github.com/jtolds/gls v4.2.0+incompatible // indirect + github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect + github.com/aws/aws-sdk-go v1.38.10 + github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 + github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/google/uuid v1.2.0 + github.com/hashicorp/go-retryablehttp v0.6.8 + github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/pkg/errors v0.9.1 // indirect - github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 + github.com/satori/go.uuid v1.2.0 github.com/sevlyar/go-daemon v0.1.5 - github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 - github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 - github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 // indirect - github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c // indirect - github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a - golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 - gopkg.in/ini.v1 v1.46.0 + github.com/shirou/gopsutil v3.21.2+incompatible + github.com/sirupsen/logrus v1.8.1 + github.com/smartystreets/goconvey v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/urfave/cli v1.22.5 + golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 + google.golang.org/api v0.43.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c + gopkg.in/ini.v1 v1.62.0 ) replace github.com/jacobsa/fuse => github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc + diff --git a/go.sum b/go.sum index a1b720d9..015a8273 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,56 @@ -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= -github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= -github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0 h1:oqqswrt4x6b9OGBnNqdssxBl1xf0rSUNjU2BR4BZar0= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v52.6.0+incompatible h1:F/feBa+/Oxbu+Zprnsiq0b6rvUVlOEx3jSqCSNdtF3U= +github.com/Azure/azure-sdk-for-go v52.6.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc= +github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= @@ -31,112 +72,533 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e h1:VpbwzFX2LQDqZIqzADiJXE3yvLG2lp/sAV337Z89aqs= -github.com/aws/aws-sdk-go v1.17.14-0.20190307201122-7e7bb328b06e/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.38.7 h1:uOu2IrTiNhcSNAjBmA21t48lTx5mgGdcFKamDjXMscA= -github.com/aws/aws-sdk-go v1.38.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= +github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/aws/aws-sdk-go v1.38.10 h1:7lQrjAlyYrTGW2+9vnBv5HPSSuv+xDMmgU1YUnNSOOo= +github.com/aws/aws-sdk-go v1.38.10/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe h1:rcf1P0fm+1l0EjG16p06mYLj9gW9X36KgdHJ/88hS4g= -github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f h1:qMeDFuReF4ZPMVRMfCESfcHD7vQuCFpeiS17Dx4HYxU= -github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= +github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291 h1:GoZXB1SHOnDgMmFNYvgQT5hNoAM8JfyoTIOw0ILj7m4= +github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jtolds/gls v4.2.0+incompatible h1:f7Me2eb3Wy6xfgSj1uyi0Ho6c4NY1L4KoX7c3auAhOI= -github.com/jtolds/gls v4.2.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc h1:wNmVtupKbdEJcTkMIF2ah/nf3NzHEeNnpTR/1riaahQ= -github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc/go.mod h1:Qk5T6wYqwRWfKHhEKkRsIXwsAtQ0GK8rBJNYf5LVdwo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 h1:lqGuhK6ejK9x8b6+GPGXm0gzrajIKB7yNKL2fx1oqSU= -github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= -github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= -github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= -github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= -github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 h1:cPg+LIQqTg4YaxZsUtAus45xKwqpSG5SQrSoOuWKUg4= -github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c h1:RlFdHEfqB3LI6Q8YfB1EQPO2+kED+w9TdVaMwrs6VPo= -github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/shirou/gopsutil v3.21.2+incompatible h1:U+YvJfjCh6MslYlIAXvPtzhW3YZEtc9uncueUNpD/0A= +github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= -github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0 h1:4sAyIHT6ZohtAQDoxws+ez7bROYmUlOVvsUscYCDTqA= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 h1:4Xw2NwItrJOFR5s6PnK98PI6Bgw1LhMP1j/rO5WP0S4= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= -gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/backend.go b/internal/backend.go index 4da7eb55..414e291d 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -26,6 +26,7 @@ import ( ) type Capabilities struct { + // set this to true to disable parallel upload NoParallelMultipart bool MaxMultipartSize uint64 // indicates that the blob store has native support for directories @@ -162,7 +163,7 @@ type MultipartBlobCommitInput struct { Parts []*string NumParts uint32 - // for GCS + // additional backend specific data backendData interface{} } @@ -278,10 +279,41 @@ func (p sortBlobItemOutput) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (l ListBlobsInput) String() string { + return fmt.Sprintf("ListBlobsInput{Prefix: %v, Delim: %v, StartAfter: %v, MaxKeys: %v, ContToken: %v}", + NilStr(l.Prefix), + NilStr(l.Delimiter), + NilStr(l.StartAfter), + NilUint32(l.MaxKeys), + NilStr(l.ContinuationToken)) +} + +func (b GetBlobInput) String() string { + return fmt.Sprintf("GetBlobInput{Key: %s, Start: %d, Count: %d}", b.Key, b.Start, b.Count) +} + func (b BlobItemOutput) String() string { return fmt.Sprintf("%v: %v", *b.Key, b.Size) } +func (b PutBlobInput) String() string { + return fmt.Sprintf("PutBlobInput{Key: %s, Size: %d, ContentType: %s}", b.Key, NilUint64(b.Size), + NilStr(b.ContentType)) +} + +func (b CopyBlobInput) String() string { + return fmt.Sprintf("CopyBlobInput{Src: %s, Dest: %s, Size: %d}", b.Source, b.Destination, b.Size) +} + +func (b MultipartBlobBeginInput) String() string { + return fmt.Sprintf("MultipartBlobBeginInput{Key: %s, ContentType: %s}", b.Key, NilStr(b.ContentType)) +} + +func (b MultipartBlobAddInput) String() string { + return fmt.Sprintf("MultipartBlobAddInput{Key: %s, Offset: %d, Size: %d, Last: %v}", + NilStr(b.Commit.Key), b.Offset, b.Size, b.Last) +} + func (b BlobPrefixOutput) String() string { return fmt.Sprintf("%v", *b.Prefix) } diff --git a/internal/backend_gcs.go b/internal/backend_gcs.go new file mode 100644 index 00000000..2b11cb25 --- /dev/null +++ b/internal/backend_gcs.go @@ -0,0 +1,464 @@ +package internal + +import ( + "github.com/kahing/goofys/api/common" + + "bytes" + "context" + "fmt" + "io" + "net" + "strings" + "syscall" + + "cloud.google.com/go/storage" + "github.com/jacobsa/fuse" + + "golang.org/x/sync/errgroup" + syncsem "golang.org/x/sync/semaphore" + + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCSBackend struct { + bucketName string + config *common.GCSConfig // stores user and bucket configuration + cap Capabilities + bucket *storage.BucketHandle // provides set of methods to operate on a bucket + logger *common.LogHandle // logger for GCS backend +} + +const ( + maxListKeys int = 1000 // the max limit for number of elements during listObjects +) + +type GCSMultipartBlobCommitInput struct { + cancel context.CancelFunc // useful to abort a multipart upload in GCS + writer *storage.Writer // used to emulate mpu under GCS, which currently used a single gcsWriter +} + +// NewGCS initializes a GCS Backend. +// It creates an authenticated client or unauthenticated client based on existing credentials in the environment. +func NewGCS(bucket string, config *common.GCSConfig) (*GCSBackend, error) { + var client *storage.Client + var err error + + // TODO: storage.NewClient has automated mechanisms to set up credentials together with HTTP settings. + // Currently, we are using config.Credentials only to differentiate between creating an authenticated or + // unauthenticated client not using it to initialize a client. + + // If config.Credentials are configured, we'll get an authenticated client. + if config.Credentials != nil { + client, err = storage.NewClient(context.Background()) + } else { + // otherwise we will get an unauthenticated client. option.WithoutAuthentication() is necessary + // because the API will generate an error if it could not find credentials and this option is unset. + client, err = storage.NewClient(context.Background(), option.WithoutAuthentication()) + } + + if err != nil { + return nil, err + } + + return &GCSBackend{ + config: config, + bucketName: bucket, + bucket: client.Bucket(bucket), + cap: Capabilities{ + MaxMultipartSize: 5 * 1024 * 1024 * 1024, + Name: "gcs", + // parallel multipart upload is not supported in GCS + NoParallelMultipart: true, + }, + logger: common.GetLogger("gcs"), + }, nil +} + +// Init checks user's access to bucket. +func (g *GCSBackend) Init(key string) error { + g.logger.Debugf("INIT GCS: %s", key) + _, err := g.bucket.Object(key).Attrs(context.Background()) + + // Currently we return successful mounts on 404 errors. However, GCS does not differentiate bucket not found and + // object not found errors. So, even though we don't know that the bucket exists, we will still mount successfully. + if err == storage.ErrObjectNotExist { + err = nil + } + + return mapGCSError(err) +} + +func (g *GCSBackend) Capabilities() *Capabilities { + return &g.cap +} + +// Bucket returns the GCSBackend's bucket name. +func (g *GCSBackend) Bucket() string { + return g.bucketName +} + +func getDebugResponseStatus(err error) string { + if err != nil { + return fmt.Sprintf("ERROR: %v", err) + } + return "SUCCESS" +} + +// HeadBlob gets the file object metadata. +func (g *GCSBackend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { + attrs, err := g.bucket.Object(param.Key).Attrs(context.Background()) + g.logger.Debugf("HEAD %v = %v", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &HeadBlobOutput{ + BlobItemOutput: BlobItemOutput{ + Key: &attrs.Name, + ETag: &attrs.Etag, + LastModified: &attrs.Updated, + Size: uint64(attrs.Size), + StorageClass: &attrs.StorageClass, + }, + ContentType: &attrs.ContentType, + IsDirBlob: strings.HasSuffix(param.Key, "/"), + Metadata: PMetadata(attrs.Metadata), + }, nil +} + +func (g *GCSBackend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { + query := storage.Query{ + Prefix: NilStr(param.Prefix), + Delimiter: NilStr(param.Delimiter), + StartOffset: NilStr(param.StartAfter), + } + objectIterator := g.bucket.Objects(context.Background(), &query) + + // Set max keys, a number > 0 is required by the SDK. + maxKeys := int(NilUint32(param.MaxKeys)) + if maxKeys == 0 { + maxKeys = maxListKeys // follow the default JSON API mechanism to return 1000 items if maxKeys is not set. + } + + pager := iterator.NewPager(objectIterator, maxKeys, NilStr(param.ContinuationToken)) + + var entries []*storage.ObjectAttrs + nextToken, err := pager.NextPage(&entries) + g.logger.Debugf("LIST %s : %s", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + var nextContToken *string + if nextToken != "" { + nextContToken = &nextToken + } + + var prefixes []BlobPrefixOutput + var items []BlobItemOutput + for _, entry := range entries { + // if blob is a prefix, then Prefix field will be set + if entry.Prefix != "" { + prefixes = append(prefixes, BlobPrefixOutput{&entry.Prefix}) + } else if entry.Name != "" { // otherwise for actual blob, Name field will set + items = append(items, BlobItemOutput{ + Key: &entry.Name, + ETag: &entry.Etag, + LastModified: &entry.Updated, + Size: uint64(entry.Size), + StorageClass: &entry.StorageClass, + }) + } else { + log.Errorf("LIST Unknown object: %v", entry) + } + } + + return &ListBlobsOutput{ + Prefixes: prefixes, + Items: items, + NextContinuationToken: nextContToken, + IsTruncated: nextContToken != nil, + }, nil +} + +func (g *GCSBackend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { + err := g.bucket.Object(param.Key).Delete(context.Background()) + + g.logger.Debugf("DELETE Object %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &DeleteBlobOutput{}, nil +} + +// DeleteBlobs deletes multiple GCS blobs. +func (g *GCSBackend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { + // The go sdk does not support batch requests: https://issuetracker.google.com/issues/142641783 + // So we're using goroutines and errorgroup to delete multiple objects + eg, rootCtx := errgroup.WithContext(context.Background()) + sem := syncsem.NewWeighted(100) + + for _, item := range param.Items { + if err := sem.Acquire(rootCtx, 1); err != nil { + return nil, err + } + curItem := item + eg.Go(func() error { + defer sem.Release(1) + return g.bucket.Object(curItem).Delete(rootCtx) + }) + } + + if err := eg.Wait(); err != nil { + return nil, mapGCSError(err) + } + + return &DeleteBlobsOutput{}, nil +} + +// RenameBlob is not supported for GCS backend. So Goofys will do a CopyBlob followed by DeleteBlob for renames. +func (g *GCSBackend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { + return nil, syscall.ENOTSUP +} + +// CopyBlob copies a source object to another destination object under the same bucket. +func (g *GCSBackend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { + src := g.bucket.Object(param.Source) + dest := g.bucket.Object(param.Destination) + + copier := dest.CopierFrom(src) + copier.StorageClass = NilStr(param.StorageClass) + copier.Etag = NilStr(param.ETag) + copier.Metadata = NilMetadata(param.Metadata) + + _, err := copier.Run(context.Background()) + g.logger.Debugf("Copy object %s = %s ", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &CopyBlobOutput{}, nil +} + +// GetBlob returns a file reader for a GCS object. +func (g *GCSBackend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { + obj := g.bucket.Object(param.Key).ReadCompressed(true) + + var reader *storage.Reader + var err error + if param.Count != 0 { + reader, err = obj.NewRangeReader(context.Background(), int64(param.Start), int64(param.Count)) + } else if param.Start != 0 { + reader, err = obj.NewRangeReader(context.Background(), int64(param.Start), -1) + } else { + // If we don't limit the range, the full object will be read + reader, err = obj.NewReader(context.Background()) + } + + g.logger.Debugf("GET Blob %s = %v", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + // Caveats: the SDK's reader object doesn't provide ETag, StorageClass, and Metadata attributes within a single + // API call, hence we're not returning these information in the output. + // Relevant GitHub issue: https://github.com/googleapis/google-cloud-go/issues/2740 + return &GetBlobOutput{ + HeadBlobOutput: HeadBlobOutput{ + BlobItemOutput: BlobItemOutput{ + Key: PString(param.Key), + LastModified: &reader.Attrs.LastModified, + Size: uint64(reader.Attrs.Size), + }, + ContentType: &reader.Attrs.ContentType, + }, + Body: reader, + }, nil +} + +// PutBlob writes a file to GCS. +func (g *GCSBackend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { + // Handle nil pointer error when param.Body is nil + body := param.Body + if body == nil { + body = bytes.NewReader([]byte("")) + } + + writer := g.bucket.Object(param.Key).NewWriter(context.Background()) + writer.ContentType = NilStr(param.ContentType) + writer.Metadata = NilMetadata(param.Metadata) + // setting chunkSize to be equal to the file size will make this a single request upload + writer.ChunkSize = int(NilUint64(param.Size)) + + _, err := io.Copy(writer, body) + g.logger.Debugf("PUT Blob (to writer) %s = %s ", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + err = writer.Close() + g.logger.Debugf("PUT Blob (Flush) %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + attrs := writer.Attrs() + + return &PutBlobOutput{ + ETag: &attrs.Etag, + //LastModified: &attrs.Updated, // this field exist in the upstream open source goofys repo + StorageClass: &attrs.StorageClass, + }, nil +} + +// MultipartBlobBegin begins a multi part blob request. +// Under GCS backend, we'll initialize the gcsWriter object and the context for the multipart blob request here. +func (g *GCSBackend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { + ctx, cancel := context.WithCancel(context.Background()) + writer := g.bucket.Object(param.Key).NewWriter(ctx) + writer.ChunkSize = g.config.ChunkSize + writer.ContentType = NilStr(param.ContentType) + writer.Metadata = NilMetadata(param.Metadata) + + g.logger.Debugf("Multipart Blob BEGIN: %s", param) + + return &MultipartBlobCommitInput{ + Key: ¶m.Key, + Metadata: param.Metadata, + backendData: &GCSMultipartBlobCommitInput{ + writer: writer, + cancel: cancel, + }, + }, nil +} + +// MultipartBlobAdd adds part of blob to the upload request. +// Under GCS backend, we'll write that blob part into the gcsWriter. +// TODO(deka): This is a temporary implementation to allow most tests to run. +// We might change this implementation in the future. +func (g *GCSBackend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { + commitData, ok := param.Commit.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + + // Handle nil pointer error when param.Body is nil + body := param.Body + if body == nil { + body = bytes.NewReader([]byte("")) + } + + n, err := io.Copy(commitData.writer, body) + g.logger.Debugf("Multipart Blob ADD %s bytesWritten: %v = %s", param, n, getDebugResponseStatus(err)) + if err != nil { + commitData.cancel() + return nil, err + } + + return &MultipartBlobAddOutput{}, nil +} + +func (g *GCSBackend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { + commitData, ok := param.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + g.logger.Debugf("Multipart Blob ABORT %v", param.Key) + commitData.cancel() + + return &MultipartBlobAbortOutput{}, nil +} + +func (g *GCSBackend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { + commitData, ok := param.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + + // Flushing a writer will make GCS to fully upload the buffer + err := commitData.writer.Close() + g.logger.Debugf("Multipart Blob COMMIT %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + commitData.cancel() + return nil, mapGCSError(err) + } + attrs := commitData.writer.Attrs() + + return &MultipartBlobCommitOutput{ + ETag: &attrs.Etag, + }, nil +} + +func (g *GCSBackend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { + // No-op: GCS expires a resumable session after 7 days automatically + return &MultipartExpireOutput{}, nil +} + +func (g *GCSBackend) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { + err := g.bucket.Delete(context.Background()) + if err != nil { + return nil, mapGCSError(err) + } + return &RemoveBucketOutput{}, nil +} + +func (g *GCSBackend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { + // Requires an authenticated credentials + err := g.bucket.Create(context.Background(), g.config.Credentials.ProjectID, nil) + if err != nil { + return nil, mapGCSError(err) + } + + return &MakeBucketOutput{}, nil +} + +func (g *GCSBackend) Delegate() interface{} { + return g +} + +// mapGCSError maps an error to syscall / fuse errors. +func mapGCSError(err error) error { + if err == nil { + return nil + } + + if err == storage.ErrObjectNotExist { + return fuse.ENOENT + } + + // this error can be returned during list operation if the bucket does not exist + if err == storage.ErrBucketNotExist { + return syscall.ENXIO + } + + if e, ok := err.(*googleapi.Error); ok { + switch e.Code { + case 409: + return fuse.EEXIST + case 404: + return fuse.ENOENT + // Retryable errors: + // https://cloud.google.com/storage/docs/json_api/v1/status-codes#429_Too_Many_Requests + // https://cloud.google.com/storage/docs/json_api/v1/status-codes#500_Internal_Server_Error + case 429, 500, 502, 503, 504: + return syscall.EAGAIN + default: + // return syscall error if it's not nil + fuseErr := mapHttpError(e.Code) + if fuseErr != nil { + return fuseErr + } + } + } + + if e, ok := err.(net.Error); ok { + if e.Timeout() { + return syscall.ETIMEDOUT + } + } + + return err +} diff --git a/internal/backend_gcs3.go b/internal/backend_gcs3.go index 7c7272f9..e3b4dc72 100644 --- a/internal/backend_gcs3.go +++ b/internal/backend_gcs3.go @@ -34,7 +34,7 @@ type GCS3 struct { *S3Backend } -type GCSMultipartBlobCommitInput struct { +type GCS3MultipartBlobCommitInput struct { Size uint64 ETag *string Prev *MultipartBlobAddInput @@ -45,7 +45,7 @@ func NewGCS3(bucket string, flags *FlagStorage, config *S3Config) (*GCS3, error) if err != nil { return nil, err } - s3Backend.Capabilities().Name = "gcs" + s3Backend.Capabilities().Name = "gcs3" s := &GCS3{S3Backend: s3Backend} s.S3Backend.gcs = true s.S3Backend.cap.NoParallelMultipart = true @@ -125,7 +125,7 @@ func (s *GCS3) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlo Metadata: param.Metadata, UploadId: &location, Parts: make([]*string, 10000), // at most 10K parts - backendData: &GCSMultipartBlobCommitInput{}, + backendData: &GCS3MultipartBlobCommitInput{}, }, nil } @@ -183,9 +183,9 @@ func (s *GCS3) uploadPart(param *MultipartBlobAddInput, totalSize uint64, last b } func (s *GCS3) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { - var commitData *GCSMultipartBlobCommitInput + var commitData *GCS3MultipartBlobCommitInput var ok bool - if commitData, ok = param.Commit.backendData.(*GCSMultipartBlobCommitInput); !ok { + if commitData, ok = param.Commit.backendData.(*GCS3MultipartBlobCommitInput); !ok { panic("Incorrect commit data type") } @@ -210,9 +210,9 @@ func (s *GCS3) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAdd } func (s *GCS3) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { - var commitData *GCSMultipartBlobCommitInput + var commitData *GCS3MultipartBlobCommitInput var ok bool - if commitData, ok = param.backendData.(*GCSMultipartBlobCommitInput); !ok { + if commitData, ok = param.backendData.(*GCS3MultipartBlobCommitInput); !ok { panic("Incorrect commit data type") } diff --git a/internal/backend_gcs_test.go b/internal/backend_gcs_test.go new file mode 100644 index 00000000..660530cd --- /dev/null +++ b/internal/backend_gcs_test.go @@ -0,0 +1,579 @@ +package internal + +import ( + "github.com/kahing/goofys/api/common" + + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "sort" + "syscall" + + "github.com/jacobsa/fuse" + "golang.org/x/oauth2/google" + "golang.org/x/sync/errgroup" + . "gopkg.in/check.v1" +) + +type GCSTestSpec struct { + objCount int // number of non-prefix blobs uploaded to the bucket + prefixCount int // number of prefix blobs uploaded to the bucket + existingObjKey string // test setup will add a blob with this name + existingObjContent string + existingObjLen uint64 + defaultObjLen uint64 // blobs created in the setup will be of this length + nonPrefixObjects []string + prefixObjects []string + prefixes []string // these are the prefixes of the prefixObjects + metadata map[string]*string // default Metadata for all blobs + contentType *string // default Content-Type for all blobs + // will be reset in SetUpTest, optionally modified on each test, and cleaned up in TearDownTest + additionalBlobs []string +} + +type GCSBackendTest struct { + gcsBackend *GCSBackend + bucketName string + testSpec GCSTestSpec + chunkSize int +} + +var _ = Suite(&GCSBackendTest{}) + +func (s *GCSBackendTest) getGCSTestConfig() (*common.GCSConfig, error) { + config := common.NewGCSConfig() + config.ChunkSize = s.chunkSize + return config, nil +} + +func (s *GCSBackendTest) getGCSTestBackend(gcsBucket string) (*GCSBackend, error) { + config, err := s.getGCSTestConfig() + if err != nil { + return nil, err + } + spec, err := ParseBucketSpec(gcsBucket) + if err != nil { + return nil, err + } + gcsBackend, err := NewGCS(spec.Bucket, config) + + return gcsBackend, err +} + +func (s *GCSBackendTest) SetUpSuite(c *C) { + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because GOOGLE_APPLICATION_CREDENTIALS variable is unset.") + } + + bktName := "goofys-test-" + RandStringBytesMaskImprSrc(16) + s.bucketName = bktName + s.testSpec = GCSTestSpec{ + objCount: 10, + prefixCount: 5, + defaultObjLen: 16, + metadata: map[string]*string{ + "foo": PString("bar"), + "bar": PString("baz"), + }, + contentType: PString("text/plain"), + } + s.chunkSize = 1 * 1024 * 1024 + s.gcsBackend, _ = s.getGCSTestBackend(fmt.Sprintf("gs://%s", bktName)) + + _, err := s.gcsBackend.MakeBucket(&MakeBucketInput{}) + c.Assert(err, IsNil) + + errGroup, _ := errgroup.WithContext(context.Background()) + objLen := s.testSpec.defaultObjLen + + // create object without prefix + // files is stored as 00_blob, 01_blob, ... 09_blob (10 items) + for i := 0; i < s.testSpec.objCount; i++ { + objKey := fmt.Sprintf("%02d_blob", i) + objContent := RandStringBytesMaskImprSrc(int(objLen)) + + errGroup.Go(func() error { + _, err := s.gcsBackend.PutBlob(&PutBlobInput{ + Key: objKey, + Body: bytes.NewReader([]byte(objContent)), + Metadata: s.testSpec.metadata, + ContentType: s.testSpec.contentType, + }) + c.Assert(err, IsNil) + + // verify object attr after a put blob + objAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(objAttr.Key), Equals, objKey) + c.Assert(objAttr.Size, Equals, uint64(int(objLen))) + c.Assert(NilStr(objAttr.ContentType), Equals, NilStr(s.testSpec.contentType)) + c.Assert(objAttr.Metadata, DeepEquals, s.testSpec.metadata) + + return err + }) + + // set the existing object to be the first object + if s.testSpec.existingObjKey == "" { + s.testSpec.existingObjKey = objKey + s.testSpec.existingObjContent = objContent + s.testSpec.existingObjLen = objLen + } + s.testSpec.nonPrefixObjects = append(s.testSpec.nonPrefixObjects, objKey) + } + + // create object with prefix + // file is stored as 00_prefix/blob, 01_prefix/blob, ..., 04_prefix/blob (5 items) + for i := 0; i < s.testSpec.prefixCount; i++ { + objKey := fmt.Sprintf("%02d_prefix/blob", i) + objContent := RandStringBytesMaskImprSrc(int(objLen)) + errGroup.Go(func() error { + _, err := s.gcsBackend.PutBlob(&PutBlobInput{ + Key: objKey, + Body: bytes.NewReader([]byte(objContent)), + Metadata: s.testSpec.metadata, + ContentType: s.testSpec.contentType, + }) + c.Assert(err, IsNil) + + objAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(objAttr.Key), Equals, objKey) + c.Assert(objAttr.Size, Equals, uint64(int(objLen))) + c.Assert(NilStr(objAttr.ContentType), Equals, NilStr(s.testSpec.contentType)) + c.Assert(objAttr.Metadata, DeepEquals, s.testSpec.metadata) + return err + }) + + s.testSpec.prefixObjects = append(s.testSpec.prefixObjects, objKey) + dir, _ := path.Split(objKey) + s.testSpec.prefixes = append(s.testSpec.prefixes, dir) + } + + err = errGroup.Wait() + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TearDownSuite(c *C) { + out, err := s.gcsBackend.ListBlobs(&ListBlobsInput{}) + c.Assert(err, IsNil) + + var items []string + for _, item := range out.Items { + items = append(items, NilStr(item.Key)) + } + + _, err = s.gcsBackend.DeleteBlobs(&DeleteBlobsInput{Items: items}) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.RemoveBucket(&RemoveBucketInput{}) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) SetUpTest(c *C) { + s.testSpec.additionalBlobs = []string{} +} + +func (s *GCSBackendTest) TearDownTest(c *C) { + _, err := s.gcsBackend.DeleteBlobs(&DeleteBlobsInput{s.testSpec.additionalBlobs}) + // successful deletion should be: nil error or not found + c.Assert(err == nil || err == syscall.ENOENT, Equals, true) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated(c *C) { + // on existing private bucket + err := s.gcsBackend.Init(s.bucketName) + c.Assert(err, IsNil) + + // on nonexistent bucket + gcsBackend, err := s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + // there is no error because we return not found errors as successful + // bucket not found has the same 404 error code as object not found + c.Assert(err, IsNil) + + // on public bucket + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Unauthenticated(c *C) { + defaultCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCredentials) + }() + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "") + + credentials, err := google.FindDefaultCredentials(context.Background()) + if credentials != nil { + c.Skip("Skipping this test because credentials still exist in the environment.") + } + + // on existing private bucket + gcsBackend, err := s.getGCSTestBackend(s.bucketName) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(15)) + c.Assert(err, Equals, syscall.EACCES) + + // on nonexistent bucket. there is no error because bucket not found is a 404 error + gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) + + // on public bucket + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated_ReadOnlyAccess(c *C) { + // READONLY_GOOGLE_APPLICATION_CREDENTIALS is a credential that is authorized to read-only access to the bucket + if os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because READONLY_GOOGLE_APPLICATION_CREDENTIALS is unset.") + } + + defaultCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCredentials) + }() + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS")) + + // on existing private bucket + gcsBackend, err := s.getGCSTestBackend(s.bucketName) + c.Assert(err, IsNil) + + // error when trying to modify object + _, err = gcsBackend.DeleteBlob(&DeleteBlobInput{Key: s.testSpec.existingObjKey}) + c.Assert(err, Equals, syscall.EACCES) + + // on nonexistent bucket. there is no error because bucket not found is a 404 error + gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) + + // on public bucket + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_HeadBlob_NotExist(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + _, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, Equals, fuse.ENOENT) +} + +func (s *GCSBackendTest) TestGCSBackend_GetBlob_NotExist(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + _, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, Equals, fuse.ENOENT) +} + +func (s *GCSBackendTest) TestGCSBackend_GetFullBlob(c *C) { + fullBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: s.testSpec.existingObjKey}) + c.Assert(err, IsNil) + defer fullBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(fullBlob.Body) + c.Assert(err, IsNil) + c.Assert(string(actualContent), Equals, s.testSpec.existingObjContent) +} + +func (s *GCSBackendTest) TestGCSBackend_GetPartialBlob(c *C) { + var startOffset uint64 = 4 + var count uint64 = 8 + partialBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{ + Key: s.testSpec.existingObjKey, + Start: startOffset, + Count: count, + }) + c.Assert(err, IsNil) + defer partialBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(partialBlob.Body) + c.Assert(err, IsNil) + c.Assert(string(actualContent), Equals, s.testSpec.existingObjContent[startOffset:startOffset+count]) +} + +func (s *GCSBackendTest) TestGCSBackend_PutBlob_NilBody(c *C) { + objKey := RandStringBytesMaskImprSrc(int(s.testSpec.defaultObjLen)) + s.addToAdditionalBlobs(objKey) + + _, err := s.gcsBackend.PutBlob(&PutBlobInput{Key: objKey, Body: nil}) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) addToAdditionalBlobs(objKey string) { + s.testSpec.additionalBlobs = append(s.testSpec.additionalBlobs, objKey) +} + +func (s *GCSBackendTest) TestGCSBackend_GetGzipEncodedBlob(c *C) { + objKey := "gzipObj" + s.addToAdditionalBlobs(objKey) + originalContent := RandStringBytesMaskImprSrc(int(s.testSpec.defaultObjLen)) + + // Goofys cannot upload a file with content-encoding: gzip. So we will use GCS sdk directly to create such blob + s.writeGzipEncodedFile(c, objKey, originalContent) + + gzipBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, IsNil) + defer gzipBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(gzipBlob.Body) + c.Assert(string(actualContent), Equals, originalContent) +} + +func (s *GCSBackendTest) writeGzipEncodedFile(c *C, objKey string, content string) { + writer := s.gcsBackend.bucket.Object(objKey).NewWriter(context.Background()) + writer.ContentEncoding = "gzip" + _, err := writer.Write([]byte(content)) + defer writer.Close() + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_NoPrefixNoDelim(c *C) { + // list all objects in bucket as items, no prefix because delim is unset + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{}) + c.Assert(err, IsNil) + + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixObjects) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_NoPrefixWithDelim(c *C) { + // list all objects but separates items and prefixes + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{Delimiter: PString("/")}) + c.Assert(err, IsNil) + // should contain non prefix objects: 00_blob, ..., 09_blob and prefixes: 00_prefix/, ..., 04_prefix/ + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixes) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_WithPrefixNoDelim(c *C) { + // list all objects that have a matching prefix as items + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{Prefix: PString("00")}) + c.Assert(err, IsNil) + // should contain all items under prefix: 00_blob & 00_prefix/blob + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[:1], s.testSpec.prefixObjects[:1]) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_WithPrefixWithDelim(c *C) { + // lists objects that have a matching prefix and separates non prefix & prefixes + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Prefix: PString("00"), + Delimiter: PString("/"), + }) + c.Assert(err, IsNil) + // should contain 00_blob and 00_prefix/ + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[:1], s.testSpec.prefixes[:1]) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_StartAfter(c *C) { + cutoffKey := "04_prefix/blob" + // list results all come right on or after StartAfter in lexicographical order + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + StartAfter: PString(cutoffKey), + }) + c.Assert(err, IsNil) + + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + // contain elements from 05_blob and 04_prefix/ + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[5:], + s.testSpec.prefixes[len(s.testSpec.prefixes)-1:]) + + // assert all items are larger or equal to StartAfter prefix (because we used delimiter) in lexicographical order + cutOffPrefix, _ := path.Split(cutoffKey) + for _, item := range actualOutputs { + c.Assert(item >= cutOffPrefix, Equals, true) + } +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_MaxKeysPaginate(c *C) { + totalKeys := s.testSpec.objCount + s.testSpec.prefixCount + maxKeys := 1 + + var nextContToken *string + var actualOutputs []string + + // iterate over pages of maxKeys + for i := 0; i < totalKeys-1; i++ { + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + MaxKeys: PUInt32(uint32(maxKeys)), + ContinuationToken: nextContToken, + }) + c.Assert(err, IsNil) + c.Assert(len(listOutputs.Items)+len(listOutputs.Prefixes), Equals, maxKeys) + c.Assert(listOutputs.NextContinuationToken, NotNil) + nextContToken = listOutputs.NextContinuationToken + actualOutputs = append(actualOutputs, s.extractAllNamesFromListOutputs(listOutputs)...) + } + // remaining object in the last page + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + MaxKeys: PUInt32(uint32(maxKeys)), + ContinuationToken: nextContToken, + }) + actualOutputs = append(actualOutputs, s.extractAllNamesFromListOutputs(listOutputs)...) + c.Assert(err, IsNil) + c.Assert(len(listOutputs.Items)+len(listOutputs.Prefixes) <= maxKeys, Equals, true) + c.Assert(listOutputs.NextContinuationToken, IsNil) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixes) +} + +func (s *GCSBackendTest) extractAllNamesFromListOutputs(listOutputs *ListBlobsOutput) []string { + var actualOutputs []string + for _, item := range listOutputs.Items { + actualOutputs = append(actualOutputs, NilStr(item.Key)) + } + for _, prefix := range listOutputs.Prefixes { + actualOutputs = append(actualOutputs, NilStr(prefix.Prefix)) + } + return actualOutputs +} + +// This check the actual outputs against list of expected outputs +func (s *GCSBackendTest) checkListOutputs(c *C, actualOutputs []string, expectedOutputsArgs ...[]string) { + var expectedResults []string + for _, expectedOutputs := range expectedOutputsArgs { + expectedResults = append(expectedResults, expectedOutputs...) + } + sort.Strings(expectedResults) + sort.Strings(actualOutputs) + + c.Assert(actualOutputs, DeepEquals, expectedResults) +} + +func (s *GCSBackendTest) TestGCSBackend_CopyBlob_PreserveMetadata(c *C) { + srcKey := s.testSpec.existingObjKey + destKey := RandStringBytesMaskImprSrc(16) + s.addToAdditionalBlobs(destKey) + + srcAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: srcKey}) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.CopyBlob(&CopyBlobInput{ + Source: srcKey, + Destination: destKey, + }) + c.Assert(err, IsNil) + + destAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: destKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(destAttr.ContentType), Equals, NilStr(srcAttr.ContentType)) + c.Assert(destAttr.Metadata, DeepEquals, srcAttr.Metadata) + c.Assert(srcAttr.ETag, Not(Equals), destAttr.ETag) + + destOut, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: srcKey}) + c.Assert(err, IsNil) + defer destOut.Body.Close() + destContent, err := ioutil.ReadAll(destOut.Body) + c.Assert(string(destContent), Equals, s.testSpec.existingObjContent) +} + +func (s *GCSBackendTest) TestGCSBackend_MultipartUpload_BeginAddCommit(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + s.addToAdditionalBlobs(objKey) + + commitInput, err := s.gcsBackend.MultipartBlobBegin(&MultipartBlobBeginInput{ + Key: objKey, + }) + c.Assert(err, IsNil) + c.Assert(commitInput.NumParts, Equals, uint32(0)) + + // We will have numFullChunks+1 chunks. That last chunk is of size lastChunkSize + var numFullChunks = 2 + lastChunkSize := s.chunkSize - 1 + fileSize := uint64((numFullChunks * s.chunkSize) + lastChunkSize) + + // generate data to simulate MPU behavior + buf := new(bytes.Buffer) + data := io.LimitReader(&SeqReader{}, int64(fileSize)) + _, err = buf.ReadFrom(data) + c.Assert(err, IsNil) + reader := bytes.NewReader(buf.Bytes()) + + // add the full chunks + for i := 0; i < numFullChunks; i++ { + sectionReader := io.NewSectionReader(reader, int64(i*s.chunkSize), int64(s.chunkSize)) + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: uint32(i + 1), + Body: sectionReader, + Size: uint64(s.chunkSize), + }) + c.Assert(err, IsNil) + c.Assert(addOut, NotNil) + } + + // add the remaining chunk + sectionReader := io.NewSectionReader(reader, int64(numFullChunks*s.chunkSize), int64(lastChunkSize)) + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: uint32(numFullChunks + 1), + Body: sectionReader, + Size: uint64(lastChunkSize), + }) + c.Assert(addOut, NotNil) + c.Assert(err, IsNil) + + commitOut, err := s.gcsBackend.MultipartBlobCommit(commitInput) + c.Assert(err, IsNil) + c.Assert(commitOut.ETag, NotNil) + + _, err = s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + + // assert uploaded content is correct + blobOutput, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, IsNil) + defer blobOutput.Body.Close() + + _, err = reader.Seek(0, io.SeekStart) // seek to the beginning of the file + c.Assert(err, IsNil) + _, err = CompareReader(reader, blobOutput.Body, 0) // assert content + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_MultipartUpload_Abort(c *C) { + src := RandStringBytesMaskImprSrc(16) + commitInput, err := s.gcsBackend.MultipartBlobBegin(&MultipartBlobBeginInput{Key: src}) + c.Assert(err, IsNil) + + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: 1, + Body: bytes.NewReader([]byte(src)), + Size: uint64(len(src)), + Last: false, + }) + c.Assert(err, IsNil) + c.Assert(addOut, NotNil) + + _, err = s.gcsBackend.MultipartBlobAbort(commitInput) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.HeadBlob(&HeadBlobInput{Key: src}) + c.Assert(err, Equals, fuse.ENOENT) +} diff --git a/internal/goofys.go b/internal/goofys.go index 9070fb54..b3321d4a 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -55,7 +55,7 @@ type Goofys struct { umask uint32 - gcs bool + gcsS3 bool rootAttrs InodeAttributes bufferPool *BufferPool @@ -114,6 +114,8 @@ func NewBackend(bucket string, flags *FlagStorage) (cloud StorageBackend, err er } else { cloud, err = NewS3(bucket, flags, config) } + } else if config, ok := flags.Backend.(*GCSConfig); ok { + cloud, err = NewGCS(bucket, config) } else { err = fmt.Errorf("Unknown backend config: %T", flags.Backend) } @@ -197,7 +199,7 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, log.Errorf("Unable to setup backend: %v", err) return nil } - _, fs.gcs = cloud.Delegate().(*GCS3) + _, fs.gcsS3 = cloud.Delegate().(*GCS3) randomObjectName := prefix + (RandStringBytesMaskImprSrc(32)) err = cloud.Init(randomObjectName) diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 3be88ead..96852ab7 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -507,7 +507,7 @@ func (s *GoofysTest) SetUpTest(t *C) { _, err = s3.ListBuckets(nil) t.Assert(err, IsNil) - } else if cloud == "gcs" { + } else if cloud == "gcs3" { conf := s.selectTestConfig(t, flags) flags.Backend = &conf @@ -614,6 +614,15 @@ func (s *GoofysTest) SetUpTest(t *C) { s.cloud, err = NewADLv2(bucket, flags, &config) t.Assert(err, IsNil) t.Assert(s.cloud, NotNil) + } else if cloud == "gcs" { + config := NewGCSConfig() + t.Assert(config, NotNil) + + flags.Backend = config + var err error + s.cloud, err = NewGCS(bucket, config) + t.Assert(err, IsNil) + t.Assert(s.cloud, NotNil) } else { t.Fatal("Unsupported backend") } @@ -1297,6 +1306,8 @@ func (s *GoofysTest) TestBackendListPagination(t *C) { itemsPerPage = 1000 case *AZBlob, *ADLv2: itemsPerPage = 5000 + case *GCSBackend: + itemsPerPage = 1000 default: t.Fatalf("unknown backend: %T", s.cloud) } @@ -3353,6 +3364,10 @@ func (s *GoofysTest) newBackend(t *C, bucket string, createBucket bool) (cloud S config, _ := s.fs.flags.Backend.(*ADLv2Config) cloud, err = NewADLv2(bucket, s.fs.flags, config) t.Assert(err, IsNil) + case *GCSBackend: + config, _ := s.fs.flags.Backend.(*GCSConfig) + cloud, err = NewGCS(bucket, config) + t.Assert(err, IsNil) default: t.Fatal("unknown backend") } @@ -3586,6 +3601,19 @@ func (s *GoofysTest) TestMountsError(t *C) { defer func() { adlCloud.client.BaseClient.Authorizer = auth }() + } else if _, ok := s.cloud.(*GCSBackend); ok { + // We'll trigger a failure on GCS mount by using an unauthenticated client to mount to a private bucket + defaultCreds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS") + + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCreds) + }() + + var err error + config := NewGCSConfig() + cloud, err = NewGCS(s.fs.bucket, config) + t.Assert(err, IsNil) } else { cloud = s.newBackend(t, bucket, false) } diff --git a/internal/parse_bucket_test.go b/internal/parse_bucket_test.go new file mode 100644 index 00000000..8882e3f3 --- /dev/null +++ b/internal/parse_bucket_test.go @@ -0,0 +1,49 @@ +package internal + +import ( + . "gopkg.in/check.v1" +) + +type ParseBucketTest struct{} + +var _ = Suite(&ParseBucketTest{}) + +func (s *ParseBucketTest) TestParseBucketSpec(t *C) { + testCases := []struct { + input string + expected BucketSpec + }{ + { + input: "s3://bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "s3", + Bucket: "bucketName", + Prefix: "hello/everyone/", + }, + }, + { + input: "gs://bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "gs", + Bucket: "bucketName", + Prefix: "hello/everyone/", + }, + }, + { + input: "bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "s3", + Bucket: "bucketName/hello/everyone", + Prefix: "", + }, + }, + } + + for _, tc := range testCases { + spec, err := ParseBucketSpec(tc.input) + if err != nil { + log.Fatal(err) + } + t.Assert(tc.expected, DeepEquals, spec) + } +} diff --git a/internal/utils.go b/internal/utils.go index 2a26187e..51e07800 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -16,6 +16,7 @@ package internal import ( "fmt" + "strings" "time" "unicode" @@ -125,6 +126,30 @@ func NilStr(v *string) string { } } +func NilUint32(v *uint32) uint32 { + if v == nil { + return 0 + } else { + return *v + } +} + +func NilInt64(v *int64) int64 { + if v == nil { + return 0 + } else { + return *v + } +} + +func NilUint64(v *uint64) uint64 { + if v == nil { + return 0 + } else { + return *v + } +} + func xattrEscape(value []byte) (s string) { for _, c := range value { if c == '%' { @@ -187,3 +212,37 @@ func GetTgid(pid uint32) (tgid *int32, err error) { } return &tgidVal, nil } + +func PMetadata(m map[string]string) map[string]*string { + metadata := make(map[string]*string) + for k, _ := range m { + k = strings.ToLower(k) + v := m[k] + metadata[k] = &v + } + return metadata +} + +func NilMetadata(m map[string]*string) map[string]string { + metadata := make(map[string]string) + for k, v := range m { + k = strings.ToLower(k) + metadata[k] = NilStr(v) + } + return metadata +} + +// The following functions are useful to debug byte sizes +func ConvertBytesToIEC(size int64) string { + var unit int64 = 1024 + if size < unit { + return fmt.Sprintf("%d %c", size, 'B') + } + // divide size by unit until it is less than unit + curSize, exp := float64(size), 0 + for curSize >= float64(unit) { + curSize /= float64(unit) + exp += 1 + } + return fmt.Sprintf("%.1f %ciB", curSize, "KMGTPEZY"[exp-1]) +} \ No newline at end of file diff --git a/test/run-tests.sh b/test/run-tests.sh index 549c2704..28741643 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -9,6 +9,7 @@ set -o nounset : ${PROXY_PID:=""} : ${TIMEOUT:="10m"} : ${MOUNT:="false"} +: ${GOOGLE_APPLICATION_CREDENTIALS:=""} export MOUNT @@ -65,6 +66,11 @@ elif [ $CLOUD == "azblob" ]; then export ENDPOINT elif [ $CLOUD == "adlv1" ]; then TIMEOUT=40m +elif [ $CLOUD == "gcs" ]; then + if [ "$GOOGLE_APPLICATION_CREDENTIALS" = "" ]; then + echo "GOOGLE_APPLICATION_CREDENTIALS must be set" 1>&2 + exit 1 + fi fi if [ "$PROXY_BIN" != "" ]; then From db33ac2aadf45ed4213c9e5b0297dbee06daf3e9 Mon Sep 17 00:00:00 2001 From: Deka Akbar Date: Tue, 25 Aug 2020 19:17:55 -0400 Subject: [PATCH 12/40] Skip TearDownSuite in GCS test if GCS credentials are not set (#83) --- internal/backend_gcs_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/backend_gcs_test.go b/internal/backend_gcs_test.go index 660530cd..44d91f84 100644 --- a/internal/backend_gcs_test.go +++ b/internal/backend_gcs_test.go @@ -158,6 +158,9 @@ func (s *GCSBackendTest) SetUpSuite(c *C) { } func (s *GCSBackendTest) TearDownSuite(c *C) { + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because GOOGLE_APPLICATION_CREDENTIALS variable is unset.") + } out, err := s.gcsBackend.ListBlobs(&ListBlobsInput{}) c.Assert(err, IsNil) From 12ba1dff75730877d279435f1401ac6ecdef99fb Mon Sep 17 00:00:00 2001 From: Deka Akbar Date: Tue, 25 Aug 2020 19:18:30 -0400 Subject: [PATCH 13/40] Change GCS Init to use List (#81) Co-authored-by: Sai Teja Suram --- README-gcs.md | 6 +----- internal/backend_gcs.go | 25 ++++++++++++++--------- internal/backend_gcs_test.go | 39 +++++++++++++++++++----------------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/README-gcs.md b/README-gcs.md index cf44f812..1de323d1 100644 --- a/README-gcs.md +++ b/README-gcs.md @@ -5,11 +5,7 @@ Service Account credentials or user authentication. Ensure that either the service account or user has the proper permissions to the Bucket / Object under GCS. -For example, read-only access should be granted the following permissions: -``` -storage.objects.get -storage.objects.list -```` +To have a successful mount, we require users to have object listing (`storage.objects.list`) permission to the bucket. ### Service Account credentials diff --git a/internal/backend_gcs.go b/internal/backend_gcs.go index 2b11cb25..fb8efdac 100644 --- a/internal/backend_gcs.go +++ b/internal/backend_gcs.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "path" "strings" "syscall" @@ -78,16 +79,20 @@ func NewGCS(bucket string, config *common.GCSConfig) (*GCSBackend, error) { // Init checks user's access to bucket. func (g *GCSBackend) Init(key string) error { - g.logger.Debugf("INIT GCS: %s", key) - _, err := g.bucket.Object(key).Attrs(context.Background()) - - // Currently we return successful mounts on 404 errors. However, GCS does not differentiate bucket not found and - // object not found errors. So, even though we don't know that the bucket exists, we will still mount successfully. - if err == storage.ErrObjectNotExist { - err = nil - } - - return mapGCSError(err) + // We will do a successful mount if the user can list on the bucket. + // This is different other backends because GCS does not differentiate between object not found and + // bucket not found. + prefix, _ := path.Split(key) + _, err := g.ListBlobs(&ListBlobsInput{ + MaxKeys: PUInt32(1), + Prefix: PString(prefix), + }) + g.logger.Debugf("INIT GCS: ListStatus = %s", getDebugResponseStatus(err)) + if err == syscall.ENXIO { + return fmt.Errorf("bucket %v does not exist", g.bucketName) + } + // Errors can be returned directly since ListBlobs converts them to syscall errors. + return err } func (g *GCSBackend) Capabilities() *Capabilities { diff --git a/internal/backend_gcs_test.go b/internal/backend_gcs_test.go index 44d91f84..6cae756e 100644 --- a/internal/backend_gcs_test.go +++ b/internal/backend_gcs_test.go @@ -187,19 +187,18 @@ func (s *GCSBackendTest) TearDownTest(c *C) { } func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated(c *C) { - // on existing private bucket + // No error when accessing existing private bucket. err := s.gcsBackend.Init(s.bucketName) c.Assert(err, IsNil) - // on nonexistent bucket - gcsBackend, err := s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err := s.getGCSTestBackend(randBktName) c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) - // there is no error because we return not found errors as successful - // bucket not found has the same 404 error code as object not found - c.Assert(err, IsNil) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) - // on public bucket + // No error when accessing public bucket. gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) @@ -218,19 +217,20 @@ func (s *GCSBackendTest) TestGCSBackend_Init_Unauthenticated(c *C) { c.Skip("Skipping this test because credentials still exist in the environment.") } - // on existing private bucket + // Access error on existing private bucket. gcsBackend, err := s.getGCSTestBackend(s.bucketName) c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(15)) c.Assert(err, Equals, syscall.EACCES) - // on nonexistent bucket. there is no error because bucket not found is a 404 error - gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err = s.getGCSTestBackend(randBktName) c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) - c.Assert(err, IsNil) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) - // on public bucket + // No error when accessing public bucket. gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) @@ -249,21 +249,24 @@ func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated_ReadOnlyAccess(c *C) }() os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS")) - // on existing private bucket + // No error when accessing existing private bucket. gcsBackend, err := s.getGCSTestBackend(s.bucketName) c.Assert(err, IsNil) + err = gcsBackend.Init(s.bucketName) + c.Assert(err, IsNil) - // error when trying to modify object + // Access error when trying to modify object. _, err = gcsBackend.DeleteBlob(&DeleteBlobInput{Key: s.testSpec.existingObjKey}) c.Assert(err, Equals, syscall.EACCES) - // on nonexistent bucket. there is no error because bucket not found is a 404 error - gcsBackend, err = s.getGCSTestBackend(RandStringBytesMaskImprSrc(16)) + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err = s.getGCSTestBackend(randBktName) c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) - c.Assert(err, IsNil) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) - // on public bucket + // No error when accessing public bucket. gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") c.Assert(err, IsNil) err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) From e56d250e6f50eb231c6f10359e4f6c9e3acc29bd Mon Sep 17 00:00:00 2001 From: Sai Teja Suram Date: Thu, 15 Apr 2021 09:59:39 -0700 Subject: [PATCH 14/40] Minimize go mod changes --- go.mod | 35 ++++----- go.sum | 232 +++++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 193 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index 942570e4..43fa0573 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.14 require ( cloud.google.com/go/storage v1.14.0 - github.com/Azure/azure-pipeline-go v0.2.3 - github.com/Azure/azure-sdk-for-go v52.6.0+incompatible - github.com/Azure/azure-storage-blob-go v0.13.0 + github.com/Azure/azure-pipeline-go v0.2.2 + github.com/Azure/azure-sdk-for-go v32.1.0+incompatible + github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/adal v0.9.13 github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 @@ -14,31 +14,26 @@ require ( github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect - github.com/aws/aws-sdk-go v1.38.10 - github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 + github.com/aws/aws-sdk-go v1.38.7 github.com/go-ole/go-ole v1.2.5 // indirect - github.com/gofrs/uuid v4.0.0+incompatible // indirect - github.com/google/uuid v1.2.0 - github.com/hashicorp/go-retryablehttp v0.6.8 - github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291 + github.com/google/uuid v1.1.2 + github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect + github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kylelemons/godebug v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 - github.com/satori/go.uuid v1.2.0 + github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b github.com/sevlyar/go-daemon v0.1.5 - github.com/shirou/gopsutil v3.21.2+incompatible - github.com/sirupsen/logrus v1.8.1 - github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/tklauser/go-sysconf v0.3.5 // indirect - github.com/urfave/cli v1.22.5 + github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 + github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 + github.com/smartystreets/assertions v1.2.0 // indirect + github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 + golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 google.golang.org/api v0.43.0 - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c - gopkg.in/ini.v1 v1.62.0 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/ini.v1 v1.51.0 ) replace github.com/jacobsa/fuse => github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc - diff --git a/go.sum b/go.sum index 015a8273..fab56c98 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,18 +40,18 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= -github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v52.6.0+incompatible h1:F/feBa+/Oxbu+Zprnsiq0b6rvUVlOEx3jSqCSNdtF3U= -github.com/Azure/azure-sdk-for-go v52.6.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc= -github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= @@ -74,11 +75,23 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/aws/aws-sdk-go v1.38.10 h1:7lQrjAlyYrTGW2+9vnBv5HPSSuv+xDMmgU1YUnNSOOo= -github.com/aws/aws-sdk-go v1.38.10/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.38.7 h1:uOu2IrTiNhcSNAjBmA21t48lTx5mgGdcFKamDjXMscA= +github.com/aws/aws-sdk-go v1.38.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -86,12 +99,17 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -101,18 +119,26 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= -github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -172,93 +198,168 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 h1:VNVgNDgai+7LqdALaNTNKDD+EriKUN30tvQH6bgYjh8= +github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026/go.mod h1:Opf9rtYVq0eTyX+aRVmRO9hE8ERAozcdrBxWG9Q6mkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= -github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291 h1:GoZXB1SHOnDgMmFNYvgQT5hNoAM8JfyoTIOw0ILj7m4= -github.com/jacobsa/fuse v0.0.0-20210330112455-9677d0392291/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc h1:wNmVtupKbdEJcTkMIF2ah/nf3NzHEeNnpTR/1riaahQ= +github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc/go.mod h1:Qk5T6wYqwRWfKHhEKkRsIXwsAtQ0GK8rBJNYf5LVdwo= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= -github.com/shirou/gopsutil v3.21.2+incompatible h1:U+YvJfjCh6MslYlIAXvPtzhW3YZEtc9uncueUNpD/0A= -github.com/shirou/gopsutil v3.21.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= -github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= -github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -267,6 +368,11 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -275,6 +381,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -312,6 +420,10 @@ golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -322,7 +434,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -370,10 +481,17 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -381,8 +499,8 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -398,7 +516,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -408,10 +525,10 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0= -golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -425,6 +542,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -441,6 +559,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -579,17 +698,22 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 08534b28a2195d30bb3c45dce0b834f3696028b8 Mon Sep 17 00:00:00 2001 From: Sai Teja Pratap Date: Thu, 15 Apr 2021 12:01:58 -0700 Subject: [PATCH 15/40] Fix TestReadMyOwnWriteFuse for gcs --- internal/goofys_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 96852ab7..292ab557 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -4200,10 +4200,12 @@ func (s *GoofysTest) testReadMyOwnWriteFuse(t *C, externalUpdate bool) { if !externalUpdate { // we flushed and ttl expired, next lookup should // realize nothing is changed and NOT invalidate the - // cache. Except ADLv1 because PUT there doesn't + // cache. Except ADLv1,GCS because PUT there doesn't // return the mtime, so the open above will think the // file is updated and not re-use cache - if _, adlv1 := s.cloud.(*ADLv1); !adlv1 { + _, adlv1 := s.cloud.(*ADLv1) + _, isGCS := s.cloud.(*GCSBackend) + if !adlv1 && !isGCS { cloud.err = fuse.EINVAL } } else { From 829d8e5ce20faa3f9f6f054077a14325e00e9249 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Thu, 21 Apr 2022 17:19:09 +0000 Subject: [PATCH 16/40] switching fuse lib to upstream --- go.mod | 5 +---- go.sum | 19 +++++++++---------- internal/dir.go | 2 +- internal/file.go | 2 +- internal/goofys.go | 9 +++++---- internal/goofys_test.go | 35 ++++++++++++++++++----------------- internal/handles.go | 2 +- 7 files changed, 36 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 43fa0573..91120719 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,8 @@ require ( github.com/go-ole/go-ole v1.2.5 // indirect github.com/google/uuid v1.1.2 github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect - github.com/jacobsa/fuse v0.0.0-20201216155545-e0296dec955f + github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/kylelemons/godebug v1.1.0 // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b github.com/sevlyar/go-daemon v0.1.5 @@ -35,5 +34,3 @@ require ( gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/ini.v1 v1.51.0 ) - -replace github.com/jacobsa/fuse => github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc diff --git a/go.sum b/go.sum index fab56c98..ed8524a0 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= @@ -203,7 +204,6 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 h1:VNVgNDgai+7LqdALaNTNKDD+EriKUN30tvQH6bgYjh8= github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026/go.mod h1:Opf9rtYVq0eTyX+aRVmRO9hE8ERAozcdrBxWG9Q6mkQ= @@ -214,7 +214,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -235,6 +234,14 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 h1:LkLYmg9wdUlhXv9H31druPMUFPSrpN9sWMxPCTvu5dI= +github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5/go.mod h1:xtZnnLxHY6QniCrfIpTwr5h8mH8zr+jsOFj0y9cfyp4= +github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= +github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= +github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= +github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb/go.mod h1:ivcmUvxXWjb27NsPEaiYK7AidlZXS7oQ5PowUS9z3I4= +github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3/go.mod h1:mPvulh9VKXvo+yOlrD4VYOOYuLdZJ36wa/5QIrtXvWs= +github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6/go.mod h1:JEWKD6V8xETMW+DEv+IQVz++f8Cn8O/X0HPeDY3qNis= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -247,8 +254,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc h1:wNmVtupKbdEJcTkMIF2ah/nf3NzHEeNnpTR/1riaahQ= -github.com/kahing/fusego v0.0.0-20200327063725-ca77844c7bcc/go.mod h1:Qk5T6wYqwRWfKHhEKkRsIXwsAtQ0GK8rBJNYf5LVdwo= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -307,7 +312,6 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= @@ -321,13 +325,11 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+D github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -379,7 +381,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -525,7 +526,6 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -710,7 +710,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/dir.go b/internal/dir.go index 687674f0..64e838f2 100644 --- a/internal/dir.go +++ b/internal/dir.go @@ -899,7 +899,7 @@ func (parent *Inode) Unlink(name string) (err error) { } func (parent *Inode) Create( - name string, metadata fuseops.OpMetadata) (inode *Inode, fh *FileHandle) { + name string, metadata fuseops.OpContext) (inode *Inode, fh *FileHandle) { parent.logFuse("Create", name) diff --git a/internal/file.go b/internal/file.go index a2ccbaa7..33350dc0 100644 --- a/internal/file.go +++ b/internal/file.go @@ -71,7 +71,7 @@ const READAHEAD_CHUNK = uint32(20 * 1024 * 1024) // NewFileHandle returns a new file handle for the given `inode` triggered by fuse // operation with the given `opMetadata` -func NewFileHandle(inode *Inode, opMetadata fuseops.OpMetadata) *FileHandle { +func NewFileHandle(inode *Inode, opMetadata fuseops.OpContext) *FileHandle { tgid, err := GetTgid(opMetadata.Pid) if err != nil { log.Debugf( diff --git a/internal/goofys.go b/internal/goofys.go index b3321d4a..79e628ce 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -34,8 +34,9 @@ import ( "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" - "github.com/sirupsen/logrus" "net/http" + + "github.com/sirupsen/logrus" ) // goofys is a Filey System written in Go. All the backend data is @@ -882,7 +883,7 @@ func (fs *Goofys) OpenFile( in := fs.getInodeOrDie(op.Inode) fs.mu.RUnlock() - fh, err := in.OpenFile(op.Metadata) + fh, err := in.OpenFile(op.OpContext) if err != nil { return } @@ -955,7 +956,7 @@ func (fs *Goofys) FlushFile( // This check helps us with scenarios like https://github.com/kahing/goofys/issues/273 // Also see goofys_test.go:TestClientForkExec. if fh.Tgid != nil { - tgid, err := GetTgid(op.Metadata.Pid) + tgid, err := GetTgid(op.OpContext.Pid) if err != nil { fh.inode.logFuse("<-- FlushFile", fmt.Sprintf("Failed to retrieve tgid from op.Metadata.Pid. FlushFileOp:%#v, err:%v", @@ -1015,7 +1016,7 @@ func (fs *Goofys) CreateFile( parent := fs.getInodeOrDie(op.Parent) fs.mu.RUnlock() - inode, fh := parent.Create(op.Name, op.Metadata) + inode, fh := parent.Create(op.Name, op.OpContext) parent.mu.Lock() diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 292ab557..f5b33284 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -57,8 +57,9 @@ import ( "github.com/sirupsen/logrus" - . "gopkg.in/check.v1" "runtime/debug" + + . "gopkg.in/check.v1" ) // so I don't get complains about unused imports @@ -914,7 +915,7 @@ func (s *GoofysTest) TestReadFiles(t *C) { in, err := parent.LookUp(en.Name) t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) buf := make([]byte, 4096) @@ -940,7 +941,7 @@ func (s *GoofysTest) TestReadOffset(t *C) { in, err := root.LookUp(f) t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) buf := make([]byte, 4096) @@ -964,7 +965,7 @@ func (s *GoofysTest) TestReadOffset(t *C) { func (s *GoofysTest) TestCreateFiles(t *C) { fileName := "testCreateFile" - _, fh := s.getRoot(t).Create(fileName, fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := s.getRoot(t).Create(fileName, fuseops.OpContext{uint32(os.Getpid())}) err := fh.FlushFile() t.Assert(err, IsNil) @@ -983,7 +984,7 @@ func (s *GoofysTest) TestCreateFiles(t *C) { inode, err := s.getRoot(t).LookUp(fileName) t.Assert(err, IsNil) - fh, err = inode.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err = inode.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) err = fh.FlushFile() @@ -1062,7 +1063,7 @@ func (s *GoofysTest) testWriteFileAt(t *C, fileName string, offset int64, size i } } else { in := s.fs.inodes[lookup.Entry.Child] - fh, err = in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err = in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) } @@ -1164,7 +1165,7 @@ func (s *GoofysTest) TestReadRandom(t *C) { in, err := s.LookUpInode(t, "testLargeFile") t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) fr := &FileHandleReader{s.fs, fh, 0} @@ -1196,7 +1197,7 @@ func (s *GoofysTest) TestMkDir(t *C) { t.Assert(err, IsNil) fileName := "file" - _, fh := inode.Create(fileName, fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := inode.Create(fileName, fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -2874,7 +2875,7 @@ func (s *GoofysTest) TestDirMtimeCreate(t *C) { m1 := attr.Mtime time.Sleep(time.Second) - _, _ = root.Create("foo", fuseops.OpMetadata{uint32(os.Getpid())}) + _, _ = root.Create("foo", fuseops.OpContext{uint32(os.Getpid())}) attr2, _ := root.GetAttributes() m2 := attr2.Mtime @@ -2937,7 +2938,7 @@ func (s *GoofysTest) TestRead403(t *C) { in, err := s.LookUpInode(t, "file1") t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) s3.awsConfig.Credentials = credentials.AnonymousCredentials @@ -3403,7 +3404,7 @@ func (s *GoofysTest) TestVFS(t *C) { _, err = in.LookUp("file5") t.Assert(err, Equals, fuse.ENOENT) - _, fh := in.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := in.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3438,7 +3439,7 @@ func (s *GoofysTest) TestVFS(t *C) { // create another file inside subdir to make sure that our // mount check is correct for dir inside the root - _, fh = subdir.Create("testfile2", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh = subdir.Create("testfile2", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3719,7 +3720,7 @@ func (s *GoofysTest) testMountsNested(t *C, cloud StorageBackend, t.Assert(*dir_dir.Name, Equals, "dir") t.Assert(dir_dir.dir.cloud == cloud, Equals, true) - _, fh := dir_in.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := dir_in.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3727,7 +3728,7 @@ func (s *GoofysTest) testMountsNested(t *C, cloud StorageBackend, t.Assert(err, IsNil) defer resp.Body.Close() - _, fh = dir_dir.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh = dir_dir.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3982,7 +3983,7 @@ func (s *GoofysTest) TestWriteListFlush(t *C) { t.Assert(err, IsNil) s.fs.insertInode(root, dir) - in, fh := dir.Create("file1", fuseops.OpMetadata{}) + in, fh := dir.Create("file1", fuseops.OpContext{}) t.Assert(in, NotNil) t.Assert(fh, NotNil) s.fs.insertInode(dir, in) @@ -4032,7 +4033,7 @@ func (s *GoofysTest) TestWriteUnlinkFlush(t *C) { t.Assert(err, IsNil) s.fs.insertInode(root, dir) - in, fh := dir.Create("deleted", fuseops.OpMetadata{}) + in, fh := dir.Create("deleted", fuseops.OpContext{}) t.Assert(in, NotNil) t.Assert(fh, NotNil) s.fs.insertInode(dir, in) @@ -4205,7 +4206,7 @@ func (s *GoofysTest) testReadMyOwnWriteFuse(t *C, externalUpdate bool) { // file is updated and not re-use cache _, adlv1 := s.cloud.(*ADLv1) _, isGCS := s.cloud.(*GCSBackend) - if !adlv1 && !isGCS { + if !adlv1 && !isGCS { cloud.err = fuse.EINVAL } } else { diff --git a/internal/handles.go b/internal/handles.go index 210f3b67..e8b308fb 100644 --- a/internal/handles.go +++ b/internal/handles.go @@ -518,7 +518,7 @@ func (inode *Inode) ListXattr() ([]string, error) { return xattrs, nil } -func (inode *Inode) OpenFile(metadata fuseops.OpMetadata) (fh *FileHandle, err error) { +func (inode *Inode) OpenFile(metadata fuseops.OpContext) (fh *FileHandle, err error) { inode.logFuse("OpenFile") inode.mu.Lock() From f960f45b9507edc7acae850f5b92af81f23def61 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 29 Sep 2022 07:43:54 +0900 Subject: [PATCH 17/40] Fix 32-bit constant overflow This prevented compilation on arm7 systems like Raspberry Pi. Fixes #724. --- internal/backend_s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 3ee2d489..1ee9c5f0 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -504,7 +504,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri } func sizeToParts(size int64) (int, int64) { - const MAX_S3_MPU_SIZE = 5 * 1024 * 1024 * 1024 * 1024 + const MAX_S3_MPU_SIZE int64 = 5 * 1024 * 1024 * 1024 * 1024 if size > MAX_S3_MPU_SIZE { panic(fmt.Sprintf("object size: %v exceeds maximum S3 MPU size: %v", size, MAX_S3_MPU_SIZE)) } From 2b06485590baff787ac7613c9bb5e4f8c3d06c66 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 27 Sep 2022 08:35:46 +0000 Subject: [PATCH 18/40] Support writing files larger than 1TB --- internal/file.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/file.go b/internal/file.go index 33350dc0..69cf5b6d 100644 --- a/internal/file.go +++ b/internal/file.go @@ -186,12 +186,14 @@ func (fh *FileHandle) waitForCreateMPU() (err error) { func (fh *FileHandle) partSize() uint64 { var size uint64 - if fh.lastPartId < 1000 { + if fh.lastPartId < 500 { size = 5 * 1024 * 1024 - } else if fh.lastPartId < 2000 { + } else if fh.lastPartId < 1000 { size = 25 * 1024 * 1024 - } else { + } else if fh.lastPartId < 2000 { size = 125 * 1024 * 1024 + } else { + size = 625 * 1024 * 1024 } maxPartSize := fh.cloud.Capabilities().MaxMultipartSize From e39a9338d8b3982d730fb0c48f66ed8b8184a977 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 27 Sep 2022 08:37:25 +0000 Subject: [PATCH 19/40] update readme --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index f55935a7..208ec503 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,6 @@ List of non-POSIX behaviors/limitations: * `unlink` returns success even if file is not present * `fsync` is ignored, files are only flushed on `close` -In addition to the items above, the following are supportable but not yet implemented: - * creating files larger than 1TB - ## Compatibility with non-AWS S3 goofys has been tested with the following non-AWS S3 providers: From 486ead0252d13d55a909f7c8af0dfa0fb43e1a74 Mon Sep 17 00:00:00 2001 From: Alessandro Rinaldi Date: Thu, 1 Sep 2022 11:46:48 +0200 Subject: [PATCH 20/40] Fix timeFormat for x-amz-date, two-digit day of month as per RFC 7231 --- internal/v2signer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/v2signer.go b/internal/v2signer.go index c8321463..53174c68 100644 --- a/internal/v2signer.go +++ b/internal/v2signer.go @@ -39,7 +39,7 @@ var ( const ( signatureVersion = "2" signatureMethod = "HmacSHA1" - timeFormat = "Mon, 2 Jan 2006 15:04:05 +0000" + timeFormat = "Mon, 02 Jan 2006 15:04:05 +0000" ) var subresources = []string{ From f8792cd9aac35837ab58715e32b611ee865bf735 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 3 Feb 2022 23:35:13 +0900 Subject: [PATCH 21/40] Set subtype to goofys This allows identifying mounts as "fuse.goofys" instead of only "fuse". --- api/api.go | 1 + internal/goofys_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/api.go b/api/api.go index f78cf082..cf606aef 100644 --- a/api/api.go +++ b/api/api.go @@ -29,6 +29,7 @@ func Mount( // Mount the file system. mountCfg := &fuse.MountConfig{ FSName: bucketName, + Subtype: "goofys", Options: flags.MountOptions, ErrorLogger: GetStdLogger(NewLogger("fuse"), logrus.ErrorLevel), DisableWritebackCaching: true, diff --git a/internal/goofys_test.go b/internal/goofys_test.go index f5b33284..70c409ca 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -1633,6 +1633,7 @@ func (s *GoofysTest) mount(t *C, mountPoint string) { // Mount the file system. mountCfg := &fuse.MountConfig{ FSName: s.fs.bucket, + Subtype: "goofys", Options: s.fs.flags.MountOptions, ErrorLogger: GetStdLogger(NewLogger("fuse"), logrus.ErrorLevel), DisableWritebackCaching: true, From fa288b6265c5da9d5ea2e3edb6ca4ae19d56b437 Mon Sep 17 00:00:00 2001 From: Simone Ripamonti Date: Wed, 2 Nov 2022 14:40:13 -0700 Subject: [PATCH 22/40] Upgrade aws-sdk-go v1.44.37 --- go.mod | 4 ++-- go.sum | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 91120719..e76a3745 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect - github.com/aws/aws-sdk-go v1.38.7 + github.com/aws/aws-sdk-go v1.44.37 github.com/go-ole/go-ole v1.2.5 // indirect github.com/google/uuid v1.1.2 github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect @@ -29,7 +29,7 @@ require ( github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e google.golang.org/api v0.43.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/ini.v1 v1.51.0 diff --git a/go.sum b/go.sum index ed8524a0..18330a8d 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.38.7 h1:uOu2IrTiNhcSNAjBmA21t48lTx5mgGdcFKamDjXMscA= -github.com/aws/aws-sdk-go v1.38.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= +github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -455,8 +455,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -527,18 +528,22 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From cc76eec5841f7bbf499e21d97f91a01e2e2f5af0 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Wed, 2 Nov 2022 16:19:06 -0700 Subject: [PATCH 23/40] fix(azure): use updated API Where the API requires new parameters, I used an empty struct. Although this compiles, it may not be approproate for the use of the code. --- api/common/conf_azure.go | 4 ++-- go.mod | 11 ++++++----- go.sum | 17 +++++++++++++++++ internal/backend_adlv1.go | 2 +- internal/backend_azblob.go | 17 ++++++++--------- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/api/common/conf_azure.go b/api/common/conf_azure.go index 2892af1a..48ce3485 100644 --- a/api/common/conf_azure.go +++ b/api/common/conf_azure.go @@ -269,7 +269,7 @@ func azureFindAccount(client azblob.AccountsClient, account string) (*azblob.End return nil, "", err } - for _, acc := range *accountsRes.Value { + for _, acc := range accountsRes.Values() { if *acc.Name == account { // /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/... parts := strings.SplitN(*acc.ID, "/", 6) @@ -370,7 +370,7 @@ func AzureBlobConfig(endpoint string, location string, storageType string) (conf if key == "" { var keysRes azblob.AccountListKeysResult - keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account) + keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account, azblob.Kerb) if err != nil || len(*keysRes.Keys) == 0 { err = fmt.Errorf("Missing key: configure via AZURE_STORAGE_KEY "+ "or %v/config", configDir) diff --git a/go.mod b/go.mod index e76a3745..cd4feaa5 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.14 require ( cloud.google.com/go/storage v1.14.0 - github.com/Azure/azure-pipeline-go v0.2.2 - github.com/Azure/azure-sdk-for-go v32.1.0+incompatible - github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 + github.com/Azure/azure-pipeline-go v0.2.3 + github.com/Azure/azure-sdk-for-go v61.4.0+incompatible + github.com/Azure/azure-storage-blob-go v0.14.0 github.com/Azure/go-autorest/autorest v0.11.18 github.com/Azure/go-autorest/autorest/adal v0.9.13 github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 @@ -16,7 +16,8 @@ require ( github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect github.com/aws/aws-sdk-go v1.44.37 github.com/go-ole/go-ole v1.2.5 // indirect - github.com/google/uuid v1.1.2 + github.com/gofrs/uuid v4.2.0+incompatible + github.com/google/uuid v1.2.0 github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 @@ -31,6 +32,6 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e google.golang.org/api v0.43.0 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/ini.v1 v1.51.0 ) diff --git a/go.sum b/go.sum index 18330a8d..e3b5e18c 100644 --- a/go.sum +++ b/go.sum @@ -43,10 +43,16 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v61.4.0+incompatible h1:BF2Pm3aQWIa6q9KmxyF1JYKYXtVw67vtvu2Wd54NGuY= +github.com/Azure/azure-sdk-for-go v61.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= @@ -136,6 +142,8 @@ github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dT github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -201,6 +209,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -263,6 +273,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -273,6 +284,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -435,6 +448,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -503,6 +517,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -518,6 +533,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -707,6 +723,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/backend_adlv1.go b/internal/backend_adlv1.go index 6b695653..3226a9ea 100644 --- a/internal/backend_adlv1.go +++ b/internal/backend_adlv1.go @@ -27,8 +27,8 @@ import ( "syscall" "time" + uuid "github.com/gofrs/uuid" "github.com/jacobsa/fuse" - uuid "github.com/satori/go.uuid" "github.com/sirupsen/logrus" adl "github.com/Azure/azure-sdk-for-go/services/datalake/store/2016-11-01/filesystem" diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index bf0a4d72..b83fedbb 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -443,7 +443,7 @@ func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { } blob := c.NewBlobURL(param.Key) - resp, err := blob.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) + resp, err := blob.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -492,7 +492,7 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { prefixes := make([]BlobPrefixOutput, 0) items := make([]BlobItemOutput, 0) - var blobItems []azblob.BlobItem + var blobItems []azblob.BlobItemInternal var nextMarker *string options := azblob.ListBlobsSegmentOptions{ @@ -711,8 +711,7 @@ func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { src := c.NewBlobURL(param.Source) dest := c.NewBlobURL(param.Destination) - resp, err := dest.StartCopyFromURL(context.TODO(), src.URL(), nilMetadata(param.Metadata), - azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}) + resp, err := dest.StartCopyFromURL(context.TODO(), src.URL(), azblob.Metadata{}, azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}) if err != nil { return nil, mapAZBError(err) } @@ -721,7 +720,7 @@ func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { time.Sleep(50 * time.Millisecond) var copy *azblob.BlobGetPropertiesResponse - for copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}); err == nil; copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) { + for copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}); err == nil; copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) { // if there's a new copy, we can only assume the last one was done if copy.CopyStatus() != azblob.CopyStatusPending || copy.CopyID() != resp.CopyID() { break @@ -753,7 +752,7 @@ func (b *AZBlob) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { ModifiedAccessConditions: azblob.ModifiedAccessConditions{ IfMatch: ifMatch, }, - }, false) + }, false, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -806,7 +805,7 @@ func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { azblob.BlobHTTPHeaders{ ContentType: NilStr(param.ContentType), }, - nilMetadata(param.Metadata), azblob.BlobAccessConditions{}) + nilMetadata(param.Metadata), azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -843,7 +842,7 @@ func (b *AZBlob) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobA atomic.AddUint32(¶m.Commit.NumParts, 1) _, err = blob.StageBlock(context.TODO(), base64BlockId, param.Body, - azblob.LeaseAccessConditions{}, nil) + azblob.LeaseAccessConditions{}, nil, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -873,7 +872,7 @@ func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Multipar resp, err := blob.CommitBlockList(context.TODO(), parts, azblob.BlobHTTPHeaders{}, nilMetadata(param.Metadata), - azblob.BlobAccessConditions{}) + azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } From 34e770928869d071bb0c0ee83a68a76d13398f45 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Wed, 18 May 2022 10:11:40 +0000 Subject: [PATCH 24/40] Add Goofys as user agent for s3 request --- internal/backend_s3.go | 4 ++++ internal/flags.go | 3 ++- main.go | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 1ee9c5f0..ae7da9f0 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -124,6 +124,10 @@ func (s *S3Backend) newS3() { s.setV2Signer(&s.S3.Handlers) } s.S3.Handlers.Sign.PushBack(addAcceptEncoding) + s.S3.Handlers.Build.PushFrontNamed(request.NamedHandler{ + Name: "UserAgentHandler", + Fn: request.MakeAddToUserAgentHandler("Goofys", VersionNumber+"-"+VersionHash), + }) } func (s *S3Backend) detectBucketLocationByHEAD() (err error, isAws bool) { diff --git a/internal/flags.go b/internal/flags.go index f18633a8..0cea21f4 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -77,6 +77,7 @@ COPYRIGHT: ` } +var VersionNumber string var VersionHash string func NewApp() (app *cli.App) { @@ -86,7 +87,7 @@ func NewApp() (app *cli.App) { app = &cli.App{ Name: "goofys", - Version: "0.24.0-" + VersionHash, + Version: VersionNumber + "-" + VersionHash, Usage: "Mount an S3 bucket locally", HideHelp: true, Writer: os.Stderr, diff --git a/main.go b/main.go index e2e06a1f..112b8595 100644 --- a/main.go +++ b/main.go @@ -134,6 +134,7 @@ func massageArg0() { var Version = "use `make build' to fill version hash correctly" func main() { + VersionNumber = "0.24.0" VersionHash = Version massagePath() From 5661a83f953a6a2352b13e782fbcf99e92089d3a Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Thu, 19 May 2022 08:37:47 +0000 Subject: [PATCH 25/40] change name to lower case --- internal/backend_s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index ae7da9f0..42da5a8f 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -126,7 +126,7 @@ func (s *S3Backend) newS3() { s.S3.Handlers.Sign.PushBack(addAcceptEncoding) s.S3.Handlers.Build.PushFrontNamed(request.NamedHandler{ Name: "UserAgentHandler", - Fn: request.MakeAddToUserAgentHandler("Goofys", VersionNumber+"-"+VersionHash), + Fn: request.MakeAddToUserAgentHandler("goofys", VersionNumber+"-"+VersionHash), }) } From 5ae08c09108d9e420e77c2ea6b22801be7f46aec Mon Sep 17 00:00:00 2001 From: Sridharan Kuppa Date: Thu, 3 Nov 2022 14:34:37 -0700 Subject: [PATCH 26/40] Escape the special char in name correctly. As per the AWS documentation, CopyRequest is sending request param and it advised to URLEncode the source name. Using the url.QueryEscape to encode the name instead of urlPahEscape since it does not cover all special char. --- internal/backend_s3.go | 4 ++-- internal/goofys.go | 7 ------- internal/goofys_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 42da5a8f..5bb4c3ba 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -479,7 +479,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri params := &s3.UploadPartCopyInput{ Bucket: &s.bucket, Key: &to, - CopySource: aws.String(pathEscape(from)), + CopySource: aws.String(url.QueryEscape(from)), UploadId: &mpuId, CopySourceRange: &bytes, CopySourceIfMatch: srcEtag, @@ -671,7 +671,7 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { params := &s3.CopyObjectInput{ Bucket: &s.bucket, - CopySource: aws.String(pathEscape(from)), + CopySource: aws.String(url.QueryEscape(from)), Key: ¶m.Destination, StorageClass: param.StorageClass, ContentType: s.flags.GetMimeType(param.Destination), diff --git a/internal/goofys.go b/internal/goofys.go index 79e628ce..4f4bd76f 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -574,13 +574,6 @@ func mapAwsError(err error) error { } } -// note that this is NOT the same as url.PathEscape in golang 1.8, -// as this preserves / and url.PathEscape converts / to %2F -func pathEscape(path string) string { - u := url.URL{Path: path} - return u.EscapedPath() -} - func (fs *Goofys) allocateInodeId() (id fuseops.InodeID) { id = fs.nextInodeID fs.nextInodeID++ diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 70c409ca..fec9aa56 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -999,6 +999,32 @@ func (s *GoofysTest) TestCreateFiles(t *C) { defer resp.Body.Close() } +func (s *GoofysTest) TestRenameWithSpecialChar(t *C) { + fileName := "foo+" + s.testWriteFile(t, fileName, 1, 128*1024) + + inode, err := s.getRoot(t).LookUp(fileName) + t.Assert(err, IsNil) + + fh, err := inode.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + t.Assert(err, IsNil) + + err = fh.FlushFile() + t.Assert(err, IsNil) + + resp, err := s.cloud.GetBlob(&GetBlobInput{Key: fileName}) + t.Assert(err, IsNil) + // ADLv1 doesn't return size when we do a GET + if _, adlv1 := s.cloud.(*ADLv1); !adlv1 { + t.Assert(resp.HeadBlobOutput.Size, Equals, uint64(1)) + } + defer resp.Body.Close() + + root := s.getRoot(t) + err = root.Rename(fileName, root, "foo") + t.Assert(err, IsNil) +} + func (s *GoofysTest) TestUnlink(t *C) { fileName := "file1" From 8f80df798c9d5bc03ed2af03eef865f01c14dd9b Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Thu, 3 Nov 2022 16:08:48 -0700 Subject: [PATCH 27/40] fix build --- go.sum | 2 ++ internal/goofys_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index e3b5e18c..83767da8 100644 --- a/go.sum +++ b/go.sum @@ -273,6 +273,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -723,6 +724,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= diff --git a/internal/goofys_test.go b/internal/goofys_test.go index fec9aa56..16b86773 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -1006,7 +1006,7 @@ func (s *GoofysTest) TestRenameWithSpecialChar(t *C) { inode, err := s.getRoot(t).LookUp(fileName) t.Assert(err, IsNil) - fh, err := inode.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := inode.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) err = fh.FlushFile() From dea41e0d608d6db5961b2fb4672b1fedd0d3b64e Mon Sep 17 00:00:00 2001 From: nambrosini Date: Wed, 16 Jun 2021 17:53:28 -0300 Subject: [PATCH 28/40] =?UTF-8?q?=F0=9F=93=9A=20Add=20note=20on=20the=20co?= =?UTF-8?q?rrect=20package=20name=20required=20to=20mount=20on=20startup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 208ec503..7bd56a4e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ close-to-open. # Installation -* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). You may also need to install fuse-utils first. +* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). +You may also need to install fuse too if you want to mount it on startup. * On macOS, install via [Homebrew](https://brew.sh/): From 8228feb2c6a79e7f31350e94c3a1b8884051108f Mon Sep 17 00:00:00 2001 From: Ka-Hing Cheung Date: Thu, 10 Nov 2022 16:57:45 -0800 Subject: [PATCH 29/40] update jacobsa/fuse fixes messages like: 2022/11/11 00:31:20.029032 fuse.ERROR writeMessage: no such file or directory [16 0 0 0 218 255 255 255 200 127 0 0 0 0 0 0] --- go.mod | 4 ++-- go.sum | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index cd4feaa5..313fdd39 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/gofrs/uuid v4.2.0+incompatible github.com/google/uuid v1.2.0 github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect - github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 + github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/mitchellh/go-homedir v1.1.0 github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b @@ -30,7 +30,7 @@ require ( github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e + golang.org/x/sys v0.2.0 google.golang.org/api v0.43.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/ini.v1 v1.51.0 diff --git a/go.sum b/go.sum index 83767da8..ccb6a0a7 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 h1:LkLYmg9wdUlhXv9H31druPMUFPSrpN9sWMxPCTvu5dI= github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5/go.mod h1:xtZnnLxHY6QniCrfIpTwr5h8mH8zr+jsOFj0y9cfyp4= +github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 h1:uv+7DeBJF6+a54ZUihuXR+uTzhv2JslllK5ByILxxbg= +github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8/go.mod h1:liOmRdJd8oTwHCQ5M9JemRE3CebdlYcZWLk+ZjQeuq0= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= @@ -398,6 +400,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -471,8 +475,11 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -546,9 +553,14 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -559,6 +571,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From e903e56038fe0325e4538007b6acc064af8164c7 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Wed, 16 Nov 2022 14:08:03 +0000 Subject: [PATCH 30/40] Implement new BatchForget op for panic logger --- api/common/panic_logger.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 8312c21d..0f22a5ad 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -60,6 +60,10 @@ func (fs FusePanicLogger) ForgetInode(ctx context.Context, op *fuseops.ForgetIno defer LogPanic(&err) return fs.Fs.ForgetInode(ctx, op) } +func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) { + defer LogPanic(&err) + return fs.Fs.BatchForget(ctx, op) +} func (fs FusePanicLogger) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err error) { defer LogPanic(&err) return fs.Fs.MkDir(ctx, op) From b667ee474bb7dc321eb0e069842053373fb2f615 Mon Sep 17 00:00:00 2001 From: Daniil Tulin Date: Tue, 22 Nov 2022 19:01:17 +0300 Subject: [PATCH 31/40] Fix stat checking for cgroup --- api/common/panic_logger.go | 7 +++++++ internal/cgroup.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 0f22a5ad..7e345b27 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -27,6 +27,8 @@ type FusePanicLogger struct { Fs fuseutil.FileSystem } +var _ fuseutil.FileSystem = FusePanicLogger{} + func LogPanic(err *error) { if e := recover(); e != nil { log.Errorf("stacktrace from panic: %v \n"+string(debug.Stack()), e) @@ -36,6 +38,11 @@ func LogPanic(err *error) { } } +func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) { + defer LogPanic(&err) + return fs.Fs.BatchForget(ctx, op) +} + func (fs FusePanicLogger) StatFS(ctx context.Context, op *fuseops.StatFSOp) (err error) { defer LogPanic(&err) return fs.Fs.StatFS(ctx, op) diff --git a/internal/cgroup.go b/internal/cgroup.go index 8f00b285..c73ff501 100644 --- a/internal/cgroup.go +++ b/internal/cgroup.go @@ -45,7 +45,7 @@ func getCgroupAvailableMem() (retVal uint64, err error) { // newer version of docker mounts the cgroup memory limit/usage files directly under // /sys/fs/cgroup/memory/ rather than /sys/fs/cgroup/memory/docker/$container_id/ - if _, err := os.Stat(filepath.Join(CGROUP_FOLDER_PREFIX, path)); os.IsExist(err) { + if _, err := os.Stat(filepath.Join(CGROUP_FOLDER_PREFIX, path)); err == nil { path = filepath.Join(CGROUP_FOLDER_PREFIX, path) } else { path = filepath.Join(CGROUP_FOLDER_PREFIX) From 61d0ef6acd4888b924745e28dc744aff7ed1506e Mon Sep 17 00:00:00 2001 From: Daniil Tulin Date: Tue, 22 Nov 2022 20:43:45 +0300 Subject: [PATCH 32/40] fmt --- api/common/panic_logger.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 7e345b27..4f23603d 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -42,7 +42,6 @@ func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForg defer LogPanic(&err) return fs.Fs.BatchForget(ctx, op) } - func (fs FusePanicLogger) StatFS(ctx context.Context, op *fuseops.StatFSOp) (err error) { defer LogPanic(&err) return fs.Fs.StatFS(ctx, op) From 37cf83563db1bd8aa8d0b02356aacdc3210c26d8 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 26 Sep 2020 11:14:00 +0900 Subject: [PATCH 33/40] Add link for Stack Overflow --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7bd56a4e..564dbb76 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://travis-ci.org/kahing/goofys.svg?branch=master)](https://travis-ci.org/kahing/goofys) [![Github All Releases](https://img.shields.io/github/downloads/kahing/goofys/total.svg)](https://github.com/kahing/goofys/releases/) [![Twitter Follow](https://img.shields.io/twitter/follow/s3goofys.svg?style=social&label=Follow)](https://twitter.com/s3goofys) +[![Stack Overflow Questions](https://img.shields.io/stackexchange/stackoverflow/t/goofys?label=Stack%20Overflow%20questions)](https://stackoverflow.com/search?q=%5Bgoofys%5D+is%3Aquestion) # Overview From 350ff312abaa1abcf21c5a06e143c7edffe9e2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9raud=20Dug=C3=A9=20de=20Bernonville?= Date: Tue, 7 Feb 2023 15:21:31 +0100 Subject: [PATCH 34/40] Fix FusePanicLogger.BatchForget redeclared in this block --- api/common/panic_logger.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 4f23603d..3c710dfa 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -66,10 +66,6 @@ func (fs FusePanicLogger) ForgetInode(ctx context.Context, op *fuseops.ForgetIno defer LogPanic(&err) return fs.Fs.ForgetInode(ctx, op) } -func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) { - defer LogPanic(&err) - return fs.Fs.BatchForget(ctx, op) -} func (fs FusePanicLogger) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err error) { defer LogPanic(&err) return fs.Fs.MkDir(ctx, op) From dbd06bad71e8cb6f0b2b4b1c0bec69c30deb4c06 Mon Sep 17 00:00:00 2001 From: Jose Matsuda Date: Tue, 9 Apr 2024 09:12:06 -0400 Subject: [PATCH 35/40] test: --- internal/backend_s3.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 5bb4c3ba..0204823a 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -661,13 +661,15 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { from := s.bucket + "/" + param.Source - if !s.gcs && *param.Size > COPY_LIMIT { - reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass) - if err != nil { - return nil, err + /* + if !s.gcs && *param.Size > COPY_LIMIT { + reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass) + if err != nil { + return nil, err + } + return &CopyBlobOutput{reqId}, nil } - return &CopyBlobOutput{reqId}, nil - } + */ params := &s3.CopyObjectInput{ Bucket: &s.bucket, From 69a13ab8887d51ee609d4f5621158a655e672be4 Mon Sep 17 00:00:00 2001 From: Jose Matsuda Date: Tue, 9 Apr 2024 15:23:01 -0400 Subject: [PATCH 36/40] forcibly inc buffer size, will definitely crash if above 625 --- internal/backend_s3.go | 21 ++++++--- internal/file.go | 102 ++++++++++++++++++++++++++++++++++------- main.go | 3 +- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 0204823a..897a9f85 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -62,8 +62,10 @@ func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, err flags: flags, config: config, cap: Capabilities{ - Name: "s3", - MaxMultipartSize: 5 * 1024 * 1024 * 1024, + Name: "s3-jose-test", + // MaxMultipartSize doesnt seem to be respected? or at least it goes to multipart right away. + MaxMultipartSize: 5 * 1024 * 1024 * 1024, + NoParallelMultipart: true, }, } @@ -507,7 +509,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri return } -func sizeToParts(size int64) (int, int64) { +func sizeToParts(size int64) (int, int64) { // this shouldnt matter, since this mpu shouldnt get called. const MAX_S3_MPU_SIZE int64 = 5 * 1024 * 1024 * 1024 * 1024 if size > MAX_S3_MPU_SIZE { panic(fmt.Sprintf("object size: %v exceeds maximum S3 MPU size: %v", size, MAX_S3_MPU_SIZE)) @@ -552,13 +554,15 @@ func (s *S3Backend) mpuCopyParts(size int64, from string, to string, mpuId strin sem.V(MAX_CONCURRENCY) } +// This shouldnt get reached, the only reference to `copyObjectMultip` was commented out (other than the test) +// Assuming that this backend_s3.go is the one that is used. func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuId string, srcEtag *string, metadata map[string]*string, storageClass *string) (requestId string, err error) { - nParts, partSize := sizeToParts(size) + nParts, partSize := sizeToParts(size) // should be unreachable, and partsize i dont need anything to do with etags := make([]*string, nParts) if mpuId == "" { - params := &s3.CreateMultipartUploadInput{ + params := &s3.CreateMultipartUploadInput{ // should be unreachable, assuming this is the one that is used Bucket: &s.bucket, Key: &to, StorageClass: storageClass, @@ -581,7 +585,7 @@ func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuI params.ACL = &s.config.ACL } - resp, err := s.CreateMultipartUpload(params) + resp, err := s.CreateMultipartUpload(params) // should be unreachable, assuming this is the one that is used. if err != nil { return "", mapAwsError(err) } @@ -820,7 +824,9 @@ func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { }, nil } +// reached from file.go func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { + // references API then mpu := s3.CreateMultipartUploadInput{ Bucket: &s.bucket, Key: ¶m.Key, @@ -843,9 +849,10 @@ func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*Multipa mpu.ACL = &s.config.ACL } + // again reference API resp, err := s.CreateMultipartUpload(&mpu) if err != nil { - s3Log.Errorf("CreateMultipartUpload %v = %v", param.Key, err) + s3Log.Errorf("Jose: CreateMultipartUpload %v = %v", param.Key, err) // this is the err i get? return nil, mapAwsError(err) } diff --git a/internal/file.go b/internal/file.go index 69cf5b6d..fc0ad565 100644 --- a/internal/file.go +++ b/internal/file.go @@ -83,13 +83,14 @@ func NewFileHandle(inode *Inode, opMetadata fuseops.OpContext) *FileHandle { return fh } -func (fh *FileHandle) initWrite() { +func (fh *FileHandle) initWrite() { // ok how is this reached? fh.writeInit.Do(func() { fh.mpuWG.Add(1) - go fh.initMPU() + go fh.initMPU() // starts multipart? }) } +// how is initMPU reached? func (fh *FileHandle) initMPU() { defer func() { fh.mpuWG.Done() @@ -98,6 +99,7 @@ func (fh *FileHandle) initMPU() { fs := fh.inode.fs fh.mpuName = &fh.key + // how is this reached, could I just do a "PutBlob" instead here (no i dont think so, given other ex in this file)? What would break? resp, err := fh.cloud.MultipartBlobBegin(&MultipartBlobBeginInput{ Key: *fh.mpuName, ContentType: fs.flags.GetMimeType(*fh.mpuName), @@ -146,6 +148,40 @@ func (fh *FileHandle) mpuPartNoSpawn(buf *MBuf, part uint32, total int64, last b return } +// JOSE ATTEMPT +// dont think we should try this at all, since it expects things to be split up +func (fh *FileHandle) uploadSingle(buf *MBuf, part uint32, total int64, last bool) (err error) { + fs := fh.inode.fs + + fs.replicators.Take(1, true) + defer fs.replicators.Return(1) + + if part == 0 || part > 10000 { + return errors.New(fmt.Sprintf("invalid part number: %v", part)) + } + + //hmmm + mpu := MultipartBlobAddInput{ + Commit: fh.mpuId, + PartNumber: part, + Body: buf, + Size: uint64(buf.Len()), + Last: last, + Offset: uint64(total - int64(buf.Len())), + } + + defer func() { + if mpu.Body != nil { + bufferLog.Debugf("Free %T", buf) + buf.Free() + } + }() + + _, err = fh.cloud.MultipartBlobAdd(&mpu) + + return +} + func (fh *FileHandle) mpuPart(buf *MBuf, part uint32, total int64) { defer func() { fh.mpuWG.Done() @@ -169,9 +205,13 @@ func (fh *FileHandle) mpuPart(buf *MBuf, part uint32, total int64) { } func (fh *FileHandle) waitForCreateMPU() (err error) { - if fh.mpuId == nil { + // JOSE: does this need to be modified further??? Like when we upload sequentially does it need to still "wait"? + // no we dont want to even SPLIT it up into parts + if fh.mpuId == nil { // only enters here if mpuId = nil, else returns nothing. + // What will happen if we comment this if out? + // THe question is when does it really start the multipart upload? Like when is it assigned "CreaMultiPartUpload" fh.mu.Unlock() - fh.initWrite() + fh.initWrite() // this calls the MPU to start the createMultipUpload fh.mpuWG.Wait() // wait for initMPU fh.mu.Lock() @@ -183,6 +223,7 @@ func (fh *FileHandle) waitForCreateMPU() (err error) { return } +// So this is actually used regardless in WriteFile under 'MBuf{}.Init(fh.poolHandle, fh.partSize(), true)' func (fh *FileHandle) partSize() uint64 { var size uint64 @@ -200,28 +241,45 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } + // Something I could test here is increasing this and then re uploading and trying things + size = 625 * 1024 * 1024 // ~625 MB return size } func (fh *FileHandle) uploadCurrentBuf(parallel bool) (err error) { - err = fh.waitForCreateMPU() - if err != nil { - return - } + // since removing MPU, remove this call, this would mean that we don't encounter / follow the trail to backend_s3.go + // Where we eventually get to 'MultipartBlobBegin' + + /* + err = fh.waitForCreateMPU() //calls the MPU upload + // this MPU is ALWAYS called, at least once it hits this func and then dies in subsequent calls + if err != nil { + return + } + */ + // the thing is we dont want to break it up, so I feel like we dont want these variable declarations at all. fh.lastPartId++ part := fh.lastPartId buf := fh.buf fh.buf = nil - if parallel { - fh.mpuWG.Add(1) - go fh.mpuPart(buf, part, fh.nextWriteOffset) - } else { - err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) - if fh.lastWriteError == nil { - fh.lastWriteError = err - } + /* + if parallel { // we dont want this if since we dont care about parallel + fh.mpuWG.Add(1) + go fh.mpuPart(buf, part, fh.nextWriteOffset) + } else { + err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) + if fh.lastWriteError == nil { + fh.lastWriteError = err + } + } */ + // We still want to upload though, so we will probably need to modify this mpuPartNoSpawn to at least not call a CreatMultiPUpload + //err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) + err = fh.uploadSingle(buf, part, fh.nextWriteOffset, false) // attempt to change this uploadSingle I created + // i dont think we should even try modifying this, since this splits it up into multi parts + if fh.lastWriteError == nil { + fh.lastWriteError = err } return @@ -267,13 +325,23 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { for { if fh.buf == nil { fh.buf = MBuf{}.Init(fh.poolHandle, fh.partSize(), true) + // I need to modify fh.buf here(?), I feel like this sets buf size to be used later on in fh.buf.Full(). + // but can i just make this infinite? + // note that in the Init, the partSize is related to something that requestsMultiple } nCopied, _ := fh.buf.Write(data) fh.nextWriteOffset += int64(nCopied) + // JOSE: if i comment out this uploadCurrentBuf, which eventually calls multipart, what happens? + // like fh.buf.Full, will it just overflow? Maybe just need to `uploadCurrentBuff` but NOT go into multipart + // / modify that uploadCurrentBuff because we will alwyas need to upload the current thing, so this shouldnt be commented out + // may need to reach a state where the buffer is never full lol... or at least to some reasonable number so if people really need to + // they can use mc or whatever... if fh.buf.Full() { err = fh.uploadCurrentBuf(!fh.cloud.Capabilities().NoParallelMultipart) + // i feel like under no 'modifiable' circumstances would trying to change `uploadCurrBuff` work, because + // it splits the file. thing is what do i do once the buffer is full then? if err != nil { return } @@ -284,7 +352,7 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { } data = data[nCopied:] - } + } // end for loop to upload fh.inode.Attributes.Size = uint64(fh.nextWriteOffset) fh.inode.Attributes.Mtime = time.Now() diff --git a/main.go b/main.go index 112b8595..4e55f900 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,8 @@ func massageArg0() { var Version = "use `make build' to fill version hash correctly" func main() { - VersionNumber = "0.24.0" + // Changing just to make sure when I push im not using regular goofys still + VersionNumber = "0.42.0" VersionHash = Version massagePath() From 8eaaec3430e6246cb5dbf4210c8a80e586bd8e7f Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Wed, 10 Apr 2024 07:21:39 -0400 Subject: [PATCH 37/40] 10Gb --- internal/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index fc0ad565..3778b2b8 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,8 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Something I could test here is increasing this and then re uploading and trying things - size = 625 * 1024 * 1024 // ~625 MB + // Try 10Gb + size = 10000 * 1024 * 1024 // ~10GB return size } From e45008f359d1c3a1ee0b864319d4e0083cd5e241 Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Wed, 10 Apr 2024 11:27:36 -0400 Subject: [PATCH 38/40] 4.5 --- internal/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index 3778b2b8..db12504b 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,8 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Try 10Gb - size = 10000 * 1024 * 1024 // ~10GB + // Try 4.5Gb + size = 45000 * 1024 * 1024 // ~2.5Gb return size } From a1fb9da08cf7fdeec2c72d7f83f3f1ed03e71106 Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Thu, 11 Apr 2024 12:36:59 -0400 Subject: [PATCH 39/40] a little bit under 5Gb --- internal/file.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index db12504b..aeb5df41 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,7 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Try 4.5Gb - size = 45000 * 1024 * 1024 // ~2.5Gb + size = 5000 * 1024 * 1024 return size } From 5a80de4ae5ec28a9aeceb9508dd0d166e1150d18 Mon Sep 17 00:00:00 2001 From: Mathis <84033116+mathis-marcotte@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:56:12 -0400 Subject: [PATCH 40/40] replaced log.fatalf with .og.printf --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 4e55f900..3101d78f 100644 --- a/main.go +++ b/main.go @@ -215,8 +215,10 @@ func main() { if !flags.Foreground { kill(os.Getppid(), syscall.SIGUSR2) } - log.Fatalf("Mounting file system: %v", err) + // log.Fatalf("Mounting file system: %v", err) // fatal also terminates itself + log.Printf("Mounting file system: %v", err) + return } else { if !flags.Foreground { kill(os.Getppid(), syscall.SIGUSR1)