diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..c65b74e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,40 @@ +version: 2.1 + +parameters: + go-version: + type: string + default: "1.20" + kurtosis-package-catalog-yaml-file-path: + type: string + default: "kurtosis-package-catalog.yml" + +# NOTE: Because CircleCI jobs run on separate machines from each other, we duplicate steps (like checkout) between jobs. This is because doing the "correct" DRY +# refactoring of, "one job for checkout, one job for build Docker image, etc." would require a) persisting files between jobs and b) persisting Docker images between +# jobs. Both are annoying (saving/loading workspaces require re-downloading the workspace over the network, and there doesn't seem to be a good way to do Docker +# images), so we run everything inside a single job. +# See also: https://discuss.circleci.com/t/can-docker-images-be-preserved-between-jobs-in-a-workflow-without-a-manual-load-save/23388/12 +jobs: + build_catalog_validator: + docker: + - image: "cimg/go:<< pipeline.parameters.go-version>>" + working_directory: /home/circleci/workspace + steps: + - checkout + + # build and run the golang app + - run: | + export GITHUB_USER_TOKEN=${KURTOSISBOT_GITHUB_TOKEN} + catalog-validator/scripts/build.sh + catalog-validator/build/catalog-validator << pipeline.parameters.kurtosis-package-catalog-yaml-file-path >> + +workflows: + build: + jobs: + - build_catalog_validator: + context: + - github-user + filters: + branches: + ignore: + - develop + - main diff --git a/catalog-validator/go.mod b/catalog-validator/go.mod new file mode 100644 index 0000000..e131b9c --- /dev/null +++ b/catalog-validator/go.mod @@ -0,0 +1,32 @@ +module github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator + +go 1.20 + +require ( + github.com/kurtosis-tech/kurtosis-package-indexer/server v0.0.0-20240102153702-2b1c60be2f40 + github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 +) + +require ( + github.com/google/go-github/v54 v54.0.0 + github.com/sirupsen/logrus v1.9.3 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/aws/aws-sdk-go v1.44.334 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/catalog-validator/go.sum b/catalog-validator/go.sum new file mode 100644 index 0000000..7497f7d --- /dev/null +++ b/catalog-validator/go.sum @@ -0,0 +1,105 @@ +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/aws/aws-sdk-go v1.44.334 h1:h2bdbGb//fez6Sv6PaYv868s9liDeoYM6hYsAqTB4MU= +github.com/aws/aws-sdk-go v1.44.334/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-github/v54 v54.0.0 h1:OZdXwow4EAD5jEo5qg+dGFH2DpkyZvVsAehjvJuUL/c= +github.com/google/go-github/v54 v54.0.0/go.mod h1:Sw1LXWHhXRZtzJ9LI5fyJg9wbQzYvFhW8W5P2yaAQ7s= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kurtosis-tech/kurtosis-package-indexer/server v0.0.0-20240102153702-2b1c60be2f40 h1:IDBwLYTNR1B1jx2Ad4vKik3/Inm+CT3HO6UZT4tHHc0= +github.com/kurtosis-tech/kurtosis-package-indexer/server v0.0.0-20240102153702-2b1c60be2f40/go.mod h1:NJRJGx1APkiySqvxWI///pRCNXlb0UFTxiVOHNx0S4o= +github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 h1:YQTATifMUwZEtZYb0LVA7DK2pj8s71iY8rzweuUQ5+g= +github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409/go.mod h1:y5weVs5d9wXXHcDA1awRxkIhhHC1xxYJN8a7aXnE6S8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/catalog-validator/importer/importer.go b/catalog-validator/importer/importer.go new file mode 100644 index 0000000..c5e6332 --- /dev/null +++ b/catalog-validator/importer/importer.go @@ -0,0 +1,27 @@ +package importer + +import ( + "os" + + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" + "github.com/kurtosis-tech/stacktrace" +) + +func ReadCatalog(kurtosisPackageCatalogYamlFilepath string) (catalog.PackageCatalog, error) { + _, err := os.Stat(kurtosisPackageCatalogYamlFilepath) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred checking for Kurtosis package catalog YAML file existence on '%s'", kurtosisPackageCatalogYamlFilepath) + } + + fileBytes, err := os.ReadFile(kurtosisPackageCatalogYamlFilepath) + if err != nil { + return nil, stacktrace.Propagate(err, "attempted to read file with path '%v' but failed", kurtosisPackageCatalogYamlFilepath) + } + + packageCatalog, err := catalog.GetPackageCatalogFromYamlFileContent(fileBytes) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred reading the Kurtosis package catalog YAML file content from '%s'", kurtosisPackageCatalogYamlFilepath) + } + + return packageCatalog, nil +} diff --git a/catalog-validator/main.go b/catalog-validator/main.go new file mode 100644 index 0000000..d4375ce --- /dev/null +++ b/catalog-validator/main.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator/importer" + "github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator/validation/rules" + "github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator/validation/validator" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "os" + "path" + "runtime" + "strings" +) + +const ( + successExitCode = 0 + failureExitCode = 1 + + forceColors = true + fullTimestamp = true + + logMethodAlongWithLogLine = true + functionPathSeparator = "." + emptyFunctionName = "" +) + +func main() { + + ctx := context.Background() + configureLogger() + + packageCatalogYamlFilepath, err := getKurtosisPackageCatalogYAMLFilepathFromArgs() + if err != nil { + exitFailure(err) + } + + logrus.Infof("Importing the Kurtosis package catalog content from '%s'...", packageCatalogYamlFilepath) + packageCatalog, err := importer.ReadCatalog(packageCatalogYamlFilepath) + if err != nil { + exitFailure(err) + } + logrus.Info("...catalog YAML file was successfully imported") + + logrus.Info("Running the validations...") + rulesToValidate, err := rules.GetAll(ctx) + if err != nil { + exitFailure(err) + } + validatorObj := validator.NewValidator(packageCatalog, rulesToValidate) + validatorResult, err := validatorObj.Validate(ctx) + if err != nil { + exitFailure(err) + } + + if !validatorResult.IsValidCatalog() { + logrus.Errorf("THE VALIDATOR REPORT FAILURES IN THE FOLLOWING RULES") + logrus.Errorf("======================================================================") + for ruleName, packagesWithFailures := range validatorResult.GetRulesResult() { + logrus.Errorf("-------------------------------------------------------------------") + logrus.Errorf("RULE: '%s'", ruleName) + logrus.Errorf("-------------------------------------------------------------------") + for packageName, failures := range packagesWithFailures { + logrus.Errorf("Package: '%s'", packageName) + for _, failure := range failures { + logrus.Errorf(" - %s", failure) + } + } + } + logrus.Errorf("========================================================================") + exitFailure(stacktrace.NewError("the current package catalog is not valid.")) + } + + logrus.Info("...all validations passed") + + logrus.Exit(successExitCode) +} + +func getKurtosisPackageCatalogYAMLFilepathFromArgs() (string, error) { + args := os.Args + if len(args) < 2 { + return "", stacktrace.NewError("expected to received the kurtosis package catalog YAML filepath as the first argument, but it was not received") + } + return args[1], nil +} + +func configureLogger() { + logrus.SetLevel(logrus.DebugLevel) + // This allows the filename & function to be reported + logrus.SetReportCaller(logMethodAlongWithLogLine) + // NOTE: we'll want to change the ForceColors to false if we ever want structured logging + logrus.SetFormatter(&logrus.TextFormatter{ + ForceColors: forceColors, + DisableColors: false, + ForceQuote: false, + DisableQuote: false, + EnvironmentOverrideColors: false, + DisableTimestamp: false, + FullTimestamp: fullTimestamp, + TimestampFormat: "", + DisableSorting: false, + SortingFunc: nil, + DisableLevelTruncation: false, + PadLevelText: false, + QuoteEmptyFields: false, + FieldMap: nil, + CallerPrettyfier: func(f *runtime.Frame) (string, string) { + fullFunctionPath := strings.Split(f.Function, functionPathSeparator) + functionName := fullFunctionPath[len(fullFunctionPath)-1] + _, filename := path.Split(f.File) + return emptyFunctionName, formatFilenameFunctionForLogs(filename, functionName) + }, + }) +} + +func formatFilenameFunctionForLogs(filename string, functionName string) string { + var output strings.Builder + output.WriteString("[") + output.WriteString(filename) + output.WriteString(":") + output.WriteString(functionName) + output.WriteString("]") + return output.String() +} + +func exitFailure(err error) { + logrus.Error(err.Error()) + logrus.Exit(failureExitCode) +} diff --git a/catalog-validator/scripts/build.sh b/catalog-validator/scripts/build.sh new file mode 100755 index 0000000..d837476 --- /dev/null +++ b/catalog-validator/scripts/build.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# 2023-12-20 WATERMARK, DO NOT REMOVE - This script was generated from the Kurtosis Bash script template + +set -euo pipefail # Bash "strict mode" +script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +app_root_dirpath="$(dirname "${script_dirpath}")" + + +# ================================================================================================== +# Constants +# ================================================================================================== +BUILD_DIRNAME="build" + +MAIN_GO_FILEPATH="${app_root_dirpath}/main.go" +MAIN_BINARY_OUTPUT_FILENAME="catalog-validator" +MAIN_BINARY_OUTPUT_FILEPATH="${app_root_dirpath}/${BUILD_DIRNAME}/${MAIN_BINARY_OUTPUT_FILENAME}" + +# ================================================================================================== +# Main Logic +# ================================================================================================== + +# Test code +echo "Running unit tests..." +if ! cd "${app_root_dirpath}"; then + echo "Couldn't cd to the server root dirpath '${app_root_dirpath}'" >&2 + exit 1 +fi +if ! CGO_ENABLED=0 go test "./..."; then + echo "Tests failed!" >&2 + exit 1 +fi +echo "Tests succeeded" + +# Build binary for packaging inside an Alpine Linux image +echo "Building server main.go '${MAIN_GO_FILEPATH}'..." +if ! CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "${MAIN_BINARY_OUTPUT_FILEPATH}" "${MAIN_GO_FILEPATH}"; then + echo "Error: An error occurred building the server code" >&2 + exit 1 +fi +echo "Successfully built server code" diff --git a/catalog-validator/validation/rules/all_rules.go b/catalog-validator/validation/rules/all_rules.go new file mode 100644 index 0000000..715dc40 --- /dev/null +++ b/catalog-validator/validation/rules/all_rules.go @@ -0,0 +1,23 @@ +package rules + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/github" + "github.com/kurtosis-tech/stacktrace" +) + +func GetAll(ctx context.Context) ([]Rule, error) { + + gitHubClient, err := github.CreateGithubClient(ctx) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred creating the GitHub client") + } + + allRules := []Rule{ + newDuplicatedPackageRule(), + newValidPackageRule(gitHubClient), + newValidPackageIconRule(gitHubClient), + } + + return allRules, nil +} diff --git a/catalog-validator/validation/rules/duplicated_package_rule.go b/catalog-validator/validation/rules/duplicated_package_rule.go new file mode 100644 index 0000000..3c0f45e --- /dev/null +++ b/catalog-validator/validation/rules/duplicated_package_rule.go @@ -0,0 +1,46 @@ +package rules + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" +) + +const ( + duplicatedPackageRuleName = "Duplicated package" +) + +// duplicatedPackageRule checks that there is not duplicated packages name in the catalog +type duplicatedPackageRule struct { + name string +} + +func newDuplicatedPackageRule() *duplicatedPackageRule { + return &duplicatedPackageRule{name: duplicatedPackageRuleName} +} + +func (duplicatedPackageRule *duplicatedPackageRule) GetName() RuleName { + return RuleName(duplicatedPackageRule.name) +} + +func (duplicatedPackageRule *duplicatedPackageRule) Check(_ context.Context, catalog catalog.PackageCatalog) *CheckResult { + + wasValidated := true + failures := map[types.PackageName][]string{} + + packageNames := map[types.PackageName]bool{} + + for _, packageData := range catalog { + packageName := packageData.GetPackageName() + if _, found := packageNames[packageName]; found { + failures[packageName] = []string{"duplicated name"} + wasValidated = false + continue + } + packageNames[packageName] = true + } + + checkResult := newCheckResult(duplicatedPackageRule.GetName(), wasValidated, failures) + + return checkResult +} diff --git a/catalog-validator/validation/rules/invalid_rule_result.go b/catalog-validator/validation/rules/invalid_rule_result.go new file mode 100644 index 0000000..592d15a --- /dev/null +++ b/catalog-validator/validation/rules/invalid_rule_result.go @@ -0,0 +1,36 @@ +package rules + +import ( + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" + "github.com/kurtosis-tech/stacktrace" +) + +type CheckResult struct { + ruleName RuleName + wasValidated bool + failures map[types.PackageName][]string +} + +func newCheckResult(ruleName RuleName, wasValidated bool, failures map[types.PackageName][]string) *CheckResult { + return &CheckResult{ruleName: ruleName, wasValidated: wasValidated, failures: failures} +} + +func (ruleReport *CheckResult) GetRuleName() RuleName { + return ruleReport.ruleName +} + +func (ruleReport *CheckResult) WasValidated() bool { + return ruleReport.wasValidated +} + +func (ruleReport *CheckResult) GetFailures() map[types.PackageName][]string { + return ruleReport.failures +} + +func (ruleReport *CheckResult) GetFailuresForPackage(packageName types.PackageName) ([]string, error) { + failures, found := ruleReport.failures[packageName] + if !found { + return nil, stacktrace.NewError("Expected to find failures for package '%s' but nothing was found, this is a bug in the catalog", packageName) + } + return failures, nil +} diff --git a/catalog-validator/validation/rules/rule.go b/catalog-validator/validation/rules/rule.go new file mode 100644 index 0000000..bf775c0 --- /dev/null +++ b/catalog-validator/validation/rules/rule.go @@ -0,0 +1,13 @@ +package rules + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" +) + +type RuleName string + +type Rule interface { + GetName() RuleName + Check(ctx context.Context, catalog catalog.PackageCatalog) *CheckResult +} diff --git a/catalog-validator/validation/rules/valid_package_icon_rule.go b/catalog-validator/validation/rules/valid_package_icon_rule.go new file mode 100644 index 0000000..e653125 --- /dev/null +++ b/catalog-validator/validation/rules/valid_package_icon_rule.go @@ -0,0 +1,145 @@ +package rules + +import ( + "context" + "errors" + "fmt" + "github.com/google/go-github/v54/github" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/consts" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "image" + _ "image/png" // need to import it to get the PNG Encoder/Decoder + "net/http" + "path" + "strings" +) + +const ( + validPackageIconRuleName = "Valid package icon" + minImageSize = 120 + maxImageSize = 1024 +) + +// validPackageIconRule checks if the package icon is valid by checking if: +// 1- if the png image exist, does not return an error if it not because it's not mandatory yet +// 2- if the image size is equal or bigger that the minImageSize +// 3- if the image size is equal or greater than maxImageSize +// 4- if the aspect ratio is 1:1 (a square image) +type validPackageIconRule struct { + name string + gitHubClient *github.Client +} + +func newValidPackageIconRule(gitHubClient *github.Client) *validPackageIconRule { + return &validPackageIconRule{name: validPackageIconRuleName, gitHubClient: gitHubClient} +} + +func (validPackageIconRule *validPackageIconRule) GetName() RuleName { + return RuleName(validPackageIconRule.name) +} + +func (validPackageIconRule *validPackageIconRule) Check(ctx context.Context, catalog catalog.PackageCatalog) *CheckResult { + + wasValidated := true + failures := map[types.PackageName][]string{} + + for _, packageData := range catalog { + packageName := packageData.GetPackageName() + logrus.Debugf("Checking if package '%s' contains a valid icon...", packageName) + repositoryOwner := packageData.GetRepositoryOwner() + repositoryName := packageData.GetRepositoryName() + repositoryPackageRootPath := packageData.GetRepositoryPackageRootPath() + packageFailures := []string{} + packageIconImageConfig, err := validPackageIconRule.getPackageIconImageConfig(ctx, packageName, repositoryOwner, repositoryName, repositoryPackageRootPath) + if err != nil { + errorFailure := fmt.Sprintf("an error occurred getting the Kurtosis package icon image config for package '%s'. Error was:\n%s", packageName, err.Error()) + packageFailures = append(packageFailures, errorFailure) + } + if err == nil && packageIconImageConfig == nil { + logrus.Debugf("package '%s' does not have an icon yet.", packageName) + continue + } + if err == nil { + packageIconWidth := packageIconImageConfig.Width + packageIconHeight := packageIconImageConfig.Height + + if packageIconWidth < minImageSize || packageIconHeight < minImageSize { + invalidMinSizeMsg := fmt.Sprintf( + "invalid image min size, it is smaller than expected. "+ + "Valid min value is '%dpx' and the current size is width: %dpx and height: %dpx", + minImageSize, + packageIconWidth, + packageIconHeight, + ) + packageFailures = append(packageFailures, invalidMinSizeMsg) + } + + if packageIconWidth > maxImageSize || packageIconHeight > maxImageSize { + invalidMaxSizeMsg := fmt.Sprintf( + "invalid image max size, it is bigger than expected. "+ + "Valid max value is '%dpx' and the current size is width: %dpx and height: %dpx", + maxImageSize, + packageIconWidth, + packageIconHeight, + ) + packageFailures = append(packageFailures, invalidMaxSizeMsg) + } + + if packageIconWidth != packageIconHeight { + invalidAspectRatioMsg := "invalid aspect ratio, the accepted aspect ration is 1:1 (a square image)." + + packageFailures = append(packageFailures, invalidAspectRatioMsg) + } + } + + if len(packageFailures) > 0 { + failures[packageName] = packageFailures + wasValidated = false + continue + } + logrus.Debugf("...package icon for '%s' successfully validated.", packageName) + } + + checkResult := newCheckResult(validPackageIconRule.GetName(), wasValidated, failures) + + return checkResult +} + +func (validPackageIconRule *validPackageIconRule) getPackageIconImageConfig(ctx context.Context, packageName types.PackageName, repositoryOwner string, repositoryName string, repositoryPackageRootPath string) (*image.Config, error) { + packageIconFilepath := path.Join(repositoryPackageRootPath, consts.KurtosisPackageIconImgName) + repoGetContentOpts := &github.RepositoryContentGetOptions{ + Ref: "", + } + + // get contents of kurtosis package icon file from GitHub + packageIconFileContentResult, _, resp, err := validPackageIconRule.gitHubClient.Repositories.GetContents(ctx, repositoryOwner, repositoryName, packageIconFilepath, repoGetContentOpts) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + // having the icon is not mandatory + return nil, nil + } + errMsj := fmt.Sprintf("an error occurred reading content of Kurtosis Package '%s' - file '%s'", packageName, packageIconFilepath) + if errors.Is(err, &github.RateLimitError{}) { + errMsj = "GitHub API rate limit exceeded." + logrus.Errorf("%s Error is:\n%v", errMsj, err.Error()) + } + return nil, stacktrace.Propagate(err, errMsj) + } + + rawPackageIconContentStr, err := packageIconFileContentResult.GetContent() + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred getting the '%s' base 64 file content in package '%s'", packageIconFilepath, packageName) + } + + packageIconContentReader := strings.NewReader(rawPackageIconContentStr) + + packageIconConfig, _, err := image.DecodeConfig(packageIconContentReader) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred while decoding the '%s' image file in package '%s'", packageIconFilepath, packageName) + } + + return &packageIconConfig, nil +} diff --git a/catalog-validator/validation/rules/valid_package_rule.go b/catalog-validator/validation/rules/valid_package_rule.go new file mode 100644 index 0000000..0c4ad05 --- /dev/null +++ b/catalog-validator/validation/rules/valid_package_rule.go @@ -0,0 +1,124 @@ +package rules + +import ( + "context" + "errors" + "fmt" + "github.com/google/go-github/v54/github" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/consts" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + "net/http" + "path" +) + +const ( + validPackageRuleName = "Valid package" +) + +type KurtosisYaml struct { + Name string `yaml:"name"` + Description string `yaml:"description"` +} + +// validPackageRule checks if the package is valid by checking if: +// 1- the package repository exist +// 2- if the package repository contains the kurtosis.yml file +// 3- if the name inside the kurtosis.yml file is the same in the package catalog +type validPackageRule struct { + name string + gitHubClient *github.Client +} + +func newValidPackageRule(gitHubClient *github.Client) *validPackageRule { + return &validPackageRule{name: validPackageRuleName, gitHubClient: gitHubClient} +} + +func (validPackageRule *validPackageRule) GetName() RuleName { + return RuleName(validPackageRule.name) +} + +func (validPackageRule *validPackageRule) Check(ctx context.Context, catalog catalog.PackageCatalog) *CheckResult { + + wasValidated := true + failures := map[types.PackageName][]string{} + + for _, packageData := range catalog { + packageName := packageData.GetPackageName() + logrus.Debugf("Checking if package '%s' is valid...", packageName) + repositoryOwner := packageData.GetRepositoryOwner() + repositoryName := packageData.GetRepositoryName() + repositoryPackageRootPath := packageData.GetRepositoryPackageRootPath() + packageFailures := []string{} + packageNameFromKurtosisYamlFile, err := validPackageRule.getPackageNameFromKurtosisYmlFile(ctx, packageName, repositoryOwner, repositoryName, repositoryPackageRootPath) + if err != nil { + errorFailure := fmt.Sprintf("the package does not exist or does not contains the '%s' file", consts.DefaultKurtosisYamlFilename) + packageFailures = append(packageFailures, errorFailure) + } else { + if packageName != packageNameFromKurtosisYamlFile { + invalidPackageNameMsg := fmt.Sprintf("package name '%s' in the catalog does not match with the name '%s' found in the package repository", packageName, packageNameFromKurtosisYamlFile) + packageFailures = append(packageFailures, invalidPackageNameMsg) + } + } + + if len(packageFailures) > 0 { + failures[packageName] = packageFailures + wasValidated = false + continue + } + logrus.Debugf("...package '%s' successfully validated.", packageName) + } + + checkResult := newCheckResult(validPackageRule.GetName(), wasValidated, failures) + + return checkResult +} + +func (validPackageRule *validPackageRule) getPackageNameFromKurtosisYmlFile(ctx context.Context, packageName types.PackageName, repositoryOwner string, repositoryName string, repositoryPackageRootPath string) (types.PackageName, error) { + kurtosisYamlFilepath := path.Join(repositoryPackageRootPath, consts.DefaultKurtosisYamlFilename) + repoGetContentOpts := &github.RepositoryContentGetOptions{ + Ref: "", + } + + // get contents of kurtosis yaml file from GitHub + kurtosisYamlFileContentResult, _, resp, err := validPackageRule.gitHubClient.Repositories.GetContents(ctx, repositoryOwner, repositoryName, kurtosisYamlFilepath, repoGetContentOpts) + if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound { + return "", stacktrace.NewError("No '%s' file for package '%s'", kurtosisYamlFilepath, packageName) + } else if err != nil { + errMsj := fmt.Sprintf("An error occurred reading content of Kurtosis Package '%s' - file '%s'", packageName, kurtosisYamlFilepath) + if errors.Is(err, &github.RateLimitError{}) { + errMsj = "GitHub API rate limit exceeded." + logrus.Errorf("%s Error is:\n%v", errMsj, err.Error()) + } + return "", stacktrace.Propagate(err, "%s", errMsj) + } + + kurtosisYaml, err := parseKurtosisYaml(kurtosisYamlFileContentResult) + if err != nil { + return "", stacktrace.Propagate(err, "an error occurred parsing the Kurtosis YAML file for '%s'", packageName) + } + + packageNameFromYamlFile := types.PackageName(kurtosisYaml.Name) + + return packageNameFromYamlFile, nil +} + +func parseKurtosisYaml(kurtosisYamlContent *github.RepositoryContent) (*KurtosisYaml, error) { + rawFileContent, err := kurtosisYamlContent.GetContent() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the content of the '%s' file", consts.DefaultKurtosisYamlFilename) + } + + kurtosisYaml := new(KurtosisYaml) + if err = yaml.Unmarshal([]byte(rawFileContent), kurtosisYaml); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred parsing YAML for '%s'", consts.DefaultKurtosisYamlFilename) + } + + if kurtosisYaml.Name == "" { + return nil, stacktrace.NewError("Kurtosis YAML file had an empty name. This is invalid.") + } + return kurtosisYaml, nil +} diff --git a/catalog-validator/validation/validator/result.go b/catalog-validator/validation/validator/result.go new file mode 100644 index 0000000..f995418 --- /dev/null +++ b/catalog-validator/validation/validator/result.go @@ -0,0 +1,23 @@ +package validator + +import ( + "github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator/validation/rules" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" +) + +type result struct { + isValidCatalog bool + rulesResult map[rules.RuleName]map[types.PackageName][]string +} + +func newResult(isValidCatalog bool, rulesResult map[rules.RuleName]map[types.PackageName][]string) *result { + return &result{isValidCatalog: isValidCatalog, rulesResult: rulesResult} +} + +func (result *result) IsValidCatalog() bool { + return result.isValidCatalog +} + +func (result *result) GetRulesResult() map[rules.RuleName]map[types.PackageName][]string { + return result.rulesResult +} diff --git a/catalog-validator/validation/validator/validator.go b/catalog-validator/validation/validator/validator.go new file mode 100644 index 0000000..e61df6d --- /dev/null +++ b/catalog-validator/validation/validator/validator.go @@ -0,0 +1,57 @@ +package validator + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-package-catalog/catalog-validator/validation/rules" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/catalog" + "github.com/kurtosis-tech/kurtosis-package-indexer/server/types" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +type Validator struct { + catalog catalog.PackageCatalog + rules []rules.Rule +} + +func NewValidator(catalog catalog.PackageCatalog, rules []rules.Rule) *Validator { + return &Validator{catalog: catalog, rules: rules} +} + +func (validator *Validator) Validate(ctx context.Context) (*result, error) { + + isValidCatalog := true + rulesResult := map[rules.RuleName]map[types.PackageName][]string{} + + for _, rule := range validator.rules { + logrus.Debugf("Checking rule '%s'", rule.GetName()) + if checkResult := rule.Check(ctx, validator.catalog); !checkResult.WasValidated() { + isValidCatalog = false + failures := checkResult.GetFailures() + var packageName types.PackageName + for packageNameInFailures := range failures { + packageName = packageNameInFailures + break + } + ruleName := checkResult.GetRuleName() + failuresByPackageForRule, found := rulesResult[ruleName] + if found { + packageFailures, err := checkResult.GetFailuresForPackage(packageName) + if err != nil { + return nil, stacktrace.Propagate(err, "an error occurred getting failures for package '%s'", packageName) + } + failuresByPackageForRule[packageName] = packageFailures + } else { + rulesResult[ruleName] = checkResult.GetFailures() + } + + logrus.Debugf("the current catalog version does not pass rule '%s'", rule.GetName()) + continue + } + logrus.Debugf("'%s' rule passed", rule.GetName()) + } + + resultObj := newResult(isValidCatalog, rulesResult) + + return resultObj, nil +}