From d124abaf1187fa6625fbbf9d6f9dd0b6dae92329 Mon Sep 17 00:00:00 2001 From: Adam Cornett Date: Mon, 12 Aug 2024 13:19:00 -0700 Subject: [PATCH] adding call to pyxis for non EOL catalogs and updating imagestream logic Signed-off-by: Adam D. Cornett --- go.mod | 1 + go.sum | 2 + internal/pyxis/pyxis.go | 72 +++++++++++++++++++ internal/pyxis/types.go | 7 ++ .../reconcilers/certified_image_stream.go | 47 ++++++++---- .../reconcilers/marketplace_image_stream.go | 47 ++++++++---- 6 files changed, 150 insertions(+), 26 deletions(-) create mode 100644 internal/pyxis/pyxis.go create mode 100644 internal/pyxis/types.go diff --git a/go.mod b/go.mod index c1e4dd1..d928bee 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/gomega v1.34.1 github.com/openshift/api v0.0.0-20240802200810-346347bccbc8 github.com/operator-framework/api v0.26.0 + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 github.com/tektoncd/pipeline v0.62.0 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 diff --git a/go.sum b/go.sum index 1043cdd..f78bdd9 100644 --- a/go.sum +++ b/go.sum @@ -343,6 +343,8 @@ github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDN github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= diff --git a/internal/pyxis/pyxis.go b/internal/pyxis/pyxis.go new file mode 100644 index 0000000..68fb54c --- /dev/null +++ b/internal/pyxis/pyxis.go @@ -0,0 +1,72 @@ +package pyxis + +import ( + "context" + "fmt" + "net/http" + + "github.com/shurcooL/graphql" +) + +const ( + DefaultPyxisHost = "catalog.redhat.com/api/containers" +) + +type PyxisClient struct { + Client *http.Client + PyxisHost string +} + +func (p *PyxisClient) getPyxisGraphqlURL() string { + return fmt.Sprintf("https://%s/graphql/", p.PyxisHost) +} + +func NewPyxisClient(pyxisHost string, httpClient *http.Client) *PyxisClient { + return &PyxisClient{ + Client: httpClient, + PyxisHost: pyxisHost, + } +} + +func (p *PyxisClient) FindOperatorIndices(ctx context.Context, organization string) ([]OperatorIndex, error) { + // our graphQL query + var query struct { + FindOperatorIndices struct { + OperatorIndex []struct { + OCPVersion graphql.String `graphql:"ocp_version"` + Organization graphql.String `graphql:"organization"` + EndOfLife graphql.String `graphql:"end_of_life"` + } `graphql:"data"` + Errors struct { + Status graphql.Int `graphql:"status"` + Detail graphql.String `graphql:"detail"` + } `graphql:"error"` + Total graphql.Int + Page graphql.Int + // filter to make sure we get exact results, end_of_life is a string, querying for `null` yields active OCP versions. + } `graphql:"find_operator_indices(filter:{and:[{organization:{eq:$organization}},{end_of_life:{eq:null}}]})"` + } + + // variables to feed to our graphql filter + variables := map[string]interface{}{ + "organization": graphql.String(organization), + } + + // make our query + client := graphql.NewClient(p.getPyxisGraphqlURL(), p.Client) + + err := client.Query(ctx, &query, variables) + if err != nil { + return nil, fmt.Errorf("error while executing remote query for %s catalogs: %v", organization, err) + } + + operatorIndices := make([]OperatorIndex, len(query.FindOperatorIndices.OperatorIndex)) + for idx, operator := range query.FindOperatorIndices.OperatorIndex { + operatorIndices[idx] = OperatorIndex{ + OCPVersion: string(operator.OCPVersion), + Organization: string(operator.Organization), + } + } + + return operatorIndices, nil +} diff --git a/internal/pyxis/types.go b/internal/pyxis/types.go new file mode 100644 index 0000000..76c7ebb --- /dev/null +++ b/internal/pyxis/types.go @@ -0,0 +1,7 @@ +package pyxis + +type OperatorIndex struct { + OCPVersion string `json:"ocp_version"` + Organization string `json:"organization"` + EndOfLife string `json:"end_of_life,omitempty"` +} diff --git a/internal/reconcilers/certified_image_stream.go b/internal/reconcilers/certified_image_stream.go index d7fdf72..90ab7a6 100644 --- a/internal/reconcilers/certified_image_stream.go +++ b/internal/reconcilers/certified_image_stream.go @@ -2,9 +2,13 @@ package reconcilers import ( "context" + "fmt" + "net/http" + "time" "github.com/redhat-openshift-ecosystem/operator-certification-operator/api/v1alpha1" "github.com/redhat-openshift-ecosystem/operator-certification-operator/internal/objects" + "github.com/redhat-openshift-ecosystem/operator-certification-operator/internal/pyxis" "github.com/go-logr/logr" imagev1 "github.com/openshift/api/image/v1" @@ -22,8 +26,9 @@ const ( type CertifiedImageStreamReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + PyxisClient *pyxis.PyxisClient } func NewCertifiedImageStreamReconciler(client client.Client, log logr.Logger, scheme *runtime.Scheme) *CertifiedImageStreamReconciler { @@ -31,11 +36,19 @@ func NewCertifiedImageStreamReconciler(client client.Client, log logr.Logger, sc Client: client, Log: log, Scheme: scheme, + PyxisClient: pyxis.NewPyxisClient( + pyxis.DefaultPyxisHost, + &http.Client{Timeout: 60 * time.Second}), } } // reconcileCertifiedImageStream will ensure that the certified operator ImageStream is present and up to date. func (r *CertifiedImageStreamReconciler) Reconcile(ctx context.Context, pipeline *v1alpha1.OperatorPipeline) (bool, error) { + operatorIndices, err := r.PyxisClient.FindOperatorIndices(ctx, "certified-operators") + if err != nil { + return true, err + } + key := types.NamespacedName{ Namespace: pipeline.Namespace, Name: certifiedIndex, @@ -62,19 +75,27 @@ func (r *CertifiedImageStreamReconciler) Reconcile(ctx context.Context, pipeline imgImport := newImageStreamImport(key) imgImport.Spec.Import = true - imgImport.Spec.Repository = &imagev1.RepositoryImportSpec{ - From: corev1.ObjectReference{ - Kind: "DockerImage", - Name: "registry.redhat.io/redhat/certified-operator-index", - }, - ImportPolicy: imagev1.TagImportPolicy{ - Scheduled: true, - }, - ReferencePolicy: imagev1.TagReferencePolicy{ - Type: imagev1.LocalTagReferencePolicy, - }, + + imageSpecs := make([]imagev1.ImageImportSpec, 0, len(operatorIndices)) + + for _, index := range operatorIndices { + imageSpec := imagev1.ImageImportSpec{ + From: corev1.ObjectReference{ + Kind: "DockerImage", + Name: fmt.Sprintf("%s:v%s", "registry.redhat.io/redhat/certified-operator-index", index.OCPVersion), + }, + ImportPolicy: imagev1.TagImportPolicy{ + Scheduled: true, + }, + ReferencePolicy: imagev1.TagReferencePolicy{ + Type: imagev1.LocalTagReferencePolicy, + }, + } + imageSpecs = append(imageSpecs, imageSpec) } + imgImport.Spec.Images = imageSpecs + log.Info("creating new certified image stream import") if err := r.Client.Create(ctx, imgImport); err != nil { return true, err diff --git a/internal/reconcilers/marketplace_image_stream.go b/internal/reconcilers/marketplace_image_stream.go index c155682..47a44dd 100644 --- a/internal/reconcilers/marketplace_image_stream.go +++ b/internal/reconcilers/marketplace_image_stream.go @@ -2,9 +2,13 @@ package reconcilers import ( "context" + "fmt" + "net/http" + "time" "github.com/redhat-openshift-ecosystem/operator-certification-operator/api/v1alpha1" "github.com/redhat-openshift-ecosystem/operator-certification-operator/internal/objects" + "github.com/redhat-openshift-ecosystem/operator-certification-operator/internal/pyxis" "github.com/go-logr/logr" imagev1 "github.com/openshift/api/image/v1" @@ -21,8 +25,9 @@ const ( type MarketplaceImageStreamReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + PyxisClient *pyxis.PyxisClient } func NewMarketplaceImageStreamReconciler(client client.Client, log logr.Logger, scheme *runtime.Scheme) *MarketplaceImageStreamReconciler { @@ -30,11 +35,19 @@ func NewMarketplaceImageStreamReconciler(client client.Client, log logr.Logger, Client: client, Log: log, Scheme: scheme, + PyxisClient: pyxis.NewPyxisClient( + pyxis.DefaultPyxisHost, + &http.Client{Timeout: 60 * time.Second}), } } // reconcileMarketplaceImageStream will ensure that the Red Hat Marketplace ImageStream is present and up to date. func (r *MarketplaceImageStreamReconciler) Reconcile(ctx context.Context, pipeline *v1alpha1.OperatorPipeline) (bool, error) { + operatorIndices, err := r.PyxisClient.FindOperatorIndices(ctx, "redhat-marketplace") + if err != nil { + return true, err + } + key := types.NamespacedName{ Namespace: pipeline.Namespace, Name: marketplaceIndex, @@ -61,19 +74,27 @@ func (r *MarketplaceImageStreamReconciler) Reconcile(ctx context.Context, pipeli imgImport := newImageStreamImport(key) imgImport.Spec.Import = true - imgImport.Spec.Repository = &imagev1.RepositoryImportSpec{ - From: corev1.ObjectReference{ - Kind: "DockerImage", - Name: "registry.redhat.io/redhat/redhat-marketplace-index", - }, - ImportPolicy: imagev1.TagImportPolicy{ - Scheduled: true, - }, - ReferencePolicy: imagev1.TagReferencePolicy{ - Type: imagev1.LocalTagReferencePolicy, - }, + + imageSpecs := make([]imagev1.ImageImportSpec, 0, len(operatorIndices)) + + for _, index := range operatorIndices { + imageSpec := imagev1.ImageImportSpec{ + From: corev1.ObjectReference{ + Kind: "DockerImage", + Name: fmt.Sprintf("%s:v%s", "registry.redhat.io/redhat/redhat-marketplace-index", index.OCPVersion), + }, + ImportPolicy: imagev1.TagImportPolicy{ + Scheduled: true, + }, + ReferencePolicy: imagev1.TagReferencePolicy{ + Type: imagev1.LocalTagReferencePolicy, + }, + } + imageSpecs = append(imageSpecs, imageSpec) } + imgImport.Spec.Images = imageSpecs + log.Info("creating new marketplace image stream import") if err := r.Client.Create(ctx, imgImport); err != nil { return true, err