diff --git a/.golangci.yml b/.golangci.yml index c450228..379a145 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,6 +24,8 @@ issues: - gocognit - funlen - lll + - gosec + - noctx - path: js\/modules\/k6\/http\/.*_test\.go linters: # k6/http module's tests are quite complex because they often have several nested levels. @@ -62,7 +64,6 @@ linters-settings: - '^(fmt\\.Print(|f|ln)|print|println)$' # Forbid everything in syscall except the uppercase constants - '^syscall\.[^A-Z_]+$(# Using anything except constants from the syscall package is forbidden )?' - - '^logrus\.Logger$' linters: disable-all: true diff --git a/apiserver.go b/apiserver.go new file mode 100644 index 0000000..90f647c --- /dev/null +++ b/apiserver.go @@ -0,0 +1,104 @@ +package k6build + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/sirupsen/logrus" +) + +// BuildRequest defines a request to the build service +type BuildRequest struct { + K6Constrains string `json:"k6:omitempty"` + Dependencies []Dependency `json:"dependencies,omitempty"` + Platform string `json:"platformomitempty"` +} + +// String returns a text serialization of the BuildRequest +func (r BuildRequest) String() string { + buffer := &bytes.Buffer{} + buffer.WriteString(fmt.Sprintf("platform: %s", r.Platform)) + buffer.WriteString(fmt.Sprintf("k6: %s", r.K6Constrains)) + for _, d := range r.Dependencies { + buffer.WriteString(fmt.Sprintf("%s:%q", d.Name, d.Constraints)) + } + return buffer.String() +} + +// BuildResponse defines the response for a BuildRequest +type BuildResponse struct { + Error string `json:"error:omitempty"` + Artifact Artifact `json:"artifact:omitempty"` +} + +// APIServerConfig defines the configuration for the APIServer +type APIServerConfig struct { + BuildService BuildService + Log *logrus.Logger +} + +// APIServer defines a k6build API server +type APIServer struct { + srv BuildService + log *logrus.Logger +} + +// NewAPIServer creates a new build service API server +// TODO: add logger +func NewAPIServer(config APIServerConfig) *APIServer { + log := config.Log + if log == nil { + log = &logrus.Logger{Out: io.Discard} + } + return &APIServer{ + srv: config.BuildService, + log: log, + } +} + +// ServeHTTP implements the request handler for the build API server +func (a *APIServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + resp := BuildResponse{} + + w.Header().Add("Content-Type", "application/json") + + // ensure errors are reported and logged + defer func() { + if resp.Error != "" { + a.log.Error(resp.Error) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson + } + }() + + req := BuildRequest{} + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + resp.Error = fmt.Sprintf("invalid request: %s", err.Error()) + return + } + + a.log.Debugf("processing request %s", req.String()) + + artifact, err := a.srv.Build( //nolint:contextcheck + context.Background(), + req.Platform, + req.K6Constrains, + req.Dependencies, + ) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + resp.Error = fmt.Sprintf("building artifact: %s", err.Error()) + return + } + + a.log.Debugf("returning artifact %s", artifact.String()) + + resp.Artifact = artifact + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson +} diff --git a/apiserver_test.go b/apiserver_test.go new file mode 100644 index 0000000..cd47237 --- /dev/null +++ b/apiserver_test.go @@ -0,0 +1,95 @@ +package k6build + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestAPIServer(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + req BuildRequest + expect BuildResponse + }{ + { + title: "build k6 v0.1.0 ", + req: BuildRequest{ + Platform: "linux/amd64", + K6Constrains: "v0.1.0", + Dependencies: []Dependency{}, + }, + expect: BuildResponse{ + Artifact: Artifact{ + Dependencies: map[string]string{"k6": "v0.1.0"}, + }, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + buildsrv, err := SetupTestLocalBuildService( + LocalBuildServiceConfig{ + CacheDir: t.TempDir(), + Catalog: "testdata/catalog.json", + }, + ) + if err != nil { + t.Fatalf("test setup %v", err) + } + + config := APIServerConfig{ + BuildService: buildsrv, + } + apiserver := httptest.NewServer(NewAPIServer(config)) + + req := bytes.Buffer{} + err = json.NewEncoder(&req).Encode(&tc.req) + if err != nil { + t.Fatalf("test setup %v", err) + } + + resp, err := http.Post(apiserver.URL, "application/json", &req) + if err != nil { + t.Fatalf("making request %v", err) + } + defer func() { + _ = resp.Body.Close() + }() + + buildResponse := BuildResponse{} + err = json.NewDecoder(resp.Body).Decode(&buildResponse) + if err != nil { + t.Fatalf("decoding response %v", err) + } + + if buildResponse.Error != tc.expect.Error { + t.Fatalf("expected error: %s got %s", tc.expect.Error, buildResponse.Error) + } + + // don't check artifact if error is expected + if tc.expect.Error != "" { + return + } + + diff := cmp.Diff( + tc.expect.Artifact.Dependencies, + buildResponse.Artifact.Dependencies, + cmpopts.SortSlices(dependencyComp)) + if diff != "" { + t.Fatalf("dependencies don't match: %s\n", diff) + } + }) + } +} diff --git a/build.go b/build.go new file mode 100644 index 0000000..94a8943 --- /dev/null +++ b/build.go @@ -0,0 +1,82 @@ +// Package k6build defines a service for building k8 binaries +package k6build + +import ( + "bytes" + "context" + "fmt" + + "github.com/grafana/k6catalog" + "github.com/grafana/k6foundry" +) + +const ( + k6Dep = "k6" +) + +// Dependency contains the properties of a k6 dependency. +type Dependency struct { + // Name is the name of the dependency. + Name string `json:"name,omitempty"` + // Constraints contains the version constraints of the dependency. + Constraints string `json:"constraints,omitempty"` +} + +// Module defines an artifact dependency +type Module struct { + Path string `json:"path,omitempty"` + Version string `json:"vesion,omitempty"` +} + +// Artifact defines a binary that can be downloaded +// TODO: add metadata (e.g. list of dependencies, checksum, date compiled) +type Artifact struct { + ID string `json:"id,omitempty"` + // URL to fetch the artifact's binary + URL string `json:"url,omitempty"` + // list of dependencies + Dependencies map[string]string `json:"dependencies,omitempty"` + // platform + Platform string `json:"platform,omitempty"` + // binary checksum (sha256) + Checksum string `json:"checksum,omitempty"` +} + +// String returns a text serialization of the Artifact +func (a Artifact) String() string { + buffer := &bytes.Buffer{} + buffer.WriteString(fmt.Sprintf(" id: %s", a.ID)) + buffer.WriteString(fmt.Sprintf("platform: %s", a.Platform)) + for dep, version := range a.Dependencies { + buffer.WriteString(fmt.Sprintf(" %s:%q", dep, version)) + } + buffer.WriteString(fmt.Sprintf(" checksum: %s", a.Checksum)) + buffer.WriteString(fmt.Sprintf(" url: %s", a.URL)) + return buffer.String() +} + +// BuildService defines the interface of a build service +type BuildService interface { + // Build returns a k6 Artifact given its dependencies and version constrain + Build(ctx context.Context, platform string, k6Constrains string, deps []Dependency) (Artifact, error) +} + +// implements the BuildService interface +type localBuildSrv struct { + catalog k6catalog.Catalog + builder k6foundry.Builder + cache Cache +} + +// NewBuildService creates a build service +func NewBuildService( + catalog k6catalog.Catalog, + builder k6foundry.Builder, + cache Cache, +) BuildService { + return &localBuildSrv{ + catalog: catalog, + builder: builder, + cache: cache, + } +} diff --git a/cache.go b/cache.go index b4336f1..c72f1de 100644 --- a/cache.go +++ b/cache.go @@ -7,18 +7,12 @@ import ( "errors" "fmt" "io" + "net/url" "os" "path/filepath" "strings" ) -var ( - ErrObjectNotFound = errors.New("object not found") //nolint:revive - ErrAccessingObject = errors.New("accessing object") //nolint:revive - ErrCreatingObject = errors.New("creating object") //nolint:revive - ErrInitializingCache = errors.New("initializing cache") //nolint:revive -) - // Object represents an object stored in the Cache // TODO: add metadata (e.g creation data, size) type Object struct { @@ -28,17 +22,28 @@ type Object struct { URL string } +func (o Object) String() string { + buffer := &bytes.Buffer{} + buffer.WriteString(fmt.Sprintf("id: %s", o.ID)) + buffer.WriteString(fmt.Sprintf(" checksum: %s", o.Checksum)) + buffer.WriteString(fmt.Sprintf("url: %s", o.URL)) + + return buffer.String() +} + // Cache defines an interface for storing blobs type Cache interface { // Get retrieves an objects if exists in the cache or an error otherwise Get(ctx context.Context, id string) (Object, error) // Store stores the object and returns the metadata Store(ctx context.Context, id string, content io.Reader) (Object, error) + // Download returns the content of the object + Download(ctx context.Context, object Object) (io.ReadCloser, error) } -// a Cache backed by a file system -type fileCache struct { - path string +// FileCache a Cache backed by a file system +type FileCache struct { + dir string } // NewTempFileCache creates a file cache using a temporary file @@ -47,19 +52,19 @@ func NewTempFileCache() (Cache, error) { } // NewFileCache creates an cached backed by a directory -func NewFileCache(path string) (Cache, error) { - err := os.MkdirAll(path, 0o750) +func NewFileCache(dir string) (Cache, error) { + err := os.MkdirAll(dir, 0o750) if err != nil { return nil, fmt.Errorf("%w: %w", ErrInitializingCache, err) } - return &fileCache{ - path: path, + return &FileCache{ + dir: dir, }, nil } // Store stores the object and returns the metadata -func (f *fileCache) Store(_ context.Context, id string, content io.Reader) (Object, error) { +func (f *FileCache) Store(_ context.Context, id string, content io.Reader) (Object, error) { if id == "" { return Object{}, fmt.Errorf("%w id cannot be empty", ErrCreatingObject) } @@ -68,7 +73,7 @@ func (f *fileCache) Store(_ context.Context, id string, content io.Reader) (Obje return Object{}, fmt.Errorf("%w id cannot contain '/'", ErrCreatingObject) } - objectDir := filepath.Join(f.path, id) + objectDir := filepath.Join(f.dir, id) // TODO: check permissions err := os.MkdirAll(objectDir, 0o750) if err != nil { @@ -105,8 +110,8 @@ func (f *fileCache) Store(_ context.Context, id string, content io.Reader) (Obje } // Get retrieves an objects if exists in the cache or an error otherwise -func (f *fileCache) Get(_ context.Context, id string) (Object, error) { - objectDir := filepath.Join(f.path, id) +func (f *FileCache) Get(_ context.Context, id string) (Object, error) { + objectDir := filepath.Join(f.dir, id) _, err := os.Stat(objectDir) if errors.Is(err, os.ErrNotExist) { @@ -128,3 +133,43 @@ func (f *fileCache) Get(_ context.Context, id string) (Object, error) { URL: fmt.Sprintf("file://%s", filepath.Join(objectDir, "data")), }, nil } + +// Download returns the content of the object given its url +func (f *FileCache) Download(_ context.Context, object Object) (io.ReadCloser, error) { + url, err := url.Parse(object.URL) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrAccessingObject, err) + } + + switch url.Scheme { + case "file": + // prevent malicious path + objectPath, err := f.sanitizePath(url.Path) + if err != nil { + return nil, err + } + + objectFile, err := os.Open(objectPath) //nolint:gosec // path is sanitized + if err != nil { + // FIXE: is the path has invalid characters, still will return ErrNotExists + if errors.Is(err, os.ErrNotExist) { + return nil, ErrObjectNotFound + } + return nil, fmt.Errorf("%w: %w", ErrAccessingObject, err) + } + + return objectFile, nil + default: + return nil, fmt.Errorf("%w unsupported schema: %s", ErrInvalidURL, url.Scheme) + } +} + +func (f *FileCache) sanitizePath(path string) (string, error) { + path = filepath.Clean(path) + + if !filepath.IsAbs(path) || !strings.HasPrefix(path, f.dir) { + return "", fmt.Errorf("%w : invalid path %s", ErrInvalidURL, path) + } + + return path, nil +} diff --git a/cache_client.go b/cache_client.go new file mode 100644 index 0000000..c4bd556 --- /dev/null +++ b/cache_client.go @@ -0,0 +1,110 @@ +package k6build + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" +) + +// CacheClientConfig defines the configuration for accessing a remote cache service +type CacheClientConfig struct { + Server string +} + +// CacheClient access blobs in a CacheServer +type CacheClient struct { + server string +} + +// NewCacheClient returns a client for a cache server +func NewCacheClient(config CacheClientConfig) (*CacheClient, error) { + if _, err := url.Parse(config.Server); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err) + } + + return &CacheClient{ + server: config.Server, + }, nil +} + +// Get retrieves an objects if exists in the cache or an error otherwise +func (c *CacheClient) Get(_ context.Context, id string) (Object, error) { + url := fmt.Sprintf("%s/%s", c.server, id) + + // TODO: use http.Request + resp, err := http.Get(url) //nolint:gosec,noctx + if err != nil { + return Object{}, fmt.Errorf("%w %w", ErrAccessingServer, err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return Object{}, fmt.Errorf("%w with status", ErrObjectNotFound) + } + return Object{}, fmt.Errorf("%w with status %s", ErrRequestFailed, resp.Status) + } + + cacheResponse := CacheServerResponse{} + err = json.NewDecoder(resp.Body).Decode(&cacheResponse) + if err != nil { + return Object{}, fmt.Errorf("%w: %s", ErrInvalidResponse, err.Error()) + } + + if cacheResponse.Error != "" { + return Object{}, fmt.Errorf("%w: %s", ErrRequestFailed, cacheResponse.Error) + } + + return cacheResponse.Object, nil +} + +// Store stores the object and returns the metadata +func (c *CacheClient) Store(_ context.Context, id string, content io.Reader) (Object, error) { + url := fmt.Sprintf("%s/%s", c.server, id) + resp, err := http.Post( //nolint:gosec,noctx + url, + "application/octet-stream", + content, + ) + if err != nil { + return Object{}, fmt.Errorf("%w %w", ErrAccessingServer, err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return Object{}, fmt.Errorf("%w with status %s", ErrRequestFailed, resp.Status) + } + + cacheResponse := CacheServerResponse{} + err = json.NewDecoder(resp.Body).Decode(&cacheResponse) + if err != nil { + return Object{}, fmt.Errorf("%w: %s", ErrInvalidResponse, err.Error()) + } + + if cacheResponse.Error != "" { + return Object{}, fmt.Errorf("%w: %s", ErrRequestFailed, cacheResponse.Error) + } + + return cacheResponse.Object, nil +} + +// Download returns the content of the object given its url +func (c *CacheClient) Download(_ context.Context, object Object) (io.ReadCloser, error) { + resp, err := http.Get(object.URL) //nolint:noctx,bodyclose + if err != nil { + return nil, fmt.Errorf("%w %w", ErrAccessingServer, err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w with status %s", ErrRequestFailed, resp.Status) + } + + return resp.Request.Body, nil +} diff --git a/cache_client_test.go b/cache_client_test.go new file mode 100644 index 0000000..f602d9f --- /dev/null +++ b/cache_client_test.go @@ -0,0 +1,190 @@ +package k6build + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +// returns a HandleFunc that returns a canned status and response +func handlerMock(status int, resp *CacheServerResponse) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/json") + + // send canned response + respBuffer := &bytes.Buffer{} + if resp != nil { + err := json.NewEncoder(respBuffer).Encode(resp) + if err != nil { + // set uncommon status code to signal something unexpected happened + w.WriteHeader(http.StatusTeapot) + return + } + } + + w.WriteHeader(status) + _, _ = w.Write(respBuffer.Bytes()) + } +} + +// returns a HandleFunc that returns a canned status and content for a download +func downloadMock(status int, content []byte) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/octet-stream") + w.WriteHeader(status) + if content != nil { + _, _ = w.Write(content) + } + } +} + +func TestCacheClientGet(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + status int + resp *CacheServerResponse + expectErr error + }{ + { + title: "normal get", + status: http.StatusOK, + resp: &CacheServerResponse{ + Error: "", + Object: Object{}, + }, + }, + { + title: "object not found", + status: http.StatusNotFound, + resp: nil, + expectErr: ErrObjectNotFound, + }, + { + title: "error accessing object", + status: http.StatusInternalServerError, + resp: &CacheServerResponse{ + Error: "Error accessing object", + Object: Object{}, + }, + expectErr: ErrRequestFailed, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(handlerMock(tc.status, tc.resp)) + + client, err := NewCacheClient(CacheClientConfig{Server: srv.URL}) + if err != nil { + t.Fatalf("test setup %v", err) + } + + _, err = client.Get(context.TODO(), "object") + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + }) + } +} + +func TestCacheClientStore(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + status int + resp *CacheServerResponse + expectErr error + }{ + { + title: "normal response", + status: http.StatusOK, + resp: &CacheServerResponse{ + Error: "", + Object: Object{}, + }, + }, + { + title: "error creating object", + status: http.StatusInternalServerError, + resp: &CacheServerResponse{ + Error: "Error creating object", + Object: Object{}, + }, + expectErr: ErrRequestFailed, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(handlerMock(tc.status, tc.resp)) + + client, err := NewCacheClient(CacheClientConfig{Server: srv.URL}) + if err != nil { + t.Fatalf("test setup %v", err) + } + + _, err = client.Store(context.TODO(), "object", bytes.NewBuffer(nil)) + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + }) + } +} + +func TestCacheClientDownload(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + status int + content []byte + expectErr error + }{ + { + title: "normal response", + status: http.StatusOK, + content: []byte("object content"), + }, + { + title: "error creating object", + status: http.StatusInternalServerError, + expectErr: ErrRequestFailed, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(downloadMock(tc.status, tc.content)) + + client, err := NewCacheClient(CacheClientConfig{Server: srv.URL}) + if err != nil { + t.Fatalf("test setup %v", err) + } + + obj := Object{ + ID: "object", + URL: srv.URL, + } + _, err = client.Download(context.TODO(), obj) + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + }) + } +} diff --git a/cache_server.go b/cache_server.go new file mode 100644 index 0000000..00150ba --- /dev/null +++ b/cache_server.go @@ -0,0 +1,181 @@ +package k6build + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/sirupsen/logrus" +) + +// CacheServerResponse is the response to a cache server request +type CacheServerResponse struct { + Error string + Object Object +} + +// CacheServer implements an http server that handles cache requests +type CacheServer struct { + baseURL string + cache Cache + log *logrus.Logger +} + +// CacheServerConfig defines the configuration for the APIServer +type CacheServerConfig struct { + BaseURL string + Cache Cache + Log *logrus.Logger +} + +// NewCacheServer returns a CacheServer backed by a cache +func NewCacheServer(config CacheServerConfig) http.Handler { + log := config.Log + if log == nil { + log = &logrus.Logger{Out: io.Discard} + } + cacheSrv := &CacheServer{ + baseURL: config.BaseURL, + cache: config.Cache, + log: log, + } + + handler := http.NewServeMux() + // FIXME: this should be PUT (used POST as http client doesn't have PUT method) + handler.HandleFunc("POST /{id}", cacheSrv.Store) + handler.HandleFunc("GET /{id}", cacheSrv.Get) + handler.HandleFunc("GET /{id}/content", cacheSrv.Download) + + return handler +} + +// Get retrieves an objects if exists in the cache or an error otherwise +func (s *CacheServer) Get(w http.ResponseWriter, r *http.Request) { + resp := CacheServerResponse{} + + w.Header().Add("Content-Type", "application/json") + + // ensure errors are reported and logged + defer func() { + if resp.Error != "" { + s.log.Error(resp.Error) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson + } + }() + + id := r.PathValue("id") + if id == "" { + w.WriteHeader(http.StatusBadRequest) + resp.Error = ErrInvalidRequest.Error() + return + } + + object, err := s.cache.Get(context.Background(), id) //nolint:contextcheck + if err != nil { + if errors.Is(err, ErrObjectNotFound) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + resp.Error = err.Error() + + return + } + + // overwrite URL with own + baseURL := s.baseURL + if baseURL == "" { + baseURL = fmt.Sprintf("http://%s%s", r.Host, r.RequestURI) + } + downloadURL := fmt.Sprintf("%s/%s/download", baseURL, id) + + resp.Object = Object{ + ID: id, + Checksum: object.Checksum, + URL: downloadURL, + } + + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson +} + +// Store stores the object and returns the metadata +func (s *CacheServer) Store(w http.ResponseWriter, r *http.Request) { + resp := CacheServerResponse{} + + w.Header().Add("Content-Type", "application/json") + + // ensure errors are reported and logged + defer func() { + if resp.Error != "" { + s.log.Error(resp.Error) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson + } + }() + + id := r.PathValue("id") + if id == "" { + w.WriteHeader(http.StatusBadRequest) + resp.Error = ErrInvalidRequest.Error() + return + } + + object, err := s.cache.Store(context.Background(), id, r.Body) //nolint:contextcheck + if err != nil { + w.WriteHeader(http.StatusBadRequest) + resp.Error = err.Error() + return + } + + // overwrite URL with own + baseURL := s.baseURL + if baseURL == "" { + baseURL = fmt.Sprintf("http://%s%s", r.Host, r.RequestURI) + } + downloadURL := fmt.Sprintf("%s/%s/download", baseURL, id) + + resp.Object = Object{ + ID: id, + Checksum: object.Checksum, + URL: downloadURL, + } + + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) //nolint:errchkjson +} + +// Download returns an object's content given its id +func (s *CacheServer) Download(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if id == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + + object, err := s.cache.Get(context.Background(), id) //nolint:contextcheck + if err != nil { + if errors.Is(err, ErrObjectNotFound) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + return + } + + objectContent, err := s.cache.Download(context.Background(), object) //nolint:contextcheck + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + defer func() { + _ = objectContent.Close() + }() + + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-Type", "application/octet-stream") + w.Header().Add("ETag", object.ID) + _, _ = io.Copy(w, objectContent) +} diff --git a/cache_server_test.go b/cache_server_test.go new file mode 100644 index 0000000..91b5214 --- /dev/null +++ b/cache_server_test.go @@ -0,0 +1,295 @@ +package k6build + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +// MemoryCache defines the state of a memory backed cache +type MemoryCache struct { + objects map[string]Object + content map[string][]byte +} + +func NewMemoryCache() *MemoryCache { + return &MemoryCache{ + objects: map[string]Object{}, + content: map[string][]byte{}, + } +} + +func (f *MemoryCache) Get(_ context.Context, id string) (Object, error) { + object, found := f.objects[id] + if !found { + return Object{}, ErrObjectNotFound + } + + return object, nil +} + +func (f *MemoryCache) Store(_ context.Context, id string, content io.Reader) (Object, error) { + buffer := bytes.Buffer{} + _, err := buffer.ReadFrom(content) + if err != nil { + return Object{}, ErrCreatingObject + } + + checksum := fmt.Sprintf("%x", sha256.Sum256(buffer.Bytes())) + object := Object{ + ID: id, + Checksum: checksum, + URL: fmt.Sprintf("memory:///%s", id), + } + + f.objects[id] = object + f.content[id] = buffer.Bytes() + + return object, nil +} + +// Download implements Cache. +func (f *MemoryCache) Download(_ context.Context, object Object) (io.ReadCloser, error) { + url, err := url.Parse(object.URL) + if err != nil { + return nil, err + } + + id, _ := strings.CutPrefix(url.Path, "/") + content, found := f.content[id] + if !found { + return nil, ErrObjectNotFound + } + + return io.NopCloser(bytes.NewBuffer(content)), nil +} + +func TestCacheServerGet(t *testing.T) { + t.Parallel() + + cache := NewMemoryCache() + objects := map[string][]byte{ + "object1": []byte("content object 1"), + } + + for id, content := range objects { + buffer := bytes.NewBuffer(content) + if _, err := cache.Store(context.TODO(), id, buffer); err != nil { + t.Fatalf("test setup: %v", err) + } + } + + config := CacheServerConfig{ + Cache: cache, + } + cacheSrv := NewCacheServer(config) + + srv := httptest.NewServer(cacheSrv) + + testCases := []struct { + title string + id string + status int + epectErr string + }{ + { + title: "return object", + id: "object1", + status: http.StatusOK, + }, + { + title: "object not found", + id: "not_found", + status: http.StatusNotFound, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + url := fmt.Sprintf("%s/%s", srv.URL, tc.id) + resp, err := http.Get(url) + if err != nil { + t.Fatalf("accessing server %v", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != tc.status { + t.Fatalf("expected %s got %s", http.StatusText(tc.status), resp.Status) + } + + cacheResponse := CacheServerResponse{} + err = json.NewDecoder(resp.Body).Decode(&cacheResponse) + if err != nil { + t.Fatalf("reading response content %v", err) + } + + if tc.status != http.StatusOK { + if cacheResponse.Error == "" { + t.Fatalf("expected error message not none") + } + return + } + + if cacheResponse.Object.ID != tc.id { + t.Fatalf("expected object id %s got %s", tc.id, cacheResponse.Object.ID) + } + }) + } +} + +func TestCacheServerStore(t *testing.T) { + t.Parallel() + + cache := NewMemoryCache() + + config := CacheServerConfig{ + Cache: cache, + } + cacheSrv := NewCacheServer(config) + + srv := httptest.NewServer(cacheSrv) + + testCases := []struct { + title string + id string + content string + status int + }{ + { + title: "create object", + id: "object1", + content: "object 1 content", + status: http.StatusOK, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + url := fmt.Sprintf("%s/%s", srv.URL, tc.id) + resp, err := http.Post( + url, + "application/octet-stream", + bytes.NewBufferString(tc.content), + ) + if err != nil { + t.Fatalf("accessing server %v", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != tc.status { + t.Fatalf("expected %s got %s", http.StatusText(tc.status), resp.Status) + } + + cacheResponse := CacheServerResponse{} + err = json.NewDecoder(resp.Body).Decode(&cacheResponse) + if err != nil { + t.Fatalf("reading response content %v", err) + } + + if tc.status != http.StatusOK { + if cacheResponse.Error == "" { + t.Fatalf("expected error message not none") + } + return + } + + if cacheResponse.Object.ID != tc.id { + t.Fatalf("expected object id %s got %s", tc.id, cacheResponse.Object.ID) + } + }) + } +} + +func TestCacheServerDownload(t *testing.T) { + t.Parallel() + + cache := NewMemoryCache() + objects := map[string][]byte{ + "object1": []byte("content object 1"), + } + + for id, content := range objects { + buffer := bytes.NewBuffer(content) + if _, err := cache.Store(context.TODO(), id, buffer); err != nil { + t.Fatalf("test setup: %v", err) + } + } + + config := CacheServerConfig{ + Cache: cache, + } + cacheSrv := NewCacheServer(config) + + srv := httptest.NewServer(cacheSrv) + + testCases := []struct { + title string + id string + status int + content []byte + }{ + { + title: "return object", + id: "object1", + status: http.StatusOK, + content: objects["object1"], + }, + { + title: "object not found", + id: "not_found", + status: http.StatusNotFound, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + url := fmt.Sprintf("%s/%s/content", srv.URL, tc.id) + resp, err := http.Get(url) + if err != nil { + t.Fatalf("accessing server %v", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != tc.status { + t.Fatalf("expected %s got %s", http.StatusText(tc.status), resp.Status) + } + + if tc.status != http.StatusOK { + return + } + + content := bytes.Buffer{} + _, err = content.ReadFrom(resp.Body) + if err != nil { + t.Fatalf("reading content %v", err) + } + + if !bytes.Equal(content.Bytes(), tc.content) { + t.Fatalf("expected got") + } + }) + } +} diff --git a/cache_test.go b/cache_test.go index 203363b..01138ae 100644 --- a/cache_test.go +++ b/cache_test.go @@ -31,7 +31,7 @@ func setupCache(path string, preload []object) (Cache, error) { return cache, nil } -func TestStoreObject(t *testing.T) { +func TestFileCahceStoreObject(t *testing.T) { t.Parallel() testCases := []struct { @@ -117,73 +117,143 @@ func TestStoreObject(t *testing.T) { } } -func TestGetObjectCache(t *testing.T) { +func TestFileCacheRetrieval(t *testing.T) { t.Parallel() - testCases := []struct { - title string - preload []object - id string - expected []byte - expectErr error - }{ - { - title: "retrieve existing object", - preload: []object{ - { - id: "object", - content: []byte("content"), - }, - }, - id: "object", - expected: []byte("content"), - expectErr: nil, - }, + preload := []object{ { - title: "retrieve non existing object", - preload: []object{ - { - id: "object", - content: []byte("content"), - }, - }, - id: "another object", - expectErr: ErrObjectNotFound, + id: "object", + content: []byte("content"), }, } - for _, tc := range testCases { - t.Run(tc.title, func(t *testing.T) { - t.Parallel() + cacheDir := t.TempDir() + cache, err := setupCache(cacheDir, preload) + if err != nil { + t.Fatalf("test setup: %v", err) + } - cache, err := setupCache(t.TempDir(), tc.preload) - if err != nil { - t.Fatalf("test setup: %v", err) - } + t.Run("TestFileCacheGet", func(t *testing.T) { + testCases := []struct { + title string + id string + expected []byte + expectErr error + }{ + { + title: "retrieve existing object", + id: "object", + expected: []byte("content"), + expectErr: nil, + }, + { + title: "retrieve non existing object", + id: "another object", + expectErr: ErrObjectNotFound, + }, + } - obj, err := cache.Get(context.TODO(), tc.id) - if !errors.Is(err, tc.expectErr) { - t.Fatalf("expected %v got %v", tc.expectErr, err) - } + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + t.Parallel() - // if expected error, don't check returned object - if tc.expectErr != nil { - return - } + obj, err := cache.Get(context.TODO(), tc.id) + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } - fileURL, err := url.Parse(obj.URL) - if err != nil { - t.Fatalf("invalid url %v", err) - } + // if expected error, don't check returned object + if tc.expectErr != nil { + return + } - data, err := os.ReadFile(fileURL.Path) - if err != nil { - t.Fatalf("reading object url %v", err) - } + fileURL, err := url.Parse(obj.URL) + if err != nil { + t.Fatalf("invalid url %v", err) + } - if !bytes.Equal(data, tc.expected) { - t.Fatalf("expected %v got %v", tc.expected, data) - } - }) - } + data, err := os.ReadFile(fileURL.Path) + if err != nil { + t.Fatalf("reading object url %v", err) + } + + if !bytes.Equal(data, tc.expected) { + t.Fatalf("expected %v got %v", tc.expected, data) + } + }) + } + }) + + // FIXME: This test is leaking how the file cache creates the URLs for the objects + t.Run("TestFileCacheDownload", func(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + object Object + expected []byte + expectErr error + }{ + { + title: "download existing object", + object: Object{ + ID: "object", + URL: fmt.Sprintf("file://%s/object/data", cacheDir), + }, + expected: []byte("content"), + expectErr: nil, + }, + { + title: "download non existing object", + object: Object{ + ID: "object", + URL: fmt.Sprintf("file://%s/another_object/data", cacheDir), + }, + expectErr: ErrObjectNotFound, + }, + { + title: "download malformed url", + object: Object{ + ID: "object", + URL: fmt.Sprintf("file://%s/invalid&path/data", cacheDir), + }, + // FIXME: this should be an ErrInvalidURL + expectErr: ErrObjectNotFound, + }, + { + title: "download malicious url", + object: Object{ + ID: "object", + URL: fmt.Sprintf("file://%s/../../data", cacheDir), + }, + expectErr: ErrInvalidURL, + }, + } + + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + content, err := cache.Download(context.TODO(), tc.object) + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + + // if expected error, don't check returned object + if tc.expectErr != nil { + return + } + + data := bytes.Buffer{} + _, err = data.ReadFrom(content) + if err != nil { + t.Fatalf("reading content: %v", err) + } + + if !bytes.Equal(data.Bytes(), tc.expected) { + t.Fatalf("expected %v got %v", tc.expected, data) + } + }) + } + }) } diff --git a/client.go b/client.go new file mode 100644 index 0000000..9c9c512 --- /dev/null +++ b/client.go @@ -0,0 +1,75 @@ +package k6build + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" +) + +// BuildServiceClientConfig defines the configuration for accessing a remote build service +type BuildServiceClientConfig struct { + URL string +} + +// NewBuildServiceClient returns a new client for a remote build service +func NewBuildServiceClient(config BuildServiceClientConfig) (BuildService, error) { + return &BuildClient{ + srv: config.URL, + }, nil +} + +// BuildClient defines a client of a build service +type BuildClient struct { + srv string +} + +// Build request building an artidact to a build service +func (r *BuildClient) Build( + _ context.Context, + platform string, + k6Constrains string, + deps []Dependency, +) (Artifact, error) { + buildRequest := BuildRequest{ + Platform: platform, + K6Constrains: k6Constrains, + Dependencies: deps, + } + marshaled := &bytes.Buffer{} + err := json.NewEncoder(marshaled).Encode(buildRequest) + if err != nil { + return Artifact{}, fmt.Errorf("%w: %w", ErrRequestFailed, err) + } + + url, err := url.Parse(r.srv) + if err != nil { + return Artifact{}, fmt.Errorf("invalid server %w", err) + } + url.Path = "/build/" + resp, err := http.Post(url.String(), "application/json", marshaled) //nolint:noctx + if err != nil { + return Artifact{}, fmt.Errorf("%w: %w", ErrRequestFailed, err) + } + defer func() { + _ = resp.Body.Close() + }() + + buildResponse := BuildResponse{} + err = json.NewDecoder(resp.Body).Decode(&buildResponse) + if err != nil { + return Artifact{}, fmt.Errorf("%w: %w", ErrRequestFailed, err) + } + + if resp.StatusCode != http.StatusOK { + return Artifact{}, fmt.Errorf("%w: %s", ErrRequestFailed, buildResponse.Error) + } + + if buildResponse.Error != "" { + return Artifact{}, fmt.Errorf("%w: %s", ErrBuildFailed, buildResponse.Error) + } + + return buildResponse.Artifact, nil +} diff --git a/client_test.go b/client_test.go new file mode 100644 index 0000000..e418e9a --- /dev/null +++ b/client_test.go @@ -0,0 +1,110 @@ +package k6build + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +type testSrv struct { + status int + response BuildResponse +} + +func (t testSrv) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + + // validate request + req := BuildRequest{} + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + // send canned response + resp := &bytes.Buffer{} + err = json.NewEncoder(resp).Encode(t.response) + if err != nil { + // set uncommon status code to signal something unexpected happened + w.WriteHeader(http.StatusTeapot) + return + } + + w.WriteHeader(t.status) + _, _ = w.Write(resp.Bytes()) +} + +func TestRemote(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + status int + resp BuildResponse + expectErr error + }{ + { + title: "normal build", + status: http.StatusOK, + resp: BuildResponse{ + Error: "", + Artifact: Artifact{}, + }, + }, + { + title: "request failed", + status: http.StatusInternalServerError, + resp: BuildResponse{ + Error: "request failed", + Artifact: Artifact{}, + }, + expectErr: ErrRequestFailed, + }, + { + title: "failed build", + status: http.StatusOK, + resp: BuildResponse{ + Error: "failed build", + Artifact: Artifact{}, + }, + expectErr: ErrBuildFailed, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(testSrv{ + status: tc.status, + response: tc.resp, + }) + + client, err := NewBuildServiceClient( + BuildServiceClientConfig{ + URL: srv.URL, + }, + ) + if err != nil { + t.Fatalf("unexpected %v", err) + } + + _, err = client.Build( + context.TODO(), + "linux/amd64", + "v0.1.0", + []Dependency{{Name: "k6/x/test", Constraints: "*"}}, + ) + + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + }) + } +} diff --git a/cmd/client.go b/cmd/client.go new file mode 100644 index 0000000..6bd8dd9 --- /dev/null +++ b/cmd/client.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/grafana/k6build" + "github.com/spf13/cobra" +) + +const ( + clientLong = ` +k6build client connects to a remote build server +` + + clientExamples = ` +# build k6 v0.51.0 with k6/x/kubernetes v0.8.0 and k6/x/output-kafka v0.7.0 +k6build client -s http://localhost:8000 \ + -k v0.51.0 \ + -p linux/amd64 + -d k6/x/kubernetes:v0.8.0 \ + -d k6/x/output-kafka:v0.7.0 + +{ + "id": "62d08b13fdef171435e2c6874eaad0bb35f2f9c7", + "url": "http://localhost:8000/cache/62d08b13fdef171435e2c6874eaad0bb35f2f9c7/download", + "dependencies": { + "k6": "v0.51.0", + "k6/x/kubernetes": "v0.9.0", + "k6/x/output-kafka": "v0.7.0" + }, + "platform": "linux/amd64", + "checksum": "f4af178bb2e29862c0fc7d481076c9ba4468572903480fe9d6c999fea75f3793" +} + +# build k6 v0.51 with k6/x/output-kafka v0.7.0 and download to 'build/k6' +k6build client -s http://localhost:8000 + -p linux/amd64 + -k v0.51.0 -d k6/x/output-kafka:v0.7.0 + -o build/k6 -q + +# check binary +build/k6 version +k6 v0.51.0 (go1.22.2, linux/amd64) +Extensions: + github.com/grafana/xk6-output-kafka v0.7.0, xk6-kafka [output] + + + +# build latest version of k6 with a version of k6/x/kubernetes greater than v0.8.0 +k6build client -s http://localhost:8000 \ + -p linux/amd64 \ + -k v0.50.0 -d 'k6/x/kubernetes:>v0.8.0' +{ + "id": "18035a12975b262430b55988ffe053098d020034", + "url": "http://localhost:8000/cache/18035a12975b262430b55988ffe053098d020034/download", + "dependencies": { + "k6": "v0.50.0", + "k6/x/kubernetes": "v0.9.0" + }, + "platform": "linux/amd64", + "checksum": "255e5d62852af5e4109a0ac6f5818936a91c986919d12d8437e97fb96919847b" +} +` +) + +// NewClient creates new cobra command for build client command. +func NewClient() *cobra.Command { //nolint:funlen + var ( + config k6build.BuildServiceClientConfig + deps []string + k6 string + output string + platform string + quiet bool + ) + + cmd := &cobra.Command{ + Use: "client", + Short: "build k6 using a remote build server", + Long: clientLong, + Example: clientExamples, + // prevent the usage help to printed to stderr when an error is reported by a subcommand + SilenceUsage: true, + // this is needed to prevent cobra to print errors reported by subcommands in the stderr + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + client, err := k6build.NewBuildServiceClient(config) + if err != nil { + return fmt.Errorf("configuring the client %w", err) + } + + buildDeps := []k6build.Dependency{} + for _, d := range deps { + name, constrains, _ := strings.Cut(d, ":") + if constrains == "" { + constrains = "*" + } + buildDeps = append(buildDeps, k6build.Dependency{Name: name, Constraints: constrains}) + } + + artifact, err := client.Build(cmd.Context(), platform, k6, buildDeps) + if err != nil { + return fmt.Errorf("building %w", err) + } + + if !quiet { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + err = encoder.Encode(artifact) + if err != nil { + return fmt.Errorf("processing response %w", err) + } + } + + if output != "" { + resp, err := http.Get(artifact.URL) //nolint:noctx + if err != nil { + return fmt.Errorf("downloading artifact %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("request failed with status %s", resp.Status) + } + + outFile, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE, 0o755) //nolint:gosec + if err != nil { + return fmt.Errorf("opening output file %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + _, err = io.Copy(outFile, resp.Body) + if err != nil { + return fmt.Errorf("copying artifact %w", err) + } + } + + return nil + }, + } + + cmd.Flags().StringVarP(&config.URL, "server", "s", "http://localhost:8000", "url for build server") + cmd.Flags().StringArrayVarP(&deps, "dependency", "d", nil, "list of dependencies in form package:constrains") + cmd.Flags().StringVarP(&k6, "k6", "k", "*", "k6 version constrains") + cmd.Flags().StringVarP(&platform, "platform", "p", "", "target platform (default GOOS/GOARCH)") + cmd.Flags().StringVarP(&output, "output", "o", "", "path to download the artifact as an executable."+ + " If not specified, the artifact is not downloaded.") + cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "don't print artifact's details") + + return cmd +} diff --git a/cmd/cmd.go b/cmd/cmd.go index 40398ae..edf5030 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -1,103 +1,14 @@ -// Package cmd contains build cobra command factory function. +// Package cmd offers commands for interacting with the build service package cmd -import ( - "encoding/json" - "errors" - "fmt" - "os" - "strings" +import "github.com/spf13/cobra" - "github.com/grafana/k6build" - "github.com/spf13/cobra" -) - -var ErrTargetPlatformUndefined = errors.New("target platform is required") //nolint:revive - -const ( - long = ` -k6 build service returns artifacts that satisfies certain dependencies -` - - example = ` -# build k6 v0.50.0 with latest version of k6/x/kubernetes -k6build -k v0.50.0 -d k6/x/kubernetes - -# build k6 v0.51.0 with k6/x/kubernetes v0.8.0 and k6/x/output-kafka v0.7.0 -k6build -k v0.51.0 \ - -d k6/x/kubernetes:v0.8.0 \ - -d k6/x/output-kafka:v0.7.0 - -# build latest version of k6 with a version of k6/x/kubernetes greater than v0.8.0 -k6build -k v0.50.0 -d 'k6/x/kubernetes:>v0.8.0' - -# build k6 v0.50.0 with latest version of k6/x/kubernetes using a custom catalog -k6build -k v0.50.0 -d k6/x/kubernetes \ - -c /path/to/catalog.json - -# build k6 v0.50.0 using a custom GOPROXY -k6build -k v0.50.0 -e GOPROXY=http://localhost:80 -` -) - -// New creates new cobra command for resolve command. +// New creates a new root command for k6build func New() *cobra.Command { - var ( - deps []string - k6 string - platform string - config k6build.LocalBuildServiceConfig - ) - - cmd := &cobra.Command{ - Use: "k6build", - Short: "k6 build service", - Long: long, - Example: example, - // prevent the usage help to printed to stderr when an error is reported by a subcommand - SilenceUsage: true, - // this is needed to prevent cobra to print errors reported by subcommands in the stderr - SilenceErrors: true, - RunE: func(cmd *cobra.Command, _ []string) error { - srv, err := k6build.NewLocalBuildService(cmd.Context(), config) - if err != nil { - return fmt.Errorf("configuring the build service %w", err) - } - - buildDeps := []k6build.Dependency{} - for _, d := range deps { - name, constrains, _ := strings.Cut(d, ":") - if constrains == "" { - constrains = "*" - } - buildDeps = append(buildDeps, k6build.Dependency{Name: name, Constraints: constrains}) - } - - artifact, err := srv.Build(cmd.Context(), platform, k6, buildDeps) - if err != nil { - return fmt.Errorf("building %w", err) - } - - encoder := json.NewEncoder(os.Stdout) - encoder.SetIndent("", " ") - err = encoder.Encode(artifact) - if err != nil { - return fmt.Errorf("processing object %w", err) - } - - return nil - }, - } - - cmd.Flags().StringArrayVarP(&deps, "dependency", "d", nil, "list of dependencies in form package:constrains") - cmd.Flags().StringVarP(&k6, "k6", "k", "*", "k6 version constrains") - cmd.Flags().StringVarP(&platform, "platform", "p", "", "target platform (default GOOS/GOARCH)") - _ = cmd.MarkFlagRequired("platform") - cmd.Flags().StringVarP(&config.Catalog, "catalog", "c", "catalog.json", "dependencies catalog") - cmd.Flags().StringVarP(&config.CacheDir, "cache-dir", "f", "/tmp/buildservice", "cache dir") - cmd.Flags().BoolVarP(&config.Verbose, "verbose", "v", false, "print build process output") - cmd.Flags().BoolVarP(&config.CopyGoEnv, "copy-go-env", "g", true, "copy go environment") - cmd.Flags().StringToStringVarP(&config.BuildEnv, "env", "e", nil, "build environment variables") + root := &cobra.Command{} + root.AddCommand(NewLocal()) + root.AddCommand(NewServer()) + root.AddCommand(NewClient()) - return cmd + return root } diff --git a/cmd/local.go b/cmd/local.go new file mode 100644 index 0000000..1094899 --- /dev/null +++ b/cmd/local.go @@ -0,0 +1,103 @@ +// Package cmd contains build cobra command factory function. +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/grafana/k6build" + "github.com/spf13/cobra" +) + +var ErrTargetPlatformUndefined = errors.New("target platform is required") //nolint:revive + +const ( + localLong = ` +k6build local builder returns artifacts that satisfies certain dependencies +` + + localExamples = ` +# build k6 v0.50.0 with latest version of k6/x/kubernetes +k6build local -k v0.50.0 -d k6/x/kubernetes + +# build k6 v0.51.0 with k6/x/kubernetes v0.8.0 and k6/x/output-kafka v0.7.0 +k6build local -k v0.51.0 \ + -d k6/x/kubernetes:v0.8.0 \ + -d k6/x/output-kafka:v0.7.0 + +# build latest version of k6 with a version of k6/x/kubernetes greater than v0.8.0 +k6build local -k v0.50.0 -d 'k6/x/kubernetes:>v0.8.0' + +# build k6 v0.50.0 with latest version of k6/x/kubernetes using a custom catalog +k6build local -k v0.50.0 -d k6/x/kubernetes \ + -c /path/to/catalog.json + +# build k6 v0.50.0 using a custom GOPROXY +k6build local -k v0.50.0 -e GOPROXY=http://localhost:80 +` +) + +// NewLocal creates new cobra command for local build command. +func NewLocal() *cobra.Command { + var ( + deps []string + k6 string + platform string + config k6build.LocalBuildServiceConfig + ) + + cmd := &cobra.Command{ + Use: "local", + Short: "build using a local builder", + Long: localLong, + Example: localExamples, + // prevent the usage help to printed to stderr when an error is reported by a subcommand + SilenceUsage: true, + // this is needed to prevent cobra to print errors reported by subcommands in the stderr + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + srv, err := k6build.NewLocalBuildService(cmd.Context(), config) + if err != nil { + return fmt.Errorf("configuring the build service %w", err) + } + + buildDeps := []k6build.Dependency{} + for _, d := range deps { + name, constrains, _ := strings.Cut(d, ":") + if constrains == "" { + constrains = "*" + } + buildDeps = append(buildDeps, k6build.Dependency{Name: name, Constraints: constrains}) + } + + artifact, err := srv.Build(cmd.Context(), platform, k6, buildDeps) + if err != nil { + return fmt.Errorf("building %w", err) + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + err = encoder.Encode(artifact) + if err != nil { + return fmt.Errorf("processing object %w", err) + } + + return nil + }, + } + + cmd.Flags().StringArrayVarP(&deps, "dependency", "d", nil, "list of dependencies in form package:constrains") + cmd.Flags().StringVarP(&k6, "k6", "k", "*", "k6 version constrains") + cmd.Flags().StringVarP(&platform, "platform", "p", "", "target platform (default GOOS/GOARCH)") + _ = cmd.MarkFlagRequired("platform") + cmd.Flags().StringVarP(&config.Catalog, "catalog", "c", "catalog.json", "dependencies catalog") + cmd.Flags().StringVarP(&config.CacheDir, "cache-dir", "f", "/tmp/buildservice", "cache dir") + cmd.Flags().BoolVarP(&config.Verbose, "verbose", "v", false, "print build process output") + cmd.Flags().BoolVarP(&config.CopyGoEnv, "copy-go-env", "g", true, "copy go environment") + cmd.Flags().StringToStringVarP(&config.BuildEnv, "env", "e", nil, "build environment variables") + + return cmd +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 0000000..e87740c --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,137 @@ +package cmd + +import ( + "fmt" + "net/http" + "os" + + "github.com/grafana/k6build" + "github.com/grafana/k6catalog" + "github.com/grafana/k6foundry" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +const ( + serverLong = ` +starts a k6build server that server +` + + serverExample = ` +# start the build server using a custom catalog +k6build server -c /path/to/catalog.json + +# start the server the build server using a custom GOPROXY +k6build server -e GOPROXY=http://localhost:80` +) + +// NewServer creates new cobra command for resolve command. +func NewServer() *cobra.Command { //nolint:funlen + var ( + buildEnv map[string]string + cacheDir string + catalog string + copyGoEnv bool + port int + verbose bool + log *logrus.Logger + logLevel string + ) + + cmd := &cobra.Command{ + Use: "server", + Short: "k6 build service", + Long: serverLong, + Example: serverExample, + // prevent the usage help to printed to stderr when an error is reported by a subcommand + SilenceUsage: true, + // this is needed to prevent cobra to print errors reported by subcommands in the stderr + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + ll, err := logrus.ParseLevel(logLevel) + if err != nil { + return fmt.Errorf("parsing log level %w", err) + } + log = &logrus.Logger{ + Out: os.Stderr, + Formatter: new(logrus.TextFormatter), + Level: ll, + } + + catalog, err := k6catalog.NewCatalogFromJSON(catalog) + if err != nil { + return fmt.Errorf("creating catalog %w", err) + } + + builderOpts := k6foundry.NativeBuilderOpts{ + Verbose: verbose, + GoOpts: k6foundry.GoOpts{ + Env: buildEnv, + CopyGoEnv: copyGoEnv, + }, + } + builder, err := k6foundry.NewNativeBuilder(cmd.Context(), builderOpts) + if err != nil { + return fmt.Errorf("creating builder %w", err) + } + + cache, err := k6build.NewFileCache(cacheDir) + if err != nil { + return fmt.Errorf("creating cache %w", err) + } + + // FIXME: this will not work across machines + cacheSrvURL := fmt.Sprintf("http://localhost:%d/cache", port) + config := k6build.CacheServerConfig{ + BaseURL: cacheSrvURL, + Cache: cache, + Log: log, + } + cacheSrv := k6build.NewCacheServer(config) + + cacheClientConfig := k6build.CacheClientConfig{ + Server: cacheSrvURL, + } + cacheClient, err := k6build.NewCacheClient(cacheClientConfig) + if err != nil { + return fmt.Errorf("creating cache client %w", err) + } + + buildSrv := k6build.NewBuildService( + catalog, + builder, + cacheClient, + ) + + apiConfig := k6build.APIServerConfig{ + BuildService: buildSrv, + Log: log, + } + buildAPI := k6build.NewAPIServer(apiConfig) + + srv := http.NewServeMux() + srv.Handle("POST /build/", http.StripPrefix("/build", buildAPI)) + srv.Handle("/cache/", http.StripPrefix("/cache", cacheSrv)) + + listerAddr := fmt.Sprintf("localhost:%d", port) + log.Infof("starting server at %s", listerAddr) + err = http.ListenAndServe(listerAddr, srv) //nolint:gosec + if err != nil { + log.Infof("server ended with error %s", err.Error()) + } + log.Info("ending server") + + return nil + }, + } + + cmd.Flags().StringVarP(&catalog, "catalog", "c", "catalog.json", "dependencies catalog") + cmd.Flags().StringVarP(&cacheDir, "cache-dir", "f", "/tmp/buildservice", "cache dir") + cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print build process output") + cmd.Flags().BoolVarP(©GoEnv, "copy-go-env", "g", true, "copy go environment") + cmd.Flags().StringToStringVarP(&buildEnv, "env", "e", nil, "build environment variables") + cmd.Flags().IntVarP(&port, "port", "p", 8000, "port server will listen") + cmd.Flags().StringVarP(&logLevel, "log-level", "l", "INFO", "log level") + + return cmd +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..72ec3b1 --- /dev/null +++ b/errors.go @@ -0,0 +1,17 @@ +package k6build + +import "errors" + +var ( + ErrAccessingObject = errors.New("accessing object") //nolint:revive + ErrAccessingServer = errors.New("making request") //nolint:revive + ErrBuildFailed = errors.New("build failed") //nolint:revive + ErrCreatingObject = errors.New("creating object") //nolint:revive + ErrInitializingCache = errors.New("initializing cache") //nolint:revive + ErrInvalidConfig = errors.New("invalid configuration") //nolint:revive + ErrInvalidRequest = errors.New("invalid request") //nolint:revive + ErrInvalidResponse = errors.New("invalid response") //nolint:revive + ErrInvalidURL = errors.New("invalid object URL") //nolint:revive + ErrObjectNotFound = errors.New("object not found") //nolint:revive + ErrRequestFailed = errors.New("request failed") //nolint:revive +) diff --git a/service.go b/local.go similarity index 69% rename from service.go rename to local.go index ce78408..2826245 100644 --- a/service.go +++ b/local.go @@ -1,4 +1,3 @@ -// Package k6build defines a service for building k8 binaries package k6build import ( @@ -13,64 +12,6 @@ import ( "github.com/grafana/k6foundry" ) -const ( - k6Dep = "k6" -) - -// Dependency contains the properties of a k6 dependency. -type Dependency struct { - // Name is the name of the dependency. - Name string `json:"name,omitempty"` - // Constraints contains the version constraints of the dependency. - Constraints string `json:"constraints,omitempty"` -} - -// Module defines an artifact dependency -type Module struct { - Path string `json:"path,omitempty"` - Version string `json:"vesion,omitempty"` -} - -// Artifact defines a binary that can be downloaded -// TODO: add metadata (e.g. list of dependencies, checksum, date compiled) -type Artifact struct { - ID string `json:"id,omitempty"` - // URL to fetch the artifact's binary - URL string `json:"url,omitempty"` - // list of dependencies - Dependencies map[string]string `json:"dependencies,omitempty"` - // platform - Platform string `json:"platform,omitempty"` - // binary checksum (sha256) - Checksum string `json:"checksum,omitempty"` -} - -// BuildService defines the interface of a build service -type BuildService interface { - // Build returns a k6 Artifact given its dependencies and version constrain - Build(ctx context.Context, platform string, k6Constrains string, deps []Dependency) (Artifact, error) -} - -// implements the BuildService interface -type buildsrv struct { - catalog k6catalog.Catalog - builder k6foundry.Builder - cache Cache -} - -// NewBuildService creates a build service -func NewBuildService( - catalog k6catalog.Catalog, - builder k6foundry.Builder, - cache Cache, -) BuildService { - return &buildsrv{ - catalog: catalog, - builder: builder, - cache: cache, - } -} - // LocalBuildServiceConfig defines the configuration for a Local build service type LocalBuildServiceConfig struct { // Set build environment variables @@ -110,14 +51,14 @@ func NewLocalBuildService(ctx context.Context, config LocalBuildServiceConfig) ( return nil, fmt.Errorf("creating cache %w", err) } - return &buildsrv{ + return &localBuildSrv{ catalog: catalog, builder: builder, cache: cache, }, nil } -// DefaultLocalBuildService creates a Local Build service with default configuration +// DefaultLocalBuildService creates a local build service with default configuration func DefaultLocalBuildService() (BuildService, error) { catalog, err := k6catalog.DefaultCatalog() if err != nil { @@ -134,14 +75,14 @@ func DefaultLocalBuildService() (BuildService, error) { return nil, fmt.Errorf("creating temp cache %w", err) } - return &buildsrv{ + return &localBuildSrv{ catalog: catalog, builder: builder, cache: cache, }, nil } -func (b *buildsrv) Build( +func (b *localBuildSrv) Build( ctx context.Context, platform string, k6Constrains string, diff --git a/service_test.go b/local_test.go similarity index 73% rename from service_test.go rename to local_test.go index 0657cda..a95f0ac 100644 --- a/service_test.go +++ b/local_test.go @@ -3,13 +3,9 @@ package k6build import ( "context" "errors" - "fmt" - "net/http/httptest" "testing" "github.com/grafana/k6catalog" - "github.com/grafana/k6foundry" - "github.com/grafana/k6foundry/pkg/testutils/goproxy" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -17,83 +13,6 @@ import ( func dependencyComp(a, b Module) bool { return a.Path < b.Path } -func setupBuildService(cacheDir string) (BuildService, error) { - modules := []struct { - path string - version string - source string - }{ - { - path: "go.k6.io/k6", - version: "v0.1.0", - source: "testdata/deps/k6", - }, - { - path: "go.k6.io/k6", - version: "v0.2.0", - source: "testdata/deps/k6", - }, - { - path: "go.k6.io/k6ext", - version: "v0.1.0", - source: "testdata/deps/k6ext", - }, - { - path: "go.k6.io/k6ext", - version: "v0.2.0", - source: "testdata/deps/k6ext", - }, - { - path: "go.k6.io/k6ext2", - version: "v0.1.0", - source: "testdata/deps/k6ext2", - }, - } - - // creates a goproxy that serves the given modules - proxy := goproxy.NewGoProxy() - for _, m := range modules { - err := proxy.AddModVersion(m.path, m.version, m.source) - if err != nil { - return nil, fmt.Errorf("setup %w", err) - } - } - - goproxySrv := httptest.NewServer(proxy) - - opts := k6foundry.NativeBuilderOpts{ - GoOpts: k6foundry.GoOpts{ - CopyGoEnv: true, - Env: map[string]string{ - "GOPROXY": goproxySrv.URL, - "GONOPROXY": "none", - "GOPRIVATE": "go.k6.io", - "GONOSUMDB": "go.k6.io", - }, - TmpCache: true, - }, - } - - builder, err := k6foundry.NewNativeBuilder(context.Background(), opts) - if err != nil { - return nil, fmt.Errorf("setting up test builder %w", err) - } - - catalog, err := k6catalog.NewCatalogFromJSON("testdata/catalog.json") - if err != nil { - return nil, fmt.Errorf("setting up test builder %w", err) - } - - cache, err := NewFileCache(cacheDir) - if err != nil { - return nil, fmt.Errorf("creating temp cache %w", err) - } - - buildsrv := NewBuildService(catalog, builder, cache) - - return buildsrv, nil -} - func TestDependencyResolution(t *testing.T) { t.Parallel() @@ -153,7 +72,12 @@ func TestDependencyResolution(t *testing.T) { t.Run(tc.title, func(t *testing.T) { t.Parallel() - buildsrv, err := setupBuildService(t.TempDir()) + buildsrv, err := SetupTestLocalBuildService( + LocalBuildServiceConfig{ + CacheDir: t.TempDir(), + Catalog: "testdata/catalog.json", + }, + ) if err != nil { t.Fatalf("test setup %v", err) } @@ -184,8 +108,12 @@ func TestDependencyResolution(t *testing.T) { func TestIdempotentBuild(t *testing.T) { t.Parallel() - - buildsrv, err := setupBuildService(t.TempDir()) + buildsrv, err := SetupTestLocalBuildService( + LocalBuildServiceConfig{ + CacheDir: t.TempDir(), + Catalog: "testdata/catalog.json", + }, + ) if err != nil { t.Fatalf("test setup %v", err) } diff --git a/testutils.go b/testutils.go new file mode 100644 index 0000000..8157d73 --- /dev/null +++ b/testutils.go @@ -0,0 +1,90 @@ +// Package k6build offers utility functions for testing +package k6build + +import ( + "context" + "fmt" + "net/http/httptest" + + "github.com/grafana/k6catalog" + "github.com/grafana/k6foundry" + "github.com/grafana/k6foundry/pkg/testutils/goproxy" +) + +// SetupTestLocalBuildService setups a local build service for testing +func SetupTestLocalBuildService(config LocalBuildServiceConfig) (BuildService, error) { + modules := []struct { + path string + version string + source string + }{ + { + path: "go.k6.io/k6", + version: "v0.1.0", + source: "testdata/deps/k6", + }, + { + path: "go.k6.io/k6", + version: "v0.2.0", + source: "testdata/deps/k6", + }, + { + path: "go.k6.io/k6ext", + version: "v0.1.0", + source: "testdata/deps/k6ext", + }, + { + path: "go.k6.io/k6ext", + version: "v0.2.0", + source: "testdata/deps/k6ext", + }, + { + path: "go.k6.io/k6ext2", + version: "v0.1.0", + source: "testdata/deps/k6ext2", + }, + } + + // creates a goproxy that serves the given modules + proxy := goproxy.NewGoProxy() + for _, m := range modules { + err := proxy.AddModVersion(m.path, m.version, m.source) + if err != nil { + return nil, fmt.Errorf("setup %w", err) + } + } + + goproxySrv := httptest.NewServer(proxy) + + opts := k6foundry.NativeBuilderOpts{ + GoOpts: k6foundry.GoOpts{ + CopyGoEnv: true, + Env: map[string]string{ + "GOPROXY": goproxySrv.URL, + "GONOPROXY": "none", + "GOPRIVATE": "go.k6.io", + "GONOSUMDB": "go.k6.io", + }, + TmpCache: true, + }, + } + + builder, err := k6foundry.NewNativeBuilder(context.Background(), opts) + if err != nil { + return nil, fmt.Errorf("setting up test builder %w", err) + } + + catalog, err := k6catalog.NewCatalogFromJSON(config.Catalog) + if err != nil { + return nil, fmt.Errorf("setting up test builder %w", err) + } + + cache, err := NewFileCache(config.CacheDir) + if err != nil { + return nil, fmt.Errorf("creating temp cache %w", err) + } + + buildsrv := NewBuildService(catalog, builder, cache) + + return buildsrv, nil +}