diff --git a/cmd/bb_portal/BUILD.bazel b/cmd/bb_portal/BUILD.bazel index 05211b0..1dc7e90 100644 --- a/cmd/bb_portal/BUILD.bazel +++ b/cmd/bb_portal/BUILD.bazel @@ -16,7 +16,6 @@ go_library( "//internal/api", "//internal/api/grpc/bes", "//internal/graphql", - "//pkg/cas", "//pkg/processing", "//pkg/proto/configuration/bb_portal", "@com_github_99designs_gqlgen//graphql/handler", diff --git a/cmd/bb_portal/main.go b/cmd/bb_portal/main.go index 5a1c1cc..21234e9 100644 --- a/cmd/bb_portal/main.go +++ b/cmd/bb_portal/main.go @@ -16,7 +16,6 @@ import ( "github.com/buildbarn/bb-portal/ent/gen/ent" "github.com/buildbarn/bb-portal/ent/gen/ent/migrate" "github.com/buildbarn/bb-portal/internal/api/grpc/bes" - "github.com/buildbarn/bb-portal/pkg/cas" "github.com/buildbarn/bb-portal/pkg/processing" "github.com/buildbarn/bb-portal/pkg/proto/configuration/bb_portal" "github.com/buildbarn/bb-storage/pkg/global" @@ -32,13 +31,11 @@ const ( ) var ( - configFile = flag.String("config-file", "", "bb_portal config file") - dsDriver = flag.String("datasource-driver", "sqlite3", "Data source driver to use") - dsURL = flag.String("datasource-url", "file:buildportal.db?_journal=WAL&_fk=1", "Data source URL for the DB") - bepFolder = flag.String("bep-folder", "./bep-files/", "Folder to watch for new BEP files") - caFile = flag.String("ca-file", "", "Custom CA certificate file") - credentialsHelperCommand = flag.String("credential_helper", "", "Path to a credential helper. Compatible with Bazel's --credential_helper") - blobArchiveFolder = flag.String("blob-archive-folder", "./blob-archive/", + configFile = flag.String("config-file", "", "bb_portal config file") + dsDriver = flag.String("datasource-driver", "sqlite3", "Data source driver to use") + dsURL = flag.String("datasource-url", "file:buildportal.db?_journal=WAL&_fk=1", "Data source URL for the DB") + bepFolder = flag.String("bep-folder", "./bep-files/", "Folder to watch for new BEP files") + blobArchiveFolder = flag.String("blob-archive-folder", "./blob-archive/", "Folder where blobs (log outputs, stdout, stderr, undeclared test outputs) referenced from failures are archived") ) @@ -83,11 +80,7 @@ func main() { runWatcher(watcher, dbClient, *bepFolder, blobArchiver) router := mux.NewRouter() - casManager := cas.NewConnectionManager(cas.ManagerParams{ - TLSCACertFile: *caFile, - CredentialsHelperCommand: *credentialsHelperCommand, - }) - newPortalService(blobArchiver, casManager, dbClient, router) + newPortalService(blobArchiver, dbClient, router) bb_http.NewServersFromConfigurationAndServe( configuration.HttpServers, bb_http.NewMetricsHandler(router, "PortalUI"), diff --git a/cmd/bb_portal/portal_service.go b/cmd/bb_portal/portal_service.go index c273cde..80afc9b 100644 --- a/cmd/bb_portal/portal_service.go +++ b/cmd/bb_portal/portal_service.go @@ -11,18 +11,16 @@ import ( "github.com/buildbarn/bb-portal/ent/gen/ent" "github.com/buildbarn/bb-portal/internal/api" "github.com/buildbarn/bb-portal/internal/graphql" - "github.com/buildbarn/bb-portal/pkg/cas" "github.com/buildbarn/bb-portal/pkg/processing" "github.com/gorilla/mux" ) -func newPortalService(archiver processing.BlobMultiArchiver, casManager *cas.ConnectionManager, dbClient *ent.Client, router *mux.Router) { +func newPortalService(archiver processing.BlobMultiArchiver, dbClient *ent.Client, router *mux.Router) { srv := handler.NewDefaultServer(graphql.NewSchema(dbClient)) srv.Use(entgql.Transactioner{TxOpener: dbClient}) router.PathPrefix("/graphql").Handler(srv) router.Handle("/graphiql", playground.Handler("GraphQL Playground", "/graphql")) - router.Handle("/api/v1/blobs/{blobID}/{name}", api.NewBlobHandler(dbClient, casManager)) router.Handle("/api/v1/bep/upload", api.NewBEPUploadHandler(dbClient, archiver)).Methods("POST") router.PathPrefix("/").Handler(frontendServer()) } diff --git a/internal/api/BUILD.bazel b/internal/api/BUILD.bazel index 6f7a2eb..568efdd 100644 --- a/internal/api/BUILD.bazel +++ b/internal/api/BUILD.bazel @@ -2,16 +2,11 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "api", - srcs = [ - "bep_upload.go", - "blob_handler.go", - ], + srcs = ["bep_upload.go"], importpath = "github.com/buildbarn/bb-portal/internal/api", visibility = ["//:__subpackages__"], deps = [ "//ent/gen/ent", - "//ent/gen/ent/blob", - "//pkg/cas", "//pkg/processing", ], ) diff --git a/internal/api/blob_handler.go b/internal/api/blob_handler.go deleted file mode 100644 index fc56c15..0000000 --- a/internal/api/blob_handler.go +++ /dev/null @@ -1,123 +0,0 @@ -package api - -import ( - "fmt" - "io" - "log/slog" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/buildbarn/bb-portal/ent/gen/ent" - "github.com/buildbarn/bb-portal/ent/gen/ent/blob" - "github.com/buildbarn/bb-portal/pkg/cas" -) - -// A struct to handle blobs. -type blobHandler struct { - client *ent.Client - casManager *cas.ConnectionManager -} - -// NewBlobHandler Constructor functio for a blob hanlder. -func NewBlobHandler(client *ent.Client, casManager *cas.ConnectionManager) http.Handler { - return &blobHandler{client: client, casManager: casManager} -} - -// ServeHTTP Serve this over http. -func (b *blobHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - blobIDPathValue := request.PathValue("blobID") - name := request.PathValue("name") - blobID, err := strconv.Atoi(blobIDPathValue) - if err != nil { - writeErr(writer, request, http.StatusBadRequest, fmt.Sprintf("Invalid blobID: %s", blobIDPathValue)) - return - } - - // TODO: We probably want semantic IDs, not row IDs. - blobRecord, err := b.client.Blob.Get(request.Context(), blobID) - if err != nil { - writeErr( - writer, - request, - http.StatusNotFound, - fmt.Sprintf("Could not find blob with blobID: %s", blobIDPathValue), - ) - return - } - - b.serveBlob(writer, request, name, blobRecord) -} - -// Serve a blob. -func (b *blobHandler) serveBlob(writer http.ResponseWriter, request *http.Request, name string, blobRecord *ent.Blob) { - if blobRecord.ArchivingStatus == blob.ArchivingStatusSUCCESS { - http.ServeFile(writer, request, blobRecord.ArchiveURL) - return - } - - // Fallback to reading original. - uri, err := url.Parse(blobRecord.URI) - if err != nil { - writeErr( - writer, - request, - http.StatusInternalServerError, - fmt.Sprintf("Blob %d had an invalid URI: %s", blobRecord.ID, blobRecord.URI), - ) - return - } - switch uri.Scheme { - case "file": - http.ServeFile(writer, request, uri.Path) - case "bytestream": - b.serveFromBytestream(writer, request, name, uri) - default: - writeErr(writer, request, http.StatusInternalServerError, fmt.Sprintf("unsupported URI scheme: %s", uri.Scheme)) - } -} - -// Serve from bytestream function. -func (b *blobHandler) serveFromBytestream(writer http.ResponseWriter, request *http.Request, name string, uri *url.URL) { - casClient, err := b.casManager.GetClientForURI(request.Context(), uri) - if err != nil { - writeErr(writer, request, http.StatusInternalServerError, err.Error()) - return - } - defer casClient.Close() - - tmpFile, err := os.CreateTemp("", filepath.Base(uri.Path)) - if err != nil { - writeErr(writer, request, http.StatusInternalServerError, err.Error()) - return - } - defer tmpFile.Close() - defer os.Remove(tmpFile.Name()) - - err = casClient.ReadBlobToFile(request.Context(), uri, tmpFile.Name()) - if err != nil { - writeErr(writer, request, http.StatusInternalServerError, err.Error()) - return - } - - if _, err = tmpFile.Seek(0, io.SeekStart); err != nil { - writeErr(writer, request, http.StatusInternalServerError, err.Error()) - return - } - http.ServeContent(writer, request, name, time.Time{}, tmpFile) -} - -// A function to write an error. -func writeErr(writer http.ResponseWriter, request *http.Request, statusCode int, msg string) { - writer.WriteHeader(statusCode) - if _, err := writer.Write([]byte(msg)); err != nil { - slog.ErrorContext( - request.Context(), - "could not write response", - "statusCode", statusCode, "msg", msg, - ) - } -} diff --git a/pkg/auth/BUILD.bazel b/pkg/auth/BUILD.bazel deleted file mode 100644 index 07ffc0e..0000000 --- a/pkg/auth/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "auth", - srcs = ["credentials_helper.go"], - importpath = "github.com/buildbarn/bb-portal/pkg/auth", - visibility = ["//visibility:public"], - deps = [ - "@com_github_google_shlex//:shlex", - "@org_golang_google_grpc//credentials", - ], -) diff --git a/pkg/auth/credentials_helper.go b/pkg/auth/credentials_helper.go deleted file mode 100644 index 47a6640..0000000 --- a/pkg/auth/credentials_helper.go +++ /dev/null @@ -1,78 +0,0 @@ -package auth - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "os/exec" - "strings" - - "github.com/google/shlex" - "google.golang.org/grpc/credentials" -) - -// A credentials Helper request struct. -type credentialHelperRequest struct { - URI string `json:"uri"` -} - -// A credential helpers response struct. -type credentialHelperResponse struct { - Headers map[string][]string `json:"headers"` -} - -// A credential Helpers struct. -type credentialsHelper struct { - command string -} - -// NewCredentialsHelper Credential Helper constructor. -func NewCredentialsHelper(command string) credentials.PerRPCCredentials { - return &credentialsHelper{command: command} -} - -// GetRequestMetadata Get the Request Metadata. -func (h credentialsHelper) GetRequestMetadata(_ context.Context, uri ...string) (map[string]string, error) { - resp, err := h.getCredentialsFromHelper(uri[0]) - if err != nil { - return nil, err - } - headers := make(map[string]string, len(resp.Headers)) - for key, values := range resp.Headers { - headers[key] = strings.Join(values, ",") - } - return headers, nil -} - -// Get credentials from helper function. -func (h credentialsHelper) getCredentialsFromHelper(uri string) (*credentialHelperResponse, error) { - reqBytes, err := json.Marshal(credentialHelperRequest{ - URI: uri, - }) - if err != nil { - return nil, fmt.Errorf("could not marshal credential helper request: %w", err) - } - var out bytes.Buffer - parts, err := shlex.Split(h.command) - if err != nil { - return nil, fmt.Errorf("could not parse command: %w", err) - } - cmd := exec.Command(parts[0], parts[1:]...) //nolint:gosec // G204 - Trusted input from local user. - cmd.Stdin = bytes.NewReader(reqBytes) - cmd.Stdout = &out - if err = cmd.Run(); err != nil { - return nil, fmt.Errorf("error running credential helper: %w", err) - } - var resp credentialHelperResponse - err = json.Unmarshal(out.Bytes(), &resp) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal credential helper response: %w", err) - } - return &resp, nil -} - -// RequireTransportSecurity Require Transport Security function. -func (h credentialsHelper) RequireTransportSecurity() bool { - return false -} diff --git a/pkg/cas/BUILD.bazel b/pkg/cas/BUILD.bazel deleted file mode 100644 index 05fd138..0000000 --- a/pkg/cas/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "cas", - srcs = ["client.go"], - importpath = "github.com/buildbarn/bb-portal/pkg/cas", - visibility = ["//visibility:public"], - deps = [ - "//pkg/auth", - "@com_github_bazelbuild_remote_apis_sdks//go/pkg/client", - "@com_github_bazelbuild_remote_apis_sdks//go/pkg/digest", - ], -) diff --git a/pkg/cas/client.go b/pkg/cas/client.go deleted file mode 100644 index dd4af18..0000000 --- a/pkg/cas/client.go +++ /dev/null @@ -1,85 +0,0 @@ -package cas - -import ( - "context" - "fmt" - "log/slog" - "net/url" - "strings" - - "github.com/bazelbuild/remote-apis-sdks/go/pkg/client" - "github.com/bazelbuild/remote-apis-sdks/go/pkg/digest" - - "github.com/buildbarn/bb-portal/pkg/auth" -) - -// ConnectionManager A connection manager. -type ConnectionManager struct { - params ManagerParams -} - -// ManagerParams A manager Params struct. -type ManagerParams struct { - // TLSCACertFile is the PEM file that contains TLS root certificates. - TLSCACertFile string - - // CredentialsHelperCommand is a command to use as a Bazel credentials helper. - CredentialsHelperCommand string -} - -// NewConnectionManager Connection Manager constructor. -func NewConnectionManager(params ManagerParams) *ConnectionManager { - return &ConnectionManager{ - params: params, - } -} - -// Client A client struct. -type Client struct { - client *client.Client -} - -// GetClientForURI A client to Get a connection for a given URI. -func (manager *ConnectionManager) GetClientForURI(ctx context.Context, uri *url.URL) (*Client, error) { - instanceName := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/")[0] - dialParms := client.DialParams{ - Service: uri.Hostname() + ":443", - TLSCACertFile: manager.params.TLSCACertFile, - } - if manager.params.CredentialsHelperCommand != "" { - credentialsHelper := auth.NewCredentialsHelper(manager.params.CredentialsHelperCommand) - dialParms.UseExternalAuthToken = true - dialParms.ExternalPerRPCCreds = &client.PerRPCCreds{Creds: credentialsHelper} - } - remoteAPIsClient, err := client.NewClient(ctx, instanceName, dialParms) - if err != nil { - return nil, fmt.Errorf("could not open a remote APIs SDK client: %w", err) - } - return &Client{ - client: remoteAPIsClient, - }, nil -} - -// ReadBlobToFile Reads a blob to a file. -func (c *Client) ReadBlobToFile(ctx context.Context, uri *url.URL, fpath string) error { - pathParts := strings.Split(uri.Path, "/") - digestPath := strings.Join(pathParts[len(pathParts)-2:], "/") - d, err := digest.NewFromString(digestPath) - if err != nil { - return fmt.Errorf("could not create digest from path %s: %w", uri.Path, err) - } - - slog.InfoContext(ctx, "reading from CAS", "digest", d.String(), "file", fpath) - - _, err = c.client.ReadBlobToFile(ctx, d, fpath) - if err != nil { - return fmt.Errorf("could not read blob at %s to file %s: %w", uri.String(), fpath, err) - } - - return nil -} - -// Close Close the connection. -func (c *Client) Close() error { - return c.client.Close() -}