From 2c5a4db82862426a3e8d6dc2983a11b733c39733 Mon Sep 17 00:00:00 2001 From: Brijeshthummar02 Date: Tue, 22 Oct 2024 21:15:12 +0530 Subject: [PATCH] PlanetScale Plugin --- pkg/plugins/planetscale/cmd/main/main.go | 273 +++++++++++++++++ pkg/plugins/planetscale/cmd/main/main_test.go | 277 ++++++++++++++++++ .../planetscale/cmd/validator/main/main.go | 175 +++++++++++ .../planetscale/config/planetscaleconfig.go | 34 +++ .../config/planetscaleconfig_test.go | 77 +++++ pkg/plugins/planetscale/go.mod | 67 +++++ pkg/plugins/planetscale/go.sum | 40 +++ pkg/plugins/planetscale/plugin/planetscale.go | 73 +++++ 8 files changed, 1016 insertions(+) create mode 100644 pkg/plugins/planetscale/cmd/main/main.go create mode 100644 pkg/plugins/planetscale/cmd/main/main_test.go create mode 100644 pkg/plugins/planetscale/cmd/validator/main/main.go create mode 100644 pkg/plugins/planetscale/config/planetscaleconfig.go create mode 100644 pkg/plugins/planetscale/config/planetscaleconfig_test.go create mode 100644 pkg/plugins/planetscale/go.mod create mode 100644 pkg/plugins/planetscale/go.sum create mode 100644 pkg/plugins/planetscale/plugin/planetscale.go diff --git a/pkg/plugins/planetscale/cmd/main/main.go b/pkg/plugins/planetscale/cmd/main/main.go new file mode 100644 index 0000000..06a153d --- /dev/null +++ b/pkg/plugins/planetscale/cmd/main/main.go @@ -0,0 +1,273 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/hashicorp/go-plugin" + "github.com/icholy/digest" + commonconfig "github.com/opencost/opencost-plugins/common/config" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/apimachinery/pkg/util/uuid" +) + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "planetscale", +} + +const costExplorerPendingInvoicesURL = "https://api.planetscale.com/v1/orgs/%s/pending-invoices" + +func main() { + log.Debug("Initializing PlanetScale plugin") + + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + planetScaleConfig, err := GetPlanetScaleConfig(configFile) + if err != nil { + log.Fatalf("error building PlanetScale config: %v", err) + } + log.SetLogLevel(planetScaleConfig.LogLevel) + + // PlanetScale API rate limits + rateLimiter := rate.NewLimiter(1.1, 2) + planetScaleCostSrc := PlanetScaleCostSource{ + rateLimiter: rateLimiter, + orgID: planetScaleConfig.OrgID, + } + planetScaleCostSrc.httpClient = getPlanetScaleClient(*planetScaleConfig) + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &planetScaleCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) +} + +func getPlanetScaleClient(planetScaleConfig PlanetScaleConfig) HTTPClient { + return &http.Client{ + Transport: &digest.Transport{ + Username: planetScaleConfig.PublicKey, + Password: planetScaleConfig.PrivateKey, + }, + } +} + +// Implementation of CustomCostSource +type PlanetScaleCostSource struct { + orgID string + rateLimiter *rate.Limiter + httpClient HTTPClient +} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +func validateRequest(req *pb.CustomCostRequest) []string { + var errors []string + now := time.Now() + // 1. Check if resolution is less than a day + if req.Resolution.AsDuration() < 24*time.Hour { + var resolutionMessage = "Resolution should be at least one day." + log.Warnf(resolutionMessage) + errors = append(errors, resolutionMessage) + } + // Get the start of the current month + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + // 2. Check if start time is before the start of the current month + if req.Start.AsTime().Before(currentMonthStart) { + var startDateMessage = "Start date cannot be before the current month. Historical costs not currently supported." + log.Warnf(startDateMessage) + errors = append(errors, startDateMessage) + } + + // 3. Check if end time is before the start of the current month + if req.End.AsTime().Before(currentMonthStart) { + var endDateMessage = "End date cannot be before the current month. Historical costs not currently supported." + log.Warnf(endDateMessage) + errors = append(errors, endDateMessage) + } + + return errors +} + +func (p *PlanetScaleCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + requestErrors := validateRequest(req) + if len(requestErrors) > 0 { + // Return empty response + return results + } + + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + lineItems, err := GetPendingInvoices(p.orgID, p.httpClient) + if err != nil { + log.Errorf("Error fetching invoices: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error fetching invoices: %v", err)}, + } + results = append(results, &errResp) + return results + } + + for _, target := range targets { + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching PlanetScale costs for window %v", target) + result := p.getPlanetScaleCostsForWindow(&target, lineItems) + + results = append(results, result) + } + + return results +} + +func filterLineItemsByWindow(win *opencost.Window, lineItems []LineItem) []*pb.CustomCost { + var filteredItems []*pb.CustomCost + + winStartUTC := win.Start().UTC() + winEndUTC := win.End().UTC() + log.Debugf("Item window %s %s", winStartUTC, winEndUTC) + + for _, item := range lineItems { + startDate, err1 := time.Parse("2006-01-02T15:04:05Z07:00", item.StartDate) + endDate, err2 := time.Parse("2006-01-02T15:04:05Z07:00", item.EndDate) + + if err1 != nil || err2 != nil { + if err1 != nil { + log.Warnf("%s", err1) + } + if err2 != nil { + log.Warnf("%s", err2) + } + continue + } + + customCost := &pb.CustomCost{ + AccountName: item.GroupName, + ChargeCategory: "Usage", + Description: fmt.Sprintf("Usage for %s", item.SKU), + ResourceName: item.SKU, + Id: string(uuid.NewUUID()), + ProviderId: fmt.Sprintf("%s/%s/%s", item.GroupId, item.ClusterName, item.SKU), + BilledCost: float32(item.TotalPriceCents) / 100.0, + ListCost: item.Quantity * item.UnitPriceDollars, + ListUnitPrice: item.UnitPriceDollars, + UsageQuantity: item.Quantity, + UsageUnit: item.Unit, + } + + if (startDate.UTC().After(winStartUTC) || startDate.UTC().Equal(winStartUTC)) && + (endDate.UTC().Before(winEndUTC) || endDate.UTC().Equal(winEndUTC)) { + filteredItems = append(filteredItems, customCost) + } + } + + return filteredItems +} + +func (p *PlanetScaleCostSource) getPlanetScaleCostsForWindow(win *opencost.Window, lineItems []LineItem) *pb.CustomCostResponse { + costsInWindow := filterLineItemsByWindow(win, lineItems) + + resp := pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v1"}, + CostSource: "data_storage", + Domain: "planetscale", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: costsInWindow, + } + return &resp +} + +func GetPendingInvoices(org string, client HTTPClient) ([]LineItem, error) { + request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerPendingInvoicesURL, org), nil) + + request.Header.Set("Accept", "application/json") + request.Header.Set("Content-Type", "application/json") + + response, err := client.Do(request) + if err != nil { + msg := fmt.Sprintf("getPendingInvoices: error from server: %v", err) + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + log.Debugf("response Body: %s", string(body)) + + var pendingInvoicesResponse PendingInvoice + respUnmarshalError := json.Unmarshal([]byte(body), &pendingInvoicesResponse) + if respUnmarshalError != nil { + msg := fmt.Sprintf("pendingInvoices: error unmarshalling response: %v", respUnmarshalError) + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + + return pendingInvoicesResponse.LineItems, nil +} + +// Define your PlanetScaleConfig, PendingInvoice, and LineItem structs below +type PlanetScaleConfig struct { + OrgID string + PublicKey string + PrivateKey string + LogLevel string +} + +type PendingInvoice struct { + LineItems []LineItem `json:"line_items"` +} + +type LineItem struct { + GroupName string `json:"group_name"` + SKU string `json:"sku"` + GroupId string `json:"group_id"` + ClusterName string `json:"cluster_name"` + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + TotalPriceCents int64 `json:"total_price_cents"` + Quantity float64 `json:"quantity"` + UnitPriceDollars float64 `json:"unit_price_dollars"` + Unit string `json:"unit"` +} diff --git a/pkg/plugins/planetscale/cmd/main/main_test.go b/pkg/plugins/planetscale/cmd/main/main_test.go new file mode 100644 index 0000000..e56f846 --- /dev/null +++ b/pkg/plugins/planetscale/cmd/main/main_test.go @@ -0,0 +1,277 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "testing" + "time" + + "github.com/icholy/digest" + atlasplugin "github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas/plugin" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/stretchr/testify/assert" +) + +// Mock HTTPClient implementation +type MockHTTPClient struct { + DoFunc func(req *http.Request) (*http.Response, error) +} + +// The MockHTTPClient's Do method uses a function defined at runtime to mock various responses +func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { + return m.DoFunc(req) +} + +// FOR INTEGRATION TESTING PURPOSES ONLY +// expects 3 env variables to be set to work +// mapuk = public key for mongodb atlas +// maprk = private key for mongodb atlas +// maOrgId = orgId to be testsed +func TestMain(t *testing.T) { + publicKey := os.Getenv("mapuk") + privateKey := os.Getenv("maprk") + orgId := os.Getenv("maorgid") + if publicKey == "" || privateKey == "" || orgId == "" { + t.Skip("Skipping integration test.") + } + + assert.NotNil(t, publicKey) + assert.NotNil(t, privateKey) + assert.NotNil(t, orgId) + + client := &http.Client{ + Transport: &digest.Transport{ + Username: publicKey, + Password: privateKey, + }, + } + + atlasCostSource := AtlasCostSource{ + orgID: "myOrg", + atlasClient: client, + } + // Define the start and end time for the window + now := time.Now() + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + customCostRequest := pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart), // Start in current month + End: timestamppb.New(currentMonthStart.Add(24 * time.Hour)), // End in current month + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution + } + + resp := atlasCostSource.GetCustomCosts(&customCostRequest) + + assert.NotEmpty(t, resp) +} + +// tests for getCosts +func TestGetCostsPendingInvoices(t *testing.T) { + pendingInvoiceResponse := atlasplugin.PendingInvoice{ + AmountBilledCents: 0, + AmountPaidCents: 0, + Created: "2024-10-01T02:00:26Z", + CreditsCents: 0, + EndDate: "2024-11-01T00:00:00Z", + Id: "66fb726b79b56205f9376437", + LineItems: []atlasplugin.LineItem{ + { + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: "2024-10-11T00:00:00Z", + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 6.035e-07, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: "2024-10-10T00:00:00Z", + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + }, + Links: []atlasplugin.Link{ + { + Href: "https://cloud.mongodb.com/api/atlas/v2/orgs/66d7254246a21a41036ff2e9", + Rel: "self", + }, + }, + OrgId: "66d7254246a21a41036ff2e9", + SalesTaxCents: 0, + StartDate: "2024-10-01T00:00:00Z", + StartingBalanceCents: 0, + StatusName: "PENDING", + SubTotalCents: 0, + Updated: "2024-10-01T02:00:26Z", + } + + mockResponseJson, _ := json.Marshal(pendingInvoiceResponse) + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify that the request method and URL are correct + if req.Method != http.MethodGet { + t.Errorf("expected GET request, got %s", req.Method) + } + expectedURL := fmt.Sprintf(costExplorerPendingInvoicesURL, "myOrg") + if req.URL.String() != expectedURL { + t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) + } + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + }, + } + lineItems, err := GetPendingInvoices("myOrg", mockClient) + assert.Nil(t, err) + assert.Equal(t, 1, len(lineItems)) + + for _, invoice := range pendingInvoiceResponse.LineItems { + assert.Equal(t, "kubecost-mongo-dev-1", invoice.ClusterName) + assert.Equal(t, "66d7254246a21a41036ff33e", invoice.GroupId) + assert.Equal(t, "Project 0", invoice.GroupName) + // TODO add more asserts on the fields + } +} + +func TestGetCostErrorFromServer(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Return a mock response with status 200 and mock JSON body + return nil, fmt.Errorf("mock error: failed to execute request") + }, + } + costs, err := GetPendingInvoices("myOrg", mockClient) + + assert.NotEmpty(t, err) + assert.Nil(t, costs) +} + +func TestGetCostsBadMessage(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: io.NopCloser(bytes.NewBufferString("No Jason No")), + }, nil + }, + } + + _, err := GetPendingInvoices("myOrd", mockClient) + assert.NotEmpty(t, err) +} + +func TestGetAtlasCostsForWindow(t *testing.T) { + atlasCostSource := AtlasCostSource{ + orgID: "myOrg", + } + // Define the start and end time for the window + day1 := time.Date(2024, time.October, 12, 0, 0, 0, 0, time.UTC) // Now + + day2 := time.Date(2024, time.October, 13, 0, 0, 0, 0, time.UTC) + day3 := time.Date(2024, time.October, 14, 0, 0, 0, 0, time.UTC) // Now + lineItems := []atlasplugin.LineItem{ + { + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: day3.Format("2006-01-02T15:04:05Z07:00"), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 6.035e-07, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: day2.Format("2006-01-02T15:04:05Z07:00"), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + { + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: day2.Format("2006-01-02T15:04:05Z07:00"), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 0.0555, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: day1.Add(-24 * time.Hour).Format("2006-01-02T15:04:05Z07:00"), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + } + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify that the request method and URL are correct + if req.Method != http.MethodGet { + t.Errorf("expected GET request, got %s", req.Method) + } + expectedURL := fmt.Sprintf(costExplorerPendingInvoicesURL, "myOrg") + if req.URL.String() != expectedURL { + t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) + } + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString("mocked response")), + }, nil + }, + } + + // Use the mockClient to test the function + costs, err := atlasCostSource.GetCustomCosts(nil) + + assert.Nil(t, err) + assert.NotEmpty(t, costs) +} + +func TestFilterInvoicesOnWindow(t *testing.T) { + now := time.Now() + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + invoices := []atlasplugin.LineItem{ + { + ClusterName: "kubecost-mongo-dev-1", + Created: currentMonthStart.Add(-1 * time.Hour).Format(time.RFC3339), + EndDate: currentMonthStart.Add(24 * time.Hour).Format(time.RFC3339), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 0.0555, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: currentMonthStart.Format(time.RFC3339), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + { + ClusterName: "kubecost-mongo-dev-1", + Created: currentMonthStart.Add(48 * time.Hour).Format(time.RFC3339), + EndDate: currentMonthStart.Add(72 * time.Hour).Format(time.RFC3339), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 1", + Quantity: 0.0555, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: currentMonthStart.Add(24 * time.Hour).Format(time.RFC3339), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + } + + startWindow := currentMonthStart + endWindow := currentMonthStart.Add(24 * time.Hour) + + filtered := FilterInvoicesOnWindow(invoices, startWindow, endWindow) + assert.Equal(t, 1, len(filtered)) // Expecting 1 line item in the window +} diff --git a/pkg/plugins/planetscale/cmd/validator/main/main.go b/pkg/plugins/planetscale/cmd/validator/main/main.go new file mode 100644 index 0000000..4cead60 --- /dev/null +++ b/pkg/plugins/planetscale/cmd/validator/main/main.go @@ -0,0 +1,175 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/encoding/protojson" +) + +// The validator is designed to allow plugin implementors to validate their plugin information +// as called by the central test harness. +// This avoids having to ask folks to re-implement the test harness over again for each plugin. + +func main() { + // First arg is the path to the daily protobuf file + if len(os.Args) < 3 { + fmt.Println("Usage: validator ") + os.Exit(1) + } + + dailyProtobufFilePath := os.Args[1] + + // Read in the daily protobuf file + data, err := os.ReadFile(dailyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading daily protobuf file: %v\n", err) + os.Exit(1) + } + + dailyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) + + // Second arg is the path to the hourly protobuf file + hourlyProtobufFilePath := os.Args[2] + + data, err = os.ReadFile(hourlyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading hourly protobuf file: %v\n", err) + os.Exit(1) + } + + // Read in the hourly protobuf file + hourlyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) + + // Validate the custom cost response data + isValid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isValid { + os.Exit(1) + } else { + fmt.Println("Validation successful") + } +} + +func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { + if len(respDaily) == 0 { + log.Errorf("No daily response received from planetscale plugin") + return false + } + + if len(respHourly) == 0 { + log.Errorf("No hourly response received from planetscale plugin") + return false + } + + var multiErr error + + // Parse the response and look for errors + for _, resp := range respDaily { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) + } + } + + for _, resp := range respHourly { + if resp.Errors != nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors)) + } + } + + // Check if any errors occurred + if multiErr != nil { + log.Errorf("Errors occurred during plugin testing for planetscale: %v", multiErr) + return false + } + + seenCosts := map[string]bool{} + nonZeroBilledCosts := 0 + // Verify that the returned costs are non-zero + for _, resp := range respDaily { + for _, cost := range resp.Costs { + seenCosts[cost.GetResourceName()] = true + if !strings.Contains(cost.GetResourceName(), "FREE") && cost.GetListCost() == 0 { + log.Errorf("Daily list cost returned by plugin planetscale is zero for cost: %v", cost) + return false + } + if cost.GetListCost() >= 0.01 && !strings.Contains(cost.GetResourceName(), "FREE") && cost.GetBilledCost() == 0 { + log.Errorf("Daily billed cost returned by plugin planetscale is zero for cost: %v", cost) + return false + } + if cost.GetBilledCost() > 0 { + nonZeroBilledCosts++ + } + } + } + + if nonZeroBilledCosts == 0 { + log.Errorf("No non-zero billed costs returned by plugin planetscale") + return false + } + + expectedCosts := []string{ + "PLANETSCALE_DATA_STORAGE", + "PLANETSCALE_DATA_TRANSFER", + "PLANETSCALE_INSTANCE", + } + + for _, cost := range expectedCosts { + if !seenCosts[cost] { + log.Errorf("Hourly cost %s not found in plugin planetscale response", cost) + return false + } + } + + if len(seenCosts) != len(expectedCosts) { + log.Errorf("Hourly costs returned by plugin planetscale do not equal expected costs") + log.Errorf("Seen costs: %v", seenCosts) + log.Errorf("Expected costs: %v", expectedCosts) + + log.Errorf("Response: %v", respHourly) + return false + } + + // Verify the domain matches the plugin name + for _, resp := range respDaily { + if resp.Domain != "planetscale" { + log.Errorf("Daily domain returned by plugin planetscale does not match plugin name") + return false + } + } + + return true +} + +func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + protoResps := make([]*pb.CustomCostResponse, len(raw)) + for i, r := range raw { + p := &pb.CustomCostResponse{} + if err := protojson.Unmarshal(r, p); err != nil { + return nil, err + } + protoResps[i] = p + } + + return protoResps, nil +} diff --git a/pkg/plugins/planetscale/config/planetscaleconfig.go b/pkg/plugins/planetscale/config/planetscaleconfig.go new file mode 100644 index 0000000..cf35299 --- /dev/null +++ b/pkg/plugins/planetscale/config/planetscaleconfig.go @@ -0,0 +1,34 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +type PlanetScaleConfig struct { + APIKey string `json:"planetscale_api_key"` + OrgID string `json:"planetscale_org_id"` + Database string `json:"planetscale_database"` + LogLevel string `json:"planetscale_plugin_log_level"` +} + +// GetPlanetscaleConfig reads the configuration from a given JSON file. +func GetPlanetscaleConfig(configFilePath string) (*PlanetScaleConfig, error) { + var result PlanetScaleConfig + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for PlanetScale config @ %s: %v", configFilePath, err) + } + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error unmarshaling JSON into PlanetScale config: %v", err) + } + + // Set default log level if not provided + if result.LogLevel == "" { + result.LogLevel = "info" + } + + return &result, nil +} diff --git a/pkg/plugins/planetscale/config/planetscaleconfig_test.go b/pkg/plugins/planetscale/config/planetscaleconfig_test.go new file mode 100644 index 0000000..7ff7b15 --- /dev/null +++ b/pkg/plugins/planetscale/config/planetscaleconfig_test.go @@ -0,0 +1,77 @@ +package config + +import ( + "fmt" + "os" + "testing" +) + +// Unit tests for the GetPlanetscaleConfig function +func TestGetPlanetscaleConfig(t *testing.T) { + // Test: Valid configuration file + t.Run("Valid configuration file", func(t *testing.T) { + configFilePath := "test_valid_config.json" + // Create a temporary valid JSON file + validConfig := `{"planetscale_plugin_log_level": "debug"}` + err := os.WriteFile(configFilePath, []byte(validConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) + + config, err := GetPlanetscaleConfig(configFilePath) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + fmt.Println(config, configFilePath) + if config.LogLevel != "debug" { + t.Errorf("expected log level to be 'debug', but got: %s", config.LogLevel) + } + }) + + // Test: Invalid file path + t.Run("Invalid file path", func(t *testing.T) { + configFilePath := "invalid_path.json" + _, err := GetPlanetscaleConfig(configFilePath) + if err == nil { + t.Errorf("expected an error, but got none") + } + }) + + // Test: Invalid JSON format + t.Run("Invalid JSON format", func(t *testing.T) { + configFilePath := "test_invalid_json.json" + // Create a temporary invalid JSON file + invalidConfig := `{"log_level": "debug"` + err := os.WriteFile(configFilePath, []byte(invalidConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) + + _, err = GetPlanetscaleConfig(configFilePath) + if err == nil { + t.Errorf("expected an error, but got none") + } + }) + + // Test: Default log level when missing + t.Run("Default log level when missing", func(t *testing.T) { + configFilePath := "test_missing_log_level.json" + // Create a temporary JSON file without log_level + missingLogLevelConfig := `{}` + err := os.WriteFile(configFilePath, []byte(missingLogLevelConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) + + config, err := GetPlanetscaleConfig(configFilePath) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + if config.LogLevel != "info" { + t.Errorf("expected log level to be 'info', but got: %s", config.LogLevel) + } + }) +} diff --git a/pkg/plugins/planetscale/go.mod b/pkg/plugins/planetscale/go.mod new file mode 100644 index 0000000..ff677a2 --- /dev/null +++ b/pkg/plugins/planetscale/go.mod @@ -0,0 +1,67 @@ +module github.com/opencost/opencost-plugins/pkg/plugins/planetscale + +go 1.22.5 + +replace github.com/opencost/opencost-plugins/common => ../../common + +require ( + github.com/hashicorp/go-plugin v1.6.1 + github.com/icholy/digest v0.1.23 + github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 + github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 + github.com/stretchr/testify v1.9.0 + golang.org/x/time v0.6.0 + google.golang.org/protobuf v1.34.2 + k8s.io/apimachinery v0.25.3 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.17.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/goccy/go-json v0.9.11 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oklog/run v1.1.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/zerolog v1.26.1 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.8.1 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/grpc v1.66.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.25.3 // indirect + k8s.io/klog/v2 v2.80.0 // indirect + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect +) diff --git a/pkg/plugins/planetscale/go.sum b/pkg/plugins/planetscale/go.sum new file mode 100644 index 0000000..78216a3 --- /dev/null +++ b/pkg/plugins/planetscale/go.sum @@ -0,0 +1,40 @@ +github.com/davecgh/go-spew v1.1.1 h1:U7EC8pBd9x5+UckYPi4BFd56cZ6+kOaZ54hF2m93FSE= +github.com/davecgh/go-spew v1.1.1/go.mod h1:0bHFBG/Fx5A2V3iPHM2ZAn7HZEm5lUF9hskK0dM2qUk= +github.com/fatih/color v1.17.0 h1:nPE82/yQCy9Op4MI59qZ1oD22Ls7+6pIs0L5Ym41l64= +github.com/fatih/color v1.17.0/go.mod h1:f3dTLKoyIj5/lNWxAn36Mw2tRYA7M1EPlQbQ38LB2W0= +github.com/fsnotify/fsnotify v1.6.0 h1:RaS2r+5By4w2H2ztMiPjAdo/gHhLkFkMzA2QxXwRSbA= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:k7wxSNTmt5ak7t5FBc3p9A5ZX1H1VOTDLqUB3PkO/so= +github.com/go-logr/logr v1.2.4 h1:0QiRnwJ1B+F1ckH7moDqV/6bE2R3HtDd12YQZmsHzDw= +github.com/go-logr/logr v1.2.4/go.mod h1:ABZhTj0J3NkTAiIVuI4PblpESXK7b0wMO+gfp8YoLSU= +github.com/goccy/go-json v0.9.11 h1:8mEwtxGWhZWoHt16FRYQy6wF38pU2gh8oA7xlTQqlIY= +github.com/goccy/go-json v0.9.11/go.mod h1:43n2Pvn0gIFg2k85qXJ3fP2bO8BJn4WCKgMi0pe5wFg= +github.com/gogo/protobuf v1.3.2 h1:MBJaf9WQ78t3kRaO/mrUQQk4Ygt2y7S1UBGoefD7Iho= +github.com/gogo/protobuf v1.3.2/go.mod h1:flDK88aFMFxXoM3H5sF3GknOwtfWJp6IHg8h3fQXY9s= +github.com/golang/protobuf v1.5.4 h1:K2UDZs5Z4AfBDccFV+nWGeL6dbfHNHdBWl+5MN8eRNY= +github.com/golang/protobuf v1.5.4/go.mod h1:dUvD9ZjyoTtS33B10N3VntoIfZcQmQ3K37wmv0+f1h0= +github.com/google/gofuzz v1.2.0 h1:ObbeBaq9q33s3IdlsnLqnpfNOdWOfmUAw5wIHV0B62Y= +github.com/google/gofuzz v1.2.0/go.mod h1:63I6KrzGqZh8oI1qf0m4Vt66lQzTClHcPV9jY3GGUOM= +github.com/google/uuid v1.6.0 h1:ZpTYnY0hUQQGmSM6kaI3O8I3F6wTxV3Ue7nZ9fiK74M= +github.com/google/uuid v1.6.0/go.mod h1:iGROHUtgMBN5RWGVzHhIYcA4OMNTDp+c8dmNQu7/68g= +github.com/hashicorp/errwrap v1.0.0 h1:Nx7zIFlWGB15PeTSdJ1NQWr56FlB3V8cJH10YYWyQTo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:mCE+0FYg6F8KwTUmqFKp4TcVnWQq2Qx7lC/lW7wI0rE= +github.com/hashicorp/go-hclog v1.6.3 h1:8DUfN40XYlA6uh7/Ssp5mWY2d+oa3FFk/IEkWw4CkdA= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:y4PggMZkiK5FLDzz+UJnTO4K8fiY0jFLfxSYjx1dpbg= +github.com/hashicorp/go-multierror v1.1.1 h1:hRqs0F1D93VdDmp03M0UtjDYPHDi9RntLu3Y3StkgUw= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:22h/ndS5w3U97P1qQKl1EjWJOD/2c5n2+DJxFZK1KmI= +github.com/hashicorp/hcl v1.0.0 h1:8F51C3WsWckvFwJ4D0VOO7RLsN1aAwBdoFgy0Dg0SNo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:3hF7PBBzXj5Klq5Zz1Ue3S/JWoBLgPCOAd3yefI8CVs= +github.com/hashicorp/yamux v0.1.1 h1:eVsqP9CskUKH5yR5wMVoBPZDOxClJHiXog95L8lboIo= +github.com/hashicorp/yamux v0.1.1/go.mod h1:xw/TC/a4OfcXq98fsTTBUh8F8Z/Tp2ptZIH0gXK+Q3g= +github.com/json-iterator/go v1.1.12 h1:Qm6jM6KeM/KMR6v5j3n3AVtSzpyiy09VG6SVyMvKyRQ= +github.com/json-iterator/go v1.1.12/go.mod h1:y2fgUBwC8c7FfPU8F3zMY5J/Uc0b8D8HLU4dYVaf9wE= +github.com/magiconair/properties v1.8.5 h1:qbFj8u7sc9bbgN3dA9kN6mEIlzKFCDeBB1o5a3h+ZC0= +github.com/magiconair/properties v1.8.5/go.mod h1:PT+3XOwFAKDHPGNVChIh0Y1QQx3WJfnxTBymxBu21m4= +github.com/mattn/go-colorable v0.1.13 h1:TT1HDZ4Vlj6zHcHSzW8WgL8NmMywIhKufVxhKwr44R4= +github.com/mattn/go-colorable v0.1.13/go.mod h1:1qxWHEhMEE/ilxvH2UdB5PVwPAzC7bN3AwR8QoMjU78= +github.com/mattn/go-isatty v0.0.20 h1:YoZTX3Xr0HPhHI9wkkCiAmf2p2SSApKZs5wiC2k2NI4= +github.com/mattn/go-isatty v0.0.20/go.mod h1:7q1lxzFbH8QQox0b3fhlw3BBNL8GhR1gDkeR6MPlZZE= +github.com/mitchellh/go-testing-interface v1.14.1 h1:TRfUuUkU4s0z5mtI8Y5udn8lOpNMMn6hFFCyLUx0HUI= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:6fYlxGVf90sRj6fXf13cxt4WzUqPOU8nAh6QLKnzPmA= +github.com/mitchellh/mapstructure v1.5.0 h1:L6wFSWIZP0kXyVKbYxi9ghMbyD8B9upk32B/EdPwv88= +github.com/mit diff --git a/pkg/plugins/planetscale/plugin/planetscale.go b/pkg/plugins/planetscale/plugin/planetscale.go new file mode 100644 index 0000000..695472f --- /dev/null +++ b/pkg/plugins/planetscale/plugin/planetscale.go @@ -0,0 +1,73 @@ +package plugin + +type CreateCostExplorerQueryPayload struct { + Clusters []string `json:"clusters"` + EndDate string `json:"endDate"` + GroupBy string `json:"groupBy"` + IncludePartialMatches bool `json:"includePartialMatches"` + Organizations []string `json:"organizations"` + Projects []string `json:"projects"` + Services []string `json:"services"` + StartDate string `json:"startDate"` +} + +type CreateCostExplorerQueryResponse struct { + Token string `json:"token"` +} + +type Invoice struct { + InvoiceId string `json:"invoiceId"` + OrganizationId string `json:"organizationId"` + OrganizationName string `json:"organizationName"` + Service string `json:"service"` + UsageAmount float32 `json:"usageAmount"` + UsageDate string `json:"usageDate"` + // Example usage detail: + // "invoiceId":"66d7254246a21a41036ff315", + // "organizationId":"66d7254246a21a41036ff2e9", + // "organizationName":"Kubecost", + // "service":"Clusters", + // "usageAmount":51.19, + // "usageDate":"2024-09-01" +} + +type CostResponse struct { + UsageDetails []Invoice `json:"usageDetails"` +} + +type PendingInvoice struct { + AmountBilledCents int32 `json:"amountBilledCents"` + AmountPaidCents int32 `json:"amountPaidCents"` + Created string `json:"created"` + CreditsCents int32 `json:"creditCents"` + Id string `json:"id"` + EndDate string `json:"endDate"` + LineItems []LineItem `json:"lineItems"` + Links []Link `json:"links"` + OrgId string `json:"orgId"` + SalesTaxCents int32 `json:"salesTaxCents"` + StartDate string `json:"startDate"` + StartingBalanceCents int32 `json:"startingBalanceCents"` + StatusName string `json:"statusName"` + SubTotalCents int32 `json:"subtotalCents"` + Updated string `json:"updated"` +} + +type Link struct { + Href string `json:"href"` + Rel string `json:"rel"` +} + +type LineItem struct { + ClusterName string `json:"clusterName"` + Created string `json:"created"` + EndDate string `json:"endDate"` + GroupId string `json:"groupId"` + GroupName string `json:"groupName"` + Quantity float32 `json:"quantity"` + SKU string `json:"sku"` + StartDate string `json:"startDate"` + TotalPriceCents int32 `json:"totalPriceCents"` + Unit string `json:"unit"` + UnitPriceDollars float32 `json:"unitPriceDollars"` +}