From e8579004804f7e94c60fca66945285b69919aa96 Mon Sep 17 00:00:00 2001 From: Sam Betts Date: Tue, 27 Jun 2023 12:11:53 +0100 Subject: [PATCH] Separate asset discovery from scanning This commit removes the concept of scopes and instead replaces it with periodic discovery of assets. These assets will be created in the system and a ScanConfig can be configured with an ODATA $filter expression to choose the assets to scan. This increases user visibility into the assets available to scan by VMClarity, it also gives the system an opportunity to estimate how many items a ScanConfig might scan based on a particular query. It also opens the door for additional features such as the ability to scan a specific asset which has never been scanned before. Assets are tracked with lastSeen timestamp which indicates when it was last seen by discovery. When they are not found in a discovery round they are marked as terminated. Terminated assets will not be included in scans even if they match the $filter configured by the user for the scan. --- api/client/client.gen.go | 309 ------------------ api/models/models.gen.go | 276 +--------------- api/models/scan.go | 4 +- api/models/scanconfigsnapshot.go | 4 +- api/openapi.yaml | 215 +----------- api/server/server.gen.go | 237 +++++--------- backend/pkg/database/demo.go | 161 +-------- backend/pkg/database/gorm/database.go | 1 - backend/pkg/database/gorm/odata.go | 93 +----- backend/pkg/database/gorm/scope.go | 92 ------ backend/pkg/database/types/database.go | 6 - backend/pkg/rest/discovery_controller.go | 49 --- .../pkg/orchestrator/discovery/scope.go | 88 ++++- .../pkg/orchestrator/scanwatcher/watcher.go | 86 ++--- runtime_scan/pkg/provider/aws/client.go | 167 ++-------- runtime_scan/pkg/provider/aws/client_test.go | 18 + runtime_scan/pkg/provider/aws/helpers.go | 108 ------ runtime_scan/pkg/provider/azure/client.go | 130 +------- runtime_scan/pkg/provider/types.go | 6 +- shared/pkg/backendclient/client.go | 19 -- ui/src/layout/AssetScans/AssetScansTable.js | 2 +- ui/src/layout/Assets/AssetDetails.js | 4 +- ui/src/layout/Assets/AssetsTable.js | 18 +- .../Scans/ConfigurationReadOnlyDisplay.js | 16 +- .../ConfigurationsTable/index.js | 56 +--- .../StepGeneralProperties.js | 168 +--------- .../Scans/ScanConfigWizardModal/index.js | 40 +-- ui/src/layout/Scans/Scans/ScansTable.js | 31 +- .../detail-displays/AssetDetails/index.js | 6 +- ui/src/utils/utils.js | 26 +- 30 files changed, 340 insertions(+), 2096 deletions(-) delete mode 100644 backend/pkg/database/gorm/scope.go delete mode 100644 backend/pkg/rest/discovery_controller.go diff --git a/api/client/client.gen.go b/api/client/client.gen.go index 3486a5e904..2973530471 100644 --- a/api/client/client.gen.go +++ b/api/client/client.gen.go @@ -90,14 +90,6 @@ func WithRequestEditorFn(fn RequestEditorFn) ClientOption { // The interface specification for the client above. type ClientInterface interface { - // GetDiscoveryScopes request - GetDiscoveryScopes(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) - - // PutDiscoveryScopes request with any body - PutDiscoveryScopesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - PutDiscoveryScopes(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetFindings request GetFindings(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -216,42 +208,6 @@ type ClientInterface interface { PutTargetsTargetID(ctx context.Context, targetID TargetID, params *PutTargetsTargetIDParams, body PutTargetsTargetIDJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) } -func (c *Client) GetDiscoveryScopes(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetDiscoveryScopesRequest(c.Server, params) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) PutDiscoveryScopesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPutDiscoveryScopesRequestWithBody(c.Server, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) PutDiscoveryScopes(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPutDiscoveryScopesRequest(c.Server, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) GetFindings(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetFindingsRequest(c.Server, params) if err != nil { @@ -780,125 +736,6 @@ func (c *Client) PutTargetsTargetID(ctx context.Context, targetID TargetID, para return c.Client.Do(req) } -// NewGetDiscoveryScopesRequest generates requests for GetDiscoveryScopes -func NewGetDiscoveryScopesRequest(server string, params *GetDiscoveryScopesParams) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/discovery/scopes") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - queryValues := queryURL.Query() - - if params.Filter != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$filter", runtime.ParamLocationQuery, *params.Filter); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.Select != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$select", runtime.ParamLocationQuery, *params.Select); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.OrderBy != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$orderby", runtime.ParamLocationQuery, *params.OrderBy); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - queryURL.RawQuery = queryValues.Encode() - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewPutDiscoveryScopesRequest calls the generic PutDiscoveryScopes builder with application/json body -func NewPutDiscoveryScopesRequest(server string, body PutDiscoveryScopesJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewPutDiscoveryScopesRequestWithBody(server, "application/json", bodyReader) -} - -// NewPutDiscoveryScopesRequestWithBody generates requests for PutDiscoveryScopes with any type of body -func NewPutDiscoveryScopesRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/discovery/scopes") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("PUT", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - // NewGetFindingsRequest generates requests for GetFindings func NewGetFindingsRequest(server string, params *GetFindingsParams) (*http.Request, error) { var err error @@ -2901,14 +2738,6 @@ func WithBaseURL(baseURL string) ClientOption { // ClientWithResponsesInterface is the interface specification for the client with responses above. type ClientWithResponsesInterface interface { - // GetDiscoveryScopes request - GetDiscoveryScopesWithResponse(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*GetDiscoveryScopesResponse, error) - - // PutDiscoveryScopes request with any body - PutDiscoveryScopesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) - - PutDiscoveryScopesWithResponse(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) - // GetFindings request GetFindingsWithResponse(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*GetFindingsResponse, error) @@ -3027,52 +2856,6 @@ type ClientWithResponsesInterface interface { PutTargetsTargetIDWithResponse(ctx context.Context, targetID TargetID, params *PutTargetsTargetIDParams, body PutTargetsTargetIDJSONRequestBody, reqEditors ...RequestEditorFn) (*PutTargetsTargetIDResponse, error) } -type GetDiscoveryScopesResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Scopes - JSONDefault *ApiResponse -} - -// Status returns HTTPResponse.Status -func (r GetDiscoveryScopesResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetDiscoveryScopesResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type PutDiscoveryScopesResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Scopes - JSONDefault *ApiResponse -} - -// Status returns HTTPResponse.Status -func (r PutDiscoveryScopesResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r PutDiscoveryScopesResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type GetFindingsResponse struct { Body []byte HTTPResponse *http.Response @@ -3795,32 +3578,6 @@ func (r PutTargetsTargetIDResponse) StatusCode() int { return 0 } -// GetDiscoveryScopesWithResponse request returning *GetDiscoveryScopesResponse -func (c *ClientWithResponses) GetDiscoveryScopesWithResponse(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*GetDiscoveryScopesResponse, error) { - rsp, err := c.GetDiscoveryScopes(ctx, params, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetDiscoveryScopesResponse(rsp) -} - -// PutDiscoveryScopesWithBodyWithResponse request with arbitrary body returning *PutDiscoveryScopesResponse -func (c *ClientWithResponses) PutDiscoveryScopesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) { - rsp, err := c.PutDiscoveryScopesWithBody(ctx, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParsePutDiscoveryScopesResponse(rsp) -} - -func (c *ClientWithResponses) PutDiscoveryScopesWithResponse(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) { - rsp, err := c.PutDiscoveryScopes(ctx, body, reqEditors...) - if err != nil { - return nil, err - } - return ParsePutDiscoveryScopesResponse(rsp) -} - // GetFindingsWithResponse request returning *GetFindingsResponse func (c *ClientWithResponses) GetFindingsWithResponse(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*GetFindingsResponse, error) { rsp, err := c.GetFindings(ctx, params, reqEditors...) @@ -4202,72 +3959,6 @@ func (c *ClientWithResponses) PutTargetsTargetIDWithResponse(ctx context.Context return ParsePutTargetsTargetIDResponse(rsp) } -// ParseGetDiscoveryScopesResponse parses an HTTP response from a GetDiscoveryScopesWithResponse call -func ParseGetDiscoveryScopesResponse(rsp *http.Response) (*GetDiscoveryScopesResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetDiscoveryScopesResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Scopes - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest ApiResponse - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSONDefault = &dest - - } - - return response, nil -} - -// ParsePutDiscoveryScopesResponse parses an HTTP response from a PutDiscoveryScopesWithResponse call -func ParsePutDiscoveryScopesResponse(rsp *http.Response) (*PutDiscoveryScopesResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &PutDiscoveryScopesResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Scopes - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest ApiResponse - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSONDefault = &dest - - } - - return response, nil -} - // ParseGetFindingsResponse parses an HTTP response from a GetFindingsWithResponse call func ParseGetFindingsResponse(rsp *http.Response) (*GetFindingsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/api/models/models.gen.go b/api/models/models.gen.go index 0ce74d1d37..9745b99d5c 100644 --- a/api/models/models.gen.go +++ b/api/models/models.gen.go @@ -119,65 +119,6 @@ type ApiResponse struct { Message *string `json:"message,omitempty"` } -// AwsAccountScope AWS cloud account scope -type AwsAccountScope struct { - ObjectType string `json:"objectType"` - Regions *[]AwsRegion `json:"regions"` -} - -// AwsRegion AWS region -type AwsRegion struct { - Name string `json:"name"` - Vpcs *[]AwsVPC `json:"vpcs"` -} - -// AwsScanScope The scope of a configured scan. -type AwsScanScope struct { - // AllRegions Scan all regions, if set will override anything set in regions. - AllRegions *bool `json:"allRegions,omitempty"` - - // InstanceTagExclusion VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - InstanceTagExclusion *[]Tag `json:"instanceTagExclusion"` - - // InstanceTagSelector VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - InstanceTagSelector *[]Tag `json:"instanceTagSelector"` - ObjectType string `json:"objectType"` - Regions *[]AwsRegion `json:"regions"` - ShouldScanStoppedInstances *bool `json:"shouldScanStoppedInstances,omitempty"` -} - -// AwsVPC AWS VPC -type AwsVPC struct { - Id string `json:"id"` - SecurityGroups *[]SecurityGroup `json:"securityGroups"` -} - -// AzureResourceGroup Azure Resource Group -type AzureResourceGroup struct { - Name string `json:"name"` -} - -// AzureScanScope The scope of a configured scan within a subscription. -type AzureScanScope struct { - // AllResourceGroups Scan all resource groups in the subscription, if set will override anything set in resourceGroups. - AllResourceGroups *bool `json:"allResourceGroups,omitempty"` - - // InstanceTagExclusion VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - InstanceTagExclusion *[]Tag `json:"instanceTagExclusion"` - - // InstanceTagSelector VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - InstanceTagSelector *[]Tag `json:"instanceTagSelector"` - ObjectType string `json:"objectType"` - ResourceGroups *[]AzureResourceGroup `json:"resourceGroups"` -} - -// AzureSubscriptionScope Azure subscription scope -type AzureSubscriptionScope struct { - ObjectType string `json:"objectType"` - ResourceGroups *[]AzureResourceGroup `json:"resourceGroups"` - SubscriptionID *string `json:"subscriptionID,omitempty"` -} - // CloudProvider defines model for CloudProvider. type CloudProvider string @@ -466,7 +407,7 @@ type ScanConfig struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -501,7 +442,7 @@ type ScanConfigRelationship struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -527,7 +468,7 @@ type ScanConfigSnapshot struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -613,11 +554,6 @@ type ScanRelationshipState string // ScanRelationshipStateReason Machine-readable, UpperCamelCase text indicating the reason for the condition's last transition. type ScanRelationshipStateReason string -// ScanScopeType defines model for ScanScopeType. -type ScanScopeType struct { - union json.RawMessage -} - // ScanSummary defines model for ScanSummary. type ScanSummary struct { JobsCompleted *int `json:"jobsCompleted,omitempty"` @@ -671,16 +607,6 @@ type Scans struct { Items *[]Scan `json:"items,omitempty"` } -// ScopeType defines model for ScopeType. -type ScopeType struct { - union json.RawMessage -} - -// Scopes Scopes discovery -type Scopes struct { - ScopeInfo *ScopeType `json:"scopeInfo,omitempty"` -} - // Secret defines model for Secret. type Secret struct { Description *string `json:"description,omitempty"` @@ -732,7 +658,7 @@ type SuccessResponse struct { Message *string `json:"message,omitempty"` } -// Tag AWS tag +// Tag general cloud tag / label type Tag struct { Key string `json:"key"` Value string `json:"value"` @@ -740,8 +666,9 @@ type Tag struct { // Target Describes a target object. type Target struct { - Id *string `json:"id,omitempty"` - Revision *int `json:"revision,omitempty"` + Id *string `json:"id,omitempty"` + LastSeen *time.Time `json:"lastSeen,omitempty"` + Revision *int `json:"revision,omitempty"` // ScansCount Total number of scans that have ever run for this target ScansCount *int `json:"scansCount,omitempty"` @@ -749,6 +676,7 @@ type Target struct { // Summary A summary of the scan findings. Summary *ScanFindingsSummary `json:"summary,omitempty"` TargetInfo *TargetType `json:"targetInfo,omitempty"` + Terminated *time.Time `json:"terminated,omitempty"` } // TargetCommon defines model for TargetCommon. @@ -999,13 +927,6 @@ type Success = SuccessResponse // UnknownError An object that is returned in all cases of failures. type UnknownError = ApiResponse -// GetDiscoveryScopesParams defines parameters for GetDiscoveryScopes. -type GetDiscoveryScopesParams struct { - Filter *OdataFilter `form:"$filter,omitempty" json:"$filter,omitempty"` - Select *OdataSelect `form:"$select,omitempty" json:"$select,omitempty"` - OrderBy *OrderBy `form:"$orderby,omitempty" json:"$orderby,omitempty"` -} - // GetFindingsParams defines parameters for GetFindings. type GetFindingsParams struct { Filter *OdataFilter `form:"$filter,omitempty" json:"$filter,omitempty"` @@ -1131,9 +1052,6 @@ type PutTargetsTargetIDParams struct { IfMatch *Ifmatch `json:"If-Match,omitempty"` } -// PutDiscoveryScopesJSONRequestBody defines body for PutDiscoveryScopes for application/json ContentType. -type PutDiscoveryScopesJSONRequestBody = Scopes - // PostFindingsJSONRequestBody defines body for PostFindings for application/json ContentType. type PostFindingsJSONRequestBody = Finding @@ -1418,184 +1336,6 @@ func (t *Finding_FindingInfo) UnmarshalJSON(b []byte) error { return err } -// AsAwsScanScope returns the union data inside the ScanScopeType as a AwsScanScope -func (t ScanScopeType) AsAwsScanScope() (AwsScanScope, error) { - var body AwsScanScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAwsScanScope overwrites any union data inside the ScanScopeType as the provided AwsScanScope -func (t *ScanScopeType) FromAwsScanScope(v AwsScanScope) error { - v.ObjectType = "AwsScanScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAwsScanScope performs a merge with any union data inside the ScanScopeType, using the provided AwsScanScope -func (t *ScanScopeType) MergeAwsScanScope(v AwsScanScope) error { - v.ObjectType = "AwsScanScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -// AsAzureScanScope returns the union data inside the ScanScopeType as a AzureScanScope -func (t ScanScopeType) AsAzureScanScope() (AzureScanScope, error) { - var body AzureScanScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAzureScanScope overwrites any union data inside the ScanScopeType as the provided AzureScanScope -func (t *ScanScopeType) FromAzureScanScope(v AzureScanScope) error { - v.ObjectType = "AzureScanScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAzureScanScope performs a merge with any union data inside the ScanScopeType, using the provided AzureScanScope -func (t *ScanScopeType) MergeAzureScanScope(v AzureScanScope) error { - v.ObjectType = "AzureScanScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -func (t ScanScopeType) Discriminator() (string, error) { - var discriminator struct { - Discriminator string `json:"objectType"` - } - err := json.Unmarshal(t.union, &discriminator) - return discriminator.Discriminator, err -} - -func (t ScanScopeType) ValueByDiscriminator() (interface{}, error) { - discriminator, err := t.Discriminator() - if err != nil { - return nil, err - } - switch discriminator { - case "AwsScanScope": - return t.AsAwsScanScope() - case "AzureScanScope": - return t.AsAzureScanScope() - default: - return nil, errors.New("unknown discriminator value: " + discriminator) - } -} - -func (t ScanScopeType) MarshalJSON() ([]byte, error) { - b, err := t.union.MarshalJSON() - return b, err -} - -func (t *ScanScopeType) UnmarshalJSON(b []byte) error { - err := t.union.UnmarshalJSON(b) - return err -} - -// AsAwsAccountScope returns the union data inside the ScopeType as a AwsAccountScope -func (t ScopeType) AsAwsAccountScope() (AwsAccountScope, error) { - var body AwsAccountScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAwsAccountScope overwrites any union data inside the ScopeType as the provided AwsAccountScope -func (t *ScopeType) FromAwsAccountScope(v AwsAccountScope) error { - v.ObjectType = "AwsAccountScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAwsAccountScope performs a merge with any union data inside the ScopeType, using the provided AwsAccountScope -func (t *ScopeType) MergeAwsAccountScope(v AwsAccountScope) error { - v.ObjectType = "AwsAccountScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -// AsAzureSubscriptionScope returns the union data inside the ScopeType as a AzureSubscriptionScope -func (t ScopeType) AsAzureSubscriptionScope() (AzureSubscriptionScope, error) { - var body AzureSubscriptionScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAzureSubscriptionScope overwrites any union data inside the ScopeType as the provided AzureSubscriptionScope -func (t *ScopeType) FromAzureSubscriptionScope(v AzureSubscriptionScope) error { - v.ObjectType = "AzureSubscriptionScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAzureSubscriptionScope performs a merge with any union data inside the ScopeType, using the provided AzureSubscriptionScope -func (t *ScopeType) MergeAzureSubscriptionScope(v AzureSubscriptionScope) error { - v.ObjectType = "AzureSubscriptionScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -func (t ScopeType) Discriminator() (string, error) { - var discriminator struct { - Discriminator string `json:"objectType"` - } - err := json.Unmarshal(t.union, &discriminator) - return discriminator.Discriminator, err -} - -func (t ScopeType) ValueByDiscriminator() (interface{}, error) { - discriminator, err := t.Discriminator() - if err != nil { - return nil, err - } - switch discriminator { - case "AwsAccountScope": - return t.AsAwsAccountScope() - case "AzureSubscriptionScope": - return t.AsAzureSubscriptionScope() - default: - return nil, errors.New("unknown discriminator value: " + discriminator) - } -} - -func (t ScopeType) MarshalJSON() ([]byte, error) { - b, err := t.union.MarshalJSON() - return b, err -} - -func (t *ScopeType) UnmarshalJSON(b []byte) error { - err := t.union.UnmarshalJSON(b) - return err -} - // AsVMInfo returns the union data inside the TargetType as a VMInfo func (t TargetType) AsVMInfo() (VMInfo, error) { var body VMInfo diff --git a/api/models/scan.go b/api/models/scan.go index 8e8de42416..e5e66c7367 100644 --- a/api/models/scan.go +++ b/api/models/scan.go @@ -39,8 +39,8 @@ func (s *Scan) GetID() (string, bool) { return id, ok } -func (s *Scan) GetScanConfigScope() (ScanScopeType, bool) { - var scope ScanScopeType +func (s *Scan) GetScanConfigScope() (string, bool) { + var scope string var ok bool if s.ScanConfigSnapshot != nil { diff --git a/api/models/scanconfigsnapshot.go b/api/models/scanconfigsnapshot.go index 27c3b62153..7834accadc 100644 --- a/api/models/scanconfigsnapshot.go +++ b/api/models/scanconfigsnapshot.go @@ -15,8 +15,8 @@ package models -func (d *ScanConfigSnapshot) GetScope() (ScanScopeType, bool) { - var scope ScanScopeType +func (d *ScanConfigSnapshot) GetScope() (string, bool) { + var scope string var ok bool if d.Scope != nil { diff --git a/api/openapi.yaml b/api/openapi.yaml index 81cc0eff9a..11f00e1b14 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -787,43 +787,6 @@ paths: $ref: '#/components/responses/UnknownError' x-codegen-request-body-name: body - /discovery/scopes: - get: - summary: Get all available scopes - operationId: GetDiscoveryScopes - parameters: - - $ref: '#/components/parameters/odataFilter' - - $ref: '#/components/parameters/odataSelect' - - $ref: '#/components/parameters/odataOrderBy' - responses: - 200: - description: Success - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - default: - $ref: '#/components/responses/UnknownError' - put: - summary: Set all available scopes - operationId: PutDiscoveryScopes - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - required: true - responses: - 200: - description: All scopes were set successfully - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - default: - $ref: '#/components/responses/UnknownError' - x-codegen-request-body-name: body - /findings/{findingID}: get: summary: Get the details for a finding. @@ -1237,7 +1200,7 @@ components: scanFamiliesConfig: $ref: '#/components/schemas/ScanFamiliesConfig' scope: - $ref: '#/components/schemas/ScanScopeType' + type: string timeoutSeconds: type: integer minimum: 0 @@ -1286,7 +1249,7 @@ components: $ref: '#/components/schemas/ScanFamiliesConfig' readOnly: true scope: - $ref: '#/components/schemas/ScanScopeType' + type: string readOnly: true timeoutSeconds: type: integer @@ -1328,7 +1291,7 @@ components: scanFamiliesConfig: $ref: '#/components/schemas/ScanFamiliesConfig' scope: - $ref: '#/components/schemas/ScanScopeType' + type: string timeoutSeconds: type: integer minimum: 0 @@ -1359,170 +1322,6 @@ components: scanConfig: $ref: '#/components/schemas/ScanConfig' - Scopes: - type: object - description: Scopes discovery - properties: - scopeInfo: - $ref: '#/components/schemas/ScopeType' - - ScanScopeType: - type: object - anyOf: - - $ref: '#/components/schemas/AwsScanScope' - - $ref: '#/components/schemas/AzureScanScope' - discriminator: - propertyName: objectType - mapping: - AwsScanScope: '#/components/schemas/AwsScanScope' - AzureScanScope: '#/components/schemas/AzureScanScope' - - AzureScanScope: - type: object - description: The scope of a configured scan within a subscription. - properties: - objectType: - type: string - allResourceGroups: - description: Scan all resource groups in the subscription, if set will override anything set in resourceGroups. - type: boolean - resourceGroups: - type: array - items: - $ref: '#/components/schemas/AzureResourceGroup' - nullable: true - instanceTagSelector: - type: array - description: VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - instanceTagExclusion: - type: array - description: VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - required: - - objectType - additionalProperties: false - - AzureResourceGroup: - type: object - description: Azure Resource Group - properties: - name: - type: string - minLength: 1 - required: - - name - additionalProperties: false - - AwsScanScope: - type: object - description: The scope of a configured scan. - properties: - objectType: - type: string - allRegions: - description: Scan all regions, if set will override anything set in regions. - type: boolean - regions: - type: array - items: - $ref: '#/components/schemas/AwsRegion' - nullable: true - shouldScanStoppedInstances: - type: boolean - instanceTagSelector: - type: array - description: VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - instanceTagExclusion: - type: array - description: VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - required: - - objectType - additionalProperties: false - - ScopeType: - type: object - anyOf: - - $ref: '#/components/schemas/AwsAccountScope' - - $ref: '#/components/schemas/AzureSubscriptionScope' - discriminator: - propertyName: objectType - mapping: - AwsAccountScope: '#/components/schemas/AwsAccountScope' - AzureSubscriptionScope: '#/components/schemas/AzureSubscriptionScope' - - AzureSubscriptionScope: - type: object - description: Azure subscription scope - properties: - objectType: - type: string - subscriptionID: - type: string - resourceGroups: - type: array - items: - $ref: '#/components/schemas/AzureResourceGroup' - nullable: true - required: - - objectType - - AwsAccountScope: - type: object - description: AWS cloud account scope - properties: - objectType: - type: string - regions: - type: array - items: - $ref: '#/components/schemas/AwsRegion' - nullable: true - required: - - objectType - - AwsRegion: - type: object - description: AWS region - properties: - name: - type: string - minLength: 1 - vpcs: - type: array - items: - $ref: '#/components/schemas/AwsVPC' - nullable: true - required: - - name - additionalProperties: false - - AwsVPC: - type: object - description: AWS VPC - properties: - id: - type: string - minLength: 1 - securityGroups: - type: array - items: - $ref: '#/components/schemas/SecurityGroup' - nullable: true - required: - - id - additionalProperties: false - SecurityGroup: type: object description: general cloud security group @@ -1536,7 +1335,7 @@ components: Tag: type: object - description: AWS tag + description: general cloud tag / label properties: key: type: string @@ -1612,6 +1411,12 @@ components: # on the client side. # # readOnly: true + terminated: + type: string + format: date-time + lastSeen: + type: string + format: date-time targetInfo: $ref: '#/components/schemas/TargetType' scansCount: diff --git a/api/server/server.gen.go b/api/server/server.gen.go index 306a164b43..427289efc3 100644 --- a/api/server/server.gen.go +++ b/api/server/server.gen.go @@ -21,12 +21,6 @@ import ( // ServerInterface represents all server handlers. type ServerInterface interface { - // Get all available scopes - // (GET /discovery/scopes) - GetDiscoveryScopes(ctx echo.Context, params GetDiscoveryScopesParams) error - // Set all available scopes - // (PUT /discovery/scopes) - PutDiscoveryScopes(ctx echo.Context) error // Get all findings. // (GET /findings) GetFindings(ctx echo.Context, params GetFindingsParams) error @@ -121,47 +115,6 @@ type ServerInterfaceWrapper struct { Handler ServerInterface } -// GetDiscoveryScopes converts echo context to params. -func (w *ServerInterfaceWrapper) GetDiscoveryScopes(ctx echo.Context) error { - var err error - - // Parameter object where we will unmarshal all parameters from the context - var params GetDiscoveryScopesParams - // ------------- Optional query parameter "$filter" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$filter", ctx.QueryParams(), ¶ms.Filter) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $filter: %s", err)) - } - - // ------------- Optional query parameter "$select" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$select", ctx.QueryParams(), ¶ms.Select) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $select: %s", err)) - } - - // ------------- Optional query parameter "$orderby" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$orderby", ctx.QueryParams(), ¶ms.OrderBy) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $orderby: %s", err)) - } - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetDiscoveryScopes(ctx, params) - return err -} - -// PutDiscoveryScopes converts echo context to params. -func (w *ServerInterfaceWrapper) PutDiscoveryScopes(ctx echo.Context) error { - var err error - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.PutDiscoveryScopes(ctx) - return err -} - // GetFindings converts echo context to params. func (w *ServerInterfaceWrapper) GetFindings(ctx echo.Context) error { var err error @@ -1079,8 +1032,6 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL Handler: si, } - router.GET(baseURL+"/discovery/scopes", wrapper.GetDiscoveryScopes) - router.PUT(baseURL+"/discovery/scopes", wrapper.PutDiscoveryScopes) router.GET(baseURL+"/findings", wrapper.GetFindings) router.POST(baseURL+"/findings", wrapper.PostFindings) router.DELETE(baseURL+"/findings/:findingID", wrapper.DeleteFindingsFindingID) @@ -1116,104 +1067,96 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9WXPbOLbwX0Hxm6pZirGT/vo+XL+5bSetam8lOcmdmnTdgkhIQocC2ABoW5PKf7+F", - "jeICkKCsxc74zRaAA+Dg7Dg4/BYldJlTgojg0cm3KIcMLpFATP03wyTFZD46l/9gEp1EORSLKI4IXKLo", - "pNIeRwz9WWCG0uhEsALFEU8WaAnlQLHKZWcuGCbz6Pv3OMKzJRTJooS6QDBFbA13NHtzpTo4wGAi0Bwx", - "BYemUMAzWhBRgvqzQGy1hvSXRLU64EwpzRAkazgXjzkkqRcQ0s3dG1OA3uNMIOYFNNPNAYBuWIrYLysv", - "JCrbp6suUHH0+GZO35gRFqCdYIIylPhxx3VzwEonX3HuByMbQ07yjvqBCNoLgyeQnFEyw36CrXUZRrNy", - "aCfcjSCOES8y0Qm37DIMuoBsjvyQy+YhUL/LzjynhCMlHyZFkiCu/kwoEUjzIczzDCdQYEqO/+CUyN/W", - "MP/C0Cw6if7f8VrwHOtWfmzgjc0cesYU8YThXIKLTuyUYIk4h3MkSfkj+UroA7lgjLKtLeU0x13LMHMC", - "pCbVp6kGSrjVsSffGiNPCaDTP1AigFhAATAHDImCEZQCTADMMpBAjjigMzCDOCsY4kdRHOWM5ogJrBFv", - "d3/yLWIIpjckW9nTc1CC/kXPKhF2+sBPEyUYJwnNXWv8PAFJRosUQN0PcNWxuQwN8m6lYbRED0NzTInq", - "iQVa8l6cP/CxGiIHkyLL4DRDjX1BxuAq0pRoyfZf1YX87t6wASxpIk2x3CfMbiubmcGMo9iBB72J1tY1", - "G32LlphcIjIXi+jkXdxGwX2eDNr/p9uzwZtXS/Fse5JAUh7ygJ3fLZA+c0mHECRKZhYMpUCKpDZBwiwb", - "r0+7wbIJ1IRt6CEGeAY4EuABZxmg94gxnCIAyUosMJmrJkxsbzlXU2XHESZcQJKgOzi/eEyygpvDrc/8", - "6QrYjlzPRqgAU6Q2oThuBsQCreT+BDTsR9VvHAEB5xz8Dd0jUvZTZguoTK41KGV/PwKjGUDLXKxiNYmA", - "X+U4IqjlIbmRIDK4g/N+GqihwK4iBANDdr//TR1OosQRX9AiSxXHCJrnKB1ZzHnMxmESSLL2cPEjRzWZ", - "DacBkoejpGBYrD4wWuThGJtUhw0WRTh17/7fBUNjxGnBEqQhD8SEBAAsBKBBbCSSg2WnnHE30hM8YCno", - "AAS8mJbDPDK1grNu0WpQM1c9pfwUchGVCYLFbnXKV+n7Kn0r0rdJjWFCuM3927bvFLNWaN1n1yoxUmWK", - "TQ3bvSEijqrL1e5ct0jrwdWZtOtvGb3HqQ5SIFIs5bjTz5PIoLIycL3pc8xGZEZVlKiGrBSzayN6W4My", - "qr0vZ2Mnloft6uIxzygW7cUl98iJtIbIdrQT3570+Z3/4mwUWGTuYQXL6qTSnrGPJXzbfm/icOZ4YJbd", - "zKKTf3VTo0XZ9/jbEOofci4dJyU1V/u0kG4MZ6n1JjbHHtcRIcdqiISXeqy/FjhzCm04kHMk+kU0myMx", - "RpniF77ASjjMGidLVgEnewuTr3COqlQhD7lryKciI4jBKc6wWA0ZeAWzB8gGzTVBCUNi0CSYWxtKYWfI", - "2DGl4iseNJ2DqyQpp1gKjCUm0Cj7Jcxzc+Cl/AmGGEcGdQMwG0dNTGyCsTgyBDKAfuLI4HEAmuNIn3Q4", - "HcRRjQ43IFbLeSutkariSfLsjBYkvXHYqp8XSFrMmAPDceABciBPXBrKKAXTlbTXpdCSUNgSym2lUKA3", - "Ai9R5NCXOHUKeUzuYYblyAELqQzSKyHoAbFh6+FG4naypoo3V0VQh6C7eMTc3BrVxN1sLQe75rLi8ntc", - "DWfWsXGu/ptK63mBkwUoCP6zQNJk5oJBTARI6HIqOVLacgksOOLK8ZHEn+FEWcgbREjN2hybS+xtU8Pj", - "owJm9sg40EFTaaUzdYaCqlXNsXRn9AUQX59ReYlRsebr4C8xFyoibCfoBR2kPStH0K8tS3HVRMlSN3ht", - "QNNu7YkAbaL5NdZXFi1k3EKxMJ6S3DDSgXTjRnFgpouCDtpMuCULwCGyg80xi949m2NmWrc5tlwfeRA9", - "rffQ69UskYApFDA8MKXcZnZlx21k8V3VSbFFqm31+s1/8+JwC5coxX5/x3j+t4aqPe1+Z4qje8SUXhxm", - "Lk3sOIkSxMUZFGhO2crtwCAuzntcI9nH54q2cd5hioRzR/Ng9s0mTZS6+aXRK9yPceyvPzCgyWXrTqWX", - "fCrBgmafX/F8UfZrg7hCKS6WHR0u6UPZ6go+NPtvy2crLeGWns/RkxAbRxkk88InKjKcIHuLvvkU3hBF", - "XrDMzbk+yXePGHezewfaNmJli/I9c/AtTd3hq81DVHGU09QjrYeFr2xQ8ExSapFPBBSoym63SBtqcTT5", - "ivMcpVEcvYc4U3+cU+KO2JUu2yAlpgd5lZBpD7HmxpWuTjJyOI3BZGQ3t2cyMtO65b/BTbjYX29iAzk9", - "rp9EKZovrm7G/4zi6LeL8fXFZRRHp7e3l6Oz07vRzbWkm9H46vPp+CKKo4/Xv13ffL7uIp5tCdpxQaRj", - "OkkWKC0yZW+uIQ+4WjNwADeA9I1aTTeoexNKshWQsNRPd3II5oAjEQMsyrsYCDgmcwvFwkzBjDLlXtQA", - "rOEmjJJLTNYglctZMIaIAGp5dgLZ8CWaMbpUv3+JpL/GBWRC38/pGaUf1/Lo7CRq2imVPk9tO5Ck64VA", - "htYrmWHGhb1qzDLACgKgcAxvbbG2bg1GbUd5WNVFlR3RbIYSge8RkJuU7vYSk+opvmverVgQbcfujNH1", - "IQD0mDPEuc0BQY9wmUv2iP4L/Az+Af4B3rmiHLXtOPz0BQIEPZbbwhysSRHoDAAgGJ7PETMBn6PACIuL", - "6ie/3FxtiYEmU7p0S51cK9RwqbPWwBtIHbsGX6AGgmWRCfxGJxhWeMqdOIRIak/qSWE1hu5xw4CphFV4", - "Tdz0uZm6ZzMUv4YxITDnCyrCYZUjJBzJ/cP2zK090CbmDM9QskqkCJOddFxEiiWD7bYFcV4GNaM4Gkle", - "nUs+k3piSpkItC3UbFe+uN2vxRKSNwzBVBKXTdYEUs9LY4vMQYoExBkHcEoLLVoyKIWW2oRgkHBssyLc", - "c48R5K6sgyuYLDBB5eQx+JjniJ3BJcrOIEdASPavrETOzRSwUuwnlGiF9Feul1VfUHljWuJLHmd6U4go", - "jm4IumFXlCF9oaMxeUcnOlRrkb8qMfyRoMccJRrONVW5GGV3m2DrPIFiuYRsFUKEE9O1khbcEWQ0nDs6", - "51ruS92ifzOaUTmeSmdxkEtVViW6auxxC85o3VDYSOgY66Ate1LMSxFch4xnQC3QKOkShlT4dpTSoIQa", - "00FrDqlplbLHwp04k3oipI+3kMEsQ9mk4tSnaAaLTEQnP7kSjJbwES+LJSDFcoqYPAMbEDABUUjUejCR", - "h6SAq9NDMFnYkzIwopOf3irFrf9554pMe33Nfsn7Hi5xhhEPl8CNEetgh03JO2NICedwkP7BJnlbEUyv", - "we41YxUU2u8UlTllNsgt4dFCTJCUOtwt5e1ZK6sFE8B1Z8OfhgSlXpEcqoxNyY6aZr+QKnFSBqZoRhkC", - "U6RYuBB0CQVOYJatpDSWMI6+kKhCD29j13OLDlb13Qwd9J5nMyugb6s1K6FTQrFKT2njw5pg0fuWP0yV", - "6QtJitL9iywPDp8gwvYqtjzLd4ixXmKpirV+sK9i7ocQc30HHZbuPHF6Co3MXdNiLzJrEQy7OVR1To2E", - "kIeKlOv+hUiExIBTwzkLSOaozP2tDE2pSiCFyldXjUhKaEzmX9RNhkbEoY2j5yFCBlg+r0z/H2jbDM0J", - "qTJIYFpIv7rpSROpzNmXKqIYNYdzdATU6Ey9nADLgqtU/Yw+SA5jAP1ZwExCkH0n+N8oOPe8TkeevfV4", - "gM/YoAzZvn9jbfHRZpC6ajD6wsgMMDMAKo9K1offjrZV8msDMiArMqqSChKQAVIZ57oSH3ITXllD9Xol", - "4FalKmGndNl7UOtYrX4+xZAIeS4lu63H3VdyEw3SQ/NuKzrBSy4m/2uyjv00XjgAExaq0kmZNtb2J4SU", - "URcVqmjLGtWlkvzl6+E6aE/f20rE2tNlXDlrT5fJ+og8PT5tfhirWtzMdx7B3h8xPp0KhzY9QfWKywAe", - "Gh/vlVEB8fIw9+blxM/75fbB4+lhS9xPfD1sLf9p8fZ+rPwA8fdeayzQ210b/cGPVGrFCfpeZDRe4/a9", - "xqgXPgiYv/3eN2wdne8NfLha001YpotL8bezXv6gU35Gl3kmycStE2WXSzQTd3RcEE8RnXYKTI+BkRt5", - "p586a3ODMoCJFs0qqwPkBcspR/zIIqGZtCKNryiOPn28vL4Yn/4yuhzd/TOKo6vTS5OqMrk4G1/cyZ9G", - "k7Ob6/ejDx/HNqNlfHNz99tINl78z+3lzejOeV826fOyG8kITavbWtz2RW+7KAt8vGU48d2SCLa6go+n", - "QqBl7rNaCo4mORVDCg+0hvh4tJo83XKkelOPdfskXOZVenvNpzrE+orOoYBjBN3Wi2zUANztF2SOCfrk", - "zWmUinqmlMB7nPns0N8IfSCfMCu4r4dZwjlm6gE47unXMdek4HnfeqTSu4NfUWiSppx1k2gF32uc4nkE", - "KDYNTWyi9mq1l8I0X+tpe4AGrJd4CluN/yn9sNUN14vU5Fs361vI38t3eKuWzFVhSZs42n3IZVjSuQDz", - "UrH1tr3nJQQi6RnNiqXn9huR1Ka6tRtnOEO3zhdOEmm1F07mcZM1n3VwwpUaM8NkjljOsIvFr6lAJ9pS", - "xFzdCOiwmCfHhomurakOvs35UbxRrq85nT2n+upZ3Tl3lfBQaFkdvYNNUmBqMaYn5xHWCvwMS76dI4IY", - "zExtOFthSNec2aRKUaCD0agKOLCkXllOj5vSgaqT7qG4QHqA2y6xdwfnGxR7ErAdsv2K3O+07mFWBFC9", - "HG47/+5cqHQsuzMXjEOqBx15DnqDvCB+1mWK1G/3zLEt4D0C6B6x8sZHiTO9QueL2gFuetvJsu56gILR", - "iPRrGN1+RpfLGkqaHZ7pTYcoyaQfB137f0q6jCHDwEyZrcU9d0ClARMfhmoDpLEeMSkL03YXcQm4ZLLm", - "t72kuGVUimm3RuvIPxpyP2XnfPLt1Np3qD3g6r2hcr336pWZA++/7Nr4lAYYKPZ5wcaVIgbemZWTCSgK", - "Hkanujaj6r8lHtmsKs/9E++VuiTkmrOetSqoC4CwozP9gza/UZoF00P3m2dhJz10GKON501CGnVGczyM", - "VhWvn/w0mou78opnw7cwNnR8TYUNBcbVh7Jl2lMcjRFMV+Ulj+eOzvPUpR9JBd9cBzbR/T22TtYGIwN1", - "oGvkUD3ogBGqnRxDQ9IzXMPC9I1j5EAB3oLgJ4ph0cBPV0EFyezb9b5+tkRjX3SwLOXYDabyaL57XXFk", - "NtK3zYFhQY3SoapAKyiHFtiFCrCTmaTbg8j8zSS9PbImbvHSVxrA3nx5Cmra5mpx0a7F1yuRVmvq+sot", - "ZLAgyWKYvnhSeYcMCjmLpxrPPgpsS3trHg49qNCvPzBaO+MK7hpnExsiqWCodjguv9Gd6fbUaGajXmC7", - "+isPx10N1pkcGXA6fRcEKeaC0UFTn+shyiV+HDTyPX7UbLJCbJR6Cs+Qr0+03vJ1zZzAV+a5t+JVYEWr", - "ugdVKWdVVeQrfymWbro5M1TSdLMEw8lwqrky41RRHVsX/In1dryTtFY9hRxNElpL1dQxKnUfpc3S0hX1", - "9cPLHCbC1967wvOS6BsOqvod5Frm82qSiElUgyBFQl2Eg0tMikeg+AdPC3eJ/NH5Jf7q8ISlKh6d/+/l", - "6LcLMMMoS4Gq/GmTpGXzMRLJMeVvGMoQ5Pqe7Qlvl9fPRPxXee0duRRWhTIa1eR1gx8a+NsS/kGVSaH+", - "OFpiQhkwAP8eVrLCW1w1+LauLpP3fGnXkoftqzvrQPowv/WCau34T2tRDodkuM7a0urCksvXge/G2g2r", - "5YgBK949eednDKvHOY40bU9C9694vgjvfUkfwjvrEnXh/a/RPMNzPM1QwJh+vDtq7J2NR3ejs9PLKI5+", - "HX34NYqjq4vz0cerKI4ubz5HcXR98eFy9GH0y+WFK26hDGrNt6YoffTp6iyD6sr29HYk3Z9S1kTvjt4e", - "vTUFdAjMcXQS/f+jt0fvIq291a6OyzyMY14mbJggall3R9od0Qckysxbk9sR174t6REh6y7H1U8p+vze", - "ZnfzPcPQ7uXnEH9vfNfup7dvt/dNO719/6fstBVpKj24YZWLO6596+57NQguca6+IQLvIVYiAJhDUvUA", - "HYd0WzgOSQpfxMUvNF3tBAX1jw1+PwjiT7PM4AY8IF04yyYHzIosW23rRCa+E4mjxzcJTdEckTcG4W+m", - "NF3Z73TKvxWs41mlELWP08pi1c+QxfTdaWjvO5qHL+QrDu9svu36rARDeWz7Ew3rd2CqdiV3CQXKqwS1", - "C3FQVh0PkQfvdjNt07Ah6KFWcN88s1eI+nmLh97zWdGRrvNfLoUXcqZyHf+9bWSYK0bHSkyHytXglmhR", - "5d0jAO0eNxCGx9/KD09/11ZqhrRJX6flc/W7peb3lY9VD5OT689c+wRCNzYq3Pzz25/3RUv2BEfnKtVT", - "WeXbOkSN2fUhHumLq279tJUD2I2asvphD/K+R9z/IAQiNY5YoPIZ4oyyBrXk9gvwDf0jf94+yx5Yi+2F", - "ihTqUFV5rE3aZ6bIfggaV/iuUnWYJvN7Y69kvwnZf8z1d5FeyX4/ZK/xPZzupQXH62V0fBZDtdrOq1P7", - "kpza6sntz6+t1jvq8W3rpLWbaFelduRePdzmzC4nt1Zz8vCObnU5O3N2W4VJXZRZWQjMGILpShfJ49v3", - "fOsVmjaQncff1v8E+cAVqp9URg4WrtVpX5QzXD3enTrEtYLTHU7xbk7k5XrH3bLrxyQat5PcpKAuR/lQ", - "VIRn6lv8O/MyhurQfdGhdbHrauvw/kaHGn0W3PLMtPnP737aF1YuBJyDFKfkrwIonjnadvih8YmDp4Ug", - "XgXKfgWKDV68CpRXgXJogVIGdjaQKNZBqTze67J8bbfX4M5LCu6032juJ8Qz4Jllf/BnTXq70DOOx657", - "DQG552+kJKOHEpsq/wmmKUotOk2VBeOL5CjBM5yY8qcH1EV6wbuLEXkeX/tUQUmNVV2gKz1o/EGSrpG2", - "/eiRQUfjlPxnt5kY13Em/Y+JMwVI9UllzEaGYzn4BcczQhjxgFENQz+7imrUqDQoinEI2tm107GZMtgv", - "Dd7ZL+ZVhJpSCrnNJjA1ml+UXngWzPRi1NOPFw7R+99KNORVMB1GMNnICGzw+TOJjbzKnVe544iaWItn", - "mLndGy95jZS8vDSYfSfA8CNwYT9wZwsG88Z3QHo+zt4bPNllzswhsmV68mSeS4LMTjNjekT6rpNhOghy", - "qBTV4YrghBhl021ozb3E9Jed5730Jrw8FeMvO73lmYWA9pfRoiP0vZqnJ0K0e+LZxyX0Ia6fezNZno1X", - "dVB3atd3zMMV7Q8XmNlOgsqrJNimJKiloLxKgldJsJ9QyZAYiVjXAfWZl7ZU6Guc5OVllOwrUmLJqDPM", - "sSak3UXeD5MV4g922C9rHD7cYd2D3aZ5+OWvuQ7dbdCj/A7IQPl3/M1+PTYgwmHo+M6MGCwY7VTbiHM8", - "EzLamxFxZ7/gu7OAi95gZ8BlewTw0rNwnk/gZYeEsVZwvdGUfVLGfq6yD3OB3eVNlRKo5U8dmtiehzr9", - "kRway3ZPjW288uUh+fLVSHkVD89APCgQiN1bni9YFp1ExzDH0fffv/9fAAAA//8AUfassMkAAA==", + "H4sIAAAAAAAC/+x9W3PbOJbwX0Hxm6r5doqxk97eh/WbYztpVftWkpPeqU7XFkQeSeiQABsAbWtS+e9b", + "uFG8QSRlXeyM32wBOADO/RwcgN+CiKUZo0ClCE6+BRnmOAUJXP83IzQmdD46V/8QGpwEGZaLIAwoTiE4", + "KbWHAYe/csIhDk4kzyEMRLSAFKuBcpmpzkJyQufB9+9hQGYpltGigLoAHANfwR3N3lzpDi1gCJUwB67h", + "sBhLfMZyKgtQf+XAlytIf4t0awucKWMJYLqCc/GYYRp7AYFpXr8xDegDSSRwL6CZae4B6IbHwN8vvZCY", + "ap8u14EKg8c3c/bGjnAA3QQTSCDy406Y5h4rnXwlmR+MauxDyTvmByJZJwwRYXrG6Iz4GbbSZRjPqqFr", + "4W4EcQwiT+RauEWXYdAl5nPwQy6ah0D9rjqLjFEBWj9M8igCof+MGJVg5BBnWUIiLAmjx38KRtVvK5h/", + "4zALToL/d7xSPMemVRxbeGM7h5kxBhFxkilwwYmbEqUgBJ6DYuVP9CtlD/SCc8a3tpTTjKxbhp0TgZ7U", + "UFMPVHDLY0++1UaeUsSmf0IkkVxgiYhAHGTOKcSIUISTBEVYgEBshmaYJDkHcRSEQcZZBlwSg3i3+5Nv", + "AQcc39Bk6ajXwgnmFzOrQthZwvL4lrN7Ehs9BTRPg5Pfg9PfJkEYnP4r5xD8ETZ1yTnhIzpj2lBU1hMT", + "fq35qkUBJcwQoLXRLOpO/9zGxSve/L3c94+WXV08Zgkjsrm46B6MGDRmr1CmpZ369iRYziM4f9/aKIlM", + "2oflPNErIhJS0T5jniR4qoZXqIk5x8t2Ytptf7Cm2JIHJ8nNLDj5fT2TO5R9D+s42xpd1lBqEmHapBaY", + "xiqe+m1ic+wJYxRaVkMVvNjjNzTAWSo04WAhQHbt5E7r5DEkWl7EgmQK5qxGWbrsQdlbHH3FcyhzhSLy", + "uiGf84QCx1OSELkcMvAKJw+YD5prAhEHOWgSIiJNo5xr7AwZO2ZMfiWDpmuRKsXKMVEKIyUUS2NrUpxl", + "luCF/ukNMQws6gZgNgzqmNgEY2FgGWQA/4SBxeMANIeBoXR/PgiDCh9uwKxO8pbGIpXVk5LZGctpfEOb", + "hvm3BVAkF0QgK3HoAQukKM7ugUOMpkuEkfLIAgWFp1htK8YS3kiSQtBiL0ncquQJvccJUSMHLKQ0yKyE", + "wgPwYesRVuOuFU3tcpZV0BpFd/FIhA0cK+puttKD6+Zy6vJ7WPZoqtg41/9NQaCHBYkWKKfkrxxQxKiQ", + "HBMqUcTSqZJIwiiKcK6cJ7nQPWYJieSRdnEHO0l2bS2bi1zAWV3oHZM4cSQTSPdCOIoY1zSUTK9qTu6B", + "IhMDihWNijgmXBm9KvhLIqR2Ct0EnaB7Wc8SCbqtZaGu6ihJTYPXB7Ttzp/oYU2MvIYmamkg4xbLhUKG", + "2vaMJGB8aeX4Y0IFstMFvQhtJ9ySB9Cisnu7Yw69e3bH7LTt7li6InkvflrtocMbUyIvsQr7e8NWK6TA", + "r9y4jTy+qyorNli1aV6/+YOvxmgOKcTEH+8IvYX41nK1p90fTAm4B67t4jB3aeLGKZSAkGdYwpzxZXsA", + "A0Ked4RGqk9rVNWK8zWuSH/pqBNm32JSR2m7vNR69Y9jWvbXKUKWXbYeVHrZp5QsqPf5hcwXRb8miCuI", + "SZ6u6XDJHorWtuRDvf+2YrbCE27Y+QyehNgwSDCd5z5VkZAIXCJt8ym8KYos50m75Po03z1w0S7ua9C2", + "kSg7lO9Zgm9Z3J6+2jxFFQYZiz3aelj6agwmqXSmODXPJhJLKIvbLRhHLQwmX0mWQRyEwQdMEv3HOaPt", + "GbsiZBtkxMwgrxGy7X28uXGpaysbtQSNvdnIbW7PbGSnbdf/Fjf91f5qExvo6XGVEoVqvri6Gf8zCINf", + "L8bXF5dBGJze3l6Ozk7vRjfXim9G46vfTscXQRh8uv71+ua363XMsy1FO86pCkwn0QLiPNH+5goyjmOi", + "RBAnt6U5ZjgRUM+9WzhIWEA6BkYV23CERjPEaLJECpb+6U4NIQIJkCEiEj2QJEFTUCE9oXMHxcGM0Yxx", + "HV5UAKzgRpzRS0JXIHXImXMOVCK9PDeBavgSzDhL9e9fAhWvCYm51E12RhXHNSI6N4medspUzFPZDqbx", + "aiGYw2olM8KFNFvS6+A5RVi2DG9ssbJuA0ZvR0dY5UUVHWE2g0iSe0BqkyrcTgktU/Fd/fjCgWgGdmec", + "rYiA4DHjIJRR0mcg8IjTTIlH8F/oZ/QP9A/0ri3LUdlOS5y+AEThsdgWEWjFikgsWJ7ESHIynwO3CZ+j", + "nhmWNq6fvL+52pIATaYsbdc6mTGo/bXOygJvoHXcGnyJGozSPJHkjTljLMmUw2UdC7Gj1JPSahzuSc2B", + "KaVVREXddIWZpmc9Fb+CMaE4Ewsm+8MqRig4SvqH7Vk4f6DJzAmZQbSMlApTnUxeRKkli+2mB3FeJDWD", + "MBgpWZ0rOVN2Ysq47Olb6NmufHm7X/IU0zcccKyYy53XImXnlbNF5ygGiUkiEJ6y3KiWBCulpTchOaZC", + "W4QjLzrGgO2ZbnXqKxwtCIVi8hB9yjLgZziF5AwLQFKJf2klam6ugRVqP2LUGKS/C7Os6oKKE9MCX4qc", + "8U0ugzC4oXDDrxgHc6BjMHnHJiZV65C/LDD8icJjBpGBc83kgtB50d2dsbdSIE9TzJd9mHBiu5YqA9Yk", + "Ga3kjs6F0fvKtpjfrGXUgae2WQJlypSVma6ce9xCMFp1FDZSOtY7aOqemIhCBVchkxnSC7RGuoChDL4b", + "pS0oZdZ1MJZDWVpt7IlcMW6h1r3KK8WPt5jjJIFkUgrqY5jhPJHByU9hi+Sn+JGkeYponk6BKxq4hIBN", + "iGKq10OoIpIGrqkHOFo4SlkYwclPb7XhNv+8a8tMe2PNbs37AackISD6a+DaiFWyY0SFxDSCMw5aOfcH", + "6R9s6zc0w3Q67F43VkNhnjhRjWG5nIDSLKJdkzt6as+EUCRMZyuDls2U7VBSqB1KJXKGL7/QMgMyjqYw", + "YxzQFLSY5pKlWJIIJ8lSaVwF4+gLDUo0fxu2VVWtEUff6c9Bz3I2s/RdW614Amu1EC/1VH48rigPs2/1", + "w1S7t5jGEO9fLXlw+AQ1tVfV5Fl+i6rqZJay6uoG+2+pyjpx+CJUWxdxa1khErdm6SatEUCtRNG2uAPK", + "SmbCbQ7KQafVCoqQoEPyL1QhJESCWWlZYDoHoRBXGxozRJlEWMfguhGUViZ0/kWfUBhEHNrpeR5qY4BH", + "8+qz/KA+y9B6jrIQ9Czp6DYjHSUepTm7yjy0MGZ4DkdIj06AzuUCpbmQysNI2IOSIo7grxwnCoLqOyH/", + "gqO+BSJVXvHsrSN6e8aOYp/t+zfWVBFNAamqf2sTrF5AMwsAPRAV9le9xWamrFQb26N6saSHSmUcPao3", + "SuPajrOHnGKX1lA+GulxIlLWolOWdhJqlWfVdRIRh+6pTP1haab7Ul2hRXrfmtmS3veyi63dmqzyNrXL", + "AcimdMp8UpR8NeMEqXTURYkrmrpGdykVbvl6tBHa0/e2lG32dBmXaO3pMlmRyNPj8+bEWFZyXj569I7q", + "qI3VdCqzHuEpSPZOx+DcdqeO6pHr7he2vJzcd7fePnguvN8S95Mb77eWf7dceTdWfoDceac31jOiLdmj", + "fpUXbcasWYXxJ5uKM5Zmidp6u55XXS5hJu/YOKeee53NkowOo5lZGVb/W4dK4Z1Qo250lQHKcp4xAeLI", + "IaFeRKEciiAMPn+6vL4Yn74fXY7u/hmEwdXppS2dmFycjS/u1E+jydnN9YfRx09jV2Exvrm5+3WkGi/+", + "5/byZnTXen4z6YoOa4fjdU/SeZHEAmjeE8SPt5xEvqy95Msr/HgqJaSZzxLnAiYZk26NwnNYXea0xhAf", + "35WLeRvBQWcprGmf9JfjUm+vS1CFWF3ROZZ4DLjdIqtGA6C9/YLOCYXP3ho7ZXxmWrF9IInPt/qVsgf6", + "mfBc+HrYJZwTDpFknHT0WzPXJBdZ13qUIr/DX6Fv0aCadZMIXOw19n4eQfeG4ba9b9W4odtRzw00PmNJ", + "nnrO8IDGrmCn2TgjCdy23tNQwlu5p2GvaDhHwoRpbQf8M0LnwDNO2hjjmkk4MTaTCJ3/NAkCT6UAl+u2", + "pjv4NudH8UYVi5Y6ey5YNLO2Vw6VAuV+nOl2sMlBfiXafnI1FEQ5J3L5kbM8G1hCOAcVKSYoSlgeKy7U", + "kNBcg6obUROCpYReajkvp5I9FPK5WrXnDQa+DVC8CyDsGwi6k+mhpUD5wtt+K+AOz5+EW4nn6BgleApJ", + "Y2Vfof3+yT1O8h5yoIa7zn+0Ll053etPa62zbgYdeUjfLN3HQk7AWL1+5VvdFRLibJ0RrJ6HWNIv8D0g", + "uAde5M+1SjR7ar1bOCDoabr3LvixGq/7Jru7wyeB6yvTRrY3LaQ0MM9YmlbQWO/wTHPNsmDGbryt2/9T", + "ChEss/esQdha5mkHnN1j4gNwej8rYEZMipd91j+B0SPN75xFlya+5UyZh3ZLuqayY8gJgZvzyecDK0+3", + "cv2l84yg7bZMp54deALh1iamrIdj5IqzN75nP/DUophMYpmLfnyqkz2m/5ZkZLM3Te6fmNlfpyFXkvWs", + "TUFVAfQjne3fa/MbHXRzM3S/J91u0kMH3U08bxKAVwWt5VqpfjLsyRdLhbwrkuwb3iRwic5rJl3iKixf", + "MyyKS8JgDDheFml2zymJ56JAN5JysbkNrKP7e+gCkA1G9rSBbSOH2sEWGH2tU8vQPgfkbcP62ZuWkQMV", + "eAOCnylcDqTfO1efr3o95+Ru/nb1cw/cdb31VDyEtx5M6crx+nWFgd1I1zbXPW7kRelQU2AMVIsV2IUJ", + "cJPZ0saD6PzNNL0jWR23JPVdrHbnNJ7nCF1z+WnGdYuvvuNYAuC9rJ7gnEaLYfbiSZfjEyzVLJ63TEoJ", + "vEEZyFLer4exlHjeH/od7ndLyZOQrdC4hLsabULLJCUMVYjTFje21xo9NYtae22t+Xam6I+7CqwzNbIH", + "dboOJmIiJGeDpj43Q3RI/Dho5AfyaMRkCXzkyf0R+vWJ3lu2enGk5x3dzPteUM/3gKoRVOkxoLIhX/of", + "sljPN2eWS+phluQkGs41V3acfpIkssb3ia+VeCdprHqKBUwiVimWMzkqfQ5m3NIiFPX1I2mGI+lr71zh", + "ecH0tQBV/44yo/NFuaTBlgphFIPUx7boktD8EWn5IdPcVeNUdzs6vyRfWyJhZYpH5/97Ofr1As0IJDHS", + "7ya6MlXVfAwyOmbiDYcEsDDne0+4+bkqxvcfITZ31GawSpxRBWUP3/3Q0P9P8Z9MuxT6j6OUUMaRBfgf", + "/fLU3qcpe58SVnXyng8LG/qweWToAkgf5rf+HFUz/9NYVEtAMtxmbWl1/cp7V4nv2tqtqGXAkVPvnsrf", + "M0709YiWQllPSe0vZL7o3/uSPfTvbB746t//GuYJmZNpAj3GdOO95YWys/HobnR2ehmEwS+jj78EYXB1", + "cT76dBWEweXNb0EYXF98vBx9HL2/vGjLW2iH2sitfdI7+Hx1lmB9VHx6O1LhT6FrgndHb4/e2udHKM5I", + "cBL859Hbo3eBsd56V8ez0qOiNnlavFai/I3gY1FYIPTA1dc4PDpj1eW4/PEJX6Bb726/ANG3uznJ6dv7", + "jmX9F/KV9O9sP9XRt3vx2Ys/at8v+Ont2619MKAgm/+jBcbdtRf626EVyzuufNXgezlbr5hEfyhgdS9A", + "v0MmWvjplokyQynLAEK+Z/Fy2xt3n4cof0viewPf73YzbV3NUnioPJ5sr1ZqRP28RaJ3fCViZN5sLpYi", + "cjVTsY7/3jYy7IFHy0psh9JBxZZ4UdesAsJuj0EYPL6JWAxzoG8su72ZsnjpvkGj/tYwCmV4/K34jtB3", + "YzMTMA5GlZfP9e+Omz+Uvj00TE+uvlrkUwjrsVGS5p/f/rwvXnIUHJ3rgjftI2yLiAazKyIemTT6evu0", + "FQLsxkw5+7AHfd+h7n8QBlEWR4VM7lrKjPEat2Tug141+6N+3r7IHtiK7YWLNOqgbDw0V83yJFk+M0P2", + "Q/C4xneZq/tZsjDI8jbHK5evbL8J23/KzDcuXtl+P2xv8D2c75UHJ6rPKvg8hvLrC69B7UsKasuU219c", + "W37/oiO2rbLWLjRk5Y2wvUa49ZnbgtzK22KHD3TLy9lZsNt4gK6NM0sLwQkHHC/Nw0hi+5Fv9cWODXTn", + "8bfyd0l7xMAlrp9Uv2g6TLlWPof6koLhMnl3GhBXHg9dExTvhiIvNzper7t+TKZpD5LrHLQuUD4UF7nv", + "ce8qyhhqQ/fFhy7Erpqtw8cba8zos5CWZ2bNf373076wciHxHMUkpn+XSMvM0bbTD7Xnqp+WgnhVKPtV", + "KC558apQXhXKoRVKkdjZQKO4AKV0lWid5+u6vSZ3XlJyp3ljbD8pngGXvrqTPyvW24Wdabl6t9cUUPv8", + "tQJJeCiw+QAcEI5jiB067Z1vG4tkEJEZiexzeAe0RWbBu8sRea6C+kxBwY1lW2DunRv8YRqvkLb97JFF", + "R41KftptpsZNnsn8Y/NMPbT6pDRmI8exGPyC8xl9BPGAWQ3LP7vKalS4tFcW4xC8s+ugYzNjsF8evHNf", + "PyopNW0UMldNYN/sfFF24VkI04sxTz9eOsTsfyvZkFfFdBjF5DIjuCbnzyQ38qp3XvVOS9bEeTzD3O3O", + "fMlrpuTllcHsuwBGHKEL91Ej92yqqL0L3/Gh3c7kyS5rZg5RLdNRJ/NcCmR2WhnTodJ3XQyzhiGHalGT", + "ruhdEKN9ug29uZdY/rLzupfOgpenYvxll7c8sxTQ/ipaTIa+0/J0ZIh2zzz7OIQ+xPFzZyXLs4mqDhpO", + "7fqMebih/eESM9spUHnVBNvUBJUSlFdN8KoJ9pMqGZIjkatXCX3upXu48DVP8vIqSvaVKXFstDbNsWKk", + "3WXeD1MV4k92uHf+D5/ucOHBbss8/PrXHofuNulRfJVgoP47/ua+Jtgjw2H5+M6OGKwY3VTbyHM8Ezba", + "mxNx577ouLOEi9ng2oTL9hjgpVfhPJ/Eyw4ZY2XgOrMp++SM/RxlH+YAe100VWigRjx1aGZ7Hub0Rwpo", + "nNg9NbfxKpeHlMtXJ+VVPTwD9aBBAL93Mp/zJDgJjnFGgu9/fP+/AAAA//8iNYEGf7cAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/backend/pkg/database/demo.go b/backend/pkg/database/demo.go index 6b677b1368..8ace2f3ac1 100644 --- a/backend/pkg/database/demo.go +++ b/backend/pkg/database/demo.go @@ -45,67 +45,10 @@ const ( awsInstanceUSEast11 = "i-instance-1-from-us-east-1" ) -var regions = []models.AwsRegion{ - { - Name: awsRegionEUCentral1, - Vpcs: utils.PointerTo([]models.AwsVPC{ - { - Id: awsVPCEUCentral11, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGEUCentral111, - }, - }), - }, - { - Id: awsVPCEUCentral12, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGEUCentral121, - }, - }), - }, - }), - }, - { - Name: awsRegionUSEast1, - Vpcs: utils.PointerTo([]models.AwsVPC{ - { - Id: awsVPCUSEast11, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGUSEast111, - }, - }), - }, - { - Id: awsVPCUSEast12, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGUSEast121, - }, - { - Id: awsSGUSEast122, - }, - }), - }, - }), - }, -} - // nolint:gomnd,maintidx,cyclop func CreateDemoData(ctx context.Context, db types.Database) { logger := log.GetLoggerFromContextOrDiscard(ctx) - // Create scopes: - scopes, err := createScopes() - if err != nil { - logger.Fatalf("failed to create scopes FromAwsScope: %v", err) - } - if _, err := db.ScopesTable().SetScopes(scopes); err != nil { - logger.Fatalf("failed to save scopes: %v", err) - } - // Create scan configs: scanConfigs := createScanConfigs(ctx) for i, scanConfig := range scanConfigs { @@ -437,17 +380,6 @@ func createVMInfo(instanceID, location, image, instanceType, platform string, return &info } -func createScopes() (models.Scopes, error) { - scopesType := models.ScopeType{} - err := scopesType.FromAwsAccountScope(models.AwsAccountScope{ - Regions: utils.PointerTo(regions), - }) - // nolint:wrapcheck - return models.Scopes{ - ScopeInfo: &scopesType, - }, err -} - func createTargets() []models.Target { return []models.Target{ { @@ -513,9 +445,7 @@ func createTargets() []models.Target { } } -func createScanConfigs(ctx context.Context) []models.ScanConfig { - logger := log.GetLoggerFromContextOrDiscard(ctx) - +func createScanConfigs(_ context.Context) []models.ScanConfig { // Scan config 1 scanFamiliesConfig1 := models.ScanFamiliesConfig{ Exploits: &models.ExploitsConfig{ @@ -540,56 +470,6 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Enabled: utils.PointerTo(true), }, } - tag1 := models.Tag{ - Key: "app", - Value: "my-app1", - } - tag2 := models.Tag{ - Key: "app", - Value: "my-app2", - } - tag3 := models.Tag{ - Key: "system", - Value: "sys1", - } - tag4 := models.Tag{ - Key: "system", - Value: "sys2", - } - ScanConfig1SecurityGroups := []models.SecurityGroup{ - { - Id: awsSGEUCentral111, - }, - } - ScanConfig1VPCs := []models.AwsVPC{ - { - Id: awsVPCEUCentral11, - SecurityGroups: &ScanConfig1SecurityGroups, - }, - } - ScanConfig1Regions := []models.AwsRegion{ - { - Name: awsRegionEUCentral1, - Vpcs: &ScanConfig1VPCs, - }, - } - scanConfig1SelectorTags := []models.Tag{tag1, tag2} - scanConfig1ExclusionTags := []models.Tag{tag3, tag4} - scope1 := models.AwsScanScope{ - AllRegions: utils.PointerTo(false), - InstanceTagExclusion: &scanConfig1ExclusionTags, - InstanceTagSelector: &scanConfig1SelectorTags, - ObjectType: "AwsScanScope", - Regions: &ScanConfig1Regions, - ShouldScanStoppedInstances: utils.PointerTo(false), - } - - var scanScopeType1 models.ScanScopeType - - err := scanScopeType1.FromAwsScanScope(scope1) - if err != nil { - logger.Fatalf("failed to convert scope1: %v", err) - } // Scan config 2 scanFamiliesConfig2 := models.ScanFamiliesConfig{ @@ -616,41 +496,6 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { }, } - ScanConfig2SecurityGroups := []models.SecurityGroup{ - { - Id: awsSGUSEast111, - }, - } - ScanConfig2VPCs := []models.AwsVPC{ - { - Id: awsVPCUSEast11, - SecurityGroups: &ScanConfig2SecurityGroups, - }, - } - ScanConfig2Regions := []models.AwsRegion{ - { - Name: awsRegionUSEast1, - Vpcs: &ScanConfig2VPCs, - }, - } - scanConfig2SelectorTags := []models.Tag{tag2} - scanConfig2ExclusionTags := []models.Tag{tag4} - scanConfig2Scope := models.AwsScanScope{ - AllRegions: utils.PointerTo(false), - InstanceTagExclusion: &scanConfig2ExclusionTags, - InstanceTagSelector: &scanConfig2SelectorTags, - ObjectType: "AwsScanScope", - Regions: &ScanConfig2Regions, - ShouldScanStoppedInstances: utils.PointerTo(true), - } - - var scanScopeType2 models.ScanScopeType - - err = scanScopeType2.FromAwsScanScope(scanConfig2Scope) - if err != nil { - logger.Fatalf("failed to convert scanConfig2Scope: %v", err) - } - return []models.ScanConfig{ { Name: utils.PointerTo("Scan Config 1"), @@ -658,7 +503,7 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Scheduled: &models.RuntimeScheduleScanConfig{ OperationTime: utils.PointerTo(time.Now().Add(5 * time.Hour)), }, - Scope: &scanScopeType1, + Scope: utils.PointerTo("startswith(targetInfo.location, 'eu-central-1')"), MaxParallelScanners: utils.PointerTo(2), }, { @@ -673,7 +518,7 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Scheduled: &models.RuntimeScheduleScanConfig{ CronLine: utils.PointerTo("0 */4 * * *"), }, - Scope: &scanScopeType2, + Scope: utils.PointerTo("startswith(targetInfo.location, 'us-east-1')"), }, } } diff --git a/backend/pkg/database/gorm/database.go b/backend/pkg/database/gorm/database.go index 229ab0acfc..0e499d6f48 100644 --- a/backend/pkg/database/gorm/database.go +++ b/backend/pkg/database/gorm/database.go @@ -74,7 +74,6 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { ScanResult{}, ScanConfig{}, Scan{}, - Scopes{}, Finding{}, ); err != nil { return nil, fmt.Errorf("failed to run auto migration: %w", err) diff --git a/backend/pkg/database/gorm/odata.go b/backend/pkg/database/gorm/odata.go index 9351d41f12..761b8529b5 100644 --- a/backend/pkg/database/gorm/odata.go +++ b/backend/pkg/database/gorm/odata.go @@ -324,6 +324,8 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ Fields: odatasql.Schema{ "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "revision": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "terminated": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "lastSeen": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "scansCount": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "targetInfo": odatasql.FieldMeta{ FieldType: odatasql.ComplexFieldType, @@ -405,11 +407,7 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ FieldType: odatasql.ComplexFieldType, ComplexFieldSchemas: []string{"RuntimeScheduleScanConfig"}, }, - "scope": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsScanScope"}, - DiscriminatorProperty: "objectType", - }, + "scope": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "maxParallelScanners": odatasql.FieldMeta{ FieldType: odatasql.PrimitiveFieldType, }, @@ -434,11 +432,7 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ FieldType: odatasql.ComplexFieldType, ComplexFieldSchemas: []string{"RuntimeScheduleScanConfig"}, }, - "scope": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsScanScope"}, - DiscriminatorProperty: "objectType", - }, + "scope": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "maxParallelScanners": odatasql.FieldMeta{ FieldType: odatasql.PrimitiveFieldType, }, @@ -531,91 +525,12 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ "enabled": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, }, }, - scopesSchemaName: { - Table: "scopes", - Fields: odatasql.Schema{ - "scopeInfo": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsAccountScope"}, - DiscriminatorProperty: "objectType", - }, - }, - }, - "AwsAccountScope": { - Fields: odatasql.Schema{ - "objectType": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "regions": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsRegion"}, - }, - }, - }, - }, - "AwsScanScope": { - Fields: odatasql.Schema{ - "objectType": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "allRegions": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "shouldScanStoppedInstances": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "instanceTagExclusion": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"Tag"}, - }, - }, - "instanceTagSelector": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"Tag"}, - }, - }, - "regions": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsRegion"}, - }, - }, - }, - }, "Tag": { Fields: odatasql.Schema{ "key": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "value": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, }, }, - "AwsRegion": { - Fields: odatasql.Schema{ - "name": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "vpcs": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsVPC"}, - }, - }, - }, - }, - "AwsVPC": { - Fields: odatasql.Schema{ - "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "securityGroups": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsSecurityGroup"}, - }, - }, - }, - }, - "AwsSecurityGroup": { - Fields: odatasql.Schema{ - "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - }, - }, "Finding": { Table: "findings", Fields: odatasql.Schema{ diff --git a/backend/pkg/database/gorm/scope.go b/backend/pkg/database/gorm/scope.go deleted file mode 100644 index 63d6db928d..0000000000 --- a/backend/pkg/database/gorm/scope.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gorm - -import ( - "encoding/json" - "errors" - "fmt" - - "gorm.io/gorm" - - "github.com/openclarity/vmclarity/api/models" - "github.com/openclarity/vmclarity/backend/pkg/database/types" -) - -const ( - scopesSchemaName = "Scopes" -) - -type Scopes struct { - ODataObject -} - -type ScopesTableHandler struct { - DB *gorm.DB -} - -func (db *Handler) ScopesTable() types.ScopesTable { - return &ScopesTableHandler{ - DB: db.DB, - } -} - -func (s ScopesTableHandler) GetScopes(params models.GetDiscoveryScopesParams) (models.Scopes, error) { - var dbScopes Scopes - err := ODataQuery(s.DB, scopesSchemaName, params.Filter, params.Select, nil, nil, nil, nil, false, &dbScopes) - if err != nil { - return models.Scopes{}, err - } - - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return models.Scopes{}, types.ErrNotFound - } - return models.Scopes{}, err - } - - var scopes models.Scopes - err = json.Unmarshal(dbScopes.Data, &scopes) - if err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert DB model to API model: %w", err) - } - - return scopes, nil -} - -func (s ScopesTableHandler) SetScopes(scopes models.Scopes) (models.Scopes, error) { - marshaled, err := json.Marshal(scopes) - if err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert API model to DB model: %w", err) - } - - var dbScopes Scopes - dbScopes.Data = marshaled - - if err = s.DB.Save(&dbScopes).Error; err != nil { - return models.Scopes{}, fmt.Errorf("failed to save scopes in db: %w", err) - } - - // TODO(sambetts) Maybe this isn't required now because the DB isn't - // creating any of the data (like the ID) so we can just return the - // target pre-marshal above. - var apiScopes models.Scopes - if err = json.Unmarshal(dbScopes.Data, &apiScopes); err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert DB model to API model: %w", err) - } - - return apiScopes, nil -} diff --git a/backend/pkg/database/types/database.go b/backend/pkg/database/types/database.go index 7e7a542a75..85a2012c0e 100644 --- a/backend/pkg/database/types/database.go +++ b/backend/pkg/database/types/database.go @@ -54,7 +54,6 @@ type Database interface { ScanConfigsTable() ScanConfigsTable ScansTable() ScansTable TargetsTable() TargetsTable - ScopesTable() ScopesTable FindingsTable() FindingsTable } @@ -102,11 +101,6 @@ type TargetsTable interface { DeleteTarget(targetID models.TargetID) error } -type ScopesTable interface { - GetScopes(params models.GetDiscoveryScopesParams) (models.Scopes, error) - SetScopes(scopes models.Scopes) (models.Scopes, error) -} - type FindingsTable interface { GetFindings(params models.GetFindingsParams) (models.Findings, error) GetFinding(findingID models.FindingID, params models.GetFindingsFindingIDParams) (models.Finding, error) diff --git a/backend/pkg/rest/discovery_controller.go b/backend/pkg/rest/discovery_controller.go deleted file mode 100644 index 54ee6a4d1c..0000000000 --- a/backend/pkg/rest/discovery_controller.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rest - -import ( - "fmt" - "net/http" - - "github.com/labstack/echo/v4" - - "github.com/openclarity/vmclarity/api/models" -) - -func (s *ServerImpl) GetDiscoveryScopes(ctx echo.Context, params models.GetDiscoveryScopesParams) error { - dbScopes, err := s.dbHandler.ScopesTable().GetScopes(params) - if err != nil { - return sendError(ctx, http.StatusInternalServerError, fmt.Sprintf("failed to get scopes from db: %v", err)) - } - - return sendResponse(ctx, http.StatusOK, dbScopes) -} - -func (s *ServerImpl) PutDiscoveryScopes(ctx echo.Context) error { - var scopes models.Scopes - err := ctx.Bind(&scopes) - if err != nil { - return sendError(ctx, http.StatusBadRequest, fmt.Errorf("failed to bind request: %v", err).Error()) - } - - updatedScopes, err := s.dbHandler.ScopesTable().SetScopes(scopes) - if err != nil { - return sendError(ctx, http.StatusInternalServerError, fmt.Errorf("failed to set scopes in db: %v", err).Error()) - } - - return sendResponse(ctx, http.StatusOK, &updatedScopes) -} diff --git a/runtime_scan/pkg/orchestrator/discovery/scope.go b/runtime_scan/pkg/orchestrator/discovery/scope.go index 1a1616cc66..1a4989decb 100644 --- a/runtime_scan/pkg/orchestrator/discovery/scope.go +++ b/runtime_scan/pkg/orchestrator/discovery/scope.go @@ -17,12 +17,16 @@ package discovery import ( "context" + "errors" + "fmt" "time" log "github.com/sirupsen/logrus" + "github.com/openclarity/vmclarity/api/models" "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/shared/pkg/backendclient" + "github.com/openclarity/vmclarity/shared/pkg/utils" ) const ( @@ -44,17 +48,12 @@ func New(config Config) *ScopeDiscoverer { func (sd *ScopeDiscoverer) Start(ctx context.Context) { go func() { for { - log.Debug("Discovering available scopes") - // nolint:contextcheck - scopes, err := sd.providerClient.DiscoverScopes(ctx) + log.Debug("Discovering available assets") + err := sd.DiscoverAndCreateAssets(ctx) if err != nil { - log.Warnf("Failed to discover scopes: %v", err) - } else { - _, err := sd.backendClient.PutDiscoveryScopes(ctx, scopes) - if err != nil { - log.Warnf("Failed to set scopes: %v", err) - } + log.Warnf("Failed to discover assets: %v", err) } + select { case <-time.After(discoveryInterval): log.Debug("Discovery interval elapsed") @@ -65,3 +64,74 @@ func (sd *ScopeDiscoverer) Start(ctx context.Context) { } }() } + +func (sd *ScopeDiscoverer) DiscoverAndCreateAssets(ctx context.Context) error { + discoveryTime := time.Now() + + assetTypes, err := sd.providerClient.DiscoverTargets(ctx) + if err != nil { + return fmt.Errorf("failed to discover assets from provider: %w", err) + } + + errs := []error{} + for _, assetType := range assetTypes { + targetData := models.Target{ + TargetInfo: utils.PointerTo(assetType), + LastSeen: &discoveryTime, + } + _, err := sd.backendClient.PostTarget(ctx, targetData) + if err == nil { + continue + } + + var conflictError backendclient.TargetConflictError + if !errors.As(err, &conflictError) { + // If there is an error, and its not a conflict telling + // us that the target already exists, then we need to + // keept track of it and log it as a failure to + // complete discovery. We don't fail instantly here + // because discovering the assets is a heavy operation + // so we want to give the best chance to create all the + // assets in the DB before failing. + errs = append(errs, fmt.Errorf("failed to post target: %v", err)) + continue + } + + // As we got a conflict it means there is an existing target + // which matches the unique properties of this target, in this + // case we'll patch the data instead. + err = sd.backendClient.PatchTarget(ctx, targetData, *conflictError.ConflictingTarget.Id) + if err != nil { + errs = append(errs, fmt.Errorf("failed to patch target: %v", err)) + } + } + + // Find all assets which were not updated or created by this discovery + // run by comparing their lastSeen time to this discovery's time stamp + // + // TODO(sambetts) when we add multiple providers/standalone support we + // need to filter these assets by provider so that we don't find assets + // which don't belong to us. We need to give the provider some kind of + // identity in this case. + targetResp, err := sd.backendClient.GetTargets(ctx, models.GetTargetsParams{ + Filter: utils.PointerTo(fmt.Sprintf("lastSeen eq null or lastSeen lt %s", discoveryTime.Format(time.RFC3339))), + Select: utils.PointerTo("id"), + }) + if err != nil { + return fmt.Errorf("failed to get existing Targets: %w", err) + } + + // Patch all assets which were not found by this discovery as terminated + for _, target := range *targetResp.Items { + targetData := models.Target{ + Terminated: &discoveryTime, + } + + err := sd.backendClient.PatchTarget(ctx, targetData, *target.Id) + if err != nil { + errs = append(errs, fmt.Errorf("failed to patch target: %v", err)) + } + } + + return errors.Join(errs...) +} diff --git a/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go b/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go index 5ab457d6ed..ab3ecb826a 100644 --- a/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go +++ b/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go @@ -191,21 +191,37 @@ func (w *Watcher) reconcilePending(ctx context.Context, scan *models.Scan) error return errors.New("invalid Scan: Id is nil") } + // We don't want to scan terminated assets these are history assets and + // we'll just get a not found error during the asset scan. + targetFilter := "terminated eq null" + scope, ok := scan.GetScanConfigScope() if !ok { return fmt.Errorf("invalid Scan: Scope is nil. ScanID=%s", scanID) } - targets, err := w.provider.DiscoverTargets(ctx, &scope) + // If the scan has a scope configured, 'and' it with the check for + // terminated to make sure that we take both into account. + if scope != "" { + targetFilter = fmt.Sprintf("(%s) and (%s)", targetFilter, scope) + } + + targets, err := w.backend.GetTargets(ctx, models.GetTargetsParams{ + Filter: &targetFilter, + Select: utils.PointerTo("id"), + }) if err != nil { return fmt.Errorf("failed to discover Targets for Scan. ScanID=%s: %w", scanID, err) } - numOfTargets := len(targets) + + numOfTargets := len(*targets.Items) if numOfTargets > 0 { - if err = w.createTargets(ctx, scan, targets); err != nil { - return fmt.Errorf("failed to create Targets for Scan. ScanID=%s: %w", scanID, err) + targetIds := []string{} + for _, target := range *targets.Items { + targetIds = append(targetIds, *target.Id) } + scan.TargetIDs = &targetIds scan.State = utils.PointerTo(models.ScanStateDiscovered) scan.StateMessage = utils.PointerTo("Targets for Scan are successfully discovered") } else { @@ -229,68 +245,6 @@ func (w *Watcher) reconcilePending(ctx context.Context, scan *models.Scan) error return nil } -func (w *Watcher) createTargets(ctx context.Context, scan *models.Scan, targetTypes []models.TargetType) error { - logger := log.GetLoggerFromContextOrDiscard(ctx) - - var creatingTargetsFailed bool - var wg sync.WaitGroup - - results := make(chan string, len(targetTypes)) - for _, t := range targetTypes { - targetType := t - - wg.Add(1) - go func() { - defer wg.Done() - - targetID, err := w.createTarget(ctx, targetType) - if err != nil { - creatingTargetsFailed = true - return - } - - logger.WithField("TargetID", targetID).Trace("Pushing Target to channel") - results <- targetID - }() - } - logger.Trace("Waiting until all Target(s) are created") - wg.Wait() - close(results) - - if creatingTargetsFailed { - return fmt.Errorf("failed to create Target(s) for Scan. ScanID=%s", *scan.Id) - } - - targetIDs := make([]string, 0) - for targetID := range results { - targetIDs = append(targetIDs, targetID) - } - scan.TargetIDs = &targetIDs - - logger.Tracef("Created Target(s): %v", targetIDs) - - return nil -} - -func (w *Watcher) createTarget(ctx context.Context, targetType models.TargetType) (string, error) { - logger := log.GetLoggerFromContextOrDiscard(ctx) - - target, err := w.backend.PostTarget(ctx, models.Target{ - TargetInfo: &targetType, - }) - if err != nil { - var conErr backendclient.TargetConflictError - if errors.As(err, &conErr) { - logger.WithField("TargetID", *conErr.ConflictingTarget.Id).Trace("Target already exist") - return *conErr.ConflictingTarget.Id, nil - } - return "", fmt.Errorf("failed to post Target: %w", err) - } - logger.WithField("TargetID", *target.Id).Debug("Target object created") - - return *target.Id, nil -} - func (w *Watcher) reconcileDiscovered(ctx context.Context, scan *models.Scan) error { logger := log.GetLoggerFromContextOrDiscard(ctx) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 74675d313f..00586f315f 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -69,97 +69,28 @@ func (c Client) Kind() models.CloudProvider { return models.AWS } -func (c *Client) DiscoverScopes(ctx context.Context) (*models.Scopes, error) { - regions, err := c.ListAllRegions(ctx, true) - if err != nil { - return nil, fmt.Errorf("failed to list all regions: %w", err) - } - - scopes := models.ScopeType{} - err = scopes.FromAwsAccountScope(models.AwsAccountScope{ - Regions: convertToAPIRegions(regions), - }) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed to cast AwsAccountScope to ScopeType: %w", err), - } - } - - return &models.Scopes{ - ScopeInfo: &scopes, - }, nil -} - // nolint:cyclop -func (c *Client) DiscoverTargets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.TargetType, error) { - var filters []ec2types.Filter - - awsScanScope, err := scanScope.AsAwsScanScope() +func (c *Client) DiscoverTargets(ctx context.Context) ([]models.TargetType, error) { + regions, err := c.ListAllRegions(ctx) if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed to cast ScanScopeType to AwsAccountScope: %w", err), - } + return nil, fmt.Errorf("failed to get regions: %w", err) } - scope := convertFromAPIScanScope(&awsScanScope) - - regions, err := c.getRegionsToScan(ctx, scope) - if err != nil { - return nil, fmt.Errorf("failed to get regions to scan: %w", err) - } - if len(regions) == 0 { - return nil, FatalError{ - Err: errors.New("no regions to scan"), - } - } - - filters = append(filters, EC2FiltersFromTags(scope.TagSelector)...) - - instanceStates := []ec2types.InstanceStateName{ec2types.InstanceStateNameRunning} - if scope.ScanStopped { - instanceStates = append(instanceStates, ec2types.InstanceStateNameStopped) - } - filters = append(filters, EC2FiltersFromInstanceState(instanceStates...)...) - targets := make([]models.TargetType, 0) for _, region := range regions { - // if no vpcs, that mean that we don't need any vpc filters - if len(region.VPCs) == 0 { - instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.Name) - if err != nil { - return nil, fmt.Errorf("failed to get instances: %w", err) - } - - for _, instance := range instances { - target, err := getVMInfoFromInstance(instance) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed convert EC2 Instance to TargetType: %w", err), - } - } - targets = append(targets, target) - } - - continue + instances, err := c.GetInstances(ctx, []ec2types.Filter{}, []models.Tag{}, region.Name) + if err != nil { + return nil, fmt.Errorf("failed to get instances: %w", err) } - // need to do a per vpc call for DescribeInstances - for _, vpc := range region.VPCs { - vpcFilters := append(filters, createVPCFilters(vpc)...) - - instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.Name) + for _, instance := range instances { + target, err := getVMInfoFromInstance(instance) if err != nil { - return nil, fmt.Errorf("failed to get instances: %w", err) - } - for _, instance := range instances { - target, err := getVMInfoFromInstance(instance) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed convert EC2 Instance to TargetType: %w", err), - } + return nil, FatalError{ + Err: fmt.Errorf("failed convert EC2 Instance to TargetType: %w", err), } - targets = append(targets, target) } + targets = append(targets, target) } } @@ -790,10 +721,14 @@ func (c *Client) RemoveTargetScan(ctx context.Context, config *provider.ScanJobC func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []models.Tag, regionID string) ([]Instance, error) { ret := make([]Instance, 0) - out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: filters, + input := &ec2.DescribeInstancesInput{ MaxResults: utils.PointerTo[int32](maxResults), // TODO what will be a good number? - }, func(options *ec2.Options) { + } + if len(filters) > 0 { + input.Filters = filters + } + + out, err := c.ec2Client.DescribeInstances(ctx, input, func(options *ec2.Options) { options.Region = regionID }) if err != nil { @@ -804,11 +739,15 @@ func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, ex // use pagination // TODO we can make it better by not saving all results in memory. See https://github.com/openclarity/vmclarity/pull/3#discussion_r1021656861 for out.NextToken != nil { - out, err = c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: filters, - MaxResults: utils.PointerTo[int32](maxResults), + input := &ec2.DescribeInstancesInput{ + MaxResults: utils.PointerTo[int32](maxResults), // TODO what will be a good number? NextToken: out.NextToken, - }, func(options *ec2.Options) { + } + if len(filters) > 0 { + input.Filters = filters + } + + out, err = c.ec2Client.DescribeInstances(ctx, input, func(options *ec2.Options) { options.Region = regionID }) if err != nil { @@ -829,6 +768,13 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(ctx context.Context, re if hasExcludeTags(excludeTags, instance.Tags) { continue } + + // Ignore terminated instances they are destroyed and + // will be garbage collected by AWS. + if instance.State.Name == ec2types.InstanceStateNameTerminated { + continue + } + if err := validateInstanceFields(instance); err != nil { logger.Errorf("Instance validation failed. instance id=%v: %v", getPointerValOrEmpty(instance.InstanceId), err) continue @@ -852,17 +798,7 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(ctx context.Context, re return ret } -func (c *Client) getRegionsToScan(ctx context.Context, scope *ScanScope) ([]Region, error) { - if scope.AllRegions { - return c.ListAllRegions(ctx, false) - } - - return scope.Regions, nil -} - -func (c *Client) ListAllRegions(ctx context.Context, isRecursive bool) ([]Region, error) { - logger := log.GetLoggerFromContextOrDiscard(ctx) - +func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { ret := make([]Region, 0) out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? @@ -877,40 +813,5 @@ func (c *Client) ListAllRegions(ctx context.Context, isRecursive bool) ([]Region }) } - if isRecursive { - for i, region := range ret { - // List region VPCs - vpcs, err := c.ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ - MaxResults: utils.PointerTo[int32](maxResults), - }, func(options *ec2.Options) { - options.Region = region.Name - }) - if err != nil { - logger.Errorf("Failed to describe vpcs. Region=%s: %v", region.Name, err) - continue - } - ret[i].VPCs = convertAwsVPCs(vpcs.Vpcs) - for i2, vpc := range ret[i].VPCs { - // List VPC's security groups - securityGroups, err := c.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - Filters: []ec2types.Filter{ - { - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpc.ID}, - }, - }, - MaxResults: utils.PointerTo[int32](maxResults), - }, func(options *ec2.Options) { - options.Region = region.Name - }) - if err != nil { - logger.Errorf("Failed to describe security groups. Region=%s, VpcID=%s: %v", region.Name, vpc.ID, err) - continue - } - ret[i].VPCs[i2].SecurityGroups = getSecurityGroupsFromEC2SecurityGroups(securityGroups.SecurityGroups) - } - } - } - return ret, nil } diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 6f20d47393..94d89d84c4 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -471,6 +471,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-1"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc1"), ImageId: utils.PointerTo("image1"), Placement: &ec2types.Placement{ @@ -493,6 +496,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-2"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc2"), ImageId: utils.PointerTo("image2"), Placement: &ec2types.Placement{ @@ -518,6 +524,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Placement: &ec2types.Placement{ AvailabilityZone: utils.PointerTo("az3"), }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, InstanceType: "t2.large", PlatformDetails: utils.PointerTo("linux"), LaunchTime: utils.PointerTo(launchTime), @@ -604,6 +613,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-1"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc1"), ImageId: utils.PointerTo("image1"), Placement: &ec2types.Placement{ @@ -626,6 +638,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-2"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc2"), ImageId: utils.PointerTo("image2"), Placement: &ec2types.Placement{ @@ -651,6 +666,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Placement: &ec2types.Placement{ AvailabilityZone: utils.PointerTo("az3"), }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, InstanceType: "t2.large", PlatformDetails: utils.PointerTo("linux"), LaunchTime: utils.PointerTo(launchTime), diff --git a/runtime_scan/pkg/provider/aws/helpers.go b/runtime_scan/pkg/provider/aws/helpers.go index 03798614e0..5121ca1dde 100644 --- a/runtime_scan/pkg/provider/aws/helpers.go +++ b/runtime_scan/pkg/provider/aws/helpers.go @@ -141,86 +141,6 @@ func instanceFromEC2Instance(i *ec2types.Instance, client *ec2.Client, region st } } -func convertFromAPIScanScope(scope *models.AwsScanScope) *ScanScope { - var tagSelector []models.Tag - if scope.InstanceTagSelector != nil { - tagSelector = *scope.InstanceTagSelector - } - - var excludeTags []models.Tag - if scope.InstanceTagExclusion != nil { - excludeTags = *scope.InstanceTagExclusion - } - - return &ScanScope{ - AllRegions: convertBool(scope.AllRegions), - Regions: convertFromAPIRegions(scope.Regions), - ScanStopped: convertBool(scope.ShouldScanStoppedInstances), - TagSelector: tagSelector, - ExcludeTags: excludeTags, - } -} - -func convertFromAPIRegions(regions *[]models.AwsRegion) []Region { - var ret []Region - if regions != nil { - for _, region := range *regions { - ret = append(ret, Region{ - Name: region.Name, - VPCs: convertFromAPIVPCs(region.Vpcs), - }) - } - } - - return ret -} - -func convertFromAPIVPCs(vpcs *[]models.AwsVPC) []VPC { - if vpcs == nil { - return nil - } - ret := make([]VPC, len(*vpcs)) - for i, vpc := range *vpcs { - ret[i] = VPC{ - ID: vpc.Id, - SecurityGroups: *vpc.SecurityGroups, - } - } - - return ret -} - -func convertToAPIRegions(regions []Region) *[]models.AwsRegion { - ret := make([]models.AwsRegion, len(regions)) - for i := range regions { - ret[i] = models.AwsRegion{ - Name: regions[i].Name, - Vpcs: convertToAPIVPCs(regions[i].VPCs), - } - } - - return &ret -} - -func convertToAPIVPCs(vpcs []VPC) *[]models.AwsVPC { - ret := make([]models.AwsVPC, len(vpcs)) - for i := range vpcs { - ret[i] = models.AwsVPC{ - Id: vpcs[i].ID, - SecurityGroups: utils.PointerTo(vpcs[i].SecurityGroups), - } - } - - return &ret -} - -func convertBool(all *bool) bool { - if all != nil { - return *all - } - return false -} - func getTagsFromECTags(tags []ec2types.Tag) []models.Tag { if len(tags) == 0 { return nil @@ -358,34 +278,6 @@ func getSecurityGroupsFromEC2GroupIdentifiers(identifiers []ec2types.GroupIdenti return ret } -func getSecurityGroupsFromEC2SecurityGroups(groups []ec2types.SecurityGroup) []models.SecurityGroup { - var ret []models.SecurityGroup - - for _, securityGroup := range groups { - if securityGroup.GroupId != nil { - ret = append(ret, models.SecurityGroup{ - Id: *securityGroup.GroupId, - }) - } - } - - return ret -} - -func convertAwsVPCs(vpcs []ec2types.Vpc) []VPC { - var ret []VPC - for _, vpc := range vpcs { - if vpc.VpcId != nil { - ret = append(ret, VPC{ - ID: *vpc.VpcId, - SecurityGroups: nil, - }) - } - } - - return ret -} - // AND logic - if excludeTags = {tag1:val1, tag2:val2}, // then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}). func hasExcludeTags(excludeTags []models.Tag, instanceTags []ec2types.Tag) bool { diff --git a/runtime_scan/pkg/provider/azure/client.go b/runtime_scan/pkg/provider/azure/client.go index 181f6d75ec..6fed290b95 100644 --- a/runtime_scan/pkg/provider/azure/client.go +++ b/runtime_scan/pkg/provider/azure/client.go @@ -178,74 +178,21 @@ func (c *Client) RemoveTargetScan(ctx context.Context, config *provider.ScanJobC return nil } -func (c *Client) DiscoverScopes(ctx context.Context) (*models.Scopes, error) { - var ret models.Scopes - ret.ScopeInfo = &models.ScopeType{} - resourceGroups := []models.AzureResourceGroup{} - - // discover all resource groups in the user subscription - res := c.rgClient.NewListPager(nil) +// nolint: cyclop +func (c *Client) DiscoverTargets(ctx context.Context) ([]models.TargetType, error) { + var ret []models.TargetType + // list all vms in all resourceGroups in the subscription + res := c.vmClient.NewListAllPager(nil) for res.More() { page, err := res.NextPage(ctx) if err != nil { return nil, fmt.Errorf("failed to get next page: %w", err) } - for _, rg := range page.Value { - resourceGroups = append(resourceGroups, models.AzureResourceGroup{Name: *rg.Name}) - } - } - - err := ret.ScopeInfo.FromAzureSubscriptionScope(models.AzureSubscriptionScope{ - ResourceGroups: &resourceGroups, - SubscriptionID: &c.azureConfig.SubscriptionID, - }) - if err != nil { - return nil, fmt.Errorf("failed to convert from azure subscription scope: %v", err) - } - - return &ret, nil -} - -// nolint: cyclop -func (c *Client) DiscoverTargets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.TargetType, error) { - var ret []models.TargetType - - azureScanScope, err := scanScope.AsAzureScanScope() - if err != nil { - return nil, fmt.Errorf("failed to convert as azure scan scope: %v", err) - } - - if azureScanScope.AllResourceGroups != nil && *azureScanScope.AllResourceGroups { - // list all vms in all resourceGroups in the subscription - res := c.vmClient.NewListAllPager(nil) - for res.More() { - page, err := res.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get next page: %w", err) - } - ts, err := processVirtualMachineListIntoTargetTypes(page.VirtualMachineListResult, azureScanScope) - if err != nil { - return nil, err - } - ret = append(ret, ts...) - } - return ret, nil - } - - // if scan scope is only for specific resource groups and not all: - for _, resourceGroup := range *azureScanScope.ResourceGroups { - res := c.vmClient.NewListPager(resourceGroup.Name, nil) - for res.More() { - page, err := res.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get next page: %w", err) - } - ts, err := processVirtualMachineListIntoTargetTypes(page.VirtualMachineListResult, azureScanScope) - if err != nil { - return nil, err - } - ret = append(ret, ts...) + ts, err := processVirtualMachineListIntoTargetTypes(page.VirtualMachineListResult) + if err != nil { + return nil, err } + ret = append(ret, ts...) } return ret, nil } @@ -263,16 +210,9 @@ func resourceGroupAndNameFromInstanceID(instanceID string) (string, string, erro return idParts[resourceGroupPartIdx], idParts[vmNamePartIdx], nil } -func processVirtualMachineListIntoTargetTypes(vmList armcompute.VirtualMachineListResult, azureScanScope models.AzureScanScope) ([]models.TargetType, error) { +func processVirtualMachineListIntoTargetTypes(vmList armcompute.VirtualMachineListResult) ([]models.TargetType, error) { ret := make([]models.TargetType, 0, len(vmList.Value)) for _, vm := range vmList.Value { - // filter by tags: - if !hasIncludeTags(vm, azureScanScope.InstanceTagSelector) { - continue - } - if hasExcludeTags(vm, azureScanScope.InstanceTagExclusion) { - continue - } info, err := getVMInfoFromVirtualMachine(vm) if err != nil { return nil, fmt.Errorf("unable to convert instance to vminfo: %w", err) @@ -303,56 +243,6 @@ func getVMInfoFromVirtualMachine(vm *armcompute.VirtualMachine) (models.TargetTy return targetType, err } -// AND logic - if tags = {tag1:val1, tag2:val2}, -// then a vm will be excluded/included only if it has ALL of these tags ({tag1:val1, tag2:val2}). -func hasIncludeTags(vm *armcompute.VirtualMachine, tags *[]models.Tag) bool { - if tags == nil { - return true - } - if len(*tags) == 0 { - return true - } - if len(vm.Tags) == 0 { - return false - } - - for _, tag := range *tags { - val, ok := vm.Tags[tag.Key] - if !ok { - return false - } - if !(strings.Compare(*val, tag.Value) == 0) { - return false - } - } - return true -} - -// AND logic - if tags = {tag1:val1, tag2:val2}, -// then a vm will be excluded/included only if it has ALL of these tags ({tag1:val1, tag2:val2}). -func hasExcludeTags(vm *armcompute.VirtualMachine, tags *[]models.Tag) bool { - if tags == nil { - return false - } - if len(*tags) == 0 { - return false - } - if len(vm.Tags) == 0 { - return false - } - - for _, tag := range *tags { - val, ok := vm.Tags[tag.Key] - if !ok { - return false - } - if !(strings.Compare(*val, tag.Value) == 0) { - return false - } - } - return true -} - func convertTags(tags map[string]*string) *[]models.Tag { ret := make([]models.Tag, 0, len(tags)) for key, val := range tags { diff --git a/runtime_scan/pkg/provider/types.go b/runtime_scan/pkg/provider/types.go index 1ba26be3bb..9d6ef70906 100644 --- a/runtime_scan/pkg/provider/types.go +++ b/runtime_scan/pkg/provider/types.go @@ -30,10 +30,8 @@ type Provider interface { } type Discoverer interface { - // DiscoverScopes returns a list of discovered models.Scopes - DiscoverScopes(ctx context.Context) (*models.Scopes, error) - // DiscoverTargets returns list of TargetType in ScanScopeType - DiscoverTargets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.TargetType, error) + // DiscoverTargets returns list of discovered TargetType + DiscoverTargets(ctx context.Context) ([]models.TargetType, error) } type Scanner interface { diff --git a/shared/pkg/backendclient/client.go b/shared/pkg/backendclient/client.go index 89846b32bf..6308114bea 100644 --- a/shared/pkg/backendclient/client.go +++ b/shared/pkg/backendclient/client.go @@ -532,25 +532,6 @@ func (b *BackendClient) GetTarget(ctx context.Context, targetID string, params m } } -func (b *BackendClient) PutDiscoveryScopes(ctx context.Context, scope *models.Scopes) (*models.Scopes, error) { - resp, err := b.apiClient.PutDiscoveryScopesWithResponse(ctx, *scope) - if err != nil { - return nil, fmt.Errorf("failed to put discovery scope: %v", err) - } - switch resp.StatusCode() { - case http.StatusOK: - if resp.JSON200 == nil { - return nil, fmt.Errorf("failed to put scopes: empty body. status code=%v", http.StatusOK) - } - return resp.JSON200, nil - default: - if resp.JSONDefault != nil && resp.JSONDefault.Message != nil { - return nil, fmt.Errorf("failed to put scopes. status code=%v: %s", resp.StatusCode(), *resp.JSONDefault.Message) - } - return nil, fmt.Errorf("failed to put scopes. status code=%v", resp.StatusCode()) - } -} - func (b *BackendClient) GetTargets(ctx context.Context, params models.GetTargetsParams) (*models.Targets, error) { resp, err := b.apiClient.GetTargetsWithResponse(ctx, ¶ms) if err != nil { diff --git a/ui/src/layout/AssetScans/AssetScansTable.js b/ui/src/layout/AssetScans/AssetScansTable.js index 6663c91e19..c6f8983ed9 100644 --- a/ui/src/layout/AssetScans/AssetScansTable.js +++ b/ui/src/layout/AssetScans/AssetScansTable.js @@ -88,4 +88,4 @@ const AssetScansTable = () => { ) } -export default AssetScansTable; \ No newline at end of file +export default AssetScansTable; diff --git a/ui/src/layout/Assets/AssetDetails.js b/ui/src/layout/Assets/AssetDetails.js index bb9eb1b2ac..70d70d585b 100644 --- a/ui/src/layout/Assets/AssetDetails.js +++ b/ui/src/layout/Assets/AssetDetails.js @@ -48,11 +48,11 @@ const AssetDetails = () => ( ({title: targetInfo?.instanceID})} detailsContent={props => } withPadding /> ) -export default AssetDetails; \ No newline at end of file +export default AssetDetails; diff --git a/ui/src/layout/Assets/AssetsTable.js b/ui/src/layout/Assets/AssetsTable.js index 1b6dd488ce..4ac5681bd8 100644 --- a/ui/src/layout/Assets/AssetsTable.js +++ b/ui/src/layout/Assets/AssetsTable.js @@ -3,7 +3,7 @@ import TablePage from 'components/TablePage'; import ExpandableList from 'components/ExpandableList'; import { APIS } from 'utils/systemConsts'; import { getFindingsColumnsConfigList, getVulnerabilitiesColumnConfigItem, getAssetColumnsFiltersConfig, - findingsColumnsFiltersConfig, vulnerabilitiesCountersColumnsFiltersConfig, formatTagsToStringsList } from 'utils/utils'; + findingsColumnsFiltersConfig, vulnerabilitiesCountersColumnsFiltersConfig, formatTagsToStringsList, formatDate} from 'utils/utils'; import { FILTER_TYPES } from 'context/FiltersProvider'; const TABLE_TITLE = "assets"; @@ -43,6 +43,18 @@ const AssetsTable = () => { sortIds: LOCATION_SORT_IDS, accessor: "targetInfo.location" }, + { + Header: "LastSeen", + id: "lastSeen", + sortIds: ["lastSeen"], + accessor: original => formatDate(original.lastSeen) + }, + { + Header: "Terminated", + id: "terminated", + sortIds: ["terminated"], + accessor: original => formatDate(original?.terminated) + }, getVulnerabilitiesColumnConfigItem(TABLE_TITLE), ...getFindingsColumnsConfigList(TABLE_TITLE) ], []); @@ -51,7 +63,7 @@ const AssetsTable = () => { { ...vulnerabilitiesCountersColumnsFiltersConfig, ...findingsColumnsFiltersConfig ]} - defaultSortBy={{sortIds: LOCATION_SORT_IDS, desc: false}} + defaultSortBy={{sortIds: ["lastSeen", "terminated"], desc: true}} withMargin /> ) diff --git a/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js b/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js index 70ebcb6826..25e5df0161 100644 --- a/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js +++ b/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js @@ -1,30 +1,18 @@ import React from 'react'; import TitleValueDisplay, { ValuesListDisplay } from 'components/TitleValueDisplay'; -import { TagsList } from 'components/Tag'; import { getEnabledScanTypesList, getScanTimeTypeTag } from 'layout/Scans/utils'; -import { cronExpressionToHuman, formatDate, formatTagsToStringsList } from 'utils/utils'; -import { ScopeDisplay } from './scopeDisplayUtils'; - -const InstancesDisplay = ({tags}) => ( - -) +import { cronExpressionToHuman, formatDate } from 'utils/utils'; const FlagPropDisplay = ({checked, label}) =>
{`${label} ${checked ? "enabled" : "disabled"}`}
const ConfigurationReadOnlyDisplay = ({configData}) => { const {scope, scanFamiliesConfig, scheduled, maxParallelScanners, scannerInstanceCreationConfig} = configData; - const {allRegions, regions, instanceTagSelector, instanceTagExclusion, shouldScanStoppedInstances} = scope; const {cronLine, operationTime} = scheduled; const {useSpotInstances} = scannerInstanceCreationConfig || {}; return ( <> - - - - - - + {scope} <> diff --git a/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js b/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js index 137e3e9f44..f5faa89213 100644 --- a/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js +++ b/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js @@ -3,13 +3,11 @@ import { isNull } from 'lodash'; import ButtonWithIcon from 'components/ButtonWithIcon'; import { ICON_NAMES } from 'components/Icon'; import EmptyDisplay from 'components/EmptyDisplay'; -import ExpandableList from 'components/ExpandableList'; import TablePage from 'components/TablePage'; import { OPERATORS } from 'components/Filter'; -import { BoldText, toCapitalized, formatDate, getScanScopeColumnFiltersConfig, formatTagsToStringsList } from 'utils/utils'; +import { BoldText, toCapitalized, formatDate } from 'utils/utils'; import { APIS } from 'utils/systemConsts'; import { getScanTimeTypeTag } from 'layout/Scans/utils'; -import { ExpandableScopeDisplay } from 'layout/Scans/scopeDisplayUtils'; import { useModalDisplayDispatch, MODAL_DISPLAY_ACTIONS } from 'layout/Scans/ScanConfigWizardModal/ModalDisplayProvider'; import { FILTER_TYPES } from 'context/FiltersProvider'; import ConfigurationActionsDisplay from '../ConfigurationActionsDisplay'; @@ -44,46 +42,10 @@ const ConfigurationsTable = () => { accessor: "name" }, { - Header: "Scope", + Header: "Asset Query", id: "scope", - sortIds: [ - "scope.allRegions", - "scope.regions" - ], - Cell: ({row}) => { - const {allRegions, regions} = row.original.scope; - - return ( - - ) - }, - alignToTop: true - }, - { - Header: "Excluded instances", - id: "instanceTagExclusion", - sortIds: ["scope.instanceTagExclusion"], - Cell: ({row}) => { - const {instanceTagExclusion} = row.original.scope; - - return ( - - ) - }, - alignToTop: true - }, - { - Header: "Included instances", - id: "instanceTagSelector", - sortIds: ["scope.instanceTagSelector"], - Cell: ({row}) => { - const {instanceTagSelector} = row.original.scope; - - return ( - - ) - }, - alignToTop: true + sortIds: ["scope"], + accessor: "scope" }, { Header: "Scan time", @@ -150,11 +112,11 @@ const ConfigurationsTable = () => { {...OPERATORS.endswith}, {...OPERATORS.contains, valueItems: [], creatable: true} ]}, - ...getScanScopeColumnFiltersConfig(), - {value: "scope.instanceTagExclusion", label: "Excluded instances", operators: [ - {...OPERATORS.contains, valueItems: [], creatable: true} - ]}, - {value: "scope.instanceTagSelector", label: "Included instances", operators: [ + {value: "scope", label: "Asset Query", operators: [ + {...OPERATORS.eq, valueItems: [], creatable: true}, + {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.startswith}, + {...OPERATORS.endswith}, {...OPERATORS.contains, valueItems: [], creatable: true} ]}, {value: "scheduled.operationTime", label: "Scan time", isDate: true, operators: [ diff --git a/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js b/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js index 1ac66e5c92..e92d0b8d08 100644 --- a/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js +++ b/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js @@ -1,137 +1,7 @@ -import React, { useEffect } from 'react'; -import { isEmpty } from 'lodash'; -import { usePrevious, useFetch } from 'hooks'; -import { TextField, RadioField, MultiselectField, SelectField, FieldsPair, useFormikContext, CheckboxField, - FieldLabel, validators } from 'components/Form'; -import { APIS } from 'utils/systemConsts'; - -export const VPCS_EMPTY_VALUE = [{id: "", securityGroups: []}] -export const REGIONS_EMPTY_VALUE = [{name: "", vpcs: VPCS_EMPTY_VALUE}]; - -export const SCOPE_ITEMS = { - ALL: {value: "ALL", label: "All"}, - DEFINED: {value: "DEFINED", label: "Define scope"} -} - -const SecurityGroupField = ({index, name, placeholder, disabled, regionData}) => { - const {name: regionId, vpcs} = regionData || {}; - const vpcId = isEmpty(vpcs) ? null : vpcs[index]?.id; - const prevVpsId = usePrevious(vpcId); - - const [{data, loading, error}, fetchSecurityGroups] = useFetch(APIS.SCOPES_DISCOVERY, {loadOnMount: false}); - - useEffect(() => { - if (!!vpcId && prevVpsId !== vpcId) { - fetchSecurityGroups({queryParams: { - "$select": `scopeInfo/regions($filter=name eq '${regionId}';$select=name,vpcs($filter=id eq '${vpcId}';$select=securityGroups))` - }}); - } - }, [prevVpsId, vpcId, regionId, fetchSecurityGroups]); - - if (error) { - return null; - } - - const regionsData = !data ? [] : data?.scopeInfo?.regions; - const vpcsData = !regionsData ? [] : regionsData[0]?.vpcs; - - return ( - ({value: id, label: id}))} - loading={loading} - /> - ) -} - -const RegionFields = ({index, name, disabled}) => { - const {values} = useFormikContext(); - const {regions} = values.scope; - - const regionId = regions[index]?.name; - const prevRegionId = usePrevious(regionId); - - const [{data, loading, error}, fetchVpcs] = useFetch(APIS.SCOPES_DISCOVERY, {loadOnMount: false}); - - useEffect(() => { - if (!!regionId && prevRegionId !== regionId) { - fetchVpcs({queryParams: { - "$select": `scopeInfo/regions($filter=name eq '${regionId}';$select=id,vpcs/id)`, - }}); - } - }, [prevRegionId, regionId, fetchVpcs]); - - if (error) { - return null; - } - - const regionsData = !data ? [] : data?.scopeInfo?.regions; - - return ( - ({value: id, label: id})), - clearable: true, - loading: loading - }} - secondFieldProps={{ - component: SecurityGroupField, - key: "securityGroups", - placeholder: "Select security group...", - emptyValue: [], - regionData: !!regionsData ? regions[index] : null - }} - /> - ); -} - -const DefinedScopeFields = () => { - const [{data, loading, error}] = useFetch(APIS.SCOPES_DISCOVERY, {queryParams: {"$select": "scopeInfo/regions/name"}}); - - if (error) { - return null; - } - - return ( - ({value: name, label: name})), - validate: validators.validateRequired, - loading: loading - }} - secondFieldProps={{ - component: RegionFields, - key: "vpcs", - emptyValue: VPCS_EMPTY_VALUE - }} - /> - ) -} +import React from 'react'; +import { TextField, validators } from 'components/Form'; const StepGeneralProperties = () => { - const {values, setFieldValue} = useFormikContext(); - const {scopeSelect} = values.scope; - - const isDefineScope = scopeSelect === SCOPE_ITEMS.DEFINED.value; - const prevIsDefineScope = usePrevious(isDefineScope); - - useEffect(() => { - if (prevIsDefineScope && !isDefineScope) { - setFieldValue("scope.regions", REGIONS_EMPTY_VALUE); - } - }, [prevIsDefineScope, isDefineScope, setFieldValue]); - return (
{ placeholder="Type a name..." validate={validators.validateRequired} /> - - {isDefineScope && } -
- Instances - - - -
) } -export default StepGeneralProperties; \ No newline at end of file +export default StepGeneralProperties; diff --git a/ui/src/layout/Scans/ScanConfigWizardModal/index.js b/ui/src/layout/Scans/ScanConfigWizardModal/index.js index e2455f4fde..4de0fa12dc 100644 --- a/ui/src/layout/Scans/ScanConfigWizardModal/index.js +++ b/ui/src/layout/Scans/ScanConfigWizardModal/index.js @@ -1,11 +1,8 @@ import React from 'react'; -import { isEmpty } from 'lodash'; import { FETCH_METHODS } from 'hooks'; import WizardModal from 'components/WizardModal'; import { APIS } from 'utils/systemConsts'; -import { formatTagsToStringsList } from 'utils/utils'; -import { formatStringInstancesToTags } from '../utils'; -import StepGeneralProperties, { REGIONS_EMPTY_VALUE, VPCS_EMPTY_VALUE, SCOPE_ITEMS } from './StepGeneralProperties'; +import StepGeneralProperties from './StepGeneralProperties'; import StepScanTypes from './StepScanTypes'; import StepTimeConfiguration, { SCHEDULE_TYPES_ITEMS, CRON_QUICK_OPTIONS } from './StepTimeConfiguration'; import StepAdvancedSettings from './StepAdvancedSettings'; @@ -16,7 +13,6 @@ const padDateTime = time => String(time).padStart(2, "0"); const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { const {id, name, scope, scanFamiliesConfig, scheduled, maxParallelScanners, scannerInstanceCreationConfig} = initialData || {}; - const {allRegions, regions, shouldScanStoppedInstances, instanceTagSelector, instanceTagExclusion} = scope || {}; const {operationTime, cronLine} = scheduled || {}; const {useSpotInstances} = scannerInstanceCreationConfig || {}; @@ -25,13 +21,7 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { const initialValues = { id: id || null, name: name || "", - scope: { - scopeSelect: (!regions || allRegions) ? SCOPE_ITEMS.ALL.value : SCOPE_ITEMS.DEFINED.value, - regions: REGIONS_EMPTY_VALUE, - shouldScanStoppedInstances: shouldScanStoppedInstances || false, - instanceTagSelector: formatTagsToStringsList(instanceTagSelector || []), - instanceTagExclusion: formatTagsToStringsList(instanceTagExclusion || []) - }, + scope: scope || "", scanFamiliesConfig: { sbom: {enabled: true}, vulnerabilities: {enabled: true}, @@ -53,14 +43,6 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { } } - if (!isEmpty(regions)) { - initialValues.scope.regions = regions.map(({name, vpcs}) => { - return {name, vpcs: isEmpty(vpcs) ? VPCS_EMPTY_VALUE : vpcs.map(({id, securityGroups}) => { - return {id: id || "", securityGroups: (securityGroups || []).map(({id}) => id)} - })} - }) - } - if (!!operationTime && !cronLine) { const dateTime = new Date(operationTime); initialValues.scheduled.scheduledSelect = SCHEDULE_TYPES_ITEMS.LATER.value; @@ -104,23 +86,7 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { initialValues={initialValues} submitUrl={APIS.SCAN_CONFIGS} getSubmitParams={formValues => { - const {id, scope, scheduled, ...submitData} = formValues; - - const {scopeSelect, regions, shouldScanStoppedInstances, instanceTagSelector, instanceTagExclusion} = scope; - const isAllScope = scopeSelect === SCOPE_ITEMS.ALL.value; - - submitData.scope = { - objectType: "AwsScanScope", - allRegions: isAllScope, - regions: isAllScope ? [] : regions.map(({name, vpcs}) => { - return {name, vpcs: vpcs.filter(({id}) => !!id).map(({id, securityGroups}) => { - return {id, securityGroups: securityGroups.map(id => ({id}))} - })} - }), - shouldScanStoppedInstances, - instanceTagSelector: formatStringInstancesToTags(instanceTagSelector), - instanceTagExclusion: formatStringInstancesToTags(instanceTagExclusion), - } + const {id, scheduled, ...submitData} = formValues; const {scheduledSelect, laterDate, laterTime, cronLine} = scheduled; const isNow = scheduledSelect === SCHEDULE_TYPES_ITEMS.NOW.value; diff --git a/ui/src/layout/Scans/Scans/ScansTable.js b/ui/src/layout/Scans/Scans/ScansTable.js index 50e8ffb254..9b880d451a 100644 --- a/ui/src/layout/Scans/Scans/ScansTable.js +++ b/ui/src/layout/Scans/Scans/ScansTable.js @@ -5,11 +5,10 @@ import { utils } from 'components/Table'; import ScanProgressBar, { SCAN_STATES } from 'components/ScanProgressBar'; import EmptyDisplay from 'components/EmptyDisplay'; import { OPERATORS } from 'components/Filter'; -import { ExpandableScopeDisplay } from 'layout/Scans/scopeDisplayUtils'; import { useModalDisplayDispatch, MODAL_DISPLAY_ACTIONS } from 'layout/Scans/ScanConfigWizardModal/ModalDisplayProvider'; import { APIS, ROUTES } from 'utils/systemConsts'; import { formatDate, getFindingsColumnsConfigList, getVulnerabilitiesColumnConfigItem, formatNumber, findingsColumnsFiltersConfig, - vulnerabilitiesCountersColumnsFiltersConfig, getScanScopeColumnFiltersConfig } from 'utils/utils'; + vulnerabilitiesCountersColumnsFiltersConfig } from 'utils/utils'; import { FILTER_TYPES } from 'context/FiltersProvider'; import ScanActionsDisplay from './ScanActionsDisplay'; import { SCANS_PATHS } from '../utils'; @@ -57,16 +56,8 @@ const ScansTable = () => { { Header: "Scope", id: "scope", - sortIds: [ - "scanConfigSnapshot.scope.allRegions", - "scanConfigSnapshot.scope.regions" - ], - Cell: ({row}) => { - const {allRegions, regions} = row.original.scanConfigSnapshot?.scope; - - return - }, - width: 260 + sortIds: ["scanConfigSnapshot.scope"], + accessor: "scanConfigSnapshot.scope" }, { Header: "Status", @@ -113,11 +104,18 @@ const ScansTable = () => { filterType={FILTER_TYPES.SCANS} filtersConfig={[ {value: "scanConfigSnapshot.name", label: "Config name", operators: [ - {...OPERATORS.eq, valueItems: [], creatable: true}, - {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.eq, valueItems: [], creatable: true}, + {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.startswith}, + {...OPERATORS.endswith}, + {...OPERATORS.contains, valueItems: [], creatable: true} + ]}, + {value: "scope", label: "Asset Query", operators: [ + {...OPERATORS.eq, valueitems: [], creatable: true}, + {...OPERATORS.ne, valueitems: [], creatable: true}, {...OPERATORS.startswith}, {...OPERATORS.endswith}, - {...OPERATORS.contains, valueItems: [], creatable: true} + {...OPERATORS.contains, valueitems: [], creatable: true} ]}, {value: "startTime", label: "Started", isDate: true, operators: [ {...OPERATORS.ge}, @@ -127,7 +125,6 @@ const ScansTable = () => { {...OPERATORS.ge}, {...OPERATORS.le}, ]}, - ...getScanScopeColumnFiltersConfig("scanConfigSnapshot.scope"), {value: "state", label: "Status", operators: [ {...OPERATORS.eq, valueItems: FILTER_SCAN_STATUS_ITEMS}, {...OPERATORS.ne, valueItems: FILTER_SCAN_STATUS_ITEMS} @@ -165,4 +162,4 @@ const ScansTable = () => { ) } -export default ScansTable; \ No newline at end of file +export default ScansTable; diff --git a/ui/src/layout/detail-displays/AssetDetails/index.js b/ui/src/layout/detail-displays/AssetDetails/index.js index 582784a2af..71b5aab5d0 100644 --- a/ui/src/layout/detail-displays/AssetDetails/index.js +++ b/ui/src/layout/detail-displays/AssetDetails/index.js @@ -51,7 +51,7 @@ const AssetScansDisplay = ({assetName, targetId}) => { const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false}) => { const navigate = useNavigate(); - const {id, targetInfo} = assetData; + const {id, targetInfo, lastSeen, terminated} = assetData; const {instanceID, objectType, location, tags, image, instanceType, platform, launchTime} = targetInfo || {}; return ( @@ -62,6 +62,8 @@ const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false} {instanceID} {objectType} + {formatDate(lastSeen)} + {formatDate(terminated)} {location} @@ -84,4 +86,4 @@ const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false} ) } -export default AssetDetails; \ No newline at end of file +export default AssetDetails; diff --git a/ui/src/utils/utils.js b/ui/src/utils/utils.js index 924d6737dd..0efe5bc92c 100644 --- a/ui/src/utils/utils.js +++ b/ui/src/utils/utils.js @@ -181,28 +181,4 @@ export const getAssetColumnsFiltersConfig = (props) => { ] } -export const getScanScopeColumnFiltersConfig = (scopePrefix="scope") => { - const SCOPE_ALL_REGIONS_FILTER_ITEMS = [ - {value: `${scopePrefix}.allRegions`, label: "All"} - ] - - const formatScanScopeToOdata = (valuesList, operator, scope) => ( - valuesList.map(value => { - if (operator === OPERATORS.contains.value) { - return `${operator}(${scope},'${value}')`; - } - - return `(${value} eq ${operator === OPERATORS.eq.value ? "true" : "false"})`; - }).join(` or `) - ) - - return [ - {value: `${scopePrefix}.regions`, label: "Scope", customOdataFormat: formatScanScopeToOdata, operators: [ - {...OPERATORS.eq, valueItems: SCOPE_ALL_REGIONS_FILTER_ITEMS}, - {...OPERATORS.ne, valueItems: SCOPE_ALL_REGIONS_FILTER_ITEMS}, - {...OPERATORS.contains, valueItems: [], creatable: true} - ]} - ] -} - -export const formatTagsToStringsList = tags => tags?.map(({key, value}) => `${key}=${value}`); \ No newline at end of file +export const formatTagsToStringsList = tags => tags?.map(({key, value}) => `${key}=${value}`);