From 2fae375733476959ac23d7748e12aad205a4f76d Mon Sep 17 00:00:00 2001 From: Sam Betts Date: Tue, 27 Jun 2023 12:11:53 +0100 Subject: [PATCH] Separate asset discovery from scanning This commit removes the concept of scopes and instead replaces it with periodic discovery of assets. These assets will be created in the system and a ScanConfig can be configured with an ODATA $filter expression to choose the assets to scan. This increases user visibility into the assets available to scan by VMClarity, it also gives the system an opportunity to estimate how many items a ScanConfig might scan based on a particular query. It also opens the door for additional features such as the ability to scan a specific asset which has never been scanned before. Assets are tracked with lastSeen timestamp which indicates when it was last seen by discovery. When they are not found in a discovery round they are marked as terminated. Terminated assets will not be included in scans even if they match the $filter configured by the user for the scan. --- api/client/client.gen.go | 309 ------------- api/models/models.gen.go | 281 +----------- api/models/scan.go | 4 +- api/models/scanconfigsnapshot.go | 4 +- api/openapi.yaml | 224 +--------- api/server/server.gen.go | 237 ++++------ backend/pkg/database/demo.go | 161 +------ backend/pkg/database/gorm/database.go | 1 - backend/pkg/database/gorm/odata.go | 100 +---- backend/pkg/database/gorm/scope.go | 92 ---- backend/pkg/database/types/database.go | 6 - backend/pkg/rest/discovery_controller.go | 49 -- .../pkg/orchestrator/discovery/discoverer.go | 141 ++++++ .../pkg/orchestrator/discovery/scope.go | 67 --- .../pkg/orchestrator/scanwatcher/watcher.go | 86 +--- runtime_scan/pkg/provider/aws/client.go | 173 ++----- runtime_scan/pkg/provider/aws/client_test.go | 422 +----------------- runtime_scan/pkg/provider/aws/helpers.go | 186 -------- runtime_scan/pkg/provider/azure/client.go | 130 +----- runtime_scan/pkg/provider/types.go | 6 +- shared/pkg/backendclient/client.go | 19 - ui/src/layout/Assets/AssetDetails.js | 2 +- ui/src/layout/Assets/AssetsTable.js | 18 +- ui/src/layout/Dashboard/index.js | 4 +- .../Scans/ConfigurationReadOnlyDisplay.js | 18 +- .../ConfigurationsTable/index.js | 56 +-- .../StepGeneralProperties.js | 166 +------ .../Scans/ScanConfigWizardModal/index.js | 40 +- ui/src/layout/Scans/Scans/ScansTable.js | 31 +- .../detail-displays/AssetDetails/index.js | 7 +- ui/src/utils/systemConsts.js | 3 +- ui/src/utils/utils.js | 26 +- 32 files changed, 421 insertions(+), 2648 deletions(-) delete mode 100644 backend/pkg/database/gorm/scope.go delete mode 100644 backend/pkg/rest/discovery_controller.go create mode 100644 runtime_scan/pkg/orchestrator/discovery/discoverer.go delete mode 100644 runtime_scan/pkg/orchestrator/discovery/scope.go diff --git a/api/client/client.gen.go b/api/client/client.gen.go index 20bde0538e..9e49fd3171 100644 --- a/api/client/client.gen.go +++ b/api/client/client.gen.go @@ -135,14 +135,6 @@ type ClientInterface interface { PutAssetsAssetID(ctx context.Context, assetID AssetID, params *PutAssetsAssetIDParams, body PutAssetsAssetIDJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetDiscoveryScopes request - GetDiscoveryScopes(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) - - // PutDiscoveryScopes request with any body - PutDiscoveryScopesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - PutDiscoveryScopes(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetFindings request GetFindings(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -420,42 +412,6 @@ func (c *Client) PutAssetsAssetID(ctx context.Context, assetID AssetID, params * return c.Client.Do(req) } -func (c *Client) GetDiscoveryScopes(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetDiscoveryScopesRequest(c.Server, params) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) PutDiscoveryScopesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPutDiscoveryScopesRequestWithBody(c.Server, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) PutDiscoveryScopes(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPutDiscoveryScopesRequest(c.Server, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) GetFindings(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetFindingsRequest(c.Server, params) if err != nil { @@ -1552,125 +1508,6 @@ func NewPutAssetsAssetIDRequestWithBody(server string, assetID AssetID, params * return req, nil } -// NewGetDiscoveryScopesRequest generates requests for GetDiscoveryScopes -func NewGetDiscoveryScopesRequest(server string, params *GetDiscoveryScopesParams) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/discovery/scopes") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - queryValues := queryURL.Query() - - if params.Filter != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$filter", runtime.ParamLocationQuery, *params.Filter); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.Select != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$select", runtime.ParamLocationQuery, *params.Select); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - if params.OrderBy != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "$orderby", runtime.ParamLocationQuery, *params.OrderBy); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - queryURL.RawQuery = queryValues.Encode() - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewPutDiscoveryScopesRequest calls the generic PutDiscoveryScopes builder with application/json body -func NewPutDiscoveryScopesRequest(server string, body PutDiscoveryScopesJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewPutDiscoveryScopesRequestWithBody(server, "application/json", bodyReader) -} - -// NewPutDiscoveryScopesRequestWithBody generates requests for PutDiscoveryScopes with any type of body -func NewPutDiscoveryScopesRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/discovery/scopes") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("PUT", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - // NewGetFindingsRequest generates requests for GetFindings func NewGetFindingsRequest(server string, params *GetFindingsParams) (*http.Request, error) { var err error @@ -2946,14 +2783,6 @@ type ClientWithResponsesInterface interface { PutAssetsAssetIDWithResponse(ctx context.Context, assetID AssetID, params *PutAssetsAssetIDParams, body PutAssetsAssetIDJSONRequestBody, reqEditors ...RequestEditorFn) (*PutAssetsAssetIDResponse, error) - // GetDiscoveryScopes request - GetDiscoveryScopesWithResponse(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*GetDiscoveryScopesResponse, error) - - // PutDiscoveryScopes request with any body - PutDiscoveryScopesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) - - PutDiscoveryScopesWithResponse(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) - // GetFindings request GetFindingsWithResponse(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*GetFindingsResponse, error) @@ -3303,52 +3132,6 @@ func (r PutAssetsAssetIDResponse) StatusCode() int { return 0 } -type GetDiscoveryScopesResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Scopes - JSONDefault *ApiResponse -} - -// Status returns HTTPResponse.Status -func (r GetDiscoveryScopesResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetDiscoveryScopesResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type PutDiscoveryScopesResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Scopes - JSONDefault *ApiResponse -} - -// Status returns HTTPResponse.Status -func (r PutDiscoveryScopesResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r PutDiscoveryScopesResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type GetFindingsResponse struct { Body []byte HTTPResponse *http.Response @@ -3942,32 +3725,6 @@ func (c *ClientWithResponses) PutAssetsAssetIDWithResponse(ctx context.Context, return ParsePutAssetsAssetIDResponse(rsp) } -// GetDiscoveryScopesWithResponse request returning *GetDiscoveryScopesResponse -func (c *ClientWithResponses) GetDiscoveryScopesWithResponse(ctx context.Context, params *GetDiscoveryScopesParams, reqEditors ...RequestEditorFn) (*GetDiscoveryScopesResponse, error) { - rsp, err := c.GetDiscoveryScopes(ctx, params, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetDiscoveryScopesResponse(rsp) -} - -// PutDiscoveryScopesWithBodyWithResponse request with arbitrary body returning *PutDiscoveryScopesResponse -func (c *ClientWithResponses) PutDiscoveryScopesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) { - rsp, err := c.PutDiscoveryScopesWithBody(ctx, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParsePutDiscoveryScopesResponse(rsp) -} - -func (c *ClientWithResponses) PutDiscoveryScopesWithResponse(ctx context.Context, body PutDiscoveryScopesJSONRequestBody, reqEditors ...RequestEditorFn) (*PutDiscoveryScopesResponse, error) { - rsp, err := c.PutDiscoveryScopes(ctx, body, reqEditors...) - if err != nil { - return nil, err - } - return ParsePutDiscoveryScopesResponse(rsp) -} - // GetFindingsWithResponse request returning *GetFindingsResponse func (c *ClientWithResponses) GetFindingsWithResponse(ctx context.Context, params *GetFindingsParams, reqEditors ...RequestEditorFn) (*GetFindingsResponse, error) { rsp, err := c.GetFindings(ctx, params, reqEditors...) @@ -4726,72 +4483,6 @@ func ParsePutAssetsAssetIDResponse(rsp *http.Response) (*PutAssetsAssetIDRespons return response, nil } -// ParseGetDiscoveryScopesResponse parses an HTTP response from a GetDiscoveryScopesWithResponse call -func ParseGetDiscoveryScopesResponse(rsp *http.Response) (*GetDiscoveryScopesResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetDiscoveryScopesResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Scopes - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest ApiResponse - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSONDefault = &dest - - } - - return response, nil -} - -// ParsePutDiscoveryScopesResponse parses an HTTP response from a PutDiscoveryScopesWithResponse call -func ParsePutDiscoveryScopesResponse(rsp *http.Response) (*PutDiscoveryScopesResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &PutDiscoveryScopesResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Scopes - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest ApiResponse - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSONDefault = &dest - - } - - return response, nil -} - // ParseGetFindingsResponse parses an HTTP response from a GetFindingsWithResponse call func ParseGetFindingsResponse(rsp *http.Response) (*GetFindingsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/api/models/models.gen.go b/api/models/models.gen.go index 83829283d3..8b85446d49 100644 --- a/api/models/models.gen.go +++ b/api/models/models.gen.go @@ -122,14 +122,17 @@ type ApiResponse struct { // Asset Describes an asset object. type Asset struct { AssetInfo *AssetType `json:"assetInfo,omitempty"` + FirstSeen *time.Time `json:"firstSeen,omitempty"` Id *string `json:"id,omitempty"` + LastSeen *time.Time `json:"lastSeen,omitempty"` Revision *int `json:"revision,omitempty"` // ScansCount Total number of scans that have ever run for this asset ScansCount *int `json:"scansCount,omitempty"` // Summary A summary of the scan findings. - Summary *ScanFindingsSummary `json:"summary,omitempty"` + Summary *ScanFindingsSummary `json:"summary,omitempty"` + TerminatedOn *time.Time `json:"terminatedOn,omitempty"` } // AssetCommon defines model for AssetCommon. @@ -234,65 +237,6 @@ type Assets struct { Items *[]Asset `json:"items,omitempty"` } -// AwsAccountScope AWS cloud account scope -type AwsAccountScope struct { - ObjectType string `json:"objectType"` - Regions *[]AwsRegion `json:"regions"` -} - -// AwsRegion AWS region -type AwsRegion struct { - Name string `json:"name"` - Vpcs *[]AwsVPC `json:"vpcs"` -} - -// AwsScanScope The scope of a configured scan. -type AwsScanScope struct { - // AllRegions Scan all regions, if set will override anything set in regions. - AllRegions *bool `json:"allRegions,omitempty"` - - // InstanceTagExclusion VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - InstanceTagExclusion *[]Tag `json:"instanceTagExclusion"` - - // InstanceTagSelector VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - InstanceTagSelector *[]Tag `json:"instanceTagSelector"` - ObjectType string `json:"objectType"` - Regions *[]AwsRegion `json:"regions"` - ShouldScanStoppedInstances *bool `json:"shouldScanStoppedInstances,omitempty"` -} - -// AwsVPC AWS VPC -type AwsVPC struct { - Id string `json:"id"` - SecurityGroups *[]SecurityGroup `json:"securityGroups"` -} - -// AzureResourceGroup Azure Resource Group -type AzureResourceGroup struct { - Name string `json:"name"` -} - -// AzureScanScope The scope of a configured scan within a subscription. -type AzureScanScope struct { - // AllResourceGroups Scan all resource groups in the subscription, if set will override anything set in resourceGroups. - AllResourceGroups *bool `json:"allResourceGroups,omitempty"` - - // InstanceTagExclusion VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - InstanceTagExclusion *[]Tag `json:"instanceTagExclusion"` - - // InstanceTagSelector VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - InstanceTagSelector *[]Tag `json:"instanceTagSelector"` - ObjectType string `json:"objectType"` - ResourceGroups *[]AzureResourceGroup `json:"resourceGroups"` -} - -// AzureSubscriptionScope Azure subscription scope -type AzureSubscriptionScope struct { - ObjectType string `json:"objectType"` - ResourceGroups *[]AzureResourceGroup `json:"resourceGroups"` - SubscriptionID *string `json:"subscriptionID,omitempty"` -} - // CloudProvider defines model for CloudProvider. type CloudProvider string @@ -580,7 +524,7 @@ type ScanConfig struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -615,7 +559,7 @@ type ScanConfigRelationship struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -641,7 +585,13 @@ type ScanConfigSnapshot struct { // Scheduled Runtime schedule scan configuration. If only operationTime is set, it will be a single scan scheduled for the operationTime. If only cronLine is set, the current time will be the "from time" to start the scheduling according to the cronLine. If both operationTime and cronLine are set, the first scan will run at operationTime and the operationTime will be the first time that the cronLine will be effective from. Scheduled *RuntimeScheduleScanConfig `json:"scheduled,omitempty"` - Scope *ScanScopeType `json:"scope,omitempty"` + + // Scope The query used to limit the scope of this scan. It uses + // the ODATA $filter query language to limit the collection of assets + // that this scan will operate over. For example + // `startswith(assetInfo.location, 'eu-west-2')` will limit this scan + // to just assets in the eu-west-2 AWS region. + Scope *string `json:"scope,omitempty"` // TimeoutSeconds The maximum time in seconds that a scan started from this config // should run for before being automatically aborted. @@ -726,11 +676,6 @@ type ScanRelationshipState string // ScanRelationshipStateReason Machine-readable, UpperCamelCase text indicating the reason for the condition's last transition. type ScanRelationshipStateReason string -// ScanScopeType defines model for ScanScopeType. -type ScanScopeType struct { - union json.RawMessage -} - // ScanSummary defines model for ScanSummary. type ScanSummary struct { JobsCompleted *int `json:"jobsCompleted,omitempty"` @@ -784,16 +729,6 @@ type Scans struct { Items *[]Scan `json:"items,omitempty"` } -// ScopeType defines model for ScopeType. -type ScopeType struct { - union json.RawMessage -} - -// Scopes Scopes discovery -type Scopes struct { - ScopeInfo *ScopeType `json:"scopeInfo,omitempty"` -} - // Secret defines model for Secret. type Secret struct { Description *string `json:"description,omitempty"` @@ -845,7 +780,7 @@ type SuccessResponse struct { Message *string `json:"message,omitempty"` } -// Tag AWS tag +// Tag general cloud tag / label type Tag struct { Key string `json:"key"` Value string `json:"value"` @@ -1051,13 +986,6 @@ type PutAssetsAssetIDParams struct { IfMatch *Ifmatch `json:"If-Match,omitempty"` } -// GetDiscoveryScopesParams defines parameters for GetDiscoveryScopes. -type GetDiscoveryScopesParams struct { - Filter *OdataFilter `form:"$filter,omitempty" json:"$filter,omitempty"` - Select *OdataSelect `form:"$select,omitempty" json:"$select,omitempty"` - OrderBy *OrderBy `form:"$orderby,omitempty" json:"$orderby,omitempty"` -} - // GetFindingsParams defines parameters for GetFindings. type GetFindingsParams struct { Filter *OdataFilter `form:"$filter,omitempty" json:"$filter,omitempty"` @@ -1147,9 +1075,6 @@ type PatchAssetsAssetIDJSONRequestBody = Asset // PutAssetsAssetIDJSONRequestBody defines body for PutAssetsAssetID for application/json ContentType. type PutAssetsAssetIDJSONRequestBody = Asset -// PutDiscoveryScopesJSONRequestBody defines body for PutDiscoveryScopes for application/json ContentType. -type PutDiscoveryScopesJSONRequestBody = Scopes - // PostFindingsJSONRequestBody defines body for PostFindings for application/json ContentType. type PostFindingsJSONRequestBody = Finding @@ -1534,181 +1459,3 @@ func (t *Finding_FindingInfo) UnmarshalJSON(b []byte) error { err := t.union.UnmarshalJSON(b) return err } - -// AsAwsScanScope returns the union data inside the ScanScopeType as a AwsScanScope -func (t ScanScopeType) AsAwsScanScope() (AwsScanScope, error) { - var body AwsScanScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAwsScanScope overwrites any union data inside the ScanScopeType as the provided AwsScanScope -func (t *ScanScopeType) FromAwsScanScope(v AwsScanScope) error { - v.ObjectType = "AwsScanScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAwsScanScope performs a merge with any union data inside the ScanScopeType, using the provided AwsScanScope -func (t *ScanScopeType) MergeAwsScanScope(v AwsScanScope) error { - v.ObjectType = "AwsScanScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -// AsAzureScanScope returns the union data inside the ScanScopeType as a AzureScanScope -func (t ScanScopeType) AsAzureScanScope() (AzureScanScope, error) { - var body AzureScanScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAzureScanScope overwrites any union data inside the ScanScopeType as the provided AzureScanScope -func (t *ScanScopeType) FromAzureScanScope(v AzureScanScope) error { - v.ObjectType = "AzureScanScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAzureScanScope performs a merge with any union data inside the ScanScopeType, using the provided AzureScanScope -func (t *ScanScopeType) MergeAzureScanScope(v AzureScanScope) error { - v.ObjectType = "AzureScanScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -func (t ScanScopeType) Discriminator() (string, error) { - var discriminator struct { - Discriminator string `json:"objectType"` - } - err := json.Unmarshal(t.union, &discriminator) - return discriminator.Discriminator, err -} - -func (t ScanScopeType) ValueByDiscriminator() (interface{}, error) { - discriminator, err := t.Discriminator() - if err != nil { - return nil, err - } - switch discriminator { - case "AwsScanScope": - return t.AsAwsScanScope() - case "AzureScanScope": - return t.AsAzureScanScope() - default: - return nil, errors.New("unknown discriminator value: " + discriminator) - } -} - -func (t ScanScopeType) MarshalJSON() ([]byte, error) { - b, err := t.union.MarshalJSON() - return b, err -} - -func (t *ScanScopeType) UnmarshalJSON(b []byte) error { - err := t.union.UnmarshalJSON(b) - return err -} - -// AsAwsAccountScope returns the union data inside the ScopeType as a AwsAccountScope -func (t ScopeType) AsAwsAccountScope() (AwsAccountScope, error) { - var body AwsAccountScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAwsAccountScope overwrites any union data inside the ScopeType as the provided AwsAccountScope -func (t *ScopeType) FromAwsAccountScope(v AwsAccountScope) error { - v.ObjectType = "AwsAccountScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAwsAccountScope performs a merge with any union data inside the ScopeType, using the provided AwsAccountScope -func (t *ScopeType) MergeAwsAccountScope(v AwsAccountScope) error { - v.ObjectType = "AwsAccountScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -// AsAzureSubscriptionScope returns the union data inside the ScopeType as a AzureSubscriptionScope -func (t ScopeType) AsAzureSubscriptionScope() (AzureSubscriptionScope, error) { - var body AzureSubscriptionScope - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAzureSubscriptionScope overwrites any union data inside the ScopeType as the provided AzureSubscriptionScope -func (t *ScopeType) FromAzureSubscriptionScope(v AzureSubscriptionScope) error { - v.ObjectType = "AzureSubscriptionScope" - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAzureSubscriptionScope performs a merge with any union data inside the ScopeType, using the provided AzureSubscriptionScope -func (t *ScopeType) MergeAzureSubscriptionScope(v AzureSubscriptionScope) error { - v.ObjectType = "AzureSubscriptionScope" - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JsonMerge(b, t.union) - t.union = merged - return err -} - -func (t ScopeType) Discriminator() (string, error) { - var discriminator struct { - Discriminator string `json:"objectType"` - } - err := json.Unmarshal(t.union, &discriminator) - return discriminator.Discriminator, err -} - -func (t ScopeType) ValueByDiscriminator() (interface{}, error) { - discriminator, err := t.Discriminator() - if err != nil { - return nil, err - } - switch discriminator { - case "AwsAccountScope": - return t.AsAwsAccountScope() - case "AzureSubscriptionScope": - return t.AsAzureSubscriptionScope() - default: - return nil, errors.New("unknown discriminator value: " + discriminator) - } -} - -func (t ScopeType) MarshalJSON() ([]byte, error) { - b, err := t.union.MarshalJSON() - return b, err -} - -func (t *ScopeType) UnmarshalJSON(b []byte) error { - err := t.union.UnmarshalJSON(b) - return err -} diff --git a/api/models/scan.go b/api/models/scan.go index 8e8de42416..e5e66c7367 100644 --- a/api/models/scan.go +++ b/api/models/scan.go @@ -39,8 +39,8 @@ func (s *Scan) GetID() (string, bool) { return id, ok } -func (s *Scan) GetScanConfigScope() (ScanScopeType, bool) { - var scope ScanScopeType +func (s *Scan) GetScanConfigScope() (string, bool) { + var scope string var ok bool if s.ScanConfigSnapshot != nil { diff --git a/api/models/scanconfigsnapshot.go b/api/models/scanconfigsnapshot.go index 27c3b62153..7834accadc 100644 --- a/api/models/scanconfigsnapshot.go +++ b/api/models/scanconfigsnapshot.go @@ -15,8 +15,8 @@ package models -func (d *ScanConfigSnapshot) GetScope() (ScanScopeType, bool) { - var scope ScanScopeType +func (d *ScanConfigSnapshot) GetScope() (string, bool) { + var scope string var ok bool if d.Scope != nil { diff --git a/api/openapi.yaml b/api/openapi.yaml index 65d2a92f01..c39dea8bf3 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -787,43 +787,6 @@ paths: $ref: '#/components/responses/UnknownError' x-codegen-request-body-name: body - /discovery/scopes: - get: - summary: Get all available scopes - operationId: GetDiscoveryScopes - parameters: - - $ref: '#/components/parameters/odataFilter' - - $ref: '#/components/parameters/odataSelect' - - $ref: '#/components/parameters/odataOrderBy' - responses: - 200: - description: Success - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - default: - $ref: '#/components/responses/UnknownError' - put: - summary: Set all available scopes - operationId: PutDiscoveryScopes - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - required: true - responses: - 200: - description: All scopes were set successfully - content: - application/json: - schema: - $ref: '#/components/schemas/Scopes' - default: - $ref: '#/components/responses/UnknownError' - x-codegen-request-body-name: body - /findings/{findingID}: get: summary: Get the details for a finding. @@ -1237,7 +1200,13 @@ components: scanFamiliesConfig: $ref: '#/components/schemas/ScanFamiliesConfig' scope: - $ref: '#/components/schemas/ScanScopeType' + description: | + The query used to limit the scope of this scan. It uses + the ODATA $filter query language to limit the collection of assets + that this scan will operate over. For example + `startswith(assetInfo.location, 'eu-west-2')` will limit this scan + to just assets in the eu-west-2 AWS region. + type: string timeoutSeconds: type: integer minimum: 0 @@ -1286,7 +1255,7 @@ components: $ref: '#/components/schemas/ScanFamiliesConfig' readOnly: true scope: - $ref: '#/components/schemas/ScanScopeType' + type: string readOnly: true timeoutSeconds: type: integer @@ -1328,7 +1297,7 @@ components: scanFamiliesConfig: $ref: '#/components/schemas/ScanFamiliesConfig' scope: - $ref: '#/components/schemas/ScanScopeType' + type: string timeoutSeconds: type: integer minimum: 0 @@ -1359,170 +1328,6 @@ components: scanConfig: $ref: '#/components/schemas/ScanConfig' - Scopes: - type: object - description: Scopes discovery - properties: - scopeInfo: - $ref: '#/components/schemas/ScopeType' - - ScanScopeType: - type: object - anyOf: - - $ref: '#/components/schemas/AwsScanScope' - - $ref: '#/components/schemas/AzureScanScope' - discriminator: - propertyName: objectType - mapping: - AwsScanScope: '#/components/schemas/AwsScanScope' - AzureScanScope: '#/components/schemas/AzureScanScope' - - AzureScanScope: - type: object - description: The scope of a configured scan within a subscription. - properties: - objectType: - type: string - allResourceGroups: - description: Scan all resource groups in the subscription, if set will override anything set in resourceGroups. - type: boolean - resourceGroups: - type: array - items: - $ref: '#/components/schemas/AzureResourceGroup' - nullable: true - instanceTagSelector: - type: array - description: VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - instanceTagExclusion: - type: array - description: VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - required: - - objectType - additionalProperties: false - - AzureResourceGroup: - type: object - description: Azure Resource Group - properties: - name: - type: string - minLength: 1 - required: - - name - additionalProperties: false - - AwsScanScope: - type: object - description: The scope of a configured scan. - properties: - objectType: - type: string - allRegions: - description: Scan all regions, if set will override anything set in regions. - type: boolean - regions: - type: array - items: - $ref: '#/components/schemas/AwsRegion' - nullable: true - shouldScanStoppedInstances: - type: boolean - instanceTagSelector: - type: array - description: VM instances will be scanned if they contain all of these tags. If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - instanceTagExclusion: - type: array - description: VM instances will not be scanned if they contain all of these tags (even if they match instanceTagSelector). If empty, not taken into account. - items: - $ref: '#/components/schemas/Tag' - nullable: true - required: - - objectType - additionalProperties: false - - ScopeType: - type: object - anyOf: - - $ref: '#/components/schemas/AwsAccountScope' - - $ref: '#/components/schemas/AzureSubscriptionScope' - discriminator: - propertyName: objectType - mapping: - AwsAccountScope: '#/components/schemas/AwsAccountScope' - AzureSubscriptionScope: '#/components/schemas/AzureSubscriptionScope' - - AzureSubscriptionScope: - type: object - description: Azure subscription scope - properties: - objectType: - type: string - subscriptionID: - type: string - resourceGroups: - type: array - items: - $ref: '#/components/schemas/AzureResourceGroup' - nullable: true - required: - - objectType - - AwsAccountScope: - type: object - description: AWS cloud account scope - properties: - objectType: - type: string - regions: - type: array - items: - $ref: '#/components/schemas/AwsRegion' - nullable: true - required: - - objectType - - AwsRegion: - type: object - description: AWS region - properties: - name: - type: string - minLength: 1 - vpcs: - type: array - items: - $ref: '#/components/schemas/AwsVPC' - nullable: true - required: - - name - additionalProperties: false - - AwsVPC: - type: object - description: AWS VPC - properties: - id: - type: string - minLength: 1 - securityGroups: - type: array - items: - $ref: '#/components/schemas/SecurityGroup' - nullable: true - required: - - id - additionalProperties: false - SecurityGroup: type: object description: general cloud security group @@ -1536,7 +1341,7 @@ components: Tag: type: object - description: AWS tag + description: general cloud tag / label properties: key: type: string @@ -1614,6 +1419,15 @@ components: # readOnly: true assetInfo: $ref: '#/components/schemas/AssetType' + firstSeen: + type: string + format: date-time + terminatedOn: + type: string + format: date-time + lastSeen: + type: string + format: date-time scansCount: description: Total number of scans that have ever run for this asset type: integer diff --git a/api/server/server.gen.go b/api/server/server.gen.go index fe66fa9a46..2e0c7ee3d2 100644 --- a/api/server/server.gen.go +++ b/api/server/server.gen.go @@ -54,12 +54,6 @@ type ServerInterface interface { // Update asset. // (PUT /assets/{assetID}) PutAssetsAssetID(ctx echo.Context, assetID AssetID, params PutAssetsAssetIDParams) error - // Get all available scopes - // (GET /discovery/scopes) - GetDiscoveryScopes(ctx echo.Context, params GetDiscoveryScopesParams) error - // Set all available scopes - // (PUT /discovery/scopes) - PutDiscoveryScopes(ctx echo.Context) error // Get all findings. // (GET /findings) GetFindings(ctx echo.Context, params GetFindingsParams) error @@ -483,47 +477,6 @@ func (w *ServerInterfaceWrapper) PutAssetsAssetID(ctx echo.Context) error { return err } -// GetDiscoveryScopes converts echo context to params. -func (w *ServerInterfaceWrapper) GetDiscoveryScopes(ctx echo.Context) error { - var err error - - // Parameter object where we will unmarshal all parameters from the context - var params GetDiscoveryScopesParams - // ------------- Optional query parameter "$filter" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$filter", ctx.QueryParams(), ¶ms.Filter) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $filter: %s", err)) - } - - // ------------- Optional query parameter "$select" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$select", ctx.QueryParams(), ¶ms.Select) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $select: %s", err)) - } - - // ------------- Optional query parameter "$orderby" ------------- - - err = runtime.BindQueryParameter("form", true, false, "$orderby", ctx.QueryParams(), ¶ms.OrderBy) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter $orderby: %s", err)) - } - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.GetDiscoveryScopes(ctx, params) - return err -} - -// PutDiscoveryScopes converts echo context to params. -func (w *ServerInterfaceWrapper) PutDiscoveryScopes(ctx echo.Context) error { - var err error - - // Invoke the callback with all the unmarshalled arguments - err = w.Handler.PutDiscoveryScopes(ctx) - return err -} - // GetFindings converts echo context to params. func (w *ServerInterfaceWrapper) GetFindings(ctx echo.Context) error { var err error @@ -1090,8 +1043,6 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.GET(baseURL+"/assets/:assetID", wrapper.GetAssetsAssetID) router.PATCH(baseURL+"/assets/:assetID", wrapper.PatchAssetsAssetID) router.PUT(baseURL+"/assets/:assetID", wrapper.PutAssetsAssetID) - router.GET(baseURL+"/discovery/scopes", wrapper.GetDiscoveryScopes) - router.PUT(baseURL+"/discovery/scopes", wrapper.PutDiscoveryScopes) router.GET(baseURL+"/findings", wrapper.GetFindings) router.POST(baseURL+"/findings", wrapper.PostFindings) router.DELETE(baseURL+"/findings/:findingID", wrapper.DeleteFindingsFindingID) @@ -1116,103 +1067,97 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xd63PcqLL/VyjdU3UepdjJ3r0frr85tpOdWr/K4yT31MnWLSwxM2wk0AKyPSfl//0U", - "Lw16IKHxzNjO+ps9ggaa5kd30w3fo4TmBSWICB4dfI8KyGCOBGLqP8g5EpNj+Scm0UFUQLGI4ojAHEUH", - "1dc4YuiPEjOURgeClSiOeLJAOZTVxLKQRblgmMyjh4dY15omkPTTNSXG0Z5hkmIy91JefR9HF89yKJJF", - "RXWBYIrYiu5k9uZMFeggg4lAc8QUHZpCAY9oSURF6o8SseWK0l8S9bWDzg2lGYJkRefkvoAk9RJC+nP/", - "wBShDzgTiHkJzfTnAEIXLEXs/dJLicrvN8s+UnF0/2ZO35galqBtYIoylPh5x/XngJ5Ov+HCT0Z+DJnJ", - "a+onIuggDZ5AckTJDPsFtlZknMzyviXGx6+uB1mYF5RwpLBhWiYJ4urPhBKBtEzDoshwAgWmZP93Ton8", - "bUXzLwzNooPov/ZXoLOvv/J9Q+/KtKFbTBFPGC4kuejANglyxDmcIykWn8g3Qu/ICWOUbawrhwXu64Zp", - "EyDVqOa1qijpunUPvjdqHhJAb35HiQBiAQXAHDAkSkZQCjABMMtAAjnigM7ADOKsZIjvRXFUMFogJrBm", - "vB39wfeIIZhekGxpZ68tBeYX3apk2KEE13bPjtV/N4gDSIACYNPTdvsa9smMDrJRFryWHZAQmnaudoZu", - "Mcd6cppLRMswr/Cy3uNrKmAGSJnfICYZpspqvi7gLQLoFjHASgJmlAGxwFwPK4q72inzHLLloIgmkHzQ", - "Wwifmip+Jh/RPK8NrPH95B5zs+m2+RvEW0nKkQbfjN4tcLIAJcF/lAgklHDBICYCJDS/wUQtEJDAUgqe", - "WKgSswzrmV9XwK5QpujyhcZZr7AB5pQEgq7ET/c6gQTcIKA3M5TuUBo9Q9+6dAa0+zhpXUH+vyQffvPN", - "oaz/GOmsycBDHKH7IqNYS3xf5RNdTjW/0un4JaMS/FHapRN5ZzSH2R1kaKjNM13Mtpljnqhdt2R6CIP1", - "GxUsIYY4LVmCjmQ3y2KIzFW9+FRAgYZRklEqvgUw9kqXs33jNzQfrDO9oXlVwQjEkNA1J56jhKHh7k1V", - "saoxAUXJgwRNVpnq4o9cHXF0W2YEMXiDM2wlvo/KZ6f4Unf9oW899WL+NIC/q5X5bLHfTgdqD1NpTOov", - "LJAWvtaKJWWWwZsMNZqFjEE1Qxnk4ppBwrHs/DXOVTszynIoooMohQK9EfLXuE2b224hUuYS/86p6i5B", - "0lC6REocojiaJguUlpn69QrBdHlNFc/jaEIuGZ0zqfjG0eENZUIVOqYEOUgazKOyQxZCcbLB7Ic4miMp", - "jNn4ioEo2VFxLFC2SYSiV7umBLA1aoWhUbviSGhoEuiVhQ4xSPoUC60iaaVCFQQwSSiT4iuVKLmU5/gW", - "EaCNdx6kV1Rrst7iKeZCqjBum/2tAUhSUMA52gOqcobIXCxAXnIhlbmM3kmliAH0RwkzSUGWneJ/Iwk7", - "VS+CYdAzMoMYXr4rXVBiL1lezKKDfw0g/ZlSMh/i/mKXNA0qd4yZLvdbHKVYsjuXcKyN2BwWhcSPg++R", - "LTdAJo5swwP9iiMzkKFhWmlcnmuPgWae1Z+7ObqWFHcI8Nakl0sz+6nEdU1RveOHieLQNKFFl0/hyxQk", - "GS1TxUHJSa4KNg0lZwK7TZ+5xe+wId3xK1VleM9u2BxOR37rHrAhLNdmmqptHmaXzmBmMOMo7uCDHkRr", - "6Nrp9T3KMTlVcxsdvOvQDm6LZNT4P18ejR686opn2GqzsJM8YuTXC6TnXMk5sBsyShVYd5jMWXa1mu2G", - "i00a3DDLDCt5DPAMKHMcZxmgt4gxnCIAyVIs5FqVnzCxpfdWSpdrmhEuIEnQNZyf3CdZaS2Zesufz4At", - "yHVrhKr1x7WCJjsiFmgpxyegcZdR9RtHQMA5B39DclXbcsplD5zGtfeYsr/vgckMoLwQy1g1IuA3WY8I", - "atdQ8Mq+hkFKa0cvQjgwZvS7H9TTIUoc8QUts1SrV7QoUDqxnPMcmYxDILm0x8OPrNVcbNovMYA8HCUl", - "w2L5kdGyCOfY1K02Gop8vp9/lwxZR4SmPJITkgCwFIAmsRYkB2OnbHE76AnusAQ6AAEvb6pqHkx1eNYP", - "rYY1c1XS6iRuA8Gw6zb5ir6v6Ougb1Maw0C4vfo3rd+pxerIuk+vVTDiLop1FdudMSKO3O7qw9d+SBvg", - "1ZHU6y8ZvcWpPqC3fqvDL9PIsLLD7RSvzMamPZZidm6gt1Upo/q0tPNjL5fHjco4+DuMxVvUybQGZHc5", - "DX1j0vN3/L7zo8Ai665WsuxRbsoH/7CN09lOD8yyAO+DZdlD/H2M9I+Zl56Z6j4Jcv2UQUtqNYj1ucd1", - "NERHb4ikl3q0vxY5MwubPN+aNSY2yK10CZNvcI5coRhyHdXOHMZUNIdcY6roM5lRjTQ8wmPqmgOqMVU6", - "FtWQS62Cn2CKcXRmHeTBnI2jJifW4VgcGQEZIT9xZPg4gs1xpGc6XA7iqCaHawhrn4dRLidakvSiQ1X9", - "skBEH5mbFQfuIAdyxqWejFJws5Tquj6vCTsX8hweY3ILMyxrjuiIU0n3hKA7xMb1Z72T1h6c8507zlYw", - "2NeWRcvneOZoD3FH+p5tWEHo8cloh3PVwCDpoM3TmYLhzfJsdZ7XiB3TH7wqoPlu1YmA3cSeCKj4whYz", - "LqFYGENJDhjpCBhjRXFgzx2DJto0uCEFoAOyg7Uxy94da2NukIpvZoOVsdUYBo2aHAmYQgHD/VLKamZn", - "tt5aCt9ZXRRbotreXr/7AyU7rMIcpdhv7hjD/9JItee735bi6BYxtS+ODByy9SRLEBdHUKA51eEsbfsF", - "cXE8YBnJMj5LtM3zHlUkfHU0J2bXy6QrFqstHF2RC2Erpz2+Yb+AFpeN25Re8XF8Bc0yv+D5oirXJnGG", - "UlzmPQVO6V31tcv30Cy/KZOt0oRb+3yBHh1TROalDyoynCAb9L5+E14PRVGyrHvl+pDvFjHevdx72LbW", - "UrYs3/EKrmIZmq2u76GKo4KmHrQe573qjNF0lpsTRfYNF4UKD/sAcdYXJ+aYbKM2MV3JuwmZ7yHa3JVT", - "tFOMOozGYDGyg9uxGLnxri2+umFnQbC/GsQaOH1Vn4kKmk/OLq7+GcXRrydX5yenURwdXl6eTo4OrycX", - "51JuJldnXw6vTqI4+nT+6/nFl/M+4dkU0F6VRBqmNghyWuUgjTxZM3QAN4T0gVptb1DHJpRkSyBpQRvR", - "CTAHHIkYYFEdxUDAMZlbKpZmagLoUZ3Aim7CKDnFZEVSmZwlY4gIoLpnG5AfvkYzRnP1+9dI2mtcQCb0", - "8ZxuUdpxLYvONqKavaHS5qkNB5J01RHI0KonM8y4sCeNWaZSAqDoqN4aYq3fmowajrKw3E5VBdFshhKB", - "bxGQg5Tmdo6JO4vvmkcrlkTbsDtidDUJAN0XDHFuQ0DQPcwLuTyi/wE/g3+Af4B3XV6O2nA67PQFAgTd", - "V8PCHKxEEegAACAYns8RMw6fvUAPS5fUT99fnG1oAVVB8y1Chd5Qw1FntQOvgTq2D/7sm7zMBH5jIzvt", - "kvKEDel038EI0ckxl+tidXrqujhG602IpONivNdONFtN/pBtq0u20hyqL1MCC76gIpxWVUMHqTOxZlx7", - "ewVleIaSZSJxUxbSzhiJhWaO22rLceVJ9ce7Dys0qrUzn7PwlzKH5A1DMJVzbxM6gVQupIZH5iBFAuKM", - "A3hDS41nGZRIqQYhqvj/PS87rhDkXZEOZzBZYIKqxmPwqSgQO4I5yo4gR0BIzHF6Ittmili11ySU6F3w", - "r1x3q96h6pS24peczvSiFFEcXRB0wc4oQ+oQSTOySi+wvF9WDP5E0H2BEk3mnKrwj6q4zcHtnIDwPJje", - "XMb69r8OlJgtv40oKeYVrtYJ4xlQcGB23oqG3MVtLbUtEmr0Ab0dqIw6uYNj0R0M48sSu7+EDGYZyqaO", - "pZ6iGSwzER381BU0lMN7nJd5I8lP1jVeTkhUfzABhSGuBAjBZGGPAwyN6OCnt2o31v+863I3ew3IYWT7", - "AHOcYcTDEa5RY+XBsGF2Rwwp8Asn6a9sEqh1ws2QFu7VTRUVOmzpVHFiVSw7zhEtxRTJVc27UdTOtVJF", - "MAFcF9YTDY0IStyWCqnSICXGapn9SlzhpAzcoBllCNwgpUuWguZQ4ARm2VKinaSx95VEjjy8jbuuD+hZ", - "qb7jnic9vFlvlx0a6iMyjWvAEphuvG3I8vDwERC2U9gaTtOwMDYoLONzsV9h7sXD3NBEh4UwTzs18UY0", - "rvliTydrbgk7OORanAYh5KQiZY9/JZIhMeDUrJwFJHNUxfM6VVOqgkKhMsDVRyQRGpP5V3U8oRnx1MrR", - "84CQEZrP66L/E+o2YwM93AWyq1RZp82nz5Wty9H4DMS+ewuegUIZMnz/wNrw0V4g9a3B7BcGM8DMEHAS", - "RVaT3wL1kXegOBg17iITp95jrzJx+jDyohEXYQNS9R0H7NhLQ5yWHnGBh7MneMWlcWNIO2sBGL+LKydV", - "LFjbnhASo04cqWhjjSriRHT5SnRNtKfspeOG9hS5cubaU2S6miJPic+PvE1l0DEVbP0RY9Mpd2PTElSZ", - "WYbw03i9+1F52As+iJQbu/Dq5XjJh3ePJ/eah3VxN170sL78ybzqw0xZz8seaMSudPnghJLaPQJD2RON", - "xNmhzIn6HQUB7bdTc8P6Mfb2EZe9wVEp3TeANTXc3+kNP6J5kSFROwB2YFEWOUUzcU2vSuK567UdrjKg", - "NxQGQHRWstYiKAOYaKxTERigKFlBOeJ7lgnNABOpU0Vx9PnT6fnJ1eH7yenk+p9RHJ0dnpqwkunJ0dXJ", - "tfxpMj26OP8w+fjpykafXF1cXP86kR9P/u/y9GJy3XnONB0ynhuBA01l2irSNvm2fd8pvL9kOPEdfgi2", - "PIP3h0KgvPApIyVH04KKMXcEtKr41qgb6NyyjwbDhPX3aTiKOKW9WlGdYr1Hx1DAKwS71QH50d6H1vX9", - "hMwxQZ+98Ydy55spWP2AM596+Suhd+QzZiX3lTBdOMZM5WrjgXI9bU1LXgz1R+4i1/AbCg2oXOe+rt3e", - "1PU87uha/3qu9ba92jVJYTtfKws9YAes38YU1ht/1vu43o3fF6mJjW5eRSF/r3Lmli3MVd7GkFt1HW9j", - "ZwdMVmErDX0gawGR9IhmZe451EYktWFp7Y8znKHLzmwkybRaNpJJRLL6qPY5dEWUzDCZI1Yw3LXEz6lA", - "B9o+wFw5+rW3yxOawkTf0FQB3+D8LF4rLtfMzo7Dcp17Xtv75cqlEHoDjh7BOtFxNdfRo2P+anfxjAuU", - "Nfd2mmvc7GVA+nqYdS4UCjQwGhfuj7ytvrqpnptb+VUhXUKtAmlTbfr2+ms4X+NeJgHbnthvqDun6hZm", - "ZYDUy+q2cBdz7S2LTanCuS+Y3+q/nhsw7Gf3NpC+lVG/OsS9BMeXIJHBkiSLcYGHj0rIyKCQrXjy53Zx", - "I1YcCTgPpx50M48fHmtz7PCuMTexERKHQ7XJ6RS3Tjf2YzGtkeHfvq6Fh/OuRutI1gyYnSE1IcVcMDqq", - "6WNdRe3p96NqfsD3epksEZuknlQx8u2ReWLFKsstMC688OaoBuag1t3uTgKqe5ay9CdP9cvNkZGS5jYg", - "GE7GS82ZqafS4OxFXo/MkPM20ur1DeRomtDaOYwOgnAeOKjOL3zlcF7ARPi+D/bwuBL6xmmH+h0UGvO5", - "6yoy/l8IUiSUOQxOMSnvgVo/+KbsvtNucnyKv3XoBVJVnhz//+nk1xMwwyhLgbqrw56Ays/7SCT7lL9h", - "KEOQa237UWkANgbEr9C3R9S1YTmS0bj+TX/wUwN/y+HvVNnE6o+9HBPKgCH497AkE+91KME6ex2Td6y6", - "t/CwrcDbMxUf5zeeAt1+gqHVqY4z4fF71oZ6F3ZyvIqsavTdLLUCMWDh3XOofMSwirzpOIP1nNb+gueL", - "8NKn9C68sE4qDy9/juYZnuObDAXUGeZ7R1b80dXkenJ0eBrF0S+Tj79EcXR2cjz5dBbF0enFlyiOzk8+", - "nk4+Tt6fnnQ97KAUar1uzS1y0eezowwqw+3wcsIjB2uid3tv996alDcCCxwdRP+993bvXaR3bzWqfVh7", - "DGCu3SZVjpzUOKKPSDhPBsS11xo9uLEqsu8+9ufzjzWLmxf3Qovr15BCS1/TIrwj33B4YfM0Ymjx6pnB", - "3xpv3P309u3mHpVbTZz/aTut9Jqsk256VQf3a2/fPbiHkVJQRrzXoNPjeYfAXVJelzi5fSAu3tN0uXnO", - "2FcG3ScJH1pT8m5bDTfw2H5Ut3qZsF910PbzJqWi/6nBib5MzJlLwEvZWNWV/908N0ywX0d3jkxwngru", - "XHVJ7Ul7m5JddYKIVg/BVWefEPACJXiGUVq9lnb/JqEpmiPyxkjmmxuaLu37ofJvRd0B1/3vzvuyD2FQ", - "e1h7kXYc6rqv2W4JdS3c7QS+BtDr57c/72pxrFbo5Fi53ZUcbhRCXRncM8a2fge4gZPy5yeRF/s0sZ78", - "JwfnHQncp0JfsFgHCeOJnpVZtnx+SP0M1sVz2y5+fvfTrphyIuAcpDglfxX6bvqN7Vdq7dclMXBjiqOi", - "7FK6SvEKJa9Q8golfzoo0bLYVDtGarnD7oNX18ELdB3s1G0Q4hLYqjvgSVwBnQgICLqzT5I/G0fAVn0A", - "fhRWnwHMGILpUucw843b/etY9taqNxZ9ijKkjyHqsnusftfSe6jLr6dUSYXKs+L7x1+zlJ+H9OxWmdiK", - "ra7nVQ9vT7+E3Lf9PXrqX7gb55m5cLbnvqnkYdBtswuZ2Il99SS2Va9dZTCnZU89sZA9h23zhzJa9GLb", - "iOfjdTXufDW+6iKvmPC0mCCV+SojZZ9XqSs+Xa7K6jVZLs/Qp7ET74AZ/o68A1kG4C3EKgwKmEnqgfP2", - "JG0Da10W7A5s/Yw/zDLDG3CH9HXfNczd1IxMfTMSvuJmzvNZvpVWPbH16jZ8SW7Datp2Bw2ri256/YeO", - "QG0DDqq30nbrQ6w12+VFdJ8JfGI/ou3K1jyJ9YcHO3piCjgXfm3Yj2jHuAYY7n+3j7mG+BOtNNt47vFm", - "S9XaJryKO9Oh7Qxu06NnJ7HXq7fRCXi5vr0e/PnxBETuOGKBqhuOdICgKy19Pr/NL9kn3sV2IkWKdcjd", - "PJ7ejeDZyH4IGTeRRSupfqxz7VXs1xF76zt7FfvdiL11Ho2Ve6nB8fo9wT6Nwb1O+NWofUlGrTtzu7Nr", - "3QudB2zbumhtx9vlPI6xUwu32XKXkVt7VOPpDV23O1szdlsvr3RJptORLUfQNK6gXgM797+v/gmygR2p", - "nzo1R4Or2+yLMobd6d2qQVx7UavHKN7OjLxc67gfu35Moek2kpsS1GcoP5UUbft0fuweuis5tCZ2fdt6", - "enujZxt9Fqvlme3mP1JiU/MNx8e5IF4BZbeAYp0Xr4DyCijPJSpoHUSxBsqgW+fVofPyHDq7duXwPXBi", - "36KzlwDzxmMZ/Y+jD7qAtun8eQq3z4DD57l4erbq4hlA7217dfzyOBZDtXsn2LHD18w/5yb1/KW5cbbu", - "vxl03DyW4y/bTfPMHDS788zoW7mG9p0Bd832ZWcXttRTWFGDDplnYzg9qcW0bVNp/Db7w7lbNuNneUWC", - "TSJBzZPyigSvSLAbP0mwg0Td+M5u7RovWRYdRPuwwNHDbw//CQAA///6+mYY6McAAA==", + "H4sIAAAAAAAC/+xd63PbOJL/V1C8rZrdLdpO5uY+nL85tpNRjV9lOcltbaZuIbIlYUICHAC0rU35f98C", + "QFDgAyIpS7Kd+JstAo1X968faADfgoilGaNApQgOvwUZ5jgFCVz/h4UAOTpRfxIaHAYZlvMgDChOITgs", + "v4YBhz9zwiEODiXPIQxENIcUq2pykamiQnJCZ8HDQ2hqjSNMV9MtSgyjPSU0JnTmpbz8PowumaZYRvOS", + "6hxwDHxJdzTdO9cFWsgQKmEGXNNhMZb4mOVUlqT+zIEvlpT+EumvLXQmjCWA6ZLO6X2GaewlBObz6oFp", + "Qu9JIoF7CU3N5x6ELnkM/N3CS4mp75PFKlJhcL83Y3tFDUvQNjCGBCL/3AnzuUdPx19J5iejPvZZyRvm", + "JyJZJw0RYXrM6JT4GbZSZBjPilUiJoZL14MqLDJGBWhsGOdRBEL/GTEqwfA0zrKERFgSRg/+EIyq35Y0", + "/8JhGhwG/3WwBJ0D81UcFPSuizZMizGIiJNMkQsObZMoBSHwDBRbfKRfKbujp5wzvrGuHGVkVTeKNhHo", + "Rs1c64qKrlv38Fut5hFFbPIHRBLJOZaICMRB5pxCjAhFOElQhAUIxKZoikmScxD7QRhknGXAJTETb0d/", + "+C3ggONLmizs6jW5oPjFtKom7EiBa7NnJ/q/CQiEKdIAXPS02b6BfTplndOoCt6oDmho5kKOAfQyTBlP", + "sQwOgxhL2JMkhSBsIgGJWwEiwUMJcbglghgOqMuhERRRgnJ1Wm6YxAmieToBrlZFlzWLN8e3gOAWOOI5", + "RVPGkZwTYeZu2Qm3nTxNMV90ykGE6Xujp8S4qKIWEnhKKJYQX/YeuXf9j1maVqaj9v30nojCHmgufa9l", + "V6QcRvUx292cRHOUU/JnDihiVEiOCZUoYulEjZYwiiKcK5mQc11imhDDlOvy/jUkmq6YGxXglQPEnZJI", + "sqVkmF5HmKIJIKNnId6UoHi43uVhz9C3ztM92n0Ejz+42uifah5+962hqv8Y7qzwwEMYwH2WMGI4flXl", + "U1NON780N8UVZ0ovQdxmrnlXNMXJHebQ1ea5KWbbTImItEGQczOEzvq1CpYQB8FyHsGx6maedZG5rhYf", + "SyyhG1s5Y/Jrj4m9NuVs38SEpZ11xhOWlhUKhuhiuvrCC4g4dHdvrIuVjUksc9GL0VSVsSn+aA1wmycU", + "OJ6QhFiOX0Xlk1N8Ybr+sEqeVmL+uMf8LiXz2WK/XQ5oDlMbc/ovIsEwX0NiaZ4keJJArVnMOV5Yy+SG", + "YyqI6vyN0sm9bRRhuwU0TxX+XTDdXQrKh7sCzQ5BGIyjOcR5on+9Bhwvbpie8zAY0SvOZlzZ5GFwNGFc", + "6kInjIKDpL3nKG/hhb44WZvshzCYgWLGZHjFnijZUnEoUDZJ9EWvZk0FYGvU6odGzYoDoaFOYCUvtLBB", + "tMqwMCaSMSp0QYSjiHHFvsqIUqI8I7dAkYkriF52RSmT1RbPiJDKhHHbXN0awjRGGZ7BPtKVE6AzOUdp", + "LqQy5hJ2p4wijuDPHCeKgio7Jv8GBTtlL3rDoGdkBWJ4513bggp76eJyGhz+swPpz7WR+RCuLnbF4l7l", + "Tgg35X4Pg5io6daOh/GvU5xlCj8OvwW2XAeZMLANd/QrDIqBdA3TcuPiwgQzzORZ+7l9Rtfi4hYG3hr3", + "CkTok7Hreqx6nLA8vuLslsQmfGhV19HnsVJB/855m+YJl5xTX5KYcLOmbW4/M7Gc1o8OC7QGw1zHwinb", + "5mAUNn4Lv9yCiac1Wq8sa5vd4BuTMalP3rV+lEQm7dVynjzKUnnwD7uwO+3y4CTpAUB2yhSyVOdsY+uy", + "YqXanUHXVOklCMtBrD97wsRqW3pDFb3YE85vkCtWYZMu7rS2sL00yxWOvuIZuEzRpT0qbseQioWfO6SK", + "ccsGNVIzCofULXzUIVVahKpLq5bw05tiGJxbG7n3zIZBfSbWmbEwKBhkAP+EQTGPA6Y5DMxK9+eDMKjw", + "4RrMusrIUOLEclpEYqtK/fMcqImaFRKH7rBAasXZLXCI0WSBsDZVg/BxcXBCb3FC4mVIuE9HnEqmJxSU", + "CTGoP+sFW1bgnC/0MF3C4Kq2LFo+x7CDjeMMND9tZLGvBzXY5iwb6CTdS3k6S9CtLM+XLn1tZ8t88JqA", + "xXdrTvTQJtYp0Lufjcm4wnKuJkMNe0oSMEHwiFGJCRXIhh56LXTR4IYMgBbI7m2N2endsTXmxql9K9vb", + "GFuOoTPoloLEMZa4N20TVePntt5aBt95lRUbrNpUr9/827gtmz0pxMTv7ggTGLwquNrz3e9LCbgFrvXi", + "wL0DW09vSAp5jCXMmIloN/0XEPKkwzNSZVqdqtY5X2GK9JeO+sLsWkzatmOazNEWvOwnOc3xdYpQwS4b", + "9ym97OPECuplfiWzeVmuSeIcYpKnKwqcsbvya1vsoV5+Uy5baQk39HwGj95WoLPcBxUJicCm5KzfhDdC", + "keU8aZdcH/LdAhft4r5i2tYSZTvlO5bgMpxZb3X9CFUYZCz2oPWw6FXrNq0jbs5G0leSZXqH6D0myaqt", + "IsdlG6TETCWvEiq+97Hmrp2irWzU4jT2ZiM7uB2zkbvl3ZhXd+epF+wvB7EGTl9XV6KE5tPzy+t/BGHw", + "2+n1xelZEAZHV1dno+Ojm9HlheKb0fX556Pr0yAMPl78dnH5+WIV82wKaK9zqhxTuw86LjMk9YrHsd57", + "xcmV08YUJwLqWXwFHSQKQtoHRhXdsI9GU8RoskCKFrabuogIJECGiEh0R5IETUC59ITOLBVLMy5yaKBK", + "YEk34oyeEbokqV3OnHOgEunu2QbUhy/BlLNU//4lUP6akJhL/aloUflxDY/ONqKbnTDl81SGg2m87Ajm", + "sOyJTtozQ9L94DlFWLZUbwyx0m9DRg9He1hup8qCMJ1CJMktIDVI5W6nhLqr+LaeX2VJNB27Y86Wi4Dg", + "PuMglFLS2ZRwj9NMiUfwP+gX9Hf0d/S2LcpRGU6Lnz4HROG+HBYRaMmKSMxZnsRIcjKbAS8CPvs9Iyxt", + "XD9+d3m+IQEq82YahDKjUPujzlIDr4E6tg/+BLw0TyTZs5u7VqTsVLYl2510bhKPToSSi4mRVJNdsb7d", + "BDQelubRI7uvPZtvufhdvq0p2ch0Kr+MKc7EnMn+tMoaJk+FyzVTW5oSlJApRItI4aYqZIIxCguLNW6a", + "LSdlJNWf8tJt0OjWzn3Bwl/zFNM9DjhWa2/TzZEyLpSFR2coBolJIhCesNzgWYIVUupByDIFaN87HdeA", + "i5T0atPnOJoTCmXjIfqYZcCPcQrJMRaApMIcpyeqba6JlbomYtRowZ+E6Va1Q+UubTlfajnjy1wGYXBJ", + "4ZKfMw56E8lMZJlhZOd+UU7wRwr3GUSGzAWTc0JnZXF7QqB1AfqnwrkJoq0gshSNdaCkUPlNRImJKHG1", + "SphMkYaDQvOWNJQWt7W0WqSssAeMOtBJtUqDEydDvE+i6P0V5jhJIBk7nnoMU5wnMjj8OWyRrBTfkzRP", + "a3m+qm4R5cRU94dQlBXENQMBjuZ2O6CgERz+/EZrY/PP27Zws9eB7Ea29zglCQHRH+FqNZYRjBEVEtMI", + "jjlo8OtP0l+5ON5hcu66rHCvbaqpMI/zp+qwXI5BSa5oR0q7ntrcIBQJU9gsJi7YTGGzMjq1lahw1PDl", + "F+oyIONoAlPGAU1A24u5ZCmWJMJJslCIpmjsf6GBs+ZvwrYDTCuk0bel86QbNOtp0q6hPuJAQQU8ep4q", + "2DYseebwETC1U2jqzsayUNXJLMOPXPxwUNY5hy8C2roWt9/JlHGrhV07wVh8sbuOlXCDHRy4nmSBCmoh", + "QfvZX6iakBAJVkjLHNMZlOmDTtWYIcokwtqx1h9BoTKhsy9628FMxFMbPc8DNgZYND+koDeXQx94RrlQ", + "XMlQQlJiI1EsqzlyaCRVQfGFqgKXJ0c3R6g4Vl6QsVscVVIRSxKItGiUabKKhg4hFcRN/MgEawAp72Qf", + "vVerbqI8X+i/NHSIOyLnfy1P5O3bWH2IfoJ87w6E3Pv5p7/9y1CzPSia+EIlQ3/kQtYydcuK6OjzGHGY", + "KQdLy9SPYOENTWlxIWNX5wKcNp/+YEBVsoanW686pPUMzOo+w/cPrAmoTQGpKstCgxYoiqYFAaQEndCq", + "bd1QcwMPfDqoPezUplPvsec2nT4MPFXp6pwe55KcUPPQE5JOS484rehoSS+71I5HNm9aQEWEyeWTMuut", + "6VVJhVGnDlc0sUYXcXLXfCXaFtpT9soJuHuKXDtr7SkyXi6Rp8SnRx4d7QzB9faBaeHZ6sBq3R9WlIpr", + "J54ovr8albvj/Z1IubHT/S9nP6Bbezz5/kC/Lu5mv6BfX36w/YPuSVlvP6GnW++omX45Je1H+OtW2x9s", + "Io6ZclNkZfvWEXVV5Aym8oZd59Rzj1Qz2aRDF2aFUGjULDQj44hQI786fwJlOc+YALFvJ6GeHqLshCAM", + "Pn08uzi9Pno3Ohvd/CMIg/OjsyIpZHx6fH16o34ajY8vL96PPny8trkj15eXN7+N1MfT/7s6uxzdtO4S", + "jbtc5Nq2f91AtMYhKQg071LC91ecRL6tC8kX5/j+SEpIM5+CzQWMMyZtH4VnG97ltEYVH9+5acoNm78z", + "ydd8H/eXDKe0V9NXKVZ7dIIlvgbcruLUR3uhQdv3UzojFD55swcVmk81VLwnic9k+o2yO/qJ8Fz4ShRd", + "OCEcIsk46Si3oq1xLrKu/ihkvMFfoW865DoH7nd71P55HLJf/3y9PUnWOHrckakOND5mSZ56NjKBxjYV", + "qflxShK4aj2BooS3cgKlOHxiNbPxvtoiSlNCZ8AzTtoY44JJODSWEhE6CGz8fk86AperhqYL+Abnn+K1", + "cjGL1dlxKqZzvU8TZZfOVT/OtCNYJyOq4kQ/Os8LopwTufjAmbneaUByZHFdC4oSlseKCzUlNNOk6krU", + "+DQpoWdazt14umeFfKZW7QrIgfcnlncniuKeSF3IlNBSoKzLTd+neINnj5pbiWfoACV4AkmjZ1+h/WTN", + "LU7yHnKgqtvCbdNtr9uo8xlJfSnd1o7y3INgP7t3QqySleoFEg4Bb5p8gnMazYelnz0qLT/BUrXiOUXl", + "CNgghHDkskeSocSz/tQVP/YBHw9gVtbYmbva2oQFkzgzVFmcVnZrDfE9FuVq57ybl3aI/nNXoXWsavZY", + "nS7DISZCcjao6RNTRWv5+0E135N7IyYL4CPP9amEfn3kaaFsedapZ3Zw5j2p2PMkYjUk6RxDdOPMC/8R", + "mtV8c1xwSV0xSE6i4VxzXtTTh6Gi4oqHR56T8jbS6PUECxhHrBKjNlvmzk2XZWzXV46kGY6k73tnD09K", + "pq9FgvXvKDOYL9yQQxEbwygGqd0qdEZofo+0/JBJbsNP1dGOTs7I1xZLQRnPo5P/Pxv9doqmBJIY6Rsb", + "7O6Q+nwAMjpgYo9DAlgY+/tRyeA2Y8Bv4jdH1KawHM6okiqcYz819NcU/8G0b6X/2E8JZRwVBP/W76iB", + "91KM3lZ8FZN3bMw38LBp0tt4s2/mN34QtnkXZ6NTLftlw3XWhnrXb1dtmYdT63shahlwZOHds+F2zInO", + "SmjZn/LsZP1KZvP+pc/YXf/C5mhx//IXMEvIjEwS6FGne95bzkYfX49uRsdHZ0EY/Dr68GsQBuenJ6OP", + "50EYnF1+DsLg4vTD2ejD6N3ZadsNn9qgNnJb3CUWfDo/TrB25Y6uRiJwsCZ4u/9m/01x8InijASHwX/v", + "v9l/GxjtrUd1gCu3Qs5MIKU8KaUsjuADSOfuyLDyoogHN5ZFDtwHKXy3K9WLF69C9C1ursXuW/qGZf07", + "8pX0L1w839G3ePkUxu+1dxh+fvNmcw8fLBfO//yCMXqLswft9MoOHlTeZ3hwN2oUowy4uNMckhYtDHfF", + "RJXjlPoAId+xeLH5mbEvYbjPZjw0luTtthqu4bH9qO92KpJE9YbNL5vkitXPYYzMlVLOWiKRq8bKrvzv", + "5mejSIRq6c5xkbikE9+WXdI6aX9TvKt3omD5IkC5h4aRyCAiUwJxeW3+/V7EYpgB3Ss4c2/C4oV940b9", + "rak74HrwzXkD6aEf1B5VXk0ahrrui0tbQl0LdzuBrw70+uXNL7sSjqWEjk50IF7z4UYh1OXB/cLZNm9V", + "1XBS/fwk/GKfzzKL/+TgvCOG+5iZa/aqIFHEpqd5kiyeH1I/A7l4buril7c/72pSTiWeoZjE9CeJtMRs", + "TF9p2a9yYk/FFAZZ3mZ05fIVSl6h5BVKfjgoMbxYNzsGWrnd4YPX0MELDB3sNGzQJySw1XDAk4QCWhEQ", + "Ubizb9M9m0DAVmMAfhTWnxFOOOB4YU68io37/et49tarLzz6GBIw2xBV3j3RvxvuPSpfWF7DqFIGlUfi", + "V4+/4ik/D+7ZrTGxFV/drKsZ3r55EmuV+nv00r/wMM4zC+FsL3xT8kNn2GYXPLET/+pJfKuVflWBOQ1/", + "6omZ7Dmoze/KaTHCtpHIx6s07lwaX22RV0x4WkxQxvzUedLFZ8OVz768BjFeUhCjXLYdhTGSxLmSYGU0", + "w2GobSiC8v2e3UY0Ks22xTTcp6ueOKphu7K1uEb1MayWnhQFnKtZNhzVsGNcAwwPvtkHBvtENyw32+zS", + "4UZU2domYhw70+h2BbcZX7CLuDLGsNEFeLmRhhX48/0xiNI4cg7lXRQmXcnlllURiM2L7BNrsZ1wkZ46", + "cJXH0zs1HkX2XfB4keew5OrHuvqvbL8O21tP/pXtd8P21pUdyvfKghPVGx19FoN78eOrU/uSnFp35Xbn", + "17pXb3b4tlXW2gZCVi5z36mHW2+5zcmtXAL/9I6u252tObuNlwLaONPpyJb382uXha6BnQfflv/08oEd", + "rh87NQeDq9vsi3KG3eXdqkNceeVlhVO8nRV5ud7xauz6Ppmm3Umuc9AqR/mpuGjbe4VDdeiu+NC62FW1", + "9fT+xgo1+iyk5Zlp8+/pmEX9XbHHhSBeAWW3gGKDF6+A8goozyVHYR1EsQ5KZ1jnNaDz8gI6uw7liH10", + "at9RspeUitq15qsf7O0MAW0z+PMUYZ+OgM9zifRsNcTTgd7bjur4+XEohprwTu/AjljzNKwoDsK+tDDO", + "1uM3nYGbx874yw7TPLMAze4iM+aOoC690xGu2T7v7MKXegovqjMg82wcpyf1mLbtKg1Xs99duGUzcZZX", + "JNgkElQiKa9I8IoEu4mT9A6Q6Pun+a2V8ZwnwWFwgDMSPPz+8J8AAAD//5LkuNMatwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/backend/pkg/database/demo.go b/backend/pkg/database/demo.go index 59f72c46d3..0753a120df 100644 --- a/backend/pkg/database/demo.go +++ b/backend/pkg/database/demo.go @@ -45,67 +45,10 @@ const ( awsInstanceUSEast11 = "i-instance-1-from-us-east-1" ) -var regions = []models.AwsRegion{ - { - Name: awsRegionEUCentral1, - Vpcs: utils.PointerTo([]models.AwsVPC{ - { - Id: awsVPCEUCentral11, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGEUCentral111, - }, - }), - }, - { - Id: awsVPCEUCentral12, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGEUCentral121, - }, - }), - }, - }), - }, - { - Name: awsRegionUSEast1, - Vpcs: utils.PointerTo([]models.AwsVPC{ - { - Id: awsVPCUSEast11, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGUSEast111, - }, - }), - }, - { - Id: awsVPCUSEast12, - SecurityGroups: utils.PointerTo([]models.SecurityGroup{ - { - Id: awsSGUSEast121, - }, - { - Id: awsSGUSEast122, - }, - }), - }, - }), - }, -} - // nolint:gomnd,maintidx,cyclop func CreateDemoData(ctx context.Context, db types.Database) { logger := log.GetLoggerFromContextOrDiscard(ctx) - // Create scopes: - scopes, err := createScopes() - if err != nil { - logger.Fatalf("failed to create scopes FromAwsScope: %v", err) - } - if _, err := db.ScopesTable().SetScopes(scopes); err != nil { - logger.Fatalf("failed to save scopes: %v", err) - } - // Create scan configs: scanConfigs := createScanConfigs(ctx) for i, scanConfig := range scanConfigs { @@ -437,17 +380,6 @@ func createVMInfo(instanceID, location, image, instanceType, platform string, return &info } -func createScopes() (models.Scopes, error) { - scopesType := models.ScopeType{} - err := scopesType.FromAwsAccountScope(models.AwsAccountScope{ - Regions: utils.PointerTo(regions), - }) - // nolint:wrapcheck - return models.Scopes{ - ScopeInfo: &scopesType, - }, err -} - func createAssets() []models.Asset { return []models.Asset{ { @@ -513,9 +445,7 @@ func createAssets() []models.Asset { } } -func createScanConfigs(ctx context.Context) []models.ScanConfig { - logger := log.GetLoggerFromContextOrDiscard(ctx) - +func createScanConfigs(_ context.Context) []models.ScanConfig { // Scan config 1 scanFamiliesConfig1 := models.ScanFamiliesConfig{ Exploits: &models.ExploitsConfig{ @@ -540,56 +470,6 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Enabled: utils.PointerTo(true), }, } - tag1 := models.Tag{ - Key: "app", - Value: "my-app1", - } - tag2 := models.Tag{ - Key: "app", - Value: "my-app2", - } - tag3 := models.Tag{ - Key: "system", - Value: "sys1", - } - tag4 := models.Tag{ - Key: "system", - Value: "sys2", - } - ScanConfig1SecurityGroups := []models.SecurityGroup{ - { - Id: awsSGEUCentral111, - }, - } - ScanConfig1VPCs := []models.AwsVPC{ - { - Id: awsVPCEUCentral11, - SecurityGroups: &ScanConfig1SecurityGroups, - }, - } - ScanConfig1Regions := []models.AwsRegion{ - { - Name: awsRegionEUCentral1, - Vpcs: &ScanConfig1VPCs, - }, - } - scanConfig1SelectorTags := []models.Tag{tag1, tag2} - scanConfig1ExclusionTags := []models.Tag{tag3, tag4} - scope1 := models.AwsScanScope{ - AllRegions: utils.PointerTo(false), - InstanceTagExclusion: &scanConfig1ExclusionTags, - InstanceTagSelector: &scanConfig1SelectorTags, - ObjectType: "AwsScanScope", - Regions: &ScanConfig1Regions, - ShouldScanStoppedInstances: utils.PointerTo(false), - } - - var scanScopeType1 models.ScanScopeType - - err := scanScopeType1.FromAwsScanScope(scope1) - if err != nil { - logger.Fatalf("failed to convert scope1: %v", err) - } // Scan config 2 scanFamiliesConfig2 := models.ScanFamiliesConfig{ @@ -616,41 +496,6 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { }, } - ScanConfig2SecurityGroups := []models.SecurityGroup{ - { - Id: awsSGUSEast111, - }, - } - ScanConfig2VPCs := []models.AwsVPC{ - { - Id: awsVPCUSEast11, - SecurityGroups: &ScanConfig2SecurityGroups, - }, - } - ScanConfig2Regions := []models.AwsRegion{ - { - Name: awsRegionUSEast1, - Vpcs: &ScanConfig2VPCs, - }, - } - scanConfig2SelectorTags := []models.Tag{tag2} - scanConfig2ExclusionTags := []models.Tag{tag4} - scanConfig2Scope := models.AwsScanScope{ - AllRegions: utils.PointerTo(false), - InstanceTagExclusion: &scanConfig2ExclusionTags, - InstanceTagSelector: &scanConfig2SelectorTags, - ObjectType: "AwsScanScope", - Regions: &ScanConfig2Regions, - ShouldScanStoppedInstances: utils.PointerTo(true), - } - - var scanScopeType2 models.ScanScopeType - - err = scanScopeType2.FromAwsScanScope(scanConfig2Scope) - if err != nil { - logger.Fatalf("failed to convert scanConfig2Scope: %v", err) - } - return []models.ScanConfig{ { Name: utils.PointerTo("Scan Config 1"), @@ -658,7 +503,7 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Scheduled: &models.RuntimeScheduleScanConfig{ OperationTime: utils.PointerTo(time.Now().Add(5 * time.Hour)), }, - Scope: &scanScopeType1, + Scope: utils.PointerTo("startswith(targetInfo.location, 'eu-central-1')"), MaxParallelScanners: utils.PointerTo(2), }, { @@ -673,7 +518,7 @@ func createScanConfigs(ctx context.Context) []models.ScanConfig { Scheduled: &models.RuntimeScheduleScanConfig{ CronLine: utils.PointerTo("0 */4 * * *"), }, - Scope: &scanScopeType2, + Scope: utils.PointerTo("startswith(targetInfo.location, 'us-east-1')"), }, } } diff --git a/backend/pkg/database/gorm/database.go b/backend/pkg/database/gorm/database.go index cd472b2d5d..a335888d90 100644 --- a/backend/pkg/database/gorm/database.go +++ b/backend/pkg/database/gorm/database.go @@ -74,7 +74,6 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { AssetScan{}, ScanConfig{}, Scan{}, - Scopes{}, Finding{}, ); err != nil { return nil, fmt.Errorf("failed to run auto migration: %w", err) diff --git a/backend/pkg/database/gorm/odata.go b/backend/pkg/database/gorm/odata.go index b7fbf2edc7..49451f023f 100644 --- a/backend/pkg/database/gorm/odata.go +++ b/backend/pkg/database/gorm/odata.go @@ -322,9 +322,12 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ assetSchemaName: { Table: "assets", Fields: odatasql.Schema{ - "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "revision": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "scansCount": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "revision": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "firstSeen": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "lastSeen": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "terminatedOn": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, + "scansCount": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "assetInfo": odatasql.FieldMeta{ FieldType: odatasql.ComplexFieldType, ComplexFieldSchemas: []string{"VMInfo"}, @@ -405,11 +408,7 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ FieldType: odatasql.ComplexFieldType, ComplexFieldSchemas: []string{"RuntimeScheduleScanConfig"}, }, - "scope": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsScanScope"}, - DiscriminatorProperty: "objectType", - }, + "scope": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "maxParallelScanners": odatasql.FieldMeta{ FieldType: odatasql.PrimitiveFieldType, }, @@ -434,11 +433,7 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ FieldType: odatasql.ComplexFieldType, ComplexFieldSchemas: []string{"RuntimeScheduleScanConfig"}, }, - "scope": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsScanScope"}, - DiscriminatorProperty: "objectType", - }, + "scope": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "maxParallelScanners": odatasql.FieldMeta{ FieldType: odatasql.PrimitiveFieldType, }, @@ -531,91 +526,12 @@ var schemaMetas = map[string]odatasql.SchemaMeta{ "enabled": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, }, }, - scopesSchemaName: { - Table: "scopes", - Fields: odatasql.Schema{ - "scopeInfo": odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsAccountScope"}, - DiscriminatorProperty: "objectType", - }, - }, - }, - "AwsAccountScope": { - Fields: odatasql.Schema{ - "objectType": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "regions": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsRegion"}, - }, - }, - }, - }, - "AwsScanScope": { - Fields: odatasql.Schema{ - "objectType": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "allRegions": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "shouldScanStoppedInstances": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "instanceTagExclusion": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"Tag"}, - }, - }, - "instanceTagSelector": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"Tag"}, - }, - }, - "regions": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsRegion"}, - }, - }, - }, - }, "Tag": { Fields: odatasql.Schema{ "key": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, "value": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, }, }, - "AwsRegion": { - Fields: odatasql.Schema{ - "name": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "vpcs": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsVPC"}, - }, - }, - }, - }, - "AwsVPC": { - Fields: odatasql.Schema{ - "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - "securityGroups": odatasql.FieldMeta{ - FieldType: odatasql.CollectionFieldType, - CollectionItemMeta: &odatasql.FieldMeta{ - FieldType: odatasql.ComplexFieldType, - ComplexFieldSchemas: []string{"AwsSecurityGroup"}, - }, - }, - }, - }, - "AwsSecurityGroup": { - Fields: odatasql.Schema{ - "id": odatasql.FieldMeta{FieldType: odatasql.PrimitiveFieldType}, - }, - }, "Finding": { Table: "findings", Fields: odatasql.Schema{ diff --git a/backend/pkg/database/gorm/scope.go b/backend/pkg/database/gorm/scope.go deleted file mode 100644 index e9412bdb40..0000000000 --- a/backend/pkg/database/gorm/scope.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gorm - -import ( - "encoding/json" - "errors" - "fmt" - - "gorm.io/gorm" - - "github.com/openclarity/vmclarity/api/models" - "github.com/openclarity/vmclarity/backend/pkg/database/types" -) - -const ( - scopesSchemaName = "Scopes" -) - -type Scopes struct { - ODataObject -} - -type ScopesTableHandler struct { - DB *gorm.DB -} - -func (db *Handler) ScopesTable() types.ScopesTable { - return &ScopesTableHandler{ - DB: db.DB, - } -} - -func (s ScopesTableHandler) GetScopes(params models.GetDiscoveryScopesParams) (models.Scopes, error) { - var dbScopes Scopes - err := ODataQuery(s.DB, scopesSchemaName, params.Filter, params.Select, nil, nil, nil, nil, false, &dbScopes) - if err != nil { - return models.Scopes{}, err - } - - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return models.Scopes{}, types.ErrNotFound - } - return models.Scopes{}, err - } - - var scopes models.Scopes - err = json.Unmarshal(dbScopes.Data, &scopes) - if err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert DB model to API model: %w", err) - } - - return scopes, nil -} - -func (s ScopesTableHandler) SetScopes(scopes models.Scopes) (models.Scopes, error) { - marshaled, err := json.Marshal(scopes) - if err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert API model to DB model: %w", err) - } - - var dbScopes Scopes - dbScopes.Data = marshaled - - if err = s.DB.Save(&dbScopes).Error; err != nil { - return models.Scopes{}, fmt.Errorf("failed to save scopes in db: %w", err) - } - - // TODO(sambetts) Maybe this isn't required now because the DB isn't - // creating any of the data (like the ID) so we can just return the - // asset pre-marshal above. - var apiScopes models.Scopes - if err = json.Unmarshal(dbScopes.Data, &apiScopes); err != nil { - return models.Scopes{}, fmt.Errorf("failed to convert DB model to API model: %w", err) - } - - return apiScopes, nil -} diff --git a/backend/pkg/database/types/database.go b/backend/pkg/database/types/database.go index 5b17fe6dc7..9abfd61f4c 100644 --- a/backend/pkg/database/types/database.go +++ b/backend/pkg/database/types/database.go @@ -54,7 +54,6 @@ type Database interface { ScanConfigsTable() ScanConfigsTable ScansTable() ScansTable AssetsTable() AssetsTable - ScopesTable() ScopesTable FindingsTable() FindingsTable } @@ -102,11 +101,6 @@ type AssetsTable interface { DeleteAsset(assetID models.AssetID) error } -type ScopesTable interface { - GetScopes(params models.GetDiscoveryScopesParams) (models.Scopes, error) - SetScopes(scopes models.Scopes) (models.Scopes, error) -} - type FindingsTable interface { GetFindings(params models.GetFindingsParams) (models.Findings, error) GetFinding(findingID models.FindingID, params models.GetFindingsFindingIDParams) (models.Finding, error) diff --git a/backend/pkg/rest/discovery_controller.go b/backend/pkg/rest/discovery_controller.go deleted file mode 100644 index 54ee6a4d1c..0000000000 --- a/backend/pkg/rest/discovery_controller.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rest - -import ( - "fmt" - "net/http" - - "github.com/labstack/echo/v4" - - "github.com/openclarity/vmclarity/api/models" -) - -func (s *ServerImpl) GetDiscoveryScopes(ctx echo.Context, params models.GetDiscoveryScopesParams) error { - dbScopes, err := s.dbHandler.ScopesTable().GetScopes(params) - if err != nil { - return sendError(ctx, http.StatusInternalServerError, fmt.Sprintf("failed to get scopes from db: %v", err)) - } - - return sendResponse(ctx, http.StatusOK, dbScopes) -} - -func (s *ServerImpl) PutDiscoveryScopes(ctx echo.Context) error { - var scopes models.Scopes - err := ctx.Bind(&scopes) - if err != nil { - return sendError(ctx, http.StatusBadRequest, fmt.Errorf("failed to bind request: %v", err).Error()) - } - - updatedScopes, err := s.dbHandler.ScopesTable().SetScopes(scopes) - if err != nil { - return sendError(ctx, http.StatusInternalServerError, fmt.Errorf("failed to set scopes in db: %v", err).Error()) - } - - return sendResponse(ctx, http.StatusOK, &updatedScopes) -} diff --git a/runtime_scan/pkg/orchestrator/discovery/discoverer.go b/runtime_scan/pkg/orchestrator/discovery/discoverer.go new file mode 100644 index 0000000000..85b0b2dea1 --- /dev/null +++ b/runtime_scan/pkg/orchestrator/discovery/discoverer.go @@ -0,0 +1,141 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "context" + "errors" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/openclarity/vmclarity/api/models" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + "github.com/openclarity/vmclarity/shared/pkg/backendclient" + "github.com/openclarity/vmclarity/shared/pkg/utils" +) + +const ( + discoveryInterval = 2 * time.Minute +) + +type Discoverer struct { + backendClient *backendclient.BackendClient + providerClient provider.Provider +} + +func New(config Config) *Discoverer { + return &Discoverer{ + backendClient: config.Backend, + providerClient: config.Provider, + } +} + +func (sd *Discoverer) Start(ctx context.Context) { + go func() { + for { + log.Debug("Discovering available assets") + err := sd.DiscoverAndCreateAssets(ctx) + if err != nil { + log.Warnf("Failed to discover assets: %v", err) + } + + select { + case <-time.After(discoveryInterval): + log.Debug("Discovery interval elapsed") + case <-ctx.Done(): + log.Infof("Stop watching scan configs.") + return + } + } + }() +} + +func (sd *Discoverer) DiscoverAndCreateAssets(ctx context.Context) error { + discoveryTime := time.Now() + + assetTypes, err := sd.providerClient.DiscoverAssets(ctx) + if err != nil { + return fmt.Errorf("failed to discover assets from provider: %w", err) + } + + errs := []error{} + for _, assetType := range assetTypes { + assetData := models.Asset{ + AssetInfo: utils.PointerTo(assetType), + LastSeen: &discoveryTime, + FirstSeen: &discoveryTime, + } + _, err := sd.backendClient.PostAsset(ctx, assetData) + if err == nil { + continue + } + + var conflictError backendclient.AssetConflictError + if !errors.As(err, &conflictError) { + // If there is an error, and its not a conflict telling + // us that the asset already exists, then we need to + // keep track of it and log it as a failure to + // complete discovery. We don't fail instantly here + // because discovering the assets is a heavy operation + // so we want to give the best chance to create all the + // assets in the DB before failing. + errs = append(errs, fmt.Errorf("failed to post asset: %v", err)) + continue + } + + // As we got a conflict it means there is an existing asset + // which matches the unique properties of this asset, in this + // case we'll patch the just AssetInfo and FirstSeen instead. + assetData.FirstSeen = nil + err = sd.backendClient.PatchAsset(ctx, assetData, *conflictError.ConflictingAsset.Id) + if err != nil { + errs = append(errs, fmt.Errorf("failed to patch asset: %v", err)) + } + } + + // Find all assets which are not already terminatedOn and were not + // updated or created by this discovery run by comparing their + // lastSeen time to this discovery's time stamp + // + // TODO(sambetts) when we add multiple providers/standalone support we + // need to filter these assets by provider so that we don't find assets + // which don't belong to us. We need to give the provider some kind of + // identity in this case. + assetResp, err := sd.backendClient.GetAssets(ctx, models.GetAssetsParams{ + Filter: utils.PointerTo(fmt.Sprintf("terminatedOn eq null and (lastSeen eq null or lastSeen lt %s)", discoveryTime.Format(time.RFC3339))), + Select: utils.PointerTo("id"), + }) + if err != nil { + return fmt.Errorf("failed to get existing Assets: %w", err) + } + + // Patch all assets which were not found by this discovery as + // terminated by setting terminatedOn. + for _, asset := range *assetResp.Items { + assetData := models.Asset{ + TerminatedOn: &discoveryTime, + } + + err := sd.backendClient.PatchAsset(ctx, assetData, *asset.Id) + if err != nil { + errs = append(errs, fmt.Errorf("failed to patch asset: %v", err)) + } + } + + return errors.Join(errs...) +} diff --git a/runtime_scan/pkg/orchestrator/discovery/scope.go b/runtime_scan/pkg/orchestrator/discovery/scope.go deleted file mode 100644 index 1a1616cc66..0000000000 --- a/runtime_scan/pkg/orchestrator/discovery/scope.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright © 2023 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package discovery - -import ( - "context" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" - "github.com/openclarity/vmclarity/shared/pkg/backendclient" -) - -const ( - discoveryInterval = 2 * time.Minute -) - -type ScopeDiscoverer struct { - backendClient *backendclient.BackendClient - providerClient provider.Provider -} - -func New(config Config) *ScopeDiscoverer { - return &ScopeDiscoverer{ - backendClient: config.Backend, - providerClient: config.Provider, - } -} - -func (sd *ScopeDiscoverer) Start(ctx context.Context) { - go func() { - for { - log.Debug("Discovering available scopes") - // nolint:contextcheck - scopes, err := sd.providerClient.DiscoverScopes(ctx) - if err != nil { - log.Warnf("Failed to discover scopes: %v", err) - } else { - _, err := sd.backendClient.PutDiscoveryScopes(ctx, scopes) - if err != nil { - log.Warnf("Failed to set scopes: %v", err) - } - } - select { - case <-time.After(discoveryInterval): - log.Debug("Discovery interval elapsed") - case <-ctx.Done(): - log.Infof("Stop watching scan configs.") - return - } - } - }() -} diff --git a/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go b/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go index 74de33837f..449300a19c 100644 --- a/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go +++ b/runtime_scan/pkg/orchestrator/scanwatcher/watcher.go @@ -191,21 +191,37 @@ func (w *Watcher) reconcilePending(ctx context.Context, scan *models.Scan) error return errors.New("invalid Scan: Id is nil") } + // We don't want to scan terminated assets. These are historic assets + // and we'll just get a not found error during the asset scan. + assetFilter := "terminatedOn eq null" + scope, ok := scan.GetScanConfigScope() if !ok { return fmt.Errorf("invalid Scan: Scope is nil. ScanID=%s", scanID) } - assets, err := w.provider.DiscoverAssets(ctx, &scope) + // If the scan has a scope configured, 'and' it with the check for + // not terminated to make sure that we take both into account. + if scope != "" { + assetFilter = fmt.Sprintf("(%s) and (%s)", assetFilter, scope) + } + + assets, err := w.backend.GetAssets(ctx, models.GetAssetsParams{ + Filter: &assetFilter, + Select: utils.PointerTo("id"), + }) if err != nil { return fmt.Errorf("failed to discover Assets for Scan. ScanID=%s: %w", scanID, err) } - numOfAssets := len(assets) + + numOfAssets := len(*assets.Items) if numOfAssets > 0 { - if err = w.createAssets(ctx, scan, assets); err != nil { - return fmt.Errorf("failed to create Assets for Scan. ScanID=%s: %w", scanID, err) + assetIds := []string{} + for _, asset := range *assets.Items { + assetIds = append(assetIds, *asset.Id) } + scan.AssetIDs = &assetIds scan.State = utils.PointerTo(models.ScanStateDiscovered) scan.StateMessage = utils.PointerTo("Assets for Scan are successfully discovered") } else { @@ -229,68 +245,6 @@ func (w *Watcher) reconcilePending(ctx context.Context, scan *models.Scan) error return nil } -func (w *Watcher) createAssets(ctx context.Context, scan *models.Scan, assetTypes []models.AssetType) error { - logger := log.GetLoggerFromContextOrDiscard(ctx) - - var creatingAssetsFailed bool - var wg sync.WaitGroup - - results := make(chan string, len(assetTypes)) - for _, t := range assetTypes { - assetType := t - - wg.Add(1) - go func() { - defer wg.Done() - - assetID, err := w.createAsset(ctx, assetType) - if err != nil { - creatingAssetsFailed = true - return - } - - logger.WithField("AssetID", assetID).Trace("Pushing Asset to channel") - results <- assetID - }() - } - logger.Trace("Waiting until all Asset(s) are created") - wg.Wait() - close(results) - - if creatingAssetsFailed { - return fmt.Errorf("failed to create Asset(s) for Scan. ScanID=%s", *scan.Id) - } - - assetIDs := make([]string, 0) - for assetID := range results { - assetIDs = append(assetIDs, assetID) - } - scan.AssetIDs = &assetIDs - - logger.Tracef("Created Asset(s): %v", assetIDs) - - return nil -} - -func (w *Watcher) createAsset(ctx context.Context, assetType models.AssetType) (string, error) { - logger := log.GetLoggerFromContextOrDiscard(ctx) - - asset, err := w.backend.PostAsset(ctx, models.Asset{ - AssetInfo: &assetType, - }) - if err != nil { - var conErr backendclient.AssetConflictError - if errors.As(err, &conErr) { - logger.WithField("AssetID", *conErr.ConflictingAsset.Id).Trace("Asset already exist") - return *conErr.ConflictingAsset.Id, nil - } - return "", fmt.Errorf("failed to post Asset: %w", err) - } - logger.WithField("AssetID", *asset.Id).Debug("Asset object created") - - return *asset.Id, nil -} - func (w *Watcher) reconcileDiscovered(ctx context.Context, scan *models.Scan) error { logger := log.GetLoggerFromContextOrDiscard(ctx) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 4bd4981a7d..ce36c4b443 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -69,97 +69,28 @@ func (c Client) Kind() models.CloudProvider { return models.AWS } -func (c *Client) DiscoverScopes(ctx context.Context) (*models.Scopes, error) { - regions, err := c.ListAllRegions(ctx, true) - if err != nil { - return nil, fmt.Errorf("failed to list all regions: %w", err) - } - - scopes := models.ScopeType{} - err = scopes.FromAwsAccountScope(models.AwsAccountScope{ - Regions: convertToAPIRegions(regions), - }) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed to cast AwsAccountScope to ScopeType: %w", err), - } - } - - return &models.Scopes{ - ScopeInfo: &scopes, - }, nil -} - // nolint:cyclop -func (c *Client) DiscoverAssets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.AssetType, error) { - var filters []ec2types.Filter - - awsScanScope, err := scanScope.AsAwsScanScope() +func (c *Client) DiscoverAssets(ctx context.Context) ([]models.AssetType, error) { + regions, err := c.ListAllRegions(ctx) if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed to cast ScanScopeType to AwsAccountScope: %w", err), - } + return nil, fmt.Errorf("failed to get regions: %w", err) } - scope := convertFromAPIScanScope(&awsScanScope) - - regions, err := c.getRegionsToScan(ctx, scope) - if err != nil { - return nil, fmt.Errorf("failed to get regions to scan: %w", err) - } - if len(regions) == 0 { - return nil, FatalError{ - Err: errors.New("no regions to scan"), - } - } - - filters = append(filters, EC2FiltersFromTags(scope.TagSelector)...) - - instanceStates := []ec2types.InstanceStateName{ec2types.InstanceStateNameRunning} - if scope.ScanStopped { - instanceStates = append(instanceStates, ec2types.InstanceStateNameStopped) - } - filters = append(filters, EC2FiltersFromInstanceState(instanceStates...)...) - assets := make([]models.AssetType, 0) for _, region := range regions { - // if no vpcs, that mean that we don't need any vpc filters - if len(region.VPCs) == 0 { - instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.Name) - if err != nil { - return nil, fmt.Errorf("failed to get instances: %w", err) - } - - for _, instance := range instances { - asset, err := getVMInfoFromInstance(instance) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed convert EC2 Instance to AssetType: %w", err), - } - } - assets = append(assets, asset) - } - - continue + instances, err := c.GetInstances(ctx, []ec2types.Filter{}, region.Name) + if err != nil { + return nil, fmt.Errorf("failed to get instances: %w", err) } - // need to do a per vpc call for DescribeInstances - for _, vpc := range region.VPCs { - vpcFilters := append(filters, createVPCFilters(vpc)...) - - instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.Name) + for _, instance := range instances { + asset, err := getVMInfoFromInstance(instance) if err != nil { - return nil, fmt.Errorf("failed to get instances: %w", err) - } - for _, instance := range instances { - asset, err := getVMInfoFromInstance(instance) - if err != nil { - return nil, FatalError{ - Err: fmt.Errorf("failed convert EC2 Instance to AssetType: %w", err), - } + return nil, FatalError{ + Err: fmt.Errorf("failed convert EC2 Instance to AssetType: %w", err), } - assets = append(assets, asset) } + assets = append(assets, asset) } } @@ -787,48 +718,59 @@ func (c *Client) RemoveAssetScan(ctx context.Context, config *provider.ScanJobCo return err } -func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []models.Tag, regionID string) ([]Instance, error) { +func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, regionID string) ([]Instance, error) { ret := make([]Instance, 0) - out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: filters, + input := &ec2.DescribeInstancesInput{ MaxResults: utils.PointerTo[int32](maxResults), // TODO what will be a good number? - }, func(options *ec2.Options) { + } + if len(filters) > 0 { + input.Filters = filters + } + + out, err := c.ec2Client.DescribeInstances(ctx, input, func(options *ec2.Options) { options.Region = regionID }) if err != nil { return nil, fmt.Errorf("failed to describe instances: %v", err) } - ret = append(ret, c.getInstancesFromDescribeInstancesOutput(ctx, out, excludeTags, regionID)...) + ret = append(ret, c.getInstancesFromDescribeInstancesOutput(ctx, out, regionID)...) // use pagination // TODO we can make it better by not saving all results in memory. See https://github.com/openclarity/vmclarity/pull/3#discussion_r1021656861 for out.NextToken != nil { - out, err = c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - Filters: filters, - MaxResults: utils.PointerTo[int32](maxResults), + input := &ec2.DescribeInstancesInput{ + MaxResults: utils.PointerTo[int32](maxResults), // TODO what will be a good number? NextToken: out.NextToken, - }, func(options *ec2.Options) { + } + if len(filters) > 0 { + input.Filters = filters + } + + out, err = c.ec2Client.DescribeInstances(ctx, input, func(options *ec2.Options) { options.Region = regionID }) if err != nil { return nil, fmt.Errorf("failed to describe instances: %v", err) } - ret = append(ret, c.getInstancesFromDescribeInstancesOutput(ctx, out, excludeTags, regionID)...) + ret = append(ret, c.getInstancesFromDescribeInstancesOutput(ctx, out, regionID)...) } return ret, nil } -func (c *Client) getInstancesFromDescribeInstancesOutput(ctx context.Context, result *ec2.DescribeInstancesOutput, excludeTags []models.Tag, regionID string) []Instance { +func (c *Client) getInstancesFromDescribeInstancesOutput(ctx context.Context, result *ec2.DescribeInstancesOutput, regionID string) []Instance { logger := log.GetLoggerFromContextOrDiscard(ctx) var ret []Instance for _, reservation := range result.Reservations { for _, instance := range reservation.Instances { - if hasExcludeTags(excludeTags, instance.Tags) { + // Ignore terminated instances they are destroyed and + // will be garbage collected by AWS. + if instance.State.Name == ec2types.InstanceStateNameTerminated { continue } + if err := validateInstanceFields(instance); err != nil { logger.Errorf("Instance validation failed. instance id=%v: %v", getPointerValOrEmpty(instance.InstanceId), err) continue @@ -852,17 +794,7 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(ctx context.Context, re return ret } -func (c *Client) getRegionsToScan(ctx context.Context, scope *ScanScope) ([]Region, error) { - if scope.AllRegions { - return c.ListAllRegions(ctx, false) - } - - return scope.Regions, nil -} - -func (c *Client) ListAllRegions(ctx context.Context, isRecursive bool) ([]Region, error) { - logger := log.GetLoggerFromContextOrDiscard(ctx) - +func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { ret := make([]Region, 0) out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? @@ -877,40 +809,5 @@ func (c *Client) ListAllRegions(ctx context.Context, isRecursive bool) ([]Region }) } - if isRecursive { - for i, region := range ret { - // List region VPCs - vpcs, err := c.ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ - MaxResults: utils.PointerTo[int32](maxResults), - }, func(options *ec2.Options) { - options.Region = region.Name - }) - if err != nil { - logger.Errorf("Failed to describe vpcs. Region=%s: %v", region.Name, err) - continue - } - ret[i].VPCs = convertAwsVPCs(vpcs.Vpcs) - for i2, vpc := range ret[i].VPCs { - // List VPC's security groups - securityGroups, err := c.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ - Filters: []ec2types.Filter{ - { - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpc.ID}, - }, - }, - MaxResults: utils.PointerTo[int32](maxResults), - }, func(options *ec2.Options) { - options.Region = region.Name - }) - if err != nil { - logger.Errorf("Failed to describe security groups. Region=%s, VpcID=%s: %v", region.Name, vpc.ID, err) - continue - } - ret[i].VPCs[i2].SecurityGroups = getSecurityGroupsFromEC2SecurityGroups(securityGroups.SecurityGroups) - } - } - } - return ret, nil } diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 6f20d47393..4b57bd5b01 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -29,287 +29,6 @@ import ( "github.com/openclarity/vmclarity/shared/pkg/utils" ) -func Test_createVPCFilters(t *testing.T) { - var ( - vpcID = "vpc-1" - sgID1 = "sg-1" - sgID2 = "sg-2" - ) - - type args struct { - vpc VPC - } - tests := []struct { - name string - args args - want []ec2types.Filter - }{ - { - name: "vpc with no security group", - args: args{ - vpc: VPC{ - ID: vpcID, - SecurityGroups: nil, - }, - }, - want: []ec2types.Filter{ - { - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpcID}, - }, - }, - }, - { - name: "vpc with one security group", - args: args{ - vpc: VPC{ - ID: vpcID, - SecurityGroups: []models.SecurityGroup{ - { - Id: sgID1, - }, - }, - }, - }, - want: []ec2types.Filter{ - { - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpcID}, - }, - { - Name: utils.PointerTo(SecurityGroupIDFilterName), - Values: []string{sgID1}, - }, - }, - }, - { - name: "vpc with two security groups", - args: args{ - vpc: VPC{ - ID: vpcID, - SecurityGroups: []models.SecurityGroup{ - { - Id: sgID1, - }, - { - Id: sgID2, - }, - }, - }, - }, - want: []ec2types.Filter{ - { - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpcID}, - }, - { - Name: utils.PointerTo(SecurityGroupIDFilterName), - Values: []string{sgID1, sgID2}, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := createVPCFilters(tt.args.vpc); !reflect.DeepEqual(got, tt.want) { - t.Errorf("createVPCFilters() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_createInclusionTagsFilters(t *testing.T) { - var ( - tagName = "foo" - filterTagName = "tag:" + tagName - tagVal = "bar" - ) - - type args struct { - tags []models.Tag - } - tests := []struct { - name string - args args - want []ec2types.Filter - }{ - { - name: "no tags", - args: args{ - tags: nil, - }, - want: []ec2types.Filter{}, - }, - { - name: "1 tag", - args: args{ - tags: []models.Tag{ - { - Key: tagName, - Value: tagVal, - }, - }, - }, - want: []ec2types.Filter{ - { - Name: &filterTagName, - Values: []string{tagVal}, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := createInclusionTagsFilters(tt.args.tags); !reflect.DeepEqual(got, tt.want) { - t.Errorf("createInclusionTagsFilters() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_hasExcludedTags(t *testing.T) { - var ( - tagName1 = "foo1" - tagName2 = "foo2" - tagVal1 = "bar1" - tagVal2 = "bar2" - ) - - type args struct { - excludeTags []models.Tag - instanceTags []ec2types.Tag - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "instance has no tags", - args: args{ - excludeTags: []models.Tag{ - { - Key: tagName1, - Value: tagVal1, - }, - { - Key: "stam1", - Value: "stam2", - }, - }, - instanceTags: nil, - }, - want: false, - }, - { - name: "empty excluded tags", - args: args{ - excludeTags: nil, - instanceTags: []ec2types.Tag{ - { - Key: &tagName1, - Value: &tagVal1, - }, - { - Key: &tagName2, - Value: &tagVal2, - }, - }, - }, - want: false, - }, - { - name: "instance does not have ALL the excluded tags (partial matching)", - args: args{ - excludeTags: []models.Tag{ - { - Key: tagName1, - Value: tagVal1, - }, - { - Key: "stam1", - Value: "stam2", - }, - }, - instanceTags: []ec2types.Tag{ - { - Key: &tagName1, - Value: &tagVal1, - }, - { - Key: &tagName2, - Value: &tagVal2, - }, - }, - }, - want: false, - }, - { - name: "instance has ALL excluded tags", - args: args{ - excludeTags: []models.Tag{ - { - Key: tagName1, - Value: tagVal1, - }, - { - Key: tagName2, - Value: tagVal2, - }, - }, - instanceTags: []ec2types.Tag{ - { - Key: &tagName1, - Value: &tagVal1, - }, - { - Key: &tagName2, - Value: &tagVal2, - }, - { - Key: utils.PointerTo("stam"), - Value: utils.PointerTo("stam"), - }, - }, - }, - want: true, - }, - { - name: "instance does not have excluded tags at all", - args: args{ - excludeTags: []models.Tag{ - { - Key: "stam1", - Value: "stam2", - }, - { - Key: "stam3", - Value: "stam4", - }, - }, - instanceTags: []ec2types.Tag{ - { - Key: &tagName1, - Value: &tagVal1, - }, - { - Key: &tagName2, - Value: &tagVal2, - }, - }, - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := hasExcludeTags(tt.args.excludeTags, tt.args.instanceTags); got != tt.want { - t.Errorf("hasExcludeTags() = %v, want %v", got, tt.want) - } - }) - } -} - func Test_getInstanceState(t *testing.T) { type args struct { result *ec2.DescribeInstancesOutput @@ -435,9 +154,8 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { type fields struct{} type args struct { - result *ec2.DescribeInstancesOutput - excludeTags []models.Tag - regionID string + result *ec2.DescribeInstancesOutput + regionID string } tests := []struct { name string @@ -451,8 +169,7 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { result: &ec2.DescribeInstancesOutput{ Reservations: []ec2types.Reservation{}, }, - excludeTags: nil, - regionID: "region-1", + regionID: "region-1", }, want: nil, }, @@ -471,6 +188,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-1"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc1"), ImageId: utils.PointerTo("image1"), Placement: &ec2types.Placement{ @@ -493,6 +213,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Value: utils.PointerTo("val-2"), }, }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, VpcId: utils.PointerTo("vpc2"), ImageId: utils.PointerTo("image2"), Placement: &ec2types.Placement{ @@ -518,6 +241,9 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { Placement: &ec2types.Placement{ AvailabilityZone: utils.PointerTo("az3"), }, + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, InstanceType: "t2.large", PlatformDetails: utils.PointerTo("linux"), LaunchTime: utils.PointerTo(launchTime), @@ -531,8 +257,7 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { }, }, }, - excludeTags: nil, - regionID: "region-1", + regionID: "region-1", }, want: []Instance{ { @@ -589,125 +314,6 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { }, }, }, - { - name: "one excluded instance", - args: args{ - result: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.PointerTo("instance-1"), - Tags: []ec2types.Tag{ - { - Key: utils.PointerTo("key-1"), - Value: utils.PointerTo("val-1"), - }, - }, - VpcId: utils.PointerTo("vpc1"), - ImageId: utils.PointerTo("image1"), - Placement: &ec2types.Placement{ - AvailabilityZone: utils.PointerTo("az1"), - }, - InstanceType: "t2.large", - PlatformDetails: utils.PointerTo("linux"), - LaunchTime: utils.PointerTo(launchTime), - SecurityGroups: []ec2types.GroupIdentifier{ - { - GroupId: utils.PointerTo("group1"), - }, - }, - }, - { - InstanceId: utils.PointerTo("instance-2"), - Tags: []ec2types.Tag{ - { - Key: utils.PointerTo("key-2"), - Value: utils.PointerTo("val-2"), - }, - }, - VpcId: utils.PointerTo("vpc2"), - ImageId: utils.PointerTo("image2"), - Placement: &ec2types.Placement{ - AvailabilityZone: utils.PointerTo("az2"), - }, - InstanceType: "t2.large", - PlatformDetails: utils.PointerTo("linux"), - LaunchTime: utils.PointerTo(launchTime), - SecurityGroups: []ec2types.GroupIdentifier{ - { - GroupId: utils.PointerTo("group2"), - }, - }, - }, - }, - }, - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.PointerTo("instance-3"), - VpcId: utils.PointerTo("vpc3"), - ImageId: utils.PointerTo("image3"), - Placement: &ec2types.Placement{ - AvailabilityZone: utils.PointerTo("az3"), - }, - InstanceType: "t2.large", - PlatformDetails: utils.PointerTo("linux"), - LaunchTime: utils.PointerTo(launchTime), - SecurityGroups: []ec2types.GroupIdentifier{ - { - GroupId: utils.PointerTo("group3"), - }, - }, - }, - }, - }, - }, - }, - excludeTags: []models.Tag{ - { - Key: "key-1", - Value: "val-1", - }, - }, - regionID: "region-1", - }, - want: []Instance{ - { - ID: "instance-2", - Region: "region-1", - VpcID: "vpc2", - SecurityGroups: []models.SecurityGroup{ - {Id: "group2"}, - }, - AvailabilityZone: "az2", - Image: "image2", - InstanceType: "t2.large", - Platform: "linux", - Tags: []models.Tag{ - { - Key: "key-2", - Value: "val-2", - }, - }, - LaunchTime: launchTime, - }, - { - ID: "instance-3", - Region: "region-1", - VpcID: "vpc3", - SecurityGroups: []models.SecurityGroup{ - {Id: "group3"}, - }, - AvailabilityZone: "az3", - Image: "image3", - InstanceType: "t2.large", - Platform: "linux", - Tags: nil, - LaunchTime: launchTime, - }, - }, - }, } ctx := context.Background() @@ -715,7 +321,7 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &Client{} - got := c.getInstancesFromDescribeInstancesOutput(ctx, tt.args.result, tt.args.excludeTags, tt.args.regionID) + got := c.getInstancesFromDescribeInstancesOutput(ctx, tt.args.result, tt.args.regionID) var gotInstances []Instance for _, instance := range got { diff --git a/runtime_scan/pkg/provider/aws/helpers.go b/runtime_scan/pkg/provider/aws/helpers.go index 38371fcd2b..49b966688b 100644 --- a/runtime_scan/pkg/provider/aws/helpers.go +++ b/runtime_scan/pkg/provider/aws/helpers.go @@ -21,7 +21,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/api/models" "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" @@ -141,86 +140,6 @@ func instanceFromEC2Instance(i *ec2types.Instance, client *ec2.Client, region st } } -func convertFromAPIScanScope(scope *models.AwsScanScope) *ScanScope { - var tagSelector []models.Tag - if scope.InstanceTagSelector != nil { - tagSelector = *scope.InstanceTagSelector - } - - var excludeTags []models.Tag - if scope.InstanceTagExclusion != nil { - excludeTags = *scope.InstanceTagExclusion - } - - return &ScanScope{ - AllRegions: convertBool(scope.AllRegions), - Regions: convertFromAPIRegions(scope.Regions), - ScanStopped: convertBool(scope.ShouldScanStoppedInstances), - TagSelector: tagSelector, - ExcludeTags: excludeTags, - } -} - -func convertFromAPIRegions(regions *[]models.AwsRegion) []Region { - var ret []Region - if regions != nil { - for _, region := range *regions { - ret = append(ret, Region{ - Name: region.Name, - VPCs: convertFromAPIVPCs(region.Vpcs), - }) - } - } - - return ret -} - -func convertFromAPIVPCs(vpcs *[]models.AwsVPC) []VPC { - if vpcs == nil { - return nil - } - ret := make([]VPC, len(*vpcs)) - for i, vpc := range *vpcs { - ret[i] = VPC{ - ID: vpc.Id, - SecurityGroups: *vpc.SecurityGroups, - } - } - - return ret -} - -func convertToAPIRegions(regions []Region) *[]models.AwsRegion { - ret := make([]models.AwsRegion, len(regions)) - for i := range regions { - ret[i] = models.AwsRegion{ - Name: regions[i].Name, - Vpcs: convertToAPIVPCs(regions[i].VPCs), - } - } - - return &ret -} - -func convertToAPIVPCs(vpcs []VPC) *[]models.AwsVPC { - ret := make([]models.AwsVPC, len(vpcs)) - for i := range vpcs { - ret[i] = models.AwsVPC{ - Id: vpcs[i].ID, - SecurityGroups: utils.PointerTo(vpcs[i].SecurityGroups), - } - } - - return &ret -} - -func convertBool(all *bool) bool { - if all != nil { - return *all - } - return false -} - func getTagsFromECTags(tags []ec2types.Tag) []models.Tag { if len(tags) == 0 { return nil @@ -295,55 +214,6 @@ func getSecurityGroupsIDs(sg []ec2types.GroupIdentifier) []models.SecurityGroup return securityGroups } -func getVPCSecurityGroupsIDs(vpc VPC) []string { - sgs := make([]string, len(vpc.SecurityGroups)) - for i, sg := range vpc.SecurityGroups { - sgs[i] = sg.Id - } - return sgs -} - -func createVPCFilters(vpc VPC) []ec2types.Filter { - ret := make([]ec2types.Filter, 0) - - // create per vpc filters - ret = append(ret, ec2types.Filter{ - Name: utils.PointerTo(VpcIDFilterName), - Values: []string{vpc.ID}, - }) - sgs := getVPCSecurityGroupsIDs(vpc) - if len(sgs) > 0 { - ret = append(ret, ec2types.Filter{ - Name: utils.PointerTo(SecurityGroupIDFilterName), - Values: sgs, - }) - } - - log.Infof("VPC filter created: %+v", ret) - - return ret -} - -func EC2FiltersFromInstanceState(states ...ec2types.InstanceStateName) []ec2types.Filter { - values := make([]string, 0, len(states)) - for _, state := range states { - values = append(values, string(state)) - } - - return []ec2types.Filter{ - { - Name: utils.PointerTo(InstanceStateFilterName), - Values: values, - }, - } -} - -// If you specify multiple filters, the filters are joined with an AND, and the request returns -// only results that match all the specified filters. -func createInclusionTagsFilters(tags []models.Tag) []ec2types.Filter { - return EC2FiltersFromTags(tags) -} - func getSecurityGroupsFromEC2GroupIdentifiers(identifiers []ec2types.GroupIdentifier) []models.SecurityGroup { var ret []models.SecurityGroup @@ -358,62 +228,6 @@ func getSecurityGroupsFromEC2GroupIdentifiers(identifiers []ec2types.GroupIdenti return ret } -func getSecurityGroupsFromEC2SecurityGroups(groups []ec2types.SecurityGroup) []models.SecurityGroup { - var ret []models.SecurityGroup - - for _, securityGroup := range groups { - if securityGroup.GroupId != nil { - ret = append(ret, models.SecurityGroup{ - Id: *securityGroup.GroupId, - }) - } - } - - return ret -} - -func convertAwsVPCs(vpcs []ec2types.Vpc) []VPC { - var ret []VPC - for _, vpc := range vpcs { - if vpc.VpcId != nil { - ret = append(ret, VPC{ - ID: *vpc.VpcId, - SecurityGroups: nil, - }) - } - } - - return ret -} - -// AND logic - if excludeTags = {tag1:val1, tag2:val2}, -// then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}). -func hasExcludeTags(excludeTags []models.Tag, instanceTags []ec2types.Tag) bool { - instanceTagsMap := make(map[string]string) - - if len(excludeTags) == 0 { - return false - } - if len(instanceTags) == 0 { - return false - } - - for _, tag := range instanceTags { - instanceTagsMap[*tag.Key] = *tag.Value - } - - for _, tag := range excludeTags { - val, ok := instanceTagsMap[tag.Key] - if !ok { - return false - } - if !(strings.Compare(val, tag.Value) == 0) { - return false - } - } - return true -} - func getVMInfoFromInstance(i Instance) (models.AssetType, error) { assetType := models.AssetType{} err := assetType.FromVMInfo(models.VMInfo{ diff --git a/runtime_scan/pkg/provider/azure/client.go b/runtime_scan/pkg/provider/azure/client.go index 294e878b29..9afcd4f1a1 100644 --- a/runtime_scan/pkg/provider/azure/client.go +++ b/runtime_scan/pkg/provider/azure/client.go @@ -178,74 +178,21 @@ func (c *Client) RemoveAssetScan(ctx context.Context, config *provider.ScanJobCo return nil } -func (c *Client) DiscoverScopes(ctx context.Context) (*models.Scopes, error) { - var ret models.Scopes - ret.ScopeInfo = &models.ScopeType{} - resourceGroups := []models.AzureResourceGroup{} - - // discover all resource groups in the user subscription - res := c.rgClient.NewListPager(nil) +// nolint: cyclop +func (c *Client) DiscoverAssets(ctx context.Context) ([]models.AssetType, error) { + var ret []models.AssetType + // list all vms in all resourceGroups in the subscription + res := c.vmClient.NewListAllPager(nil) for res.More() { page, err := res.NextPage(ctx) if err != nil { return nil, fmt.Errorf("failed to get next page: %w", err) } - for _, rg := range page.Value { - resourceGroups = append(resourceGroups, models.AzureResourceGroup{Name: *rg.Name}) - } - } - - err := ret.ScopeInfo.FromAzureSubscriptionScope(models.AzureSubscriptionScope{ - ResourceGroups: &resourceGroups, - SubscriptionID: &c.azureConfig.SubscriptionID, - }) - if err != nil { - return nil, fmt.Errorf("failed to convert from azure subscription scope: %v", err) - } - - return &ret, nil -} - -// nolint: cyclop -func (c *Client) DiscoverAssets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.AssetType, error) { - var ret []models.AssetType - - azureScanScope, err := scanScope.AsAzureScanScope() - if err != nil { - return nil, fmt.Errorf("failed to convert as azure scan scope: %v", err) - } - - if azureScanScope.AllResourceGroups != nil && *azureScanScope.AllResourceGroups { - // list all vms in all resourceGroups in the subscription - res := c.vmClient.NewListAllPager(nil) - for res.More() { - page, err := res.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get next page: %w", err) - } - ts, err := processVirtualMachineListIntoAssetTypes(page.VirtualMachineListResult, azureScanScope) - if err != nil { - return nil, err - } - ret = append(ret, ts...) - } - return ret, nil - } - - // if scan scope is only for specific resource groups and not all: - for _, resourceGroup := range *azureScanScope.ResourceGroups { - res := c.vmClient.NewListPager(resourceGroup.Name, nil) - for res.More() { - page, err := res.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get next page: %w", err) - } - ts, err := processVirtualMachineListIntoAssetTypes(page.VirtualMachineListResult, azureScanScope) - if err != nil { - return nil, err - } - ret = append(ret, ts...) + ts, err := processVirtualMachineListIntoAssetTypes(page.VirtualMachineListResult) + if err != nil { + return nil, err } + ret = append(ret, ts...) } return ret, nil } @@ -263,16 +210,9 @@ func resourceGroupAndNameFromInstanceID(instanceID string) (string, string, erro return idParts[resourceGroupPartIdx], idParts[vmNamePartIdx], nil } -func processVirtualMachineListIntoAssetTypes(vmList armcompute.VirtualMachineListResult, azureScanScope models.AzureScanScope) ([]models.AssetType, error) { +func processVirtualMachineListIntoAssetTypes(vmList armcompute.VirtualMachineListResult) ([]models.AssetType, error) { ret := make([]models.AssetType, 0, len(vmList.Value)) for _, vm := range vmList.Value { - // filter by tags: - if !hasIncludeTags(vm, azureScanScope.InstanceTagSelector) { - continue - } - if hasExcludeTags(vm, azureScanScope.InstanceTagExclusion) { - continue - } info, err := getVMInfoFromVirtualMachine(vm) if err != nil { return nil, fmt.Errorf("unable to convert instance to vminfo: %w", err) @@ -303,56 +243,6 @@ func getVMInfoFromVirtualMachine(vm *armcompute.VirtualMachine) (models.AssetTyp return assetType, err } -// AND logic - if tags = {tag1:val1, tag2:val2}, -// then a vm will be excluded/included only if it has ALL of these tags ({tag1:val1, tag2:val2}). -func hasIncludeTags(vm *armcompute.VirtualMachine, tags *[]models.Tag) bool { - if tags == nil { - return true - } - if len(*tags) == 0 { - return true - } - if len(vm.Tags) == 0 { - return false - } - - for _, tag := range *tags { - val, ok := vm.Tags[tag.Key] - if !ok { - return false - } - if !(strings.Compare(*val, tag.Value) == 0) { - return false - } - } - return true -} - -// AND logic - if tags = {tag1:val1, tag2:val2}, -// then a vm will be excluded/included only if it has ALL of these tags ({tag1:val1, tag2:val2}). -func hasExcludeTags(vm *armcompute.VirtualMachine, tags *[]models.Tag) bool { - if tags == nil { - return false - } - if len(*tags) == 0 { - return false - } - if len(vm.Tags) == 0 { - return false - } - - for _, tag := range *tags { - val, ok := vm.Tags[tag.Key] - if !ok { - return false - } - if !(strings.Compare(*val, tag.Value) == 0) { - return false - } - } - return true -} - func convertTags(tags map[string]*string) *[]models.Tag { ret := make([]models.Tag, 0, len(tags)) for key, val := range tags { diff --git a/runtime_scan/pkg/provider/types.go b/runtime_scan/pkg/provider/types.go index 00f0d027b6..8840d98e64 100644 --- a/runtime_scan/pkg/provider/types.go +++ b/runtime_scan/pkg/provider/types.go @@ -30,10 +30,8 @@ type Provider interface { } type Discoverer interface { - // DiscoverScopes returns a list of discovered models.Scopes - DiscoverScopes(ctx context.Context) (*models.Scopes, error) - // DiscoverAssets returns list of AssetType in ScanScopeType - DiscoverAssets(ctx context.Context, scanScope *models.ScanScopeType) ([]models.AssetType, error) + // DiscoverAssets returns list of discovered AssetType + DiscoverAssets(ctx context.Context) ([]models.AssetType, error) } type Scanner interface { diff --git a/shared/pkg/backendclient/client.go b/shared/pkg/backendclient/client.go index 5262293c34..30e54144d7 100644 --- a/shared/pkg/backendclient/client.go +++ b/shared/pkg/backendclient/client.go @@ -532,25 +532,6 @@ func (b *BackendClient) GetAsset(ctx context.Context, assetID string, params mod } } -func (b *BackendClient) PutDiscoveryScopes(ctx context.Context, scope *models.Scopes) (*models.Scopes, error) { - resp, err := b.apiClient.PutDiscoveryScopesWithResponse(ctx, *scope) - if err != nil { - return nil, fmt.Errorf("failed to put discovery scope: %v", err) - } - switch resp.StatusCode() { - case http.StatusOK: - if resp.JSON200 == nil { - return nil, fmt.Errorf("failed to put scopes: empty body. status code=%v", http.StatusOK) - } - return resp.JSON200, nil - default: - if resp.JSONDefault != nil && resp.JSONDefault.Message != nil { - return nil, fmt.Errorf("failed to put scopes. status code=%v: %s", resp.StatusCode(), *resp.JSONDefault.Message) - } - return nil, fmt.Errorf("failed to put scopes. status code=%v", resp.StatusCode()) - } -} - func (b *BackendClient) GetAssets(ctx context.Context, params models.GetAssetsParams) (*models.Assets, error) { resp, err := b.apiClient.GetAssetsWithResponse(ctx, ¶ms) if err != nil { diff --git a/ui/src/layout/Assets/AssetDetails.js b/ui/src/layout/Assets/AssetDetails.js index 6edee81eb6..2957508e1c 100644 --- a/ui/src/layout/Assets/AssetDetails.js +++ b/ui/src/layout/Assets/AssetDetails.js @@ -48,7 +48,7 @@ const AssetDetails = () => ( ({title: assetInfo?.instanceID})} detailsContent={props => } withPadding diff --git a/ui/src/layout/Assets/AssetsTable.js b/ui/src/layout/Assets/AssetsTable.js index abdeb82845..b2b68a7051 100644 --- a/ui/src/layout/Assets/AssetsTable.js +++ b/ui/src/layout/Assets/AssetsTable.js @@ -3,7 +3,7 @@ import TablePage from 'components/TablePage'; import ExpandableList from 'components/ExpandableList'; import { APIS } from 'utils/systemConsts'; import { getFindingsColumnsConfigList, getVulnerabilitiesColumnConfigItem, getAssetColumnsFiltersConfig, - findingsColumnsFiltersConfig, vulnerabilitiesCountersColumnsFiltersConfig, formatTagsToStringsList } from 'utils/utils'; + findingsColumnsFiltersConfig, vulnerabilitiesCountersColumnsFiltersConfig, formatTagsToStringsList, formatDate} from 'utils/utils'; import { FILTER_TYPES } from 'context/FiltersProvider'; const TABLE_TITLE = "assets"; @@ -43,6 +43,18 @@ const AssetsTable = () => { sortIds: LOCATION_SORT_IDS, accessor: "assetInfo.location" }, + { + Header: "Last Seen", + id: "lastSeen", + sortIds: ["lastSeen"], + accessor: original => formatDate(original.lastSeen) + }, + { + Header: "Terminated On", + id: "terminatedOn", + sortIds: ["terminatedOn"], + accessor: original => formatDate(original?.terminatedOn) + }, getVulnerabilitiesColumnConfigItem(TABLE_TITLE), ...getFindingsColumnsConfigList(TABLE_TITLE) ], []); @@ -51,7 +63,7 @@ const AssetsTable = () => { { ...vulnerabilitiesCountersColumnsFiltersConfig, ...findingsColumnsFiltersConfig ]} - defaultSortBy={{sortIds: LOCATION_SORT_IDS, desc: false}} + defaultSortBy={{sortIds: ["lastSeen", "terminatedOn"], desc: true}} withMargin /> ) diff --git a/ui/src/layout/Dashboard/index.js b/ui/src/layout/Dashboard/index.js index f541c24f25..6a65c033ca 100644 --- a/ui/src/layout/Dashboard/index.js +++ b/ui/src/layout/Dashboard/index.js @@ -15,7 +15,7 @@ import './dashboard.scss'; const COUNTERS_CONFIG = [ {url: APIS.SCANS, title: "Completed scans", background: COLORS["color-gradient-green"]}, - {url: APIS.ASSETS, title: "Scanned assets", background: COLORS["color-gradient-blue"]}, + {url: APIS.ASSETS, title: "Total assets", background: COLORS["color-gradient-blue"]}, {url: APIS.FINDINGS, title: "Risky findings", background: COLORS["color-gradient-yellow"]} ]; @@ -49,4 +49,4 @@ const Dashboard = () => { ) } -export default Dashboard; \ No newline at end of file +export default Dashboard; diff --git a/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js b/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js index 70ebcb6826..f27acc1f0b 100644 --- a/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js +++ b/ui/src/layout/Scans/ConfigurationReadOnlyDisplay.js @@ -1,30 +1,18 @@ import React from 'react'; import TitleValueDisplay, { ValuesListDisplay } from 'components/TitleValueDisplay'; -import { TagsList } from 'components/Tag'; import { getEnabledScanTypesList, getScanTimeTypeTag } from 'layout/Scans/utils'; -import { cronExpressionToHuman, formatDate, formatTagsToStringsList } from 'utils/utils'; -import { ScopeDisplay } from './scopeDisplayUtils'; - -const InstancesDisplay = ({tags}) => ( - -) +import { cronExpressionToHuman, formatDate } from 'utils/utils'; const FlagPropDisplay = ({checked, label}) =>
{`${label} ${checked ? "enabled" : "disabled"}`}
const ConfigurationReadOnlyDisplay = ({configData}) => { const {scope, scanFamiliesConfig, scheduled, maxParallelScanners, scannerInstanceCreationConfig} = configData; - const {allRegions, regions, instanceTagSelector, instanceTagExclusion, shouldScanStoppedInstances} = scope; const {cronLine, operationTime} = scheduled; const {useSpotInstances} = scannerInstanceCreationConfig || {}; return ( <> - - - - - - + {scope} <> @@ -40,4 +28,4 @@ const ConfigurationReadOnlyDisplay = ({configData}) => { ) } -export default ConfigurationReadOnlyDisplay; \ No newline at end of file +export default ConfigurationReadOnlyDisplay; diff --git a/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js b/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js index 137e3e9f44..84a0f96b11 100644 --- a/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js +++ b/ui/src/layout/Scans/Configurations/ConfigurationsTable/index.js @@ -3,13 +3,11 @@ import { isNull } from 'lodash'; import ButtonWithIcon from 'components/ButtonWithIcon'; import { ICON_NAMES } from 'components/Icon'; import EmptyDisplay from 'components/EmptyDisplay'; -import ExpandableList from 'components/ExpandableList'; import TablePage from 'components/TablePage'; import { OPERATORS } from 'components/Filter'; -import { BoldText, toCapitalized, formatDate, getScanScopeColumnFiltersConfig, formatTagsToStringsList } from 'utils/utils'; +import { BoldText, toCapitalized, formatDate } from 'utils/utils'; import { APIS } from 'utils/systemConsts'; import { getScanTimeTypeTag } from 'layout/Scans/utils'; -import { ExpandableScopeDisplay } from 'layout/Scans/scopeDisplayUtils'; import { useModalDisplayDispatch, MODAL_DISPLAY_ACTIONS } from 'layout/Scans/ScanConfigWizardModal/ModalDisplayProvider'; import { FILTER_TYPES } from 'context/FiltersProvider'; import ConfigurationActionsDisplay from '../ConfigurationActionsDisplay'; @@ -46,44 +44,8 @@ const ConfigurationsTable = () => { { Header: "Scope", id: "scope", - sortIds: [ - "scope.allRegions", - "scope.regions" - ], - Cell: ({row}) => { - const {allRegions, regions} = row.original.scope; - - return ( - - ) - }, - alignToTop: true - }, - { - Header: "Excluded instances", - id: "instanceTagExclusion", - sortIds: ["scope.instanceTagExclusion"], - Cell: ({row}) => { - const {instanceTagExclusion} = row.original.scope; - - return ( - - ) - }, - alignToTop: true - }, - { - Header: "Included instances", - id: "instanceTagSelector", - sortIds: ["scope.instanceTagSelector"], - Cell: ({row}) => { - const {instanceTagSelector} = row.original.scope; - - return ( - - ) - }, - alignToTop: true + sortIds: ["scope"], + accessor: "scope" }, { Header: "Scan time", @@ -150,11 +112,11 @@ const ConfigurationsTable = () => { {...OPERATORS.endswith}, {...OPERATORS.contains, valueItems: [], creatable: true} ]}, - ...getScanScopeColumnFiltersConfig(), - {value: "scope.instanceTagExclusion", label: "Excluded instances", operators: [ - {...OPERATORS.contains, valueItems: [], creatable: true} - ]}, - {value: "scope.instanceTagSelector", label: "Included instances", operators: [ + {value: "scope", label: "Asset Query", operators: [ + {...OPERATORS.eq, valueItems: [], creatable: true}, + {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.startswith}, + {...OPERATORS.endswith}, {...OPERATORS.contains, valueItems: [], creatable: true} ]}, {value: "scheduled.operationTime", label: "Scan time", isDate: true, operators: [ @@ -199,4 +161,4 @@ const ConfigurationsTable = () => { ) } -export default ConfigurationsTable; \ No newline at end of file +export default ConfigurationsTable; diff --git a/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js b/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js index 1ac66e5c92..c25e8efac5 100644 --- a/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js +++ b/ui/src/layout/Scans/ScanConfigWizardModal/StepGeneralProperties.js @@ -1,137 +1,7 @@ -import React, { useEffect } from 'react'; -import { isEmpty } from 'lodash'; -import { usePrevious, useFetch } from 'hooks'; -import { TextField, RadioField, MultiselectField, SelectField, FieldsPair, useFormikContext, CheckboxField, - FieldLabel, validators } from 'components/Form'; -import { APIS } from 'utils/systemConsts'; - -export const VPCS_EMPTY_VALUE = [{id: "", securityGroups: []}] -export const REGIONS_EMPTY_VALUE = [{name: "", vpcs: VPCS_EMPTY_VALUE}]; - -export const SCOPE_ITEMS = { - ALL: {value: "ALL", label: "All"}, - DEFINED: {value: "DEFINED", label: "Define scope"} -} - -const SecurityGroupField = ({index, name, placeholder, disabled, regionData}) => { - const {name: regionId, vpcs} = regionData || {}; - const vpcId = isEmpty(vpcs) ? null : vpcs[index]?.id; - const prevVpsId = usePrevious(vpcId); - - const [{data, loading, error}, fetchSecurityGroups] = useFetch(APIS.SCOPES_DISCOVERY, {loadOnMount: false}); - - useEffect(() => { - if (!!vpcId && prevVpsId !== vpcId) { - fetchSecurityGroups({queryParams: { - "$select": `scopeInfo/regions($filter=name eq '${regionId}';$select=name,vpcs($filter=id eq '${vpcId}';$select=securityGroups))` - }}); - } - }, [prevVpsId, vpcId, regionId, fetchSecurityGroups]); - - if (error) { - return null; - } - - const regionsData = !data ? [] : data?.scopeInfo?.regions; - const vpcsData = !regionsData ? [] : regionsData[0]?.vpcs; - - return ( - ({value: id, label: id}))} - loading={loading} - /> - ) -} - -const RegionFields = ({index, name, disabled}) => { - const {values} = useFormikContext(); - const {regions} = values.scope; - - const regionId = regions[index]?.name; - const prevRegionId = usePrevious(regionId); - - const [{data, loading, error}, fetchVpcs] = useFetch(APIS.SCOPES_DISCOVERY, {loadOnMount: false}); - - useEffect(() => { - if (!!regionId && prevRegionId !== regionId) { - fetchVpcs({queryParams: { - "$select": `scopeInfo/regions($filter=name eq '${regionId}';$select=id,vpcs/id)`, - }}); - } - }, [prevRegionId, regionId, fetchVpcs]); - - if (error) { - return null; - } - - const regionsData = !data ? [] : data?.scopeInfo?.regions; - - return ( - ({value: id, label: id})), - clearable: true, - loading: loading - }} - secondFieldProps={{ - component: SecurityGroupField, - key: "securityGroups", - placeholder: "Select security group...", - emptyValue: [], - regionData: !!regionsData ? regions[index] : null - }} - /> - ); -} - -const DefinedScopeFields = () => { - const [{data, loading, error}] = useFetch(APIS.SCOPES_DISCOVERY, {queryParams: {"$select": "scopeInfo/regions/name"}}); - - if (error) { - return null; - } - - return ( - ({value: name, label: name})), - validate: validators.validateRequired, - loading: loading - }} - secondFieldProps={{ - component: RegionFields, - key: "vpcs", - emptyValue: VPCS_EMPTY_VALUE - }} - /> - ) -} +import React from 'react'; +import { TextField, validators } from 'components/Form'; const StepGeneralProperties = () => { - const {values, setFieldValue} = useFormikContext(); - const {scopeSelect} = values.scope; - - const isDefineScope = scopeSelect === SCOPE_ITEMS.DEFINED.value; - const prevIsDefineScope = usePrevious(isDefineScope); - - useEffect(() => { - if (prevIsDefineScope && !isDefineScope) { - setFieldValue("scope.regions", REGIONS_EMPTY_VALUE); - } - }, [prevIsDefineScope, isDefineScope, setFieldValue]); - return (
{ placeholder="Type a name..." validate={validators.validateRequired} /> - - {isDefineScope && } -
- Instances - - - -
) } -export default StepGeneralProperties; \ No newline at end of file +export default StepGeneralProperties; diff --git a/ui/src/layout/Scans/ScanConfigWizardModal/index.js b/ui/src/layout/Scans/ScanConfigWizardModal/index.js index e2455f4fde..4de0fa12dc 100644 --- a/ui/src/layout/Scans/ScanConfigWizardModal/index.js +++ b/ui/src/layout/Scans/ScanConfigWizardModal/index.js @@ -1,11 +1,8 @@ import React from 'react'; -import { isEmpty } from 'lodash'; import { FETCH_METHODS } from 'hooks'; import WizardModal from 'components/WizardModal'; import { APIS } from 'utils/systemConsts'; -import { formatTagsToStringsList } from 'utils/utils'; -import { formatStringInstancesToTags } from '../utils'; -import StepGeneralProperties, { REGIONS_EMPTY_VALUE, VPCS_EMPTY_VALUE, SCOPE_ITEMS } from './StepGeneralProperties'; +import StepGeneralProperties from './StepGeneralProperties'; import StepScanTypes from './StepScanTypes'; import StepTimeConfiguration, { SCHEDULE_TYPES_ITEMS, CRON_QUICK_OPTIONS } from './StepTimeConfiguration'; import StepAdvancedSettings from './StepAdvancedSettings'; @@ -16,7 +13,6 @@ const padDateTime = time => String(time).padStart(2, "0"); const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { const {id, name, scope, scanFamiliesConfig, scheduled, maxParallelScanners, scannerInstanceCreationConfig} = initialData || {}; - const {allRegions, regions, shouldScanStoppedInstances, instanceTagSelector, instanceTagExclusion} = scope || {}; const {operationTime, cronLine} = scheduled || {}; const {useSpotInstances} = scannerInstanceCreationConfig || {}; @@ -25,13 +21,7 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { const initialValues = { id: id || null, name: name || "", - scope: { - scopeSelect: (!regions || allRegions) ? SCOPE_ITEMS.ALL.value : SCOPE_ITEMS.DEFINED.value, - regions: REGIONS_EMPTY_VALUE, - shouldScanStoppedInstances: shouldScanStoppedInstances || false, - instanceTagSelector: formatTagsToStringsList(instanceTagSelector || []), - instanceTagExclusion: formatTagsToStringsList(instanceTagExclusion || []) - }, + scope: scope || "", scanFamiliesConfig: { sbom: {enabled: true}, vulnerabilities: {enabled: true}, @@ -53,14 +43,6 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { } } - if (!isEmpty(regions)) { - initialValues.scope.regions = regions.map(({name, vpcs}) => { - return {name, vpcs: isEmpty(vpcs) ? VPCS_EMPTY_VALUE : vpcs.map(({id, securityGroups}) => { - return {id: id || "", securityGroups: (securityGroups || []).map(({id}) => id)} - })} - }) - } - if (!!operationTime && !cronLine) { const dateTime = new Date(operationTime); initialValues.scheduled.scheduledSelect = SCHEDULE_TYPES_ITEMS.LATER.value; @@ -104,23 +86,7 @@ const ScanConfigWizardModal = ({initialData, onClose, onSubmitSuccess}) => { initialValues={initialValues} submitUrl={APIS.SCAN_CONFIGS} getSubmitParams={formValues => { - const {id, scope, scheduled, ...submitData} = formValues; - - const {scopeSelect, regions, shouldScanStoppedInstances, instanceTagSelector, instanceTagExclusion} = scope; - const isAllScope = scopeSelect === SCOPE_ITEMS.ALL.value; - - submitData.scope = { - objectType: "AwsScanScope", - allRegions: isAllScope, - regions: isAllScope ? [] : regions.map(({name, vpcs}) => { - return {name, vpcs: vpcs.filter(({id}) => !!id).map(({id, securityGroups}) => { - return {id, securityGroups: securityGroups.map(id => ({id}))} - })} - }), - shouldScanStoppedInstances, - instanceTagSelector: formatStringInstancesToTags(instanceTagSelector), - instanceTagExclusion: formatStringInstancesToTags(instanceTagExclusion), - } + const {id, scheduled, ...submitData} = formValues; const {scheduledSelect, laterDate, laterTime, cronLine} = scheduled; const isNow = scheduledSelect === SCHEDULE_TYPES_ITEMS.NOW.value; diff --git a/ui/src/layout/Scans/Scans/ScansTable.js b/ui/src/layout/Scans/Scans/ScansTable.js index 50e8ffb254..9b880d451a 100644 --- a/ui/src/layout/Scans/Scans/ScansTable.js +++ b/ui/src/layout/Scans/Scans/ScansTable.js @@ -5,11 +5,10 @@ import { utils } from 'components/Table'; import ScanProgressBar, { SCAN_STATES } from 'components/ScanProgressBar'; import EmptyDisplay from 'components/EmptyDisplay'; import { OPERATORS } from 'components/Filter'; -import { ExpandableScopeDisplay } from 'layout/Scans/scopeDisplayUtils'; import { useModalDisplayDispatch, MODAL_DISPLAY_ACTIONS } from 'layout/Scans/ScanConfigWizardModal/ModalDisplayProvider'; import { APIS, ROUTES } from 'utils/systemConsts'; import { formatDate, getFindingsColumnsConfigList, getVulnerabilitiesColumnConfigItem, formatNumber, findingsColumnsFiltersConfig, - vulnerabilitiesCountersColumnsFiltersConfig, getScanScopeColumnFiltersConfig } from 'utils/utils'; + vulnerabilitiesCountersColumnsFiltersConfig } from 'utils/utils'; import { FILTER_TYPES } from 'context/FiltersProvider'; import ScanActionsDisplay from './ScanActionsDisplay'; import { SCANS_PATHS } from '../utils'; @@ -57,16 +56,8 @@ const ScansTable = () => { { Header: "Scope", id: "scope", - sortIds: [ - "scanConfigSnapshot.scope.allRegions", - "scanConfigSnapshot.scope.regions" - ], - Cell: ({row}) => { - const {allRegions, regions} = row.original.scanConfigSnapshot?.scope; - - return - }, - width: 260 + sortIds: ["scanConfigSnapshot.scope"], + accessor: "scanConfigSnapshot.scope" }, { Header: "Status", @@ -113,11 +104,18 @@ const ScansTable = () => { filterType={FILTER_TYPES.SCANS} filtersConfig={[ {value: "scanConfigSnapshot.name", label: "Config name", operators: [ - {...OPERATORS.eq, valueItems: [], creatable: true}, - {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.eq, valueItems: [], creatable: true}, + {...OPERATORS.ne, valueItems: [], creatable: true}, + {...OPERATORS.startswith}, + {...OPERATORS.endswith}, + {...OPERATORS.contains, valueItems: [], creatable: true} + ]}, + {value: "scope", label: "Asset Query", operators: [ + {...OPERATORS.eq, valueitems: [], creatable: true}, + {...OPERATORS.ne, valueitems: [], creatable: true}, {...OPERATORS.startswith}, {...OPERATORS.endswith}, - {...OPERATORS.contains, valueItems: [], creatable: true} + {...OPERATORS.contains, valueitems: [], creatable: true} ]}, {value: "startTime", label: "Started", isDate: true, operators: [ {...OPERATORS.ge}, @@ -127,7 +125,6 @@ const ScansTable = () => { {...OPERATORS.ge}, {...OPERATORS.le}, ]}, - ...getScanScopeColumnFiltersConfig("scanConfigSnapshot.scope"), {value: "state", label: "Status", operators: [ {...OPERATORS.eq, valueItems: FILTER_SCAN_STATUS_ITEMS}, {...OPERATORS.ne, valueItems: FILTER_SCAN_STATUS_ITEMS} @@ -165,4 +162,4 @@ const ScansTable = () => { ) } -export default ScansTable; \ No newline at end of file +export default ScansTable; diff --git a/ui/src/layout/detail-displays/AssetDetails/index.js b/ui/src/layout/detail-displays/AssetDetails/index.js index 4f0071395b..608e14dbaf 100644 --- a/ui/src/layout/detail-displays/AssetDetails/index.js +++ b/ui/src/layout/detail-displays/AssetDetails/index.js @@ -51,7 +51,7 @@ const AssetScansDisplay = ({assetName, assetId}) => { const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false}) => { const navigate = useNavigate(); - const {id, assetInfo} = assetData; + const {id, assetInfo, firstSeen, lastSeen, terminatedOn} = assetData; const {instanceID, objectType, location, tags, image, instanceType, platform, launchTime} = assetInfo || {}; return ( @@ -62,6 +62,9 @@ const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false} {instanceID} {objectType} + {formatDate(firstSeen)} + {formatDate(lastSeen)} + {formatDate(terminatedOn)} {location} @@ -84,4 +87,4 @@ const AssetDetails = ({assetData, withAssetLink=false, withAssetScansLink=false} ) } -export default AssetDetails; \ No newline at end of file +export default AssetDetails; diff --git a/ui/src/utils/systemConsts.js b/ui/src/utils/systemConsts.js index f9e84ed42b..5080b94e63 100644 --- a/ui/src/utils/systemConsts.js +++ b/ui/src/utils/systemConsts.js @@ -15,7 +15,6 @@ export const APIS = { SCAN_CONFIGS: "scanConfigs", ASSETS: "assets", ASSET_SCANS: "assetScans", - SCOPES_DISCOVERY: "discovery/scopes", FINDINGS: "findings", DASHBOARD_RISKIEST_REGIONS: "dashboard/riskiestRegions", DASHBOARD_RISKIEST_ASSETS: "dashboard/riskiestAssets", @@ -90,4 +89,4 @@ export const VULNERABIITY_FINDINGS_ITEM = { icon: ICON_NAMES.SHIELD, color: COLORS["color-main-dark"], darkColor: COLORS["color-main-dark-variation-dark"], -} \ No newline at end of file +} diff --git a/ui/src/utils/utils.js b/ui/src/utils/utils.js index 12c9b571d0..740185fe09 100644 --- a/ui/src/utils/utils.js +++ b/ui/src/utils/utils.js @@ -181,28 +181,4 @@ export const getAssetColumnsFiltersConfig = (props) => { ] } -export const getScanScopeColumnFiltersConfig = (scopePrefix="scope") => { - const SCOPE_ALL_REGIONS_FILTER_ITEMS = [ - {value: `${scopePrefix}.allRegions`, label: "All"} - ] - - const formatScanScopeToOdata = (valuesList, operator, scope) => ( - valuesList.map(value => { - if (operator === OPERATORS.contains.value) { - return `${operator}(${scope},'${value}')`; - } - - return `(${value} eq ${operator === OPERATORS.eq.value ? "true" : "false"})`; - }).join(` or `) - ) - - return [ - {value: `${scopePrefix}.regions`, label: "Scope", customOdataFormat: formatScanScopeToOdata, operators: [ - {...OPERATORS.eq, valueItems: SCOPE_ALL_REGIONS_FILTER_ITEMS}, - {...OPERATORS.ne, valueItems: SCOPE_ALL_REGIONS_FILTER_ITEMS}, - {...OPERATORS.contains, valueItems: [], creatable: true} - ]} - ] -} - -export const formatTagsToStringsList = tags => tags?.map(({key, value}) => `${key}=${value}`); \ No newline at end of file +export const formatTagsToStringsList = tags => tags?.map(({key, value}) => `${key}=${value}`);