From 41c1f8dca18e566dbfaefe2f90a953f1b6db926d Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Wed, 26 Jul 2023 14:39:37 +0200 Subject: [PATCH 1/7] feat(cli): create asset and asset scan --- backend/pkg/database/gorm/asset.go | 18 ++ cli/cmd/asset_create.go | 129 ++++++++ cli/cmd/asset_scan_create.go | 89 ++++++ cli/cmd/root.go | 256 +-------------- cli/cmd/scan.go | 297 ++++++++++++++++++ cli/cmd/{root_test.go => scan_test.go} | 0 docs/assets/dir-asset.json | 5 + docs/assets/vm-asset.json | 20 ++ .../provider/cloudinit/cloud-init.tmpl.yaml | 1 + .../cloudinit/testdata/cloud-init.yaml | 1 + 10 files changed, 561 insertions(+), 255 deletions(-) create mode 100644 cli/cmd/asset_create.go create mode 100644 cli/cmd/asset_scan_create.go create mode 100644 cli/cmd/scan.go rename cli/cmd/{root_test.go => scan_test.go} (100%) create mode 100644 docs/assets/dir-asset.json create mode 100644 docs/assets/vm-asset.json diff --git a/backend/pkg/database/gorm/asset.go b/backend/pkg/database/gorm/asset.go index a78447131..2c0893c02 100644 --- a/backend/pkg/database/gorm/asset.go +++ b/backend/pkg/database/gorm/asset.go @@ -310,6 +310,24 @@ func (t *AssetsTableHandler) checkUniqueness(asset models.Asset) (*models.Asset, } } return nil, nil // nolint:nilnil + case models.DirInfo: + var assets []Asset + // In the case of creating or updating a asset, needs to be checked whether other asset exists with same DirName and Location. + filter := fmt.Sprintf("id ne '%s' and assetInfo/dirName eq '%s' and assetInfo/location eq '%s'", *asset.Id, *info.DirName, *info.Location) + err = ODataQuery(t.DB, assetSchemaName, &filter, nil, nil, nil, nil, nil, true, &assets) + if err != nil { + return nil, err + } + if len(assets) > 0 { + var apiAsset models.Asset + if err := json.Unmarshal(assets[0].Data, &apiAsset); err != nil { + return nil, fmt.Errorf("failed to convert DB model to API model: %w", err) + } + return &apiAsset, &common.ConflictError{ + Reason: fmt.Sprintf("Asset directory exists with same name=%q and location=%q", *info.DirName, *info.Location), + } + } + return nil, nil // nolint:nilnil default: return nil, fmt.Errorf("asset type is not supported (%T): %w", discriminator, err) } diff --git a/cli/cmd/asset_create.go b/cli/cmd/asset_create.go new file mode 100644 index 000000000..8944b2efb --- /dev/null +++ b/cli/cmd/asset_create.go @@ -0,0 +1,129 @@ +/* +Copyright © 2023 NAME HERE + +*/ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/openclarity/vmclarity/api/models" + "github.com/openclarity/vmclarity/shared/pkg/backendclient" +) + +// standaloneCmd represents the standalone command +var assetCreateCmd = &cobra.Command{ + Use: "asset-create", + Short: "Create asset", + Long: `It creates asset. It's useful in the CI/CD mode without VMClarity orchestration`, + Run: func(cmd *cobra.Command, args []string) { + logger.Infof("Creating asset...") + filename, err := cmd.Flags().GetString("from-json-file") + if err != nil { + logger.Fatalf("Unable to get asset json file name: %v", err) + } + server, err := cmd.Flags().GetString("server") + if err != nil { + logger.Fatalf("Unable to get VMClarity server address: %v", err) + } + + assetType, err := getAssetFromJsonFile(filename) + if err != nil { + logger.Fatalf("Failed to get asset from json file: %v", err) + } + + _, err = assetType.ValueByDiscriminator() + if err != nil { + logger.Fatalf("Failed to determine asset type: %v", err) + } + + assetID, err := createAsset(context.TODO(), assetType, server) + if err != nil { + logger.Fatalf("Failed to create asset: %v", err) + } + fmt.Println(assetID) + }, +} + +func init() { + rootCmd.AddCommand(assetCreateCmd) + + assetCreateCmd.Flags().String("from-json-file", "", "asset json filename") + assetCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") + assetCreateCmd.MarkFlagRequired("from-json-file") + assetCreateCmd.MarkFlagRequired("server") + +} + +func getAssetFromJsonFile(filename string) (*models.AssetType, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + // get the file size + stat, err := file.Stat() + if err != nil { + return nil, err + } + // read the file + bs := make([]byte, stat.Size()) + _, err = file.Read(bs) + if err != nil { + return nil, err + } + + assetType := &models.AssetType{} + if err := assetType.UnmarshalJSON(bs); err != nil { + return nil, fmt.Errorf("failed to unmarshal asset into AssetType %v", err) + } + + return assetType, nil +} + +func createAsset(ctx context.Context, assetType *models.AssetType, server string) (string, error) { + client, err := backendclient.Create(server) + if err != nil { + return "", fmt.Errorf("failed to create VMClarity API client: %w", err) + } + + creationTime := time.Now() + assetData := models.Asset{ + AssetInfo: assetType, + LastSeen: &creationTime, + FirstSeen: &creationTime, + } + asset, err := client.PostAsset(ctx, assetData) + if err == nil { + return *asset.Id, nil + } + var conflictError backendclient.AssetConflictError + if !errors.As(err, &conflictError) { + // If there is an error, and it's not a conflict telling + // us that the asset already exists, then we need to + // keep track of it and log it as a failure to + // complete discovery. We don't fail instantly here + // because discovering the assets is a heavy operation, + // so we want to give the best chance to create all the + // assets in the DB before failing. + return "", fmt.Errorf("failed to post asset: %v", err) + } + + // As we got a conflict it means there is an existing asset + // which matches the unique properties of this asset, in this + // case we'll patch the just AssetInfo and FirstSeen instead. + assetData.FirstSeen = nil + err = client.PatchAsset(ctx, assetData, *conflictError.ConflictingAsset.Id) + if err != nil { + return "", fmt.Errorf("failed to patch asset: %v", err) + } + + return *conflictError.ConflictingAsset.Id, nil +} diff --git a/cli/cmd/asset_scan_create.go b/cli/cmd/asset_scan_create.go new file mode 100644 index 000000000..8eb214b3a --- /dev/null +++ b/cli/cmd/asset_scan_create.go @@ -0,0 +1,89 @@ +/* +Copyright © 2023 NAME HERE + +*/ +package cmd + +import ( + "context" + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/openclarity/vmclarity/api/models" + "github.com/openclarity/vmclarity/shared/pkg/backendclient" + "github.com/openclarity/vmclarity/shared/pkg/utils" +) + +// standaloneCmd represents the standalone command +var assetScanCreateCmd = &cobra.Command{ + Use: "asset-scan-create", + Short: "Create asset scan", + Long: `It creates asset scan. It's useful in the CI/CD mode without VMClarity orchestration`, + Run: func(cmd *cobra.Command, args []string) { + logger.Infof("asset-scan-create called") + assetID, err := cmd.Flags().GetString("asset-id") + if err != nil { + logger.Fatalf("Unable to get asset id: %v", err) + } + server, err := cmd.Flags().GetString("server") + if err != nil { + logger.Fatalf("Unable to get VMClarity server address: %v", err) + } + assetScanID, err := createAssetScan(context.TODO(), server, assetID) + if err != nil { + logger.Fatalf("Failed to create asset scan: %v", err) + } + fmt.Println(assetScanID) + }, +} + +func init() { + rootCmd.AddCommand(assetScanCreateCmd) + assetScanCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") + assetScanCreateCmd.Flags().String("asset-id", "", "Asset ID for asset scan") + assetScanCreateCmd.MarkFlagRequired("server") + assetScanCreateCmd.MarkFlagRequired("asset-id") +} + +func createAssetScan(ctx context.Context, server, assetID string) (string, error) { + client, err := backendclient.Create(server) + if err != nil { + return "", fmt.Errorf("failed to create VMClarity API client: %w", err) + } + + asset, err := client.GetAsset(ctx, assetID, models.GetAssetsAssetIDParams{}) + if err != nil { + return "", fmt.Errorf("failed to get asset %s: %w", assetID, err) + } + assetScanData := createEmptyAssetScanForAsset(asset) + + assetScan, err := client.PostAssetScan(ctx, assetScanData) + if err != nil { + var conErr backendclient.AssetScanConflictError + if errors.As(err, &conErr) { + assetScanID := *conErr.ConflictingAssetScan.Id + logger.WithField("AssetScanID", assetScanID).Debug("AssetScan already exist.") + return *conErr.ConflictingAssetScan.Id, nil + } + return "", fmt.Errorf("failed to post AssetScan to backend API: %w", err) + } + + return *assetScan.Id, nil +} + +func createEmptyAssetScanForAsset(asset models.Asset) models.AssetScan { + return models.AssetScan{ + Asset: &models.AssetRelationship{ + AssetInfo: asset.AssetInfo, + FirstSeen: asset.FirstSeen, + Id: *asset.Id, + }, + Status: &models.AssetScanStatus{ + General: &models.AssetScanState{ + State: utils.PointerTo(models.AssetScanStateStateReadyToScan), + }, + }, + } +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 65b8ffec9..af2c0198a 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -16,48 +16,17 @@ package cmd import ( - "context" - "errors" - "fmt" "os" - "time" - "github.com/ghodss/yaml" - kubeclarityutils "github.com/openclarity/kubeclarity/shared/pkg/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/openclarity/vmclarity/cli/pkg" - "github.com/openclarity/vmclarity/cli/pkg/cli" - "github.com/openclarity/vmclarity/cli/pkg/presenter" - "github.com/openclarity/vmclarity/cli/pkg/state" - "github.com/openclarity/vmclarity/shared/pkg/backendclient" - "github.com/openclarity/vmclarity/shared/pkg/families" - "github.com/openclarity/vmclarity/shared/pkg/families/malware" - misconfigurationTypes "github.com/openclarity/vmclarity/shared/pkg/families/misconfiguration/types" - "github.com/openclarity/vmclarity/shared/pkg/families/rootkits" - "github.com/openclarity/vmclarity/shared/pkg/families/sbom" - "github.com/openclarity/vmclarity/shared/pkg/families/secrets" - "github.com/openclarity/vmclarity/shared/pkg/families/vulnerabilities" "github.com/openclarity/vmclarity/shared/pkg/log" - "github.com/openclarity/vmclarity/shared/pkg/utils" -) - -const ( - DefaultWatcherInterval = 2 * time.Minute - DefaultMountTimeout = 10 * time.Minute ) var ( - cfgFile string - config *families.Config - logger *logrus.Entry - output string - - server string - assetScanID string - mountVolume bool + logger *logrus.Entry ) // rootCmd represents the base command when called without any subcommands. @@ -67,68 +36,6 @@ var rootCmd = &cobra.Command{ Long: `VMClarity`, Version: pkg.GitRevision, SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - logger.Infof("Running...") - - // Main context which remains active even if the scan is aborted allowing post-processing operations - // like updating asset scan state - ctx := log.SetLoggerForContext(cmd.Context(), logger) - - cli, err := newCli() - if err != nil { - return fmt.Errorf("failed to initialize CLI: %w", err) - } - - // Create context used to signal to operations that the scan is aborted - abortCtx, cancel := context.WithCancel(ctx) - defer cancel() - - // Start watching for abort event - cli.WatchForAbort(ctx, cancel, DefaultWatcherInterval) - - if err := cli.WaitForReadyState(abortCtx); err != nil { - err = fmt.Errorf("failed to wait for AssetScan being ready to scan: %w", err) - if e := cli.MarkDone(ctx, []error{err}); e != nil { - logger.Errorf("Failed to update AssetScan status to completed with errors: %v", e) - } - return err - } - - if mountVolume { - // Set timeout for mounting volumes - mountCtx, mountCancel := context.WithTimeout(abortCtx, DefaultMountTimeout) - defer mountCancel() - - mountPoints, err := cli.MountVolumes(mountCtx) - if err != nil { - err = fmt.Errorf("failed to mount attached volume: %w", err) - if e := cli.MarkDone(ctx, []error{err}); e != nil { - logger.Errorf("Failed to update asset scan stat to completed with errors: %v", e) - } - return err - } - setMountPointsForFamiliesInput(mountPoints, config) - } - - err = cli.MarkInProgress(ctx) - if err != nil { - return fmt.Errorf("failed to inform server %v scan has started: %w", server, err) - } - - logger.Infof("Running scanners...") - runErrors := families.New(config).Run(abortCtx, cli) - - err = cli.MarkDone(ctx, runErrors) - if err != nil { - return fmt.Errorf("failed to inform the server %v the scan was completed: %w", server, err) - } - - if len(runErrors) > 0 { - logger.Errorf("Errors when running families: %+v", runErrors) - } - - return nil - }, } // Execute adds all child commands to the root command and sets flags appropriately. @@ -141,171 +48,10 @@ func Execute() { func init() { cobra.OnInitialize( initLogger, - initConfig, ) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.vmclarity.yaml)") - rootCmd.PersistentFlags().StringVar(&output, "output", "", "set output directory path. Stdout is used if not set.") - rootCmd.PersistentFlags().StringVar(&server, "server", "", "VMClarity server to export asset scans to, for example: http://localhost:9999/api") - rootCmd.PersistentFlags().StringVar(&assetScanID, "asset-scan-id", "", "the AssetScan ID to monitor and report results to") - rootCmd.PersistentFlags().BoolVar(&mountVolume, "mount-attached-volume", false, "discover for an attached volume and mount it before the scan") - - // TODO(sambetts) we may have to change this to our own validation when - // we add the CI/CD scenario and there isn't an existing asset-scan-id - // in the backend to PATCH - rootCmd.MarkFlagsRequiredTogether("server", "asset-scan-id") -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - logger.Infof("Initializing configuration...") - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := os.UserHomeDir() - cobra.CheckErr(err) - - // Search config in home directory OR current directory with name ".families" (without extension). - viper.AddConfigPath(home) - viper.AddConfigPath(".") - viper.SetConfigType("yaml") - viper.SetConfigName(".families") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - err := viper.ReadInConfig() - cobra.CheckErr(err) - - // Load config - config = &families.Config{} - err = viper.Unmarshal(config) - cobra.CheckErr(err) - - if logrus.IsLevelEnabled(logrus.InfoLevel) { - configB, err := yaml.Marshal(config) - cobra.CheckErr(err) - logger.Infof("Using config file (%s):\n%s", viper.ConfigFileUsed(), string(configB)) - } } func initLogger() { log.InitLogger(logrus.InfoLevel.String(), os.Stderr) logger = logrus.WithField("app", "vmclarity") } - -func newCli() (*cli.CLI, error) { - var manager state.Manager - var presenters []presenter.Presenter - var err error - - if config == nil { - return nil, errors.New("families config must not be nil") - } - - if server != "" { - var client *backendclient.BackendClient - var p presenter.Presenter - - client, err = backendclient.Create(server) - if err != nil { - return nil, fmt.Errorf("failed to create VMClarity API client: %w", err) - } - - manager, err = state.NewVMClarityState(client, assetScanID) - if err != nil { - return nil, fmt.Errorf("failed to create VMClarity state manager: %w", err) - } - - p, err = presenter.NewVMClarityPresenter(client, assetScanID) - if err != nil { - return nil, fmt.Errorf("failed to create VMClarity presenter: %w", err) - } - presenters = append(presenters, p) - } else { - manager, err = state.NewLocalState() - if err != nil { - return nil, fmt.Errorf("failed to create local state: %w", err) - } - } - - if output != "" { - presenters = append(presenters, presenter.NewFilePresenter(output, config)) - } else { - presenters = append(presenters, presenter.NewConsolePresenter(os.Stdout, config)) - } - - var p presenter.Presenter - if len(presenters) == 1 { - p = presenters[0] - } else { - p = &presenter.MultiPresenter{Presenters: presenters} - } - - return &cli.CLI{Manager: manager, Presenter: p, FamiliesConfig: config}, nil -} - -func setMountPointsForFamiliesInput(mountPoints []string, familiesConfig *families.Config) *families.Config { - // update families inputs with the mount point as rootfs - for _, mountDir := range mountPoints { - if familiesConfig.SBOM.Enabled { - familiesConfig.SBOM.Inputs = append(familiesConfig.SBOM.Inputs, sbom.Input{ - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }) - } - - if familiesConfig.Vulnerabilities.Enabled { - if familiesConfig.SBOM.Enabled { - familiesConfig.Vulnerabilities.InputFromSbom = true - } else { - familiesConfig.Vulnerabilities.Inputs = append(familiesConfig.Vulnerabilities.Inputs, vulnerabilities.Input{ - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }) - } - } - - if familiesConfig.Secrets.Enabled { - familiesConfig.Secrets.Inputs = append(familiesConfig.Secrets.Inputs, secrets.Input{ - StripPathFromResult: utils.PointerTo(true), - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }) - } - - if familiesConfig.Malware.Enabled { - familiesConfig.Malware.Inputs = append(familiesConfig.Malware.Inputs, malware.Input{ - StripPathFromResult: utils.PointerTo(true), - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }) - } - - if familiesConfig.Rootkits.Enabled { - familiesConfig.Rootkits.Inputs = append(familiesConfig.Rootkits.Inputs, rootkits.Input{ - StripPathFromResult: utils.PointerTo(true), - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }) - } - - if familiesConfig.Misconfiguration.Enabled { - familiesConfig.Misconfiguration.Inputs = append( - familiesConfig.Misconfiguration.Inputs, - misconfigurationTypes.Input{ - StripPathFromResult: utils.PointerTo(true), - Input: mountDir, - InputType: string(kubeclarityutils.ROOTFS), - }, - ) - } - } - return familiesConfig -} diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go new file mode 100644 index 000000000..f55331683 --- /dev/null +++ b/cli/cmd/scan.go @@ -0,0 +1,297 @@ +/* +Copyright © 2023 NAME HERE + +*/ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/ghodss/yaml" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + kubeclarityutils "github.com/openclarity/kubeclarity/shared/pkg/utils" + + "github.com/openclarity/vmclarity/cli/pkg/cli" + "github.com/openclarity/vmclarity/cli/pkg/presenter" + "github.com/openclarity/vmclarity/cli/pkg/state" + "github.com/openclarity/vmclarity/shared/pkg/backendclient" + "github.com/openclarity/vmclarity/shared/pkg/families" + "github.com/openclarity/vmclarity/shared/pkg/families/malware" + misconfigurationTypes "github.com/openclarity/vmclarity/shared/pkg/families/misconfiguration/types" + "github.com/openclarity/vmclarity/shared/pkg/families/rootkits" + "github.com/openclarity/vmclarity/shared/pkg/families/sbom" + "github.com/openclarity/vmclarity/shared/pkg/families/secrets" + "github.com/openclarity/vmclarity/shared/pkg/families/vulnerabilities" + "github.com/openclarity/vmclarity/shared/pkg/log" + "github.com/openclarity/vmclarity/shared/pkg/utils" +) + +const ( + DefaultWatcherInterval = 2 * time.Minute + DefaultMountTimeout = 10 * time.Minute +) + +// scanCmd represents the scan command +var scanCmd = &cobra.Command{ + Use: "scan", + Short: "Scan", + Long: `Run scanner families`, + RunE: func(cmd *cobra.Command, args []string) error { + logger.Infof("Running...") + + // Main context which remains active even if the scan is aborted allowing post-processing operations + // like updating asset scan state + ctx := log.SetLoggerForContext(cmd.Context(), logger) + + cfgFile, err := cmd.Flags().GetString("config") + if err != nil { + logger.Fatalf("Unable to get asset json file name: %v", err) + } + server, err := cmd.Flags().GetString("server") + if err != nil { + logger.Fatalf("Unable to get VMClarity server address: %v", err) + } + output, err := cmd.Flags().GetString("output") + if err != nil { + logger.Fatalf("Unable to get output file name: %v", err) + } + assetScanID, err := cmd.Flags().GetString("asset-scan-id") + if err != nil { + logger.Fatalf("Unable to get asset scan ID: %v", err) + } + mountVolume, err := cmd.Flags().GetBool("mount-attached-volume") + if err != nil { + logger.Fatalf("Unable to get mount attached volume flag: %v", err) + } + + config := loadConfig(cfgFile) + cli, err := newCli(config, server, assetScanID, output) + if err != nil { + return fmt.Errorf("failed to initialize CLI: %w", err) + } + + // Create context used to signal to operations that the scan is aborted + abortCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Start watching for abort event + cli.WatchForAbort(ctx, cancel, DefaultWatcherInterval) + + if err := cli.WaitForReadyState(abortCtx); err != nil { + err = fmt.Errorf("failed to wait for AssetScan being ready to scan: %w", err) + if e := cli.MarkDone(ctx, []error{err}); e != nil { + logger.Errorf("Failed to update AssetScan status to completed with errors: %v", e) + } + return err + } + + if mountVolume { + // Set timeout for mounting volumes + mountCtx, mountCancel := context.WithTimeout(abortCtx, DefaultMountTimeout) + defer mountCancel() + + mountPoints, err := cli.MountVolumes(mountCtx) + if err != nil { + err = fmt.Errorf("failed to mount attached volume: %w", err) + if e := cli.MarkDone(ctx, []error{err}); e != nil { + logger.Errorf("Failed to update asset scan stat to completed with errors: %v", e) + } + return err + } + setMountPointsForFamiliesInput(mountPoints, config) + } + + err = cli.MarkInProgress(ctx) + if err != nil { + return fmt.Errorf("failed to inform server %v scan has started: %w", server, err) + } + + logger.Infof("Running scanners...") + runErrors := families.New(config).Run(abortCtx, cli) + + err = cli.MarkDone(ctx, runErrors) + if err != nil { + return fmt.Errorf("failed to inform the server %v the scan was completed: %w", server, err) + } + + if len(runErrors) > 0 { + logger.Errorf("Errors when running families: %+v", runErrors) + } + + return nil + }, +} + +// nolint: gochecknoinits +func init() { + rootCmd.AddCommand(scanCmd) + + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + scanCmd.Flags().String("config", "", "config file (default is $HOME/.vmclarity.yaml)") + scanCmd.Flags().String("output", "", "set output directory path. Stdout is used if not set.") + scanCmd.Flags().String("server", "", "VMClarity server to export asset scans to, for example: http://localhost:9999/api") + scanCmd.Flags().String("asset-scan-id", "", "the AssetScan ID to monitor and report results to") + scanCmd.Flags().Bool("mount-attached-volume", false, "discover for an attached volume and mount it before the scan") + + // TODO(sambetts) we may have to change this to our own validation when + // we add the CI/CD scenario and there isn't an existing asset-scan-id + // in the backend to PATCH + scanCmd.MarkFlagsRequiredTogether("server", "asset-scan-id") +} + +// loadConfig reads in config file and ENV variables if set. +func loadConfig(cfgFile string) *families.Config { + logger.Infof("Initializing configuration...") + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := os.UserHomeDir() + cobra.CheckErr(err) + + // Search config in home directory OR current directory with name ".families" (without extension). + viper.AddConfigPath(home) + viper.AddConfigPath(".") + viper.SetConfigType("yaml") + viper.SetConfigName(".families") + } + + viper.AutomaticEnv() // read in environment variables that match + + // If a config file is found, read it in. + err := viper.ReadInConfig() + cobra.CheckErr(err) + + // Load config + config := &families.Config{} + err = viper.Unmarshal(config) + cobra.CheckErr(err) + + if logrus.IsLevelEnabled(logrus.InfoLevel) { + configB, err := yaml.Marshal(config) + cobra.CheckErr(err) + logger.Infof("Using config file (%s):\n%s", viper.ConfigFileUsed(), string(configB)) + } + + return config +} + +func newCli(config *families.Config, server, assetScanID, output string) (*cli.CLI, error) { + var manager state.Manager + var presenters []presenter.Presenter + var err error + + if config == nil { + return nil, errors.New("families config must not be nil") + } + + if server != "" { + var client *backendclient.BackendClient + var p presenter.Presenter + + client, err = backendclient.Create(server) + if err != nil { + return nil, fmt.Errorf("failed to create VMClarity API client: %w", err) + } + + manager, err = state.NewVMClarityState(client, assetScanID) + if err != nil { + return nil, fmt.Errorf("failed to create VMClarity state manager: %w", err) + } + + p, err = presenter.NewVMClarityPresenter(client, assetScanID) + if err != nil { + return nil, fmt.Errorf("failed to create VMClarity presenter: %w", err) + } + presenters = append(presenters, p) + } else { + manager, err = state.NewLocalState() + if err != nil { + return nil, fmt.Errorf("failed to create local state: %w", err) + } + } + + if output != "" { + presenters = append(presenters, presenter.NewFilePresenter(output, config)) + } else { + presenters = append(presenters, presenter.NewConsolePresenter(os.Stdout, config)) + } + + var p presenter.Presenter + if len(presenters) == 1 { + p = presenters[0] + } else { + p = &presenter.MultiPresenter{Presenters: presenters} + } + + return &cli.CLI{Manager: manager, Presenter: p, FamiliesConfig: config}, nil +} + +func setMountPointsForFamiliesInput(mountPoints []string, familiesConfig *families.Config) *families.Config { + // update families inputs with the mount point as rootfs + for _, mountDir := range mountPoints { + if familiesConfig.SBOM.Enabled { + familiesConfig.SBOM.Inputs = append(familiesConfig.SBOM.Inputs, sbom.Input{ + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }) + } + + if familiesConfig.Vulnerabilities.Enabled { + if familiesConfig.SBOM.Enabled { + familiesConfig.Vulnerabilities.InputFromSbom = true + } else { + familiesConfig.Vulnerabilities.Inputs = append(familiesConfig.Vulnerabilities.Inputs, vulnerabilities.Input{ + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }) + } + } + + if familiesConfig.Secrets.Enabled { + familiesConfig.Secrets.Inputs = append(familiesConfig.Secrets.Inputs, secrets.Input{ + StripPathFromResult: utils.PointerTo(true), + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }) + } + + if familiesConfig.Malware.Enabled { + familiesConfig.Malware.Inputs = append(familiesConfig.Malware.Inputs, malware.Input{ + StripPathFromResult: utils.PointerTo(true), + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }) + } + + if familiesConfig.Rootkits.Enabled { + familiesConfig.Rootkits.Inputs = append(familiesConfig.Rootkits.Inputs, rootkits.Input{ + StripPathFromResult: utils.PointerTo(true), + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }) + } + + if familiesConfig.Misconfiguration.Enabled { + familiesConfig.Misconfiguration.Inputs = append( + familiesConfig.Misconfiguration.Inputs, + misconfigurationTypes.Input{ + StripPathFromResult: utils.PointerTo(true), + Input: mountDir, + InputType: string(kubeclarityutils.ROOTFS), + }, + ) + } + } + return familiesConfig +} diff --git a/cli/cmd/root_test.go b/cli/cmd/scan_test.go similarity index 100% rename from cli/cmd/root_test.go rename to cli/cmd/scan_test.go diff --git a/docs/assets/dir-asset.json b/docs/assets/dir-asset.json new file mode 100644 index 000000000..a741b5183 --- /dev/null +++ b/docs/assets/dir-asset.json @@ -0,0 +1,5 @@ +{ + "objectType": "DirInfo", + "dirName": "/test/path/test", + "location": "test-location" +} diff --git a/docs/assets/vm-asset.json b/docs/assets/vm-asset.json new file mode 100644 index 000000000..151161e11 --- /dev/null +++ b/docs/assets/vm-asset.json @@ -0,0 +1,20 @@ +{ + "objectType": "VMInfo", + "instanceID": "test-instance-id", + "instanceProvider": "AWS", + "location": "eu-west-2", + "tags": [ + { + "key": "test", + "value": "test" + } + ], + "securityGroups": [ + { + "id": "test-sg" + } + ], + "image": "test-image", + "instanceType": "t2-large", + "launchTime": "2023-07-26T10:33:09.080Z" +} diff --git a/runtime_scan/pkg/provider/cloudinit/cloud-init.tmpl.yaml b/runtime_scan/pkg/provider/cloudinit/cloud-init.tmpl.yaml index 89dfcb74c..e1d40a41d 100644 --- a/runtime_scan/pkg/provider/cloudinit/cloud-init.tmpl.yaml +++ b/runtime_scan/pkg/provider/cloudinit/cloud-init.tmpl.yaml @@ -27,6 +27,7 @@ write_files: -v /run:/run \ -v /var/opt/vmclarity:/var/opt/vmclarity \ {{ .ScannerImage }} \ + scan \ --config /opt/vmclarity/scanconfig.yaml \ --server {{ .VMClarityAddress }} \ --mount-attached-volume \ diff --git a/runtime_scan/pkg/provider/cloudinit/testdata/cloud-init.yaml b/runtime_scan/pkg/provider/cloudinit/testdata/cloud-init.yaml index a3400696a..95f521880 100644 --- a/runtime_scan/pkg/provider/cloudinit/testdata/cloud-init.yaml +++ b/runtime_scan/pkg/provider/cloudinit/testdata/cloud-init.yaml @@ -37,6 +37,7 @@ write_files: -v /run:/run \ -v /var/opt/vmclarity:/var/opt/vmclarity \ ghcr.io/openclarity/vmclarity-cli:latest \ + scan \ --config /opt/vmclarity/scanconfig.yaml \ --server 10.1.1.1:8888 \ --mount-attached-volume \ From 0ceee4381e108c39322979df369ac12182074bc1 Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Wed, 26 Jul 2023 14:48:00 +0200 Subject: [PATCH 2/7] docs: add cli usage --- docs/command_line.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/command_line.md diff --git a/docs/command_line.md b/docs/command_line.md new file mode 100644 index 000000000..c0055d070 --- /dev/null +++ b/docs/command_line.md @@ -0,0 +1,21 @@ +# Initiate scan using the cli: + +## Reporting results into file: +``` +./cli/bin/vmclarity-cli scan --config ~/testConf.yaml -o outputfile +``` + +If we want to report results to the VMClarity backend, we need to create asset and asset scan object before scan because it requires asset-scan-id + +## Reporting results to VMClarity backand: + +``` +ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api) +ASSET_SCAN_ID=$(./cli/bin/vmclarity-cli asset-scan-create --asset-id $ASSET_ID --server http://localhost:8888/api) +./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id $ASSET_SCAN_ID +``` + +Using one-liner: +``` +./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api | xargs -I{} ./cli/bin/vmclarity-cli asset-scan-create --asset-id {} --server http://localhost:8888/api | xargs -I{} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id {} +``` \ No newline at end of file From 8e4c3098284633a96dd6f85efefb7a4d887759d3 Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Wed, 26 Jul 2023 15:07:04 +0200 Subject: [PATCH 3/7] fix: typos and lint erros --- backend/pkg/database/gorm/asset.go | 1 + cli/cmd/asset_create.go | 38 ++++++++++++++++++++---------- cli/cmd/asset_scan_create.go | 27 ++++++++++++++++----- cli/cmd/root.go | 4 +--- cli/cmd/scan.go | 19 +++++++++++---- docs/command_line.md | 2 +- 6 files changed, 65 insertions(+), 26 deletions(-) diff --git a/backend/pkg/database/gorm/asset.go b/backend/pkg/database/gorm/asset.go index 2c0893c02..c7c19d956 100644 --- a/backend/pkg/database/gorm/asset.go +++ b/backend/pkg/database/gorm/asset.go @@ -285,6 +285,7 @@ func (t *AssetsTableHandler) DeleteAsset(assetID models.AssetID) error { return nil } +// nolint: cyclop func (t *AssetsTableHandler) checkUniqueness(asset models.Asset) (*models.Asset, error) { discriminator, err := asset.AssetInfo.ValueByDiscriminator() if err != nil { diff --git a/cli/cmd/asset_create.go b/cli/cmd/asset_create.go index 8944b2efb..d23167d3f 100644 --- a/cli/cmd/asset_create.go +++ b/cli/cmd/asset_create.go @@ -1,7 +1,18 @@ -/* -Copyright © 2023 NAME HERE +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -*/ package cmd import ( @@ -17,7 +28,7 @@ import ( "github.com/openclarity/vmclarity/shared/pkg/backendclient" ) -// standaloneCmd represents the standalone command +// assetCreateCmd represents the standalone command. var assetCreateCmd = &cobra.Command{ Use: "asset-create", Short: "Create asset", @@ -33,7 +44,7 @@ var assetCreateCmd = &cobra.Command{ logger.Fatalf("Unable to get VMClarity server address: %v", err) } - assetType, err := getAssetFromJsonFile(filename) + assetType, err := getAssetFromJSONFile(filename) if err != nil { logger.Fatalf("Failed to get asset from json file: %v", err) } @@ -56,28 +67,31 @@ func init() { assetCreateCmd.Flags().String("from-json-file", "", "asset json filename") assetCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") - assetCreateCmd.MarkFlagRequired("from-json-file") - assetCreateCmd.MarkFlagRequired("server") - + if err := assetCreateCmd.MarkFlagRequired("from-json-file"); err != nil { + logger.Fatalf("Failed to mark from-json-file flag as required: %v", err) + } + if err := assetCreateCmd.MarkFlagRequired("server"); err != nil { + logger.Fatalf("Failed to mark server flag as required: %v", err) + } } -func getAssetFromJsonFile(filename string) (*models.AssetType, error) { +func getAssetFromJSONFile(filename string) (*models.AssetType, error) { file, err := os.Open(filename) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open file: %v", err) } defer file.Close() // get the file size stat, err := file.Stat() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get file stat: %v", err) } // read the file bs := make([]byte, stat.Size()) _, err = file.Read(bs) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read file: %v", err) } assetType := &models.AssetType{} diff --git a/cli/cmd/asset_scan_create.go b/cli/cmd/asset_scan_create.go index 8eb214b3a..6801f9277 100644 --- a/cli/cmd/asset_scan_create.go +++ b/cli/cmd/asset_scan_create.go @@ -1,7 +1,18 @@ -/* -Copyright © 2023 NAME HERE +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -*/ package cmd import ( @@ -16,7 +27,7 @@ import ( "github.com/openclarity/vmclarity/shared/pkg/utils" ) -// standaloneCmd represents the standalone command +// assetScanCreateCmd represents the standalone command. var assetScanCreateCmd = &cobra.Command{ Use: "asset-scan-create", Short: "Create asset scan", @@ -43,8 +54,12 @@ func init() { rootCmd.AddCommand(assetScanCreateCmd) assetScanCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") assetScanCreateCmd.Flags().String("asset-id", "", "Asset ID for asset scan") - assetScanCreateCmd.MarkFlagRequired("server") - assetScanCreateCmd.MarkFlagRequired("asset-id") + if err := assetScanCreateCmd.MarkFlagRequired("server"); err != nil { + logger.Fatalf("Failed to mark server flag as required: %v", err) + } + if err := assetScanCreateCmd.MarkFlagRequired("asset-id"); err != nil { + logger.Fatalf("Failed to mark asset-id flag as required: %v", err) + } } func createAssetScan(ctx context.Context, server, assetID string) (string, error) { diff --git a/cli/cmd/root.go b/cli/cmd/root.go index af2c0198a..34f39a538 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -25,9 +25,7 @@ import ( "github.com/openclarity/vmclarity/shared/pkg/log" ) -var ( - logger *logrus.Entry -) +var logger *logrus.Entry // rootCmd represents the base command when called without any subcommands. var rootCmd = &cobra.Command{ diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go index f55331683..5aeead995 100644 --- a/cli/cmd/scan.go +++ b/cli/cmd/scan.go @@ -1,7 +1,18 @@ -/* -Copyright © 2023 NAME HERE +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -*/ package cmd import ( @@ -38,7 +49,7 @@ const ( DefaultMountTimeout = 10 * time.Minute ) -// scanCmd represents the scan command +// scanCmd represents the scan command. var scanCmd = &cobra.Command{ Use: "scan", Short: "Scan", diff --git a/docs/command_line.md b/docs/command_line.md index c0055d070..f79e6c75b 100644 --- a/docs/command_line.md +++ b/docs/command_line.md @@ -7,7 +7,7 @@ If we want to report results to the VMClarity backend, we need to create asset and asset scan object before scan because it requires asset-scan-id -## Reporting results to VMClarity backand: +## Reporting results to VMClarity backend: ``` ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api) From 63521cd9a9c280680c9978c1409e2f77844280a0 Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Fri, 28 Jul 2023 13:10:13 +0200 Subject: [PATCH 4/7] review --- cli/cmd/asset_create.go | 45 ++++++++++++++++++++---------------- cli/cmd/asset_scan_create.go | 26 ++++++++++++++------- cli/pkg/utils/json_helper.go | 30 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 cli/pkg/utils/json_helper.go diff --git a/cli/cmd/asset_create.go b/cli/cmd/asset_create.go index d23167d3f..09b499e5e 100644 --- a/cli/cmd/asset_create.go +++ b/cli/cmd/asset_create.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/cobra" "github.com/openclarity/vmclarity/api/models" + cliutils "github.com/openclarity/vmclarity/cli/pkg/utils" "github.com/openclarity/vmclarity/shared/pkg/backendclient" ) @@ -43,22 +44,32 @@ var assetCreateCmd = &cobra.Command{ if err != nil { logger.Fatalf("Unable to get VMClarity server address: %v", err) } - assetType, err := getAssetFromJSONFile(filename) if err != nil { logger.Fatalf("Failed to get asset from json file: %v", err) } + updateIfExist, err := cmd.Flags().GetBool("update-if-exists") + if err != nil { + logger.Fatalf("Unable to get update-if-exists flag vaule: %v", err) + } + jsonPath, err := cmd.Flags().GetString("jsonpath") + if err != nil { + logger.Fatalf("Unable to get jsonpath: %v", err) + } _, err = assetType.ValueByDiscriminator() if err != nil { logger.Fatalf("Failed to determine asset type: %v", err) } - assetID, err := createAsset(context.TODO(), assetType, server) + asset, err := createAsset(context.TODO(), assetType, server, updateIfExist) if err != nil { logger.Fatalf("Failed to create asset: %v", err) } - fmt.Println(assetID) + + if err := cliutils.PrintJSONData(asset, jsonPath); err != nil { + logger.Fatalf("Failed to print jsonpath: %v", err) + } }, } @@ -67,6 +78,8 @@ func init() { assetCreateCmd.Flags().String("from-json-file", "", "asset json filename") assetCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") + assetCreateCmd.Flags().Bool("update-if-exists", false, "the asset will be updated the asset if it exists") + assetCreateCmd.Flags().String("jsonpath", "", "print selected value of asset") if err := assetCreateCmd.MarkFlagRequired("from-json-file"); err != nil { logger.Fatalf("Failed to mark from-json-file flag as required: %v", err) } @@ -102,10 +115,10 @@ func getAssetFromJSONFile(filename string) (*models.AssetType, error) { return assetType, nil } -func createAsset(ctx context.Context, assetType *models.AssetType, server string) (string, error) { +func createAsset(ctx context.Context, assetType *models.AssetType, server string, updateIfExist bool) (*models.Asset, error) { client, err := backendclient.Create(server) if err != nil { - return "", fmt.Errorf("failed to create VMClarity API client: %w", err) + return nil, fmt.Errorf("failed to create VMClarity API client: %w", err) } creationTime := time.Now() @@ -116,28 +129,20 @@ func createAsset(ctx context.Context, assetType *models.AssetType, server string } asset, err := client.PostAsset(ctx, assetData) if err == nil { - return *asset.Id, nil + return asset, nil } var conflictError backendclient.AssetConflictError - if !errors.As(err, &conflictError) { - // If there is an error, and it's not a conflict telling - // us that the asset already exists, then we need to - // keep track of it and log it as a failure to - // complete discovery. We don't fail instantly here - // because discovering the assets is a heavy operation, - // so we want to give the best chance to create all the - // assets in the DB before failing. - return "", fmt.Errorf("failed to post asset: %v", err) - } - // As we got a conflict it means there is an existing asset // which matches the unique properties of this asset, in this - // case we'll patch the just AssetInfo and FirstSeen instead. + // case if the update-if-exists flag is set we'll patch the just AssetInfo and FirstSeen instead. + if !(errors.As(err, &conflictError) && updateIfExist) { + return nil, fmt.Errorf("failed to post asset: %v", err) + } assetData.FirstSeen = nil err = client.PatchAsset(ctx, assetData, *conflictError.ConflictingAsset.Id) if err != nil { - return "", fmt.Errorf("failed to patch asset: %v", err) + return nil, fmt.Errorf("failed to patch asset: %v", err) } - return *conflictError.ConflictingAsset.Id, nil + return conflictError.ConflictingAsset, nil } diff --git a/cli/cmd/asset_scan_create.go b/cli/cmd/asset_scan_create.go index 6801f9277..c22650ea5 100644 --- a/cli/cmd/asset_scan_create.go +++ b/cli/cmd/asset_scan_create.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/cobra" "github.com/openclarity/vmclarity/api/models" + cliutils "github.com/openclarity/vmclarity/cli/pkg/utils" "github.com/openclarity/vmclarity/shared/pkg/backendclient" "github.com/openclarity/vmclarity/shared/pkg/utils" ) @@ -42,11 +43,19 @@ var assetScanCreateCmd = &cobra.Command{ if err != nil { logger.Fatalf("Unable to get VMClarity server address: %v", err) } - assetScanID, err := createAssetScan(context.TODO(), server, assetID) + jsonPath, err := cmd.Flags().GetString("jsonpath") + if err != nil { + logger.Fatalf("Unable to get jsonpath: %v", err) + } + + assetScan, err := createAssetScan(context.TODO(), server, assetID) if err != nil { logger.Fatalf("Failed to create asset scan: %v", err) } - fmt.Println(assetScanID) + + if err := cliutils.PrintJSONData(assetScan, jsonPath); err != nil { + logger.Fatalf("Failed to print jsonpath: %v", err) + } }, } @@ -54,6 +63,7 @@ func init() { rootCmd.AddCommand(assetScanCreateCmd) assetScanCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") assetScanCreateCmd.Flags().String("asset-id", "", "Asset ID for asset scan") + assetScanCreateCmd.Flags().String("jsonpath", "", "print selected value of asset scan") if err := assetScanCreateCmd.MarkFlagRequired("server"); err != nil { logger.Fatalf("Failed to mark server flag as required: %v", err) } @@ -62,15 +72,15 @@ func init() { } } -func createAssetScan(ctx context.Context, server, assetID string) (string, error) { +func createAssetScan(ctx context.Context, server, assetID string) (*models.AssetScan, error) { client, err := backendclient.Create(server) if err != nil { - return "", fmt.Errorf("failed to create VMClarity API client: %w", err) + return nil, fmt.Errorf("failed to create VMClarity API client: %w", err) } asset, err := client.GetAsset(ctx, assetID, models.GetAssetsAssetIDParams{}) if err != nil { - return "", fmt.Errorf("failed to get asset %s: %w", assetID, err) + return nil, fmt.Errorf("failed to get asset %s: %w", assetID, err) } assetScanData := createEmptyAssetScanForAsset(asset) @@ -80,12 +90,12 @@ func createAssetScan(ctx context.Context, server, assetID string) (string, error if errors.As(err, &conErr) { assetScanID := *conErr.ConflictingAssetScan.Id logger.WithField("AssetScanID", assetScanID).Debug("AssetScan already exist.") - return *conErr.ConflictingAssetScan.Id, nil + return conErr.ConflictingAssetScan, nil } - return "", fmt.Errorf("failed to post AssetScan to backend API: %w", err) + return nil, fmt.Errorf("failed to post AssetScan to backend API: %w", err) } - return *assetScan.Id, nil + return assetScan, nil } func createEmptyAssetScanForAsset(asset models.Asset) models.AssetScan { diff --git a/cli/pkg/utils/json_helper.go b/cli/pkg/utils/json_helper.go new file mode 100644 index 000000000..2266df01d --- /dev/null +++ b/cli/pkg/utils/json_helper.go @@ -0,0 +1,30 @@ +package utils + +import ( + "encoding/json" + "fmt" + "os" + + "k8s.io/client-go/util/jsonpath" +) + +func PrintJSONData(data interface{}, fields string) error { + // If jsonpath is not set it will print the whole data as json format. + if fields == "" { + dataB, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data: %v", err) + } + fmt.Printf("asset: %s", string(dataB)) + return nil + } + j := jsonpath.New("parser") + if err := j.Parse(fields); err != nil { + return fmt.Errorf("failed to parse jsonpath: %v", err) + } + err := j.Execute(os.Stdout, data) + if err != nil { + return fmt.Errorf("failed to execute jsonpath: %v", err) + } + return nil +} From 303996ecf8a0ff1d88f45728d9ca50aa3c204f8c Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Fri, 28 Jul 2023 13:13:03 +0200 Subject: [PATCH 5/7] update docs --- docs/command_line.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/command_line.md b/docs/command_line.md index f79e6c75b..e28f32704 100644 --- a/docs/command_line.md +++ b/docs/command_line.md @@ -10,12 +10,12 @@ If we want to report results to the VMClarity backend, we need to create asset a ## Reporting results to VMClarity backend: ``` -ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api) -ASSET_SCAN_ID=$(./cli/bin/vmclarity-cli asset-scan-create --asset-id $ASSET_ID --server http://localhost:8888/api) +ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api) --jsonpath {.id} +ASSET_SCAN_ID=$(./cli/bin/vmclarity-cli asset-scan-create --asset-id $ASSET_ID --server http://localhost:8888/api) --jsonpath {.id} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id $ASSET_SCAN_ID ``` Using one-liner: ``` -./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api | xargs -I{} ./cli/bin/vmclarity-cli asset-scan-create --asset-id {} --server http://localhost:8888/api | xargs -I{} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id {} -``` \ No newline at end of file +./cli/bin/vmclarity-cli asset-create --from-json-file docs/assets/dir-asset.json --server http://localhost:8888/api --update-if-exists --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli asset-scan-create --asset-id {} --server http://localhost:8888/api --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id {} +``` From 47fa6d72b885bf687164c77b2b246b6a5c433544 Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Fri, 28 Jul 2023 13:49:59 +0200 Subject: [PATCH 6/7] fix issing header --- cli/pkg/utils/json_helper.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cli/pkg/utils/json_helper.go b/cli/pkg/utils/json_helper.go index 2266df01d..82fc82c5c 100644 --- a/cli/pkg/utils/json_helper.go +++ b/cli/pkg/utils/json_helper.go @@ -1,3 +1,18 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( From 33f74f1aace7d96aa26b500a4be4533128e7b331 Mon Sep 17 00:00:00 2001 From: Peter Balogh Date: Fri, 28 Jul 2023 16:33:29 +0200 Subject: [PATCH 7/7] updates --- cli/cmd/asset_create.go | 17 +++++++++-------- cli/cmd/asset_scan_create.go | 4 +--- cli/cmd/scan.go | 2 +- cli/pkg/utils/json_helper.go | 2 +- docs/command_line.md | 4 ++-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cli/cmd/asset_create.go b/cli/cmd/asset_create.go index 33bedc4d2..7548ec728 100644 --- a/cli/cmd/asset_create.go +++ b/cli/cmd/asset_create.go @@ -36,7 +36,7 @@ var assetCreateCmd = &cobra.Command{ Long: `It creates asset. It's useful in the CI/CD mode without VMClarity orchestration`, Run: func(cmd *cobra.Command, args []string) { logger.Infof("Creating asset...") - filename, err := cmd.Flags().GetString("from-json-file") + filename, err := cmd.Flags().GetString("file") if err != nil { logger.Fatalf("Unable to get asset json file name: %v", err) } @@ -48,7 +48,7 @@ var assetCreateCmd = &cobra.Command{ if err != nil { logger.Fatalf("Failed to get asset from json file: %v", err) } - updateIfExist, err := cmd.Flags().GetBool("update-if-exists") + updateIfExists, err := cmd.Flags().GetBool("update-if-exists") if err != nil { logger.Fatalf("Unable to get update-if-exists flag vaule: %v", err) } @@ -62,7 +62,7 @@ var assetCreateCmd = &cobra.Command{ logger.Fatalf("Failed to determine asset type: %v", err) } - asset, err := createAsset(context.TODO(), assetType, server, updateIfExist) + asset, err := createAsset(context.TODO(), assetType, server, updateIfExists) if err != nil { logger.Fatalf("Failed to create asset: %v", err) } @@ -76,12 +76,12 @@ var assetCreateCmd = &cobra.Command{ func init() { rootCmd.AddCommand(assetCreateCmd) - assetCreateCmd.Flags().String("from-json-file", "", "asset json filename") + assetCreateCmd.Flags().String("file", "", "asset json filename") assetCreateCmd.Flags().String("server", "", "VMClarity server to create asset to, for example: http://localhost:9999/api") assetCreateCmd.Flags().Bool("update-if-exists", false, "the asset will be updated the asset if it exists") assetCreateCmd.Flags().String("jsonpath", "", "print selected value of asset") - if err := assetCreateCmd.MarkFlagRequired("from-json-file"); err != nil { - logger.Fatalf("Failed to mark from-json-file flag as required: %v", err) + if err := assetCreateCmd.MarkFlagRequired("file"); err != nil { + logger.Fatalf("Failed to mark file flag as required: %v", err) } if err := assetCreateCmd.MarkFlagRequired("server"); err != nil { logger.Fatalf("Failed to mark server flag as required: %v", err) @@ -100,6 +100,7 @@ func getAssetFromJSONFile(filename string) (*models.AssetType, error) { if err != nil { return nil, fmt.Errorf("failed to get file stat: %v", err) } + // read the file bs := make([]byte, stat.Size()) _, err = file.Read(bs) @@ -115,7 +116,7 @@ func getAssetFromJSONFile(filename string) (*models.AssetType, error) { return assetType, nil } -func createAsset(ctx context.Context, assetType *models.AssetType, server string, updateIfExist bool) (*models.Asset, error) { +func createAsset(ctx context.Context, assetType *models.AssetType, server string, updateIfExists bool) (*models.Asset, error) { client, err := backendclient.Create(server) if err != nil { return nil, fmt.Errorf("failed to create VMClarity API client: %w", err) @@ -135,7 +136,7 @@ func createAsset(ctx context.Context, assetType *models.AssetType, server string // As we got a conflict it means there is an existing asset // which matches the unique properties of this asset, in this // case if the update-if-exists flag is set we'll patch the just AssetInfo and FirstSeen instead. - if !(errors.As(err, &conflictError) && updateIfExist) { + if !errors.As(err, &conflictError) || !updateIfExists { return nil, fmt.Errorf("failed to post asset: %v", err) } assetData.FirstSeen = nil diff --git a/cli/cmd/asset_scan_create.go b/cli/cmd/asset_scan_create.go index c9a484bde..737a90d95 100644 --- a/cli/cmd/asset_scan_create.go +++ b/cli/cmd/asset_scan_create.go @@ -101,9 +101,7 @@ func createAssetScan(ctx context.Context, server, assetID string) (*models.Asset func createEmptyAssetScanForAsset(asset models.Asset) models.AssetScan { return models.AssetScan{ Asset: &models.AssetRelationship{ - AssetInfo: asset.AssetInfo, - FirstSeen: asset.FirstSeen, - Id: *asset.Id, + Id: *asset.Id, }, Status: &models.AssetScanStatus{ General: &models.AssetScanState{ diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go index 1fbe89526..04d6fc085 100644 --- a/cli/cmd/scan.go +++ b/cli/cmd/scan.go @@ -54,7 +54,7 @@ var scanCmd = &cobra.Command{ cfgFile, err := cmd.Flags().GetString("config") if err != nil { - logger.Fatalf("Unable to get asset json file name: %v", err) + logger.Fatalf("Unable to get config file name: %v", err) } server, err := cmd.Flags().GetString("server") if err != nil { diff --git a/cli/pkg/utils/json_helper.go b/cli/pkg/utils/json_helper.go index 82fc82c5c..013337978 100644 --- a/cli/pkg/utils/json_helper.go +++ b/cli/pkg/utils/json_helper.go @@ -30,7 +30,7 @@ func PrintJSONData(data interface{}, fields string) error { if err != nil { return fmt.Errorf("failed to marshal data: %v", err) } - fmt.Printf("asset: %s", string(dataB)) + fmt.Println(string(dataB)) return nil } j := jsonpath.New("parser") diff --git a/docs/command_line.md b/docs/command_line.md index e28f32704..22b6bd451 100644 --- a/docs/command_line.md +++ b/docs/command_line.md @@ -10,12 +10,12 @@ If we want to report results to the VMClarity backend, we need to create asset a ## Reporting results to VMClarity backend: ``` -ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --from-json-file assets/dir-asset.json --server http://localhost:8888/api) --jsonpath {.id} +ASSET_ID=$(./cli/bin/vmclarity-cli asset-create --file assets/dir-asset.json --server http://localhost:8888/api) --jsonpath {.id} ASSET_SCAN_ID=$(./cli/bin/vmclarity-cli asset-scan-create --asset-id $ASSET_ID --server http://localhost:8888/api) --jsonpath {.id} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id $ASSET_SCAN_ID ``` Using one-liner: ``` -./cli/bin/vmclarity-cli asset-create --from-json-file docs/assets/dir-asset.json --server http://localhost:8888/api --update-if-exists --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli asset-scan-create --asset-id {} --server http://localhost:8888/api --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id {} +./cli/bin/vmclarity-cli asset-create --file docs/assets/dir-asset.json --server http://localhost:8888/api --update-if-exists --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli asset-scan-create --asset-id {} --server http://localhost:8888/api --jsonpath {.id} | xargs -I{} ./cli/bin/vmclarity-cli scan --config ~/testConf.yaml --server http://localhost:8888/api --asset-scan-id {} ```