From 7218c69a8093ed930322cea1ae80d4d1779e77d9 Mon Sep 17 00:00:00 2001 From: tiulpin Date: Fri, 27 Oct 2023 00:52:08 +0200 Subject: [PATCH] :recycle: Refactor CI/CD, Test and IDE related code Refactor and Extract CI/CD-related code in 'core/container.go' and 'core/core_test.go' files. Test file modifications are extensive and mainly include renaming, method extraction, and cleanup to improve readability and organization of test conditions. The CI/CD code extraction made the 'PrepareContainerEnvSettings' method cleaner and more focused. Consequentially, the 'cmd/pull.go' file is updated to use the refactored method. Changes also include extracting the 'QODANA_*' environment variables to a separate function for better modularity. Revisions in the '.github/workflows/ci.yml' file includes replacing 'QODANA_TOKEN' with 'QODANA_LICENSE_ONLY_TOKEN' to reflect changes in token handling. Additionally, 'core/ide.go' and 'cmd/pull.go' have minor changes to reflect the decoupling of Qodana environment and CI/CD related settings. --- .github/workflows/ci.yml | 4 +- cmd/cmd_test.go | 144 +----------------- cmd/pull.go | 25 +--- cmd/scan.go | 38 ++--- core/container.go | 62 +------- core/core_test.go | 313 +++++++++++++++++++++++++++++++-------- core/env.go | 65 +++++++- core/ide.go | 2 +- core/installers_test.go | 4 + core/product_info.go | 1 + core/system.go | 9 +- core/utils.go | 16 ++ 12 files changed, 364 insertions(+), 319 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6985398d..2a80ac12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,9 @@ jobs: registry: registry.jetbrains.team username: ${{ secrets.SPACE_USERNAME }} password: ${{ secrets.SPACE_PASSWORD }} - - run: go test -v ./... -coverprofile cover.out + - run: go test -v ./... -coverprofile cover.out -coverpkg=./... env: - QODANA_TOKEN: ${{ secrets.TEST_QODANA_TOKEN }} + QODANA_LICENSE_ONLY_TOKEN: ${{ secrets.QODANA_LICENSE_ONLY_TOKEN }} - if: startsWith(matrix.os, 'ubuntu') uses: JetBrains/qodana-action@main env: diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 9f2afa98..d4b27625 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -27,12 +27,9 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strings" "testing" - "github.com/stretchr/testify/assert" - log "github.com/sirupsen/logrus" "github.com/JetBrains/qodana-cli/v2023/core" @@ -52,6 +49,10 @@ func createProject(t *testing.T, name string) string { if err != nil { t.Fatal(err) } + err = os.MkdirAll(location+"/.idea", 0o755) + if err != nil { + t.Fatal(err) + } return location } @@ -210,7 +211,7 @@ func TestContributorsCommand(t *testing.T) { } func TestAllCommandsWithContainer(t *testing.T) { - linter := "jetbrains/qodana-python-community:2023.2" + linter := "registry.jetbrains.team/p/sa/containers/qodana-python-community:latest" if os.Getenv("GITHUB_ACTIONS") == "true" { //goland:noinspection GoBoolExpressions @@ -249,6 +250,7 @@ func TestAllCommandsWithContainer(t *testing.T) { "-i", projectPath, "-o", resultsPath, "--cache-dir", filepath.Join(projectPath, "cache"), + "-v", filepath.Join(projectPath, ".idea") + ":/data/some", "--fail-threshold", "5", "--print-problems", "--apply-fixes", @@ -332,7 +334,7 @@ func TestAllCommandsWithContainer(t *testing.T) { func TestScanWithIde(t *testing.T) { log.SetLevel(log.DebugLevel) ide := "QDPY" - token := os.Getenv("TESTS_QODANA_TOKEN") + token := os.Getenv("QODANA_LICENSE_ONLY_TOKEN") if //goland:noinspection GoBoolExpressions token == "" { t.Skip("set your token here to run the test") @@ -359,135 +361,3 @@ func TestScanWithIde(t *testing.T) { t.Fatal(err) } } - -func propertiesFixture(enableStats bool, additionalProperties []string) []string { - properties := []string{ - "-Dfus.internal.reduce.initial.delay=true", - fmt.Sprintf("-Didea.application.info.value=%s", filepath.Join(os.TempDir(), "entrypoint", "QodanaAppInfo.xml")), - "-Didea.class.before.app=com.jetbrains.rider.protocol.EarlyBackendStarter", - fmt.Sprintf("-Didea.config.path=%s", filepath.Join(os.TempDir(), "entrypoint")), - fmt.Sprintf("-Didea.headless.enable.statistics=%t", enableStats), - "-Didea.headless.statistics.device.id=FAKE", - "-Didea.headless.statistics.max.files.to.send=5000", - "-Didea.headless.statistics.salt=FAKE", - fmt.Sprintf("-Didea.log.path=%s", filepath.Join(os.TempDir(), "entrypoint", "log")), - "-Didea.parent.prefix=Rider", - "-Didea.platform.prefix=Qodana", - fmt.Sprintf("-Didea.plugins.path=%s", filepath.Join(os.TempDir(), "entrypoint", "plugins", "master")), - "-Didea.qodana.thirdpartyplugins.accept=true", - fmt.Sprintf("-Didea.system.path=%s", filepath.Join(os.TempDir(), "entrypoint", "idea", "master")), - "-Dinspect.save.project.settings=true", - "-Djava.awt.headless=true", - "-Djava.net.useSystemProxies=true", - "-Djdk.attach.allowAttachSelf=true", - `-Djdk.http.auth.tunneling.disabledSchemes=""`, - "-Djdk.module.illegalAccess.silent=true", - "-Dkotlinx.coroutines.debug=off", - "-Dqodana.automation.guid=FAKE", - "-Didea.job.launcher.without.timeout=true", - "-Dqodana.coverage.input=/data/coverage", - "-Drider.collect.full.container.statistics=true", - "-Drider.suppress.std.redirect=true", - "-Dsun.io.useCanonCaches=false", - "-Dsun.tools.attach.tmp.only=true", - "-XX:+HeapDumpOnOutOfMemoryError", - "-XX:+UseG1GC", - "-XX:-OmitStackTraceInFastThrow", - "-XX:CICompilerCount=2", - "-XX:MaxJavaStackTraceDepth=10000", - "-XX:MaxRAMPercentage=70", - "-XX:ReservedCodeCacheSize=512m", - "-XX:SoftRefLRUPolicyMSPerMB=50", - fmt.Sprintf("-Xlog:gc*:%s", filepath.Join(os.TempDir(), "entrypoint", "log", "gc.log")), - "-ea", - } - properties = append(properties, additionalProperties...) - sort.Strings(properties) - return properties -} - -func Test_Properties(t *testing.T) { - opts := &core.QodanaOptions{} - tmpDir := filepath.Join(os.TempDir(), "entrypoint") - opts.ProjectDir = tmpDir - opts.ResultsDir = opts.ProjectDir - opts.CacheDir = opts.ProjectDir - opts.CoverageDir = "/data/coverage" - opts.AnalysisId = "FAKE" - - core.Prod.BaseScriptName = "rider" - core.Prod.Code = "QDNET" - core.Prod.Version = "main" - - err := os.Setenv(core.QodanaDistEnv, opts.ProjectDir) - if err != nil { - t.Fatal(err) - } - err = os.Setenv(core.QodanaConfEnv, opts.ProjectDir) - if err != nil { - t.Fatal(err) - } - err = os.Setenv("DEVICEID", "FAKE") - if err != nil { - t.Fatal(err) - } - err = os.Setenv("SALT", "FAKE") - if err != nil { - t.Fatal(err) - } - err = os.MkdirAll(opts.ProjectDir, 0o755) - if err != nil { - t.Fatal(err) - } - - for _, tc := range []struct { - name string - cliProperties []string - qodanaYaml string - expected []string - }{ - { - name: "no overrides, just defaults and .NET project", - cliProperties: []string{}, - qodanaYaml: "dotnet:\n project: project.csproj", - expected: propertiesFixture(true, []string{"-Dqodana.net.project=project.csproj"}), - }, - { - name: "add one CLI property and .NET solution settings", - cliProperties: []string{"-xa", "idea.some.custom.property=1"}, - qodanaYaml: "dotnet:\n solution: solution.sln\n configuration: Release\n platform: x64", - expected: append( - propertiesFixture(true, []string{"-Dqodana.net.solution=solution.sln", "-Dqodana.net.configuration=Release", "-Dqodana.net.platform=x64", "-Didea.some.custom.property=1"}), - "-xa", - ), - }, - { - name: "override options from CLI, YAML should be ignored", - cliProperties: []string{"-Dfus.internal.reduce.initial.delay=false", "-Didea.application.info.value=0", "idea.headless.enable.statistics=false"}, - qodanaYaml: "" + - "version: \"1.0\"\n" + - "properties:\n" + - " fus.internal.reduce.initial.delay: true\n" + - " idea.application.info.value: 0\n", - expected: append([]string{ - "-Dfus.internal.reduce.initial.delay=false", - "-Didea.application.info.value=0", - }, propertiesFixture(false, []string{})[2:]...), - }, - } { - t.Run(tc.name, func(t *testing.T) { - err = os.WriteFile(filepath.Join(opts.ProjectDir, "qodana.yml"), []byte(tc.qodanaYaml), 0o600) - if err != nil { - t.Fatal(err) - } - opts.Property = tc.cliProperties - core.Config = core.GetQodanaYaml(opts.ProjectDir) - actual := core.GetProperties(opts, core.Config.Properties, core.Config.DotNet, []string{}) - assert.Equal(t, tc.expected, actual) - }) - } - err = os.RemoveAll(opts.ProjectDir) - if err != nil { - t.Fatal(err) - } -} diff --git a/cmd/pull.go b/cmd/pull.go index c960b182..43993b34 100644 --- a/cmd/pull.go +++ b/cmd/pull.go @@ -23,37 +23,18 @@ import ( "github.com/spf13/cobra" ) -// pullOptions represents pull command options. -type pullOptions struct { - Linter string - ProjectDir string - YamlName string -} - // newPullCommand returns a new instance of the show command. func newPullCommand() *cobra.Command { - options := &pullOptions{} + options := &core.QodanaOptions{} cmd := &cobra.Command{ Use: "pull", Short: "Pull latest version of linter", Long: `An alternative to pull an image.`, PreRun: func(cmd *cobra.Command, args []string) { - core.PrepairContainerEnvSettings() + core.PrepareContainerEnvSettings() }, Run: func(cmd *cobra.Command, args []string) { - if options.Linter == "" { - qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName) - if qodanaYaml.Linter == "" { - core.WarningMessage( - "No valid qodana.yaml found. Have you run %s? Running that for you...", - core.PrimaryBold("qodana init"), - ) - options.Linter = core.GetLinter(options.ProjectDir, options.YamlName) - core.EmptyMessage() - } else { - options.Linter = qodanaYaml.Linter - } - } + fetchAnalyzerSetting(options) containerClient, err := client.NewClientWithOpts() if err != nil { log.Fatal("couldn't connect to container engine ", err) diff --git a/cmd/scan.go b/cmd/scan.go index 1f0c0d93..40686334 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -43,23 +43,7 @@ But you can always override qodana.yaml options with the following command-line Run: func(cmd *cobra.Command, args []string) { ctx := cmd.Context() checkProjectDir(options.ProjectDir) - if options.Linter == "" && options.Ide == "" { - qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName) - if qodanaYaml.Linter == "" && qodanaYaml.Ide == "" { - core.WarningMessage( - "No valid `linter:` field found in %s. Have you run %s? Running that for you...", - core.PrimaryBold(options.YamlName), - core.PrimaryBold("qodana init"), - ) - options.Linter = core.GetLinter(options.ProjectDir, options.YamlName) - core.EmptyMessage() - } else { - options.Linter = qodanaYaml.Linter - } - if options.Ide == "" { - options.Ide = qodanaYaml.Ide - } - } + fetchAnalyzerSetting(options) exitCode := core.RunAnalysis(ctx, options) checkExitCode(exitCode, options.ResultsDir) @@ -155,6 +139,26 @@ But you can always override qodana.yaml options with the following command-line return cmd } +func fetchAnalyzerSetting(options *core.QodanaOptions) { + if options.Linter == "" && options.Ide == "" { + qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName) + if qodanaYaml.Linter == "" && qodanaYaml.Ide == "" { + core.WarningMessage( + "No valid `linter:` field found in %s. Have you run %s? Running that for you...", + core.PrimaryBold(options.YamlName), + core.PrimaryBold("qodana init"), + ) + options.Linter = core.GetLinter(options.ProjectDir, options.YamlName) + core.EmptyMessage() + } else { + options.Linter = qodanaYaml.Linter + } + if options.Ide == "" { + options.Ide = qodanaYaml.Ide + } + } +} + func checkProjectDir(projectDir string) { if core.IsInteractive() && core.IsHomeDirectory(projectDir) { core.WarningMessage( diff --git a/core/container.go b/core/container.go index f95e7f8c..f2150306 100644 --- a/core/container.go +++ b/core/container.go @@ -24,7 +24,6 @@ import ( "fmt" "github.com/pterm/pterm" "io" - "net/url" "os" "os/exec" "path/filepath" @@ -33,8 +32,6 @@ import ( cliconfig "github.com/docker/cli/cli/config" - "github.com/cucumber/ci-environment/go" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" @@ -107,66 +104,13 @@ func encodeAuthToBase64(authConfig types.AuthConfig) (string, error) { return base64.URLEncoding.EncodeToString(buf), nil } -// extractQodanaEnvironmentForDocker extracts Qodana env variables QODANA_* to the given environment array. -func extractQodanaEnvironmentForDocker(opts *QodanaOptions) { - ci := cienvironment.DetectCIEnvironment() - qEnv := "cli" - if ci != nil { - qEnv = strings.ReplaceAll(strings.ToLower(ci.Name), " ", "-") - opts.setenv(qodanaJobUrl, validateJobUrl(ci.URL, qEnv)) - if ci.Git != nil { - opts.setenv(qodanaRemoteUrl, validateRemoteUrl(ci.Git.Remote)) - opts.setenv(qodanaBranch, validateBranch(ci.Git.Branch, qEnv)) - opts.setenv(qodanaRevision, ci.Git.Revision) - } - } - opts.setenv(qodanaEnv, fmt.Sprintf("%s:%s", qEnv, Version)) -} - -func validateRemoteUrl(remote string) string { - _, err := url.ParseRequestURI(remote) - if remote == "" || err != nil { - log.Warnf("Unable to parse git remote URL, set %s env variable for proper qodana.cloud reporting", qodanaBranch) - return "" - } - return remote -} - -func validateBranch(branch string, env string) string { - if branch == "" { - if env == "github-actions" { - branch = os.Getenv("GITHUB_REF") - } else if env == "azure-pipelines" { - branch = os.Getenv("BUILD_SOURCEBRANCHNAME") - } else if env == "jenkins" { - branch = os.Getenv("GIT_BRANCH") - } - } - if branch == "" { - log.Warnf("Unable to parse git branch, set %s env variable for proper qodana.cloud reporting", qodanaBranch) - return "" - } - return branch -} - -func validateJobUrl(ciUrl string, qEnv string) string { - if strings.HasPrefix(qEnv, "azure") { // temporary workaround for Azure Pipelines - return getAzureJobUrl() - } - _, err := url.ParseRequestURI(ciUrl) - if err != nil { - return "" - } - return ciUrl -} - func checkRequiredToolInstalled(tool string) bool { _, err := exec.LookPath(tool) return err == nil } -// PrepairContainerEnvSettings checks if the host is ready to run Qodana container images. -func PrepairContainerEnvSettings() { +// PrepareContainerEnvSettings checks if the host is ready to run Qodana container images. +func PrepareContainerEnvSettings() { var tool string if os.Getenv(qodanaCliUsePodman) == "" && checkRequiredToolInstalled("docker") { tool = "docker" @@ -308,7 +252,7 @@ func CheckContainerEngineMemory() { // getDockerOptions returns qodana docker container options. func getDockerOptions(opts *QodanaOptions) *types.ContainerCreateConfig { cmdOpts := getIdeArgs(opts) - extractQodanaEnvironmentForDocker(opts) + ExtractQodanaEnvironment(opts.setenv) cachePath, err := filepath.Abs(opts.CacheDir) if err != nil { log.Fatal("couldn't get abs path for cache", err) diff --git a/core/core_test.go b/core/core_test.go index f709bb64..b35e54f9 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "github.com/JetBrains/qodana-cli/v2023/cloud" + "golang.org/x/exp/maps" "net/http" "net/http/httptest" "os" @@ -29,7 +30,7 @@ import ( "path/filepath" "reflect" "runtime" - "strings" + "sort" "testing" "time" @@ -138,34 +139,40 @@ func TestCloudUrl(t *testing.T) { } } +func unsetGitHubVariables() { + variables := []string{ + "GITHUB_SERVER_URL", + "GITHUB_REPOSITORY", + "GITHUB_RUN_ID", + "GITHUB_HEAD_REF", + "GITHUB_REF", + } + for _, v := range variables { + _ = os.Unsetenv(v) + } +} + func Test_ExtractEnvironmentVariables(t *testing.T) { revisionExpected := "1234567890abcdef1234567890abcdef12345678" branchExpected := "refs/heads/main" if os.Getenv("GITHUB_ACTIONS") == "true" { - variables := []string{ - "GITHUB_SERVER_URL", - "GITHUB_REPOSITORY", - "GITHUB_RUN_ID", - "GITHUB_HEAD_REF", - "GITHUB_REF", - } - for _, v := range variables { - _ = os.Unsetenv(v) - } + unsetGitHubVariables() } for _, tc := range []struct { - ci string - variables map[string]string - qodanaJobUrlExpected string - qodanaEnvExpected string - qodanaRemoteUrlExpected string + ci string + variables map[string]string + jobUrlExpected string + envExpected string + remoteUrlExpected string + revisionExpected string + branchExpected string }{ { - ci: "no CI detected", - variables: map[string]string{}, - qodanaEnvExpected: "cli:dev", + ci: "no CI detected", + variables: map[string]string{}, + envExpected: "cli:dev", }, { ci: "User defined", @@ -176,9 +183,27 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { qodanaBranch: branchExpected, qodanaRevision: revisionExpected, }, - qodanaEnvExpected: "user-defined", - qodanaRemoteUrlExpected: "https://qodana.jetbrains.com/never-gonna-give-you-up", - qodanaJobUrlExpected: "https://qodana.jetbrains.com/never-gonna-give-you-up", + envExpected: "user-defined", + remoteUrlExpected: "https://qodana.jetbrains.com/never-gonna-give-you-up", + jobUrlExpected: "https://qodana.jetbrains.com/never-gonna-give-you-up", + revisionExpected: revisionExpected, + branchExpected: branchExpected, + }, + { + ci: "Space", + variables: map[string]string{ + "JB_SPACE_EXECUTION_URL": "https://space.jetbrains.com/never-gonna-give-you-up", + "JB_SPACE_GIT_BRANCH": branchExpected, + "JB_SPACE_GIT_REVISION": revisionExpected, + "JB_SPACE_API_URL": "jetbrains.team", + "JB_SPACE_PROJECT_KEY": "sa", + "JB_SPACE_GIT_REPOSITORY_NAME": "entrypoint", + }, + envExpected: fmt.Sprintf("space:%s", Version), + remoteUrlExpected: "ssh://git@git.jetbrains.team/sa/entrypoint.git", + jobUrlExpected: "https://space.jetbrains.com/never-gonna-give-you-up", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "GitLab", @@ -188,9 +213,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "CI_COMMIT_SHA": revisionExpected, "CI_REPOSITORY_URL": "https://gitlab.jetbrains.com/sa/entrypoint.git", }, - qodanaEnvExpected: fmt.Sprintf("gitlab:%s", Version), - qodanaRemoteUrlExpected: "https://gitlab.jetbrains.com/sa/entrypoint.git", - qodanaJobUrlExpected: "https://gitlab.jetbrains.com/never-gonna-give-you-up", + envExpected: fmt.Sprintf("gitlab:%s", Version), + remoteUrlExpected: "https://gitlab.jetbrains.com/sa/entrypoint.git", + jobUrlExpected: "https://gitlab.jetbrains.com/never-gonna-give-you-up", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "Jenkins", @@ -200,9 +227,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "GIT_COMMIT": revisionExpected, "GIT_URL": "https://git.jetbrains.com/sa/entrypoint.git", }, - qodanaEnvExpected: fmt.Sprintf("jenkins:%s", Version), - qodanaJobUrlExpected: "https://jenkins.jetbrains.com/never-gonna-give-you-up", - qodanaRemoteUrlExpected: "https://git.jetbrains.com/sa/entrypoint.git", + envExpected: fmt.Sprintf("jenkins:%s", Version), + jobUrlExpected: "https://jenkins.jetbrains.com/never-gonna-give-you-up", + remoteUrlExpected: "https://git.jetbrains.com/sa/entrypoint.git", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "GitHub", @@ -213,9 +242,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "GITHUB_SHA": revisionExpected, "GITHUB_HEAD_REF": branchExpected, }, - qodanaEnvExpected: fmt.Sprintf("github-actions:%s", Version), - qodanaJobUrlExpected: "https://github.jetbrains.com/sa/entrypoint/actions/runs/123456789", - qodanaRemoteUrlExpected: "https://github.jetbrains.com/sa/entrypoint.git", + envExpected: fmt.Sprintf("github-actions:%s", Version), + jobUrlExpected: "https://github.jetbrains.com/sa/entrypoint/actions/runs/123456789", + remoteUrlExpected: "https://github.jetbrains.com/sa/entrypoint.git", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "GitHub push", @@ -226,9 +257,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "GITHUB_SHA": revisionExpected, "GITHUB_REF": branchExpected, }, - qodanaEnvExpected: fmt.Sprintf("github-actions:%s", Version), - qodanaJobUrlExpected: "https://github.jetbrains.com/sa/entrypoint/actions/runs/123456789", - qodanaRemoteUrlExpected: "https://github.jetbrains.com/sa/entrypoint.git", + envExpected: fmt.Sprintf("github-actions:%s", Version), + jobUrlExpected: "https://github.jetbrains.com/sa/entrypoint/actions/runs/123456789", + remoteUrlExpected: "https://github.jetbrains.com/sa/entrypoint.git", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "CircleCI", @@ -238,9 +271,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "CIRCLE_BRANCH": branchExpected, "CIRCLE_REPOSITORY_URL": "https://circleci.jetbrains.com/sa/entrypoint.git", }, - qodanaEnvExpected: fmt.Sprintf("circleci:%s", Version), - qodanaJobUrlExpected: "https://circleci.jetbrains.com/never-gonna-give-you-up", - qodanaRemoteUrlExpected: "https://circleci.jetbrains.com/sa/entrypoint.git", + envExpected: fmt.Sprintf("circleci:%s", Version), + jobUrlExpected: "https://circleci.jetbrains.com/never-gonna-give-you-up", + remoteUrlExpected: "https://circleci.jetbrains.com/sa/entrypoint.git", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, { ci: "Azure Pipelines", @@ -253,9 +288,11 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { "BUILD_SOURCEBRANCH": "refs/heads/" + branchExpected, "BUILD_REPOSITORY_URI": "https://dev.azure.com/jetbrains/sa/entrypoint.git", }, - qodanaEnvExpected: fmt.Sprintf("azure-pipelines:%s", Version), - qodanaJobUrlExpected: "https://dev.azure.com/jetbrains/sa/_build/results?buildId=123456789", - qodanaRemoteUrlExpected: "https://dev.azure.com/jetbrains/sa/entrypoint.git", + envExpected: fmt.Sprintf("azure-pipelines:%s", Version), + jobUrlExpected: "https://dev.azure.com/jetbrains/sa/_build/results?buildId=123456789", + remoteUrlExpected: "https://dev.azure.com/jetbrains/sa/entrypoint.git", + revisionExpected: revisionExpected, + branchExpected: branchExpected, }, } { t.Run(tc.ci, func(t *testing.T) { @@ -268,36 +305,50 @@ func Test_ExtractEnvironmentVariables(t *testing.T) { opts.setenv(k, v) } - extractQodanaEnvironmentForDocker(opts) - currentQodanaEnv := opts.getenv(qodanaEnv) - if currentQodanaEnv != tc.qodanaEnvExpected { - t.Errorf("Expected %s, got %s", tc.qodanaEnvExpected, currentQodanaEnv) - } - if !strings.HasPrefix(currentQodanaEnv, "cli:") { - if opts.getenv(qodanaJobUrl) != tc.qodanaJobUrlExpected { - t.Errorf("Expected %s, got %s", tc.qodanaJobUrlExpected, opts.getenv(qodanaJobUrl)) - } - if opts.getenv(qodanaRemoteUrl) != tc.qodanaRemoteUrlExpected { - t.Errorf("Expected %s, got %s", tc.qodanaRemoteUrlExpected, opts.getenv(qodanaRemoteUrl)) - } - if opts.getenv(qodanaRevision) != revisionExpected { - t.Errorf("Expected %s, got %s", revisionExpected, opts.getenv(qodanaRevision)) - } - if opts.getenv(qodanaBranch) != branchExpected { - t.Errorf("Expected %s, got %s", branchExpected, opts.getenv(qodanaBranch)) - } - } - for _, k := range []string{qodanaJobUrl, qodanaEnv, qodanaRemoteUrl, qodanaRevision, qodanaBranch} { - err := os.Unsetenv(k) - if err != nil { - t.Fatal(err) - } + for _, environment := range []struct { + name string + set func(string, string) + unset func(string) + get func(string) string + }{ + { + name: "Container", + set: opts.setenv, + get: opts.getenv, + }, + { + name: "Local", + set: setEnv, + get: os.Getenv, + }, + } { + t.Run(environment.name, func(t *testing.T) { + ExtractQodanaEnvironment(environment.set) + currentQodanaEnv := environment.get(qodanaEnv) + if currentQodanaEnv != tc.envExpected { + t.Errorf("%s: Expected %s, got %s", environment.name, tc.envExpected, currentQodanaEnv) + } + if environment.get(qodanaJobUrl) != tc.jobUrlExpected { + t.Errorf("%s: Expected %s, got %s", environment.name, tc.jobUrlExpected, environment.get(qodanaJobUrl)) + } + if environment.get(qodanaRemoteUrl) != tc.remoteUrlExpected { + t.Errorf("%s: Expected %s, got %s", environment.name, tc.remoteUrlExpected, environment.get(qodanaRemoteUrl)) + } + if environment.get(qodanaRevision) != tc.revisionExpected { + t.Errorf("%s: Expected %s, got %s", environment.name, revisionExpected, environment.get(qodanaRevision)) + } + if environment.get(qodanaBranch) != tc.branchExpected { + t.Errorf("%s: Expected %s, got %s", environment.name, branchExpected, environment.get(qodanaBranch)) + } + }) } - for k := range tc.variables { + + for _, k := range append(maps.Keys(tc.variables), []string{qodanaJobUrl, qodanaEnv, qodanaRemoteUrl, qodanaRevision, qodanaBranch}...) { err := os.Unsetenv(k) if err != nil { t.Fatal(err) } + opts.unsetenv(k) } }) } @@ -1117,3 +1168,135 @@ func TestQodanaOptions_RequiresToken(t *testing.T) { } } } + +func propertiesFixture(enableStats bool, additionalProperties []string) []string { + properties := []string{ + "-Dfus.internal.reduce.initial.delay=true", + fmt.Sprintf("-Didea.application.info.value=%s", filepath.Join(os.TempDir(), "entrypoint", "QodanaAppInfo.xml")), + "-Didea.class.before.app=com.jetbrains.rider.protocol.EarlyBackendStarter", + fmt.Sprintf("-Didea.config.path=%s", filepath.Join(os.TempDir(), "entrypoint")), + fmt.Sprintf("-Didea.headless.enable.statistics=%t", enableStats), + "-Didea.headless.statistics.device.id=FAKE", + "-Didea.headless.statistics.max.files.to.send=5000", + "-Didea.headless.statistics.salt=FAKE", + fmt.Sprintf("-Didea.log.path=%s", filepath.Join(os.TempDir(), "entrypoint", "log")), + "-Didea.parent.prefix=Rider", + "-Didea.platform.prefix=Qodana", + fmt.Sprintf("-Didea.plugins.path=%s", filepath.Join(os.TempDir(), "entrypoint", "plugins", "master")), + "-Didea.qodana.thirdpartyplugins.accept=true", + fmt.Sprintf("-Didea.system.path=%s", filepath.Join(os.TempDir(), "entrypoint", "idea", "master")), + "-Dinspect.save.project.settings=true", + "-Djava.awt.headless=true", + "-Djava.net.useSystemProxies=true", + "-Djdk.attach.allowAttachSelf=true", + `-Djdk.http.auth.tunneling.disabledSchemes=""`, + "-Djdk.module.illegalAccess.silent=true", + "-Dkotlinx.coroutines.debug=off", + "-Dqodana.automation.guid=FAKE", + "-Didea.job.launcher.without.timeout=true", + "-Dqodana.coverage.input=/data/coverage", + "-Drider.collect.full.container.statistics=true", + "-Drider.suppress.std.redirect=true", + "-Dsun.io.useCanonCaches=false", + "-Dsun.tools.attach.tmp.only=true", + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:+UseG1GC", + "-XX:-OmitStackTraceInFastThrow", + "-XX:CICompilerCount=2", + "-XX:MaxJavaStackTraceDepth=10000", + "-XX:MaxRAMPercentage=70", + "-XX:ReservedCodeCacheSize=512m", + "-XX:SoftRefLRUPolicyMSPerMB=50", + fmt.Sprintf("-Xlog:gc*:%s", filepath.Join(os.TempDir(), "entrypoint", "log", "gc.log")), + "-ea", + } + properties = append(properties, additionalProperties...) + sort.Strings(properties) + return properties +} + +func Test_Properties(t *testing.T) { + opts := &QodanaOptions{} + tmpDir := filepath.Join(os.TempDir(), "entrypoint") + opts.ProjectDir = tmpDir + opts.ResultsDir = opts.ProjectDir + opts.CacheDir = opts.ProjectDir + opts.CoverageDir = "/data/coverage" + opts.AnalysisId = "FAKE" + + Prod.BaseScriptName = "rider" + Prod.Code = "QDNET" + Prod.Version = "main" + + err := os.Setenv(QodanaDistEnv, opts.ProjectDir) + if err != nil { + t.Fatal(err) + } + err = os.Setenv(QodanaConfEnv, opts.ProjectDir) + if err != nil { + t.Fatal(err) + } + err = os.Setenv("DEVICEID", "FAKE") + if err != nil { + t.Fatal(err) + } + err = os.Setenv("SALT", "FAKE") + if err != nil { + t.Fatal(err) + } + err = os.MkdirAll(opts.ProjectDir, 0o755) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name string + cliProperties []string + qodanaYaml string + expected []string + }{ + { + name: "no overrides, just defaults and .NET project", + cliProperties: []string{}, + qodanaYaml: "dotnet:\n project: project.csproj", + expected: propertiesFixture(true, []string{"-Dqodana.net.project=project.csproj"}), + }, + { + name: "add one CLI property and .NET solution settings", + cliProperties: []string{"-xa", "idea.some.custom.property=1"}, + qodanaYaml: "dotnet:\n solution: solution.sln\n configuration: Release\n platform: x64", + expected: append( + propertiesFixture(true, []string{"-Dqodana.net.solution=solution.sln", "-Dqodana.net.configuration=Release", "-Dqodana.net.platform=x64", "-Didea.some.custom.property=1"}), + "-xa", + ), + }, + { + name: "override options from CLI, YAML should be ignored", + cliProperties: []string{"-Dfus.internal.reduce.initial.delay=false", "-Didea.application.info.value=0", "idea.headless.enable.statistics=false"}, + qodanaYaml: "" + + "version: \"1.0\"\n" + + "properties:\n" + + " fus.internal.reduce.initial.delay: true\n" + + " idea.application.info.value: 0\n", + expected: append([]string{ + "-Dfus.internal.reduce.initial.delay=false", + "-Didea.application.info.value=0", + }, propertiesFixture(false, []string{})[2:]...), + }, + } { + t.Run(tc.name, func(t *testing.T) { + err = os.WriteFile(filepath.Join(opts.ProjectDir, "qodana.yml"), []byte(tc.qodanaYaml), 0o600) + if err != nil { + t.Fatal(err) + } + opts.Property = tc.cliProperties + Config = GetQodanaYaml(opts.ProjectDir) + actual := GetProperties(opts, Config.Properties, Config.DotNet, []string{}) + assert.Equal(t, tc.expected, actual) + }) + } + err = os.RemoveAll(opts.ProjectDir) + if err != nil { + t.Fatal(err) + } +} diff --git a/core/env.go b/core/env.go index b92fff40..97c34c7f 100644 --- a/core/env.go +++ b/core/env.go @@ -20,6 +20,7 @@ import ( "fmt" cienvironment "github.com/cucumber/ci-environment/go" log "github.com/sirupsen/logrus" + "net/url" "os" "runtime" "strings" @@ -49,20 +50,66 @@ const ( qodanaClearKeyring = "QODANA_CLEAR_KEYRING" ) -func ExtractQodanaEnvironment() { +// ExtractQodanaEnvironment extracts Qodana environment variables from the current environment. +func ExtractQodanaEnvironment(setEnvironmentFunc func(string, string)) { ci := cienvironment.DetectCIEnvironment() - qEnv := "qodana" + qEnv := "cli" if ci != nil { qEnv = strings.ReplaceAll(strings.ToLower(ci.Name), " ", "-") - setEnv(qodanaJobUrl, validateJobUrl(ci.URL, qEnv)) + setEnvironmentFunc(qodanaJobUrl, validateJobUrl(ci.URL, qEnv)) if ci.Git != nil { - setEnv(qodanaRemoteUrl, validateRemoteUrl(ci.Git.Remote)) - setEnv(qodanaBranch, validateBranch(ci.Git.Branch, qEnv)) - setEnv(qodanaRevision, ci.Git.Revision) + setEnvironmentFunc(qodanaRemoteUrl, validateRemoteUrl(ci.Git.Remote, qEnv)) + setEnvironmentFunc(qodanaBranch, validateBranch(ci.Git.Branch, qEnv)) + setEnvironmentFunc(qodanaRevision, ci.Git.Revision) } + } else if space := os.Getenv("JB_SPACE_API_URL"); space != "" { + qEnv = "space" + setEnvironmentFunc(qodanaJobUrl, os.Getenv("JB_SPACE_EXECUTION_URL")) + setEnvironmentFunc(qodanaRemoteUrl, getSpaceRemoteUrl()) + setEnvironmentFunc(qodanaBranch, os.Getenv("JB_SPACE_GIT_BRANCH")) + setEnvironmentFunc(qodanaRevision, os.Getenv("JB_SPACE_GIT_REVISION")) } - setEnv(qodanaEnv, fmt.Sprintf("%s:%s", qEnv, Prod.Version)) - setEnv(QodanaDistEnv, Prod.Home) + setEnvironmentFunc(qodanaEnv, fmt.Sprintf("%s:%s", qEnv, Version)) +} + +func validateRemoteUrl(remote string, qEnv string) string { + if strings.HasPrefix(qEnv, "space") { + return getSpaceRemoteUrl() + } + _, err := url.ParseRequestURI(remote) + if remote == "" || err != nil { + log.Warnf("Unable to parse git remote URL, set %s env variable for proper qodana.cloud reporting", qodanaBranch) + return "" + } + return remote +} + +func validateBranch(branch string, env string) string { + if branch == "" { + if env == "github-actions" { + branch = os.Getenv("GITHUB_REF") + } else if env == "azure-pipelines" { + branch = os.Getenv("BUILD_SOURCEBRANCHNAME") + } else if env == "jenkins" { + branch = os.Getenv("GIT_BRANCH") + } + } + if branch == "" { + log.Warnf("Unable to parse git branch, set %s env variable for proper qodana.cloud reporting", qodanaBranch) + return "" + } + return branch +} + +func validateJobUrl(ciUrl string, qEnv string) string { + if strings.HasPrefix(qEnv, "azure") { // temporary workaround for Azure Pipelines + return getAzureJobUrl() + } + _, err := url.ParseRequestURI(ciUrl) + if err != nil { + return "" + } + return ciUrl } // bootstrap takes the given command (from CLI or qodana.yaml) and runs it. @@ -88,10 +135,12 @@ func bootstrap(command string, project string) { } func setEnv(key string, value string) { + log.Debugf("Setting %s=%s", key, value) if os.Getenv(key) == "" && value != "" { err := os.Setenv(key, value) if err != nil { return } + log.Debugf("Set %s=%s", key, value) } } diff --git a/core/ide.go b/core/ide.go index 3459eb82..580d74d9 100644 --- a/core/ide.go +++ b/core/ide.go @@ -439,7 +439,7 @@ func readAppInfoXml(ideDir string) appInfo { func prepareLocalIdeSettings(opts *QodanaOptions) { guessProduct(opts) - ExtractQodanaEnvironment() + ExtractQodanaEnvironment(setEnv) SetupLicenseToken(opts) SetupLicense(cloud.Token.Token) prepareDirectories( diff --git a/core/installers_test.go b/core/installers_test.go index b86d94ed..4f009cbd 100644 --- a/core/installers_test.go +++ b/core/installers_test.go @@ -37,6 +37,10 @@ func TestGetIde(t *testing.T) { } func TestDownloadAndInstallIDE(t *testing.T) { + if os.Getenv("GITHUB_ACTIONS") != "true" { + t.Skip("Skipping IDE download test") + } + ides := []string{"QDNET-EAP"} // QDPY requires exe on Windows, QDNET - does not for _, ide := range ides { DownloadAndInstallIDE(ide, t) diff --git a/core/product_info.go b/core/product_info.go index 10091e8f..95069792 100644 --- a/core/product_info.go +++ b/core/product_info.go @@ -282,6 +282,7 @@ func guessProduct(opts *QodanaOptions) { } log.Debug(Prod) + setEnv(QodanaDistEnv, Prod.Home) } // temporary solution to fix runs in the native mode diff --git a/core/system.go b/core/system.go index 6695facc..568e29b9 100644 --- a/core/system.go +++ b/core/system.go @@ -193,13 +193,6 @@ func prepareHost(opts *QodanaOptions) { if opts.RequiresToken() { opts.ValidateToken(false) } - - if opts.ClearCache { - err := os.RemoveAll(opts.CacheDir) - if err != nil { - log.Errorf("Could not clear local Qodana cache: %s", err) - } - } if err := os.MkdirAll(opts.CacheDir, os.ModePerm); err != nil { log.Fatal("couldn't create a directory ", err.Error()) } @@ -207,7 +200,7 @@ func prepareHost(opts *QodanaOptions) { log.Fatal("couldn't create a directory ", err.Error()) } if opts.Linter != "" { - PrepairContainerEnvSettings() + PrepareContainerEnvSettings() } if opts.Ide != "" { if Contains(allCodes, strings.TrimSuffix(opts.Ide, EapSuffix)) || strings.HasPrefix(opts.Ide, "https://") { diff --git a/core/utils.go b/core/utils.go index 005e8382..8ae62934 100644 --- a/core/utils.go +++ b/core/utils.go @@ -116,6 +116,22 @@ func getAzureJobUrl() string { return "" } +// getSpaceJobUrl returns the Space job URL. +func getSpaceRemoteUrl() string { + if server := os.Getenv("JB_SPACE_API_URL"); server != "" { + return strings.Join([]string{ + "ssh://git@git.", + server, + "/", + os.Getenv("JB_SPACE_PROJECT_KEY"), + "/", + os.Getenv("JB_SPACE_GIT_REPOSITORY_NAME"), + ".git", + }, "") + } + return "" +} + // findProcess using gopsutil to find process by name. func findProcess(processName string) bool { if IsContainer() {