Skip to content

Commit

Permalink
♻️ Refactor CI/CD, Test and IDE related code
Browse files Browse the repository at this point in the history 
Refactor and Extract CI/CD-related code in 'core/container.go' and 'core/core_test.go' files. Test file modifications are extensive and mainly include renaming, method extraction, and cleanup to improve readability and organization of test conditions. The CI/CD code extraction made the 'PrepareContainerEnvSettings' method cleaner and more focused. Consequentially, the 'cmd/pull.go' file is updated to use the refactored method. Changes also include extracting the 'QODANA_*' environment variables to a separate function for better modularity. Revisions in the '.github/workflows/ci.yml' file includes replacing 'QODANA_TOKEN' with 'QODANA_LICENSE_ONLY_TOKEN' to reflect changes in token handling. Additionally, 'core/ide.go' and 'cmd/pull.go' have minor changes to reflect the decoupling of Qodana environment and CI/CD related settings.
  • Loading branch information
@tiulpin
tiulpin committed Oct 26, 2023
1 parent a807251 commit 7218c69
Show file tree
Hide file tree
Showing 12 changed files with 364 additions and 319 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ jobs:
registry: registry.jetbrains.team
username: ${{ secrets.SPACE_USERNAME }}
password: ${{ secrets.SPACE_PASSWORD }}
- run: go test -v ./... -coverprofile cover.out
- run: go test -v ./... -coverprofile cover.out -coverpkg=./...
env:
QODANA_TOKEN: ${{ secrets.TEST_QODANA_TOKEN }}
QODANA_LICENSE_ONLY_TOKEN: ${{ secrets.QODANA_LICENSE_ONLY_TOKEN }}
- if: startsWith(matrix.os, 'ubuntu')
uses: JetBrains/qodana-action@main
env:
Expand Down
144 changes: 7 additions & 137 deletions cmd/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,9 @@ import (
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"testing"

"github.com/stretchr/testify/assert"

log "github.com/sirupsen/logrus"

"github.com/JetBrains/qodana-cli/v2023/core"
Expand All @@ -52,6 +49,10 @@ func createProject(t *testing.T, name string) string {
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(location+"/.idea", 0o755)
if err != nil {
t.Fatal(err)
}
return location
}

Expand Down Expand Up @@ -210,7 +211,7 @@ func TestContributorsCommand(t *testing.T) {
}

func TestAllCommandsWithContainer(t *testing.T) {
linter := "jetbrains/qodana-python-community:2023.2"
linter := "registry.jetbrains.team/p/sa/containers/qodana-python-community:latest"

if os.Getenv("GITHUB_ACTIONS") == "true" {
//goland:noinspection GoBoolExpressions
Expand Down Expand Up @@ -249,6 +250,7 @@ func TestAllCommandsWithContainer(t *testing.T) {
"-i", projectPath,
"-o", resultsPath,
"--cache-dir", filepath.Join(projectPath, "cache"),
"-v", filepath.Join(projectPath, ".idea") + ":/data/some",
"--fail-threshold", "5",
"--print-problems",
"--apply-fixes",
Expand Down Expand Up @@ -332,7 +334,7 @@ func TestAllCommandsWithContainer(t *testing.T) {
func TestScanWithIde(t *testing.T) {
log.SetLevel(log.DebugLevel)
ide := "QDPY"
token := os.Getenv("TESTS_QODANA_TOKEN")
token := os.Getenv("QODANA_LICENSE_ONLY_TOKEN")
if //goland:noinspection GoBoolExpressions
token == "" {
t.Skip("set your token here to run the test")
Expand All @@ -359,135 +361,3 @@ func TestScanWithIde(t *testing.T) {
t.Fatal(err)
}
}

func propertiesFixture(enableStats bool, additionalProperties []string) []string {
properties := []string{
"-Dfus.internal.reduce.initial.delay=true",
fmt.Sprintf("-Didea.application.info.value=%s", filepath.Join(os.TempDir(), "entrypoint", "QodanaAppInfo.xml")),
"-Didea.class.before.app=com.jetbrains.rider.protocol.EarlyBackendStarter",
fmt.Sprintf("-Didea.config.path=%s", filepath.Join(os.TempDir(), "entrypoint")),
fmt.Sprintf("-Didea.headless.enable.statistics=%t", enableStats),
"-Didea.headless.statistics.device.id=FAKE",
"-Didea.headless.statistics.max.files.to.send=5000",
"-Didea.headless.statistics.salt=FAKE",
fmt.Sprintf("-Didea.log.path=%s", filepath.Join(os.TempDir(), "entrypoint", "log")),
"-Didea.parent.prefix=Rider",
"-Didea.platform.prefix=Qodana",
fmt.Sprintf("-Didea.plugins.path=%s", filepath.Join(os.TempDir(), "entrypoint", "plugins", "master")),
"-Didea.qodana.thirdpartyplugins.accept=true",
fmt.Sprintf("-Didea.system.path=%s", filepath.Join(os.TempDir(), "entrypoint", "idea", "master")),
"-Dinspect.save.project.settings=true",
"-Djava.awt.headless=true",
"-Djava.net.useSystemProxies=true",
"-Djdk.attach.allowAttachSelf=true",
`-Djdk.http.auth.tunneling.disabledSchemes=""`,
"-Djdk.module.illegalAccess.silent=true",
"-Dkotlinx.coroutines.debug=off",
"-Dqodana.automation.guid=FAKE",
"-Didea.job.launcher.without.timeout=true",
"-Dqodana.coverage.input=/data/coverage",
"-Drider.collect.full.container.statistics=true",
"-Drider.suppress.std.redirect=true",
"-Dsun.io.useCanonCaches=false",
"-Dsun.tools.attach.tmp.only=true",
"-XX:+HeapDumpOnOutOfMemoryError",
"-XX:+UseG1GC",
"-XX:-OmitStackTraceInFastThrow",
"-XX:CICompilerCount=2",
"-XX:MaxJavaStackTraceDepth=10000",
"-XX:MaxRAMPercentage=70",
"-XX:ReservedCodeCacheSize=512m",
"-XX:SoftRefLRUPolicyMSPerMB=50",
fmt.Sprintf("-Xlog:gc*:%s", filepath.Join(os.TempDir(), "entrypoint", "log", "gc.log")),
"-ea",
}
properties = append(properties, additionalProperties...)
sort.Strings(properties)
return properties
}

func Test_Properties(t *testing.T) {
opts := &core.QodanaOptions{}
tmpDir := filepath.Join(os.TempDir(), "entrypoint")
opts.ProjectDir = tmpDir
opts.ResultsDir = opts.ProjectDir
opts.CacheDir = opts.ProjectDir
opts.CoverageDir = "/data/coverage"
opts.AnalysisId = "FAKE"

core.Prod.BaseScriptName = "rider"
core.Prod.Code = "QDNET"
core.Prod.Version = "main"

err := os.Setenv(core.QodanaDistEnv, opts.ProjectDir)
if err != nil {
t.Fatal(err)
}
err = os.Setenv(core.QodanaConfEnv, opts.ProjectDir)
if err != nil {
t.Fatal(err)
}
err = os.Setenv("DEVICEID", "FAKE")
if err != nil {
t.Fatal(err)
}
err = os.Setenv("SALT", "FAKE")
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(opts.ProjectDir, 0o755)
if err != nil {
t.Fatal(err)
}

for _, tc := range []struct {
name string
cliProperties []string
qodanaYaml string
expected []string
}{
{
name: "no overrides, just defaults and .NET project",
cliProperties: []string{},
qodanaYaml: "dotnet:\n project: project.csproj",
expected: propertiesFixture(true, []string{"-Dqodana.net.project=project.csproj"}),
},
{
name: "add one CLI property and .NET solution settings",
cliProperties: []string{"-xa", "idea.some.custom.property=1"},
qodanaYaml: "dotnet:\n solution: solution.sln\n configuration: Release\n platform: x64",
expected: append(
propertiesFixture(true, []string{"-Dqodana.net.solution=solution.sln", "-Dqodana.net.configuration=Release", "-Dqodana.net.platform=x64", "-Didea.some.custom.property=1"}),
"-xa",
),
},
{
name: "override options from CLI, YAML should be ignored",
cliProperties: []string{"-Dfus.internal.reduce.initial.delay=false", "-Didea.application.info.value=0", "idea.headless.enable.statistics=false"},
qodanaYaml: "" +
"version: \"1.0\"\n" +
"properties:\n" +
" fus.internal.reduce.initial.delay: true\n" +
" idea.application.info.value: 0\n",
expected: append([]string{
"-Dfus.internal.reduce.initial.delay=false",
"-Didea.application.info.value=0",
}, propertiesFixture(false, []string{})[2:]...),
},
} {
t.Run(tc.name, func(t *testing.T) {
err = os.WriteFile(filepath.Join(opts.ProjectDir, "qodana.yml"), []byte(tc.qodanaYaml), 0o600)
if err != nil {
t.Fatal(err)
}
opts.Property = tc.cliProperties
core.Config = core.GetQodanaYaml(opts.ProjectDir)
actual := core.GetProperties(opts, core.Config.Properties, core.Config.DotNet, []string{})
assert.Equal(t, tc.expected, actual)
})
}
err = os.RemoveAll(opts.ProjectDir)
if err != nil {
t.Fatal(err)
}
}
25 changes: 3 additions & 22 deletions cmd/pull.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,37 +23,18 @@ import (
"github.com/spf13/cobra"
)

// pullOptions represents pull command options.
type pullOptions struct {
Linter string
ProjectDir string
YamlName string
}

// newPullCommand returns a new instance of the show command.
func newPullCommand() *cobra.Command {
options := &pullOptions{}
options := &core.QodanaOptions{}
cmd := &cobra.Command{
Use: "pull",
Short: "Pull latest version of linter",
Long: `An alternative to pull an image.`,
PreRun: func(cmd *cobra.Command, args []string) {
core.PrepairContainerEnvSettings()
core.PrepareContainerEnvSettings()
},
Run: func(cmd *cobra.Command, args []string) {
if options.Linter == "" {
qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName)
if qodanaYaml.Linter == "" {
core.WarningMessage(
"No valid qodana.yaml found. Have you run %s? Running that for you...",
core.PrimaryBold("qodana init"),
)
options.Linter = core.GetLinter(options.ProjectDir, options.YamlName)
core.EmptyMessage()
} else {
options.Linter = qodanaYaml.Linter
}
}
fetchAnalyzerSetting(options)
containerClient, err := client.NewClientWithOpts()
if err != nil {
log.Fatal("couldn't connect to container engine ", err)
Expand Down
38 changes: 21 additions & 17 deletions cmd/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,7 @@ But you can always override qodana.yaml options with the following command-line
Run: func(cmd *cobra.Command, args []string) {
ctx := cmd.Context()
checkProjectDir(options.ProjectDir)
if options.Linter == "" && options.Ide == "" {
qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName)
if qodanaYaml.Linter == "" && qodanaYaml.Ide == "" {
core.WarningMessage(
"No valid `linter:` field found in %s. Have you run %s? Running that for you...",
core.PrimaryBold(options.YamlName),
core.PrimaryBold("qodana init"),
)
options.Linter = core.GetLinter(options.ProjectDir, options.YamlName)
core.EmptyMessage()
} else {
options.Linter = qodanaYaml.Linter
}
if options.Ide == "" {
options.Ide = qodanaYaml.Ide
}
}
fetchAnalyzerSetting(options)
exitCode := core.RunAnalysis(ctx, options)

checkExitCode(exitCode, options.ResultsDir)
Expand Down Expand Up @@ -155,6 +139,26 @@ But you can always override qodana.yaml options with the following command-line
return cmd
}

func fetchAnalyzerSetting(options *core.QodanaOptions) {
if options.Linter == "" && options.Ide == "" {
qodanaYaml := core.LoadQodanaYaml(options.ProjectDir, options.YamlName)
if qodanaYaml.Linter == "" && qodanaYaml.Ide == "" {
core.WarningMessage(
"No valid `linter:` field found in %s. Have you run %s? Running that for you...",
core.PrimaryBold(options.YamlName),
core.PrimaryBold("qodana init"),
)
options.Linter = core.GetLinter(options.ProjectDir, options.YamlName)
core.EmptyMessage()
} else {
options.Linter = qodanaYaml.Linter
}
if options.Ide == "" {
options.Ide = qodanaYaml.Ide
}
}
}

func checkProjectDir(projectDir string) {
if core.IsInteractive() && core.IsHomeDirectory(projectDir) {
core.WarningMessage(
Expand Down
62 changes: 3 additions & 59 deletions core/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"fmt"
"github.com/pterm/pterm"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
Expand All @@ -33,8 +32,6 @@ import (

cliconfig "github.com/docker/cli/cli/config"

"github.com/cucumber/ci-environment/go"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
Expand Down Expand Up @@ -107,66 +104,13 @@ func encodeAuthToBase64(authConfig types.AuthConfig) (string, error) {
return base64.URLEncoding.EncodeToString(buf), nil
}

// extractQodanaEnvironmentForDocker extracts Qodana env variables QODANA_* to the given environment array.
func extractQodanaEnvironmentForDocker(opts *QodanaOptions) {
ci := cienvironment.DetectCIEnvironment()
qEnv := "cli"
if ci != nil {
qEnv = strings.ReplaceAll(strings.ToLower(ci.Name), " ", "-")
opts.setenv(qodanaJobUrl, validateJobUrl(ci.URL, qEnv))
if ci.Git != nil {
opts.setenv(qodanaRemoteUrl, validateRemoteUrl(ci.Git.Remote))
opts.setenv(qodanaBranch, validateBranch(ci.Git.Branch, qEnv))
opts.setenv(qodanaRevision, ci.Git.Revision)
}
}
opts.setenv(qodanaEnv, fmt.Sprintf("%s:%s", qEnv, Version))
}

func validateRemoteUrl(remote string) string {
_, err := url.ParseRequestURI(remote)
if remote == "" || err != nil {
log.Warnf("Unable to parse git remote URL, set %s env variable for proper qodana.cloud reporting", qodanaBranch)
return ""
}
return remote
}

func validateBranch(branch string, env string) string {
if branch == "" {
if env == "github-actions" {
branch = os.Getenv("GITHUB_REF")
} else if env == "azure-pipelines" {
branch = os.Getenv("BUILD_SOURCEBRANCHNAME")
} else if env == "jenkins" {
branch = os.Getenv("GIT_BRANCH")
}
}
if branch == "" {
log.Warnf("Unable to parse git branch, set %s env variable for proper qodana.cloud reporting", qodanaBranch)
return ""
}
return branch
}

func validateJobUrl(ciUrl string, qEnv string) string {
if strings.HasPrefix(qEnv, "azure") { // temporary workaround for Azure Pipelines
return getAzureJobUrl()
}
_, err := url.ParseRequestURI(ciUrl)
if err != nil {
return ""
}
return ciUrl
}

func checkRequiredToolInstalled(tool string) bool {
_, err := exec.LookPath(tool)
return err == nil
}

// PrepairContainerEnvSettings checks if the host is ready to run Qodana container images.
func PrepairContainerEnvSettings() {
// PrepareContainerEnvSettings checks if the host is ready to run Qodana container images.
func PrepareContainerEnvSettings() {
var tool string
if os.Getenv(qodanaCliUsePodman) == "" && checkRequiredToolInstalled("docker") {
tool = "docker"
Expand Down Expand Up @@ -308,7 +252,7 @@ func CheckContainerEngineMemory() {
// getDockerOptions returns qodana docker container options.
func getDockerOptions(opts *QodanaOptions) *types.ContainerCreateConfig {
cmdOpts := getIdeArgs(opts)
extractQodanaEnvironmentForDocker(opts)
ExtractQodanaEnvironment(opts.setenv)
cachePath, err := filepath.Abs(opts.CacheDir)
if err != nil {
log.Fatal("couldn't get abs path for cache", err)
Expand Down
Loading

0 comments on commit 7218c69

Please sign in to comment.