From 58b020b5c5a507dc5fd51b130086f3fd82beac1c Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 22 May 2024 13:02:47 +0300 Subject: [PATCH 1/2] Improve logs masking (#252) --- .github/workflows/test.yml | 9 +++- utils/goutils.go | 11 +---- utils/masking.go | 15 +++++++ utils/pythonutils/utils.go | 56 ++++++++++++++++++++++- utils/pythonutils/utils_test.go | 78 +++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 utils/masking.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e6156c4a..292fe97b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,11 +14,12 @@ concurrency: jobs: unit-test: name: ${{ matrix.os }}, node ${{ matrix.node }}, python ${{ matrix.python }} - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ ubuntu, windows, macos ] + # macos-latest = arm64, macos-14-large = amd64 + os: [ ubuntu-latest, windows-latest, macos-latest, macos-14-large ] node: [ 14, 16, 16.9 ] include: - node: "14" @@ -27,6 +28,10 @@ jobs: python: "3.9" - node: "16.9" python: "3.x" + exclude: + # MacOS with ARM does not support node 14 + - os: macos-latest + node: 14 steps: - uses: actions/checkout@v4 diff --git a/utils/goutils.go b/utils/goutils.go index 1b988dbd..6d805945 100644 --- a/utils/goutils.go +++ b/utils/goutils.go @@ -16,9 +16,6 @@ import ( gofrogcmd "github.com/jfrog/gofrog/io" ) -// #nosec G101 -- False positive - no hardcoded credentials. -const credentialsInUrlRegexp = `(http|https|git)://.+@` - // Minimum go version, which its output does not require masking passwords in URLs. const minGoVersionForMasking = "go1.13" @@ -237,7 +234,7 @@ func getGoVersion() (string, error) { func prepareGlobalRegExp() error { var err error if protocolRegExp == nil { - protocolRegExp, err = initRegExp(credentialsInUrlRegexp, removeCredentials) + protocolRegExp, err = initRegExp(CredentialsInUrlRegexp, RemoveCredentials) if err != nil { return err } @@ -260,12 +257,6 @@ func initRegExp(regex string, execFunc func(pattern *gofrogcmd.CmdOutputPattern) return outputPattern, nil } -// Remove the credentials information from the line. -func removeCredentials(pattern *gofrogcmd.CmdOutputPattern) (string, error) { - splitResult := strings.Split(pattern.MatchedResults[0], "//") - return strings.Replace(pattern.Line, pattern.MatchedResults[0], splitResult[0]+"//", 1), nil -} - // GetCachePath returns the location of downloads dir inside the GOMODCACHE func GetCachePath() (string, error) { goModCachePath, err := GetGoModCachePath() diff --git a/utils/masking.go b/utils/masking.go new file mode 100644 index 00000000..8921a1fc --- /dev/null +++ b/utils/masking.go @@ -0,0 +1,15 @@ +package utils + +import ( + gofrogcmd "github.com/jfrog/gofrog/io" + "strings" +) + +// #nosec G101 -- False positive - no hardcoded credentials. +const CredentialsInUrlRegexp = `(?:http|https|git)://.+@` + +// Remove the credentials information from the line. +func RemoveCredentials(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + splitResult := strings.Split(pattern.MatchedResults[0], "//") + return strings.ReplaceAll(pattern.Line, pattern.MatchedResults[0], splitResult[0]+"//***@"), nil +} diff --git a/utils/pythonutils/utils.go b/utils/pythonutils/utils.go index 8c676e61..483b5e03 100644 --- a/utils/pythonutils/utils.go +++ b/utils/pythonutils/utils.go @@ -28,6 +28,11 @@ const ( type PythonTool string +var ( + credentialsInUrlRegexp = regexp.MustCompile(utils.CredentialsInUrlRegexp) + catchAllRegexp = regexp.MustCompile(".*") +) + // Parse pythonDependencyPackage list to dependencies map. (mapping dependency to his child deps) // Also returns a list of project's root dependencies func parseDependenciesToGraph(packages []pythonDependencyPackage) (map[string][]string, []string, error) { @@ -177,7 +182,7 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup, // Create a parser for multi line pattern matches. lineBuffer := "" collectingMultiLineValue := false - parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: regexp.MustCompile(".*"), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: catchAllRegexp, ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { // Check if the line matches the startCollectingPattern. if !collectingMultiLineValue && startCollectionRegexp.MatchString(pattern.Line) { // Start collecting lines. @@ -207,6 +212,53 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup, return } +// Mask the pre-known credentials that are provided as command arguments from logs. +// This function creates a log parser for each credentials argument. +func maskPreKnownCredentials(args []string) (parsers []*gofrogcmd.CmdOutputPattern) { + for _, arg := range args { + // If this argument is a credentials argument, create a log parser that masks it. + if credentialsInUrlRegexp.MatchString(arg) { + parsers = append(parsers, maskCredentialsArgument(arg, credentialsInUrlRegexp)...) + } + } + return +} + +// Creates a log parser that masks a pre-known credentials argument from logs. +// Support both multiline (using the line buffer) and single line credentials. +func maskCredentialsArgument(credentialsArgument string, credentialsRegex *regexp.Regexp) (parsers []*gofrogcmd.CmdOutputPattern) { + lineBuffer := "" + parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: catchAllRegexp, ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) { + return handlePotentialCredentialsInLogLine(pattern.Line, credentialsArgument, &lineBuffer, credentialsRegex) + }}) + + return +} + +func handlePotentialCredentialsInLogLine(patternLine, credentialsArgument string, lineBuffer *string, credentialsRegex *regexp.Regexp) (string, error) { + patternLine = strings.TrimSpace(patternLine) + if patternLine == "" { + return patternLine, nil + } + + *lineBuffer += patternLine + // If the accumulated line buffer is not a prefix of the credentials argument, reset the buffer and return the line unchanged. + if !strings.HasPrefix(credentialsArgument, *lineBuffer) { + *lineBuffer = "" + return patternLine, nil + } + + // When the whole credential was found (aggregated multiline or single line), return it filtered. + if credentialsRegex.MatchString(*lineBuffer) { + filteredLine, err := utils.RemoveCredentials(&gofrogcmd.CmdOutputPattern{Line: *lineBuffer, MatchedResults: credentialsRegex.FindStringSubmatch(*lineBuffer)}) + *lineBuffer = "" + return filteredLine, err + } + + // Avoid logging parts of the credentials till they are fully found. + return "", nil +} + func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, srcPath string) (map[string]entities.Dependency, error) { if tool == Pipenv { // Add verbosity flag to pipenv commands to collect necessary data @@ -272,6 +324,8 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, // Extract cached file, stored in Artifactory. (value at log may be split into multiple lines) parsers = append(parsers, getMultilineSplitCaptureOutputPattern(startUsingCachedPattern, usingCacheCaptureGroup, endPattern, saveCaptureGroupAsDependencyInfo)...) + parsers = append(parsers, maskPreKnownCredentials(commandArgs)...) + // Extract already installed packages names. parsers = append(parsers, &gofrogcmd.CmdOutputPattern{ RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`), diff --git a/utils/pythonutils/utils_test.go b/utils/pythonutils/utils_test.go index ea2f7881..9db46fe0 100644 --- a/utils/pythonutils/utils_test.go +++ b/utils/pythonutils/utils_test.go @@ -2,6 +2,8 @@ package pythonutils import ( "fmt" + "github.com/jfrog/build-info-go/utils" + "regexp" "strings" "testing" @@ -139,3 +141,79 @@ func runDummyTextStream(t *testing.T, txt string, parsers []*gofrogcmd.CmdOutput } } } + +func TestMaskPreKnownCredentials(t *testing.T) { + tests := []struct { + name string + inputText string + credentialsArgument string + }{ + { + name: "Single line credentials", + inputText: ` +Preparing Installation of "toml==0.10.2; python_version >= '2.6' and +python_version not in '3.0, 3.1, 3.2' +--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b +--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" +$ +/usr/local/Cellar/pipenv/2023.12.1/libexec/lib/python3.12/site-packages/pipenv/p +atched/pip/__pip-runner__.py install -i +https://user:not.an.actual.token@myplatform.jfrog.io/artifactory/api/pypi/cli-pipenv-pypi-virtual-1715766379/simple +--no-input --upgrade --no-deps -r +/var/folders/2c/cdvww2550p90b0sdbz6w6jqc0000gn/T/pipenv-bs956chg-requirements/pi +penv-hejkfcsj-hashed-reqs.txt`, + credentialsArgument: "https://user:not.an.actual.token@myplatform.jfrog.io/artifactory/api/pypi/cli-pipenv-pypi-virtual-1715766379/simple", + }, + { + name: "Multiline credentials", + inputText: ` +Preparing Installation of "toml==0.10.2; python_version >= '2.6' and +python_version not in '3.0, 3.1, 3.2' +--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b +--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" +$ +/usr/local/Cellar/pipenv/2023.12.1/libexec/lib/python3.12/site-packages/pipenv/p +atched/pip/__pip-runner__.py install -i +https://user:not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an. +actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an. +actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an. +actual.token@myplatform.jfrog.io/artifactory/api/pypi/cli-pipenv-pypi-virtual-17 +15766379/simple +--no-input --upgrade --no-deps -r +/var/folders/2c/cdvww2550p90b0sdbz6w6jqc0000gn/T/pipenv-bs956chg-requirements/pi +penv-hejkfcsj-hashed-reqs.txt`, + credentialsArgument: "https://user:not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." + + "actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." + + "actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." + + "actual.token@myplatform.jfrog.io/artifactory/api/pypi/cli-pipenv-pypi-virtual-17" + + "15766379/simple", + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + assert.Contains(t, getOnelinerText(testCase.inputText), testCase.credentialsArgument) + outputText := maskCredentialsInText(t, testCase.inputText, testCase.credentialsArgument) + assert.NotContains(t, getOnelinerText(outputText), testCase.credentialsArgument) + }) + } +} + +// This method mimics RunCmdWithOutputParser, in which the masking parsers will be used. +func maskCredentialsInText(t *testing.T, text, credentialsArgument string) string { + lines := strings.Split(text, "\n") + credentialsRegex := regexp.MustCompile(utils.CredentialsInUrlRegexp) + lineBuffer := "" + outputText := "" + + for _, line := range lines { + outputLine, err := handlePotentialCredentialsInLogLine(line, credentialsArgument, &lineBuffer, credentialsRegex) + assert.NoError(t, err) + outputText += outputLine + "\n" + } + return outputText +} + +func getOnelinerText(inputText string) string { + return strings.ReplaceAll(inputText, "\n", "") +} From bde59ff4e10831174ff6e746b9118100194915bc Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Wed, 22 May 2024 15:03:41 +0300 Subject: [PATCH 2/2] update MavenExtractorDependencyVersion (#254) --- build/maven.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/maven.go b/build/maven.go index 36bc29ab..65c8bba2 100644 --- a/build/maven.go +++ b/build/maven.go @@ -20,7 +20,7 @@ const ( classworldsConfFileName = "classworlds.conf" PropertiesTempFolderName = "properties" MavenExtractorRemotePath = "org/jfrog/buildinfo/build-info-extractor-maven3/%s" - MavenExtractorDependencyVersion = "2.41.14" + MavenExtractorDependencyVersion = "2.41.16" ClassworldsConf = `main is org.apache.maven.cli.MavenCli from plexus.core