Improve logs masking

.github/workflows/test.yml

@@ -14,11 +14,12 @@ concurrency:
 jobs:
   unit-test:
     name: ${{ matrix.os }}, node ${{ matrix.node }}, python ${{ matrix.python }}
-    runs-on: ${{ matrix.os }}-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu, windows, macos ]
+        # macos-latest = arm64, macos-14-large = amd64
+        os: [ ubuntu-latest, windows-latest, macos-latest, macos-14-large ]
         node: [ 14, 16, 16.9 ]
         include:
           - node: "14"
@@ -27,6 +28,10 @@ jobs:
             python: "3.9"
           - node: "16.9"
             python: "3.x"
+        exclude:
+          # MacOS with ARM does not support node 14
+          - os: macos-latest
+            node: 14
 
     steps:
       - uses: actions/checkout@v4

utils/goutils.go

@@ -16,9 +16,6 @@ import (
 	gofrogcmd "github.com/jfrog/gofrog/io"
 )
 
-// #nosec G101 -- False positive - no hardcoded credentials.
-const credentialsInUrlRegexp = `(http|https|git)://.+@`
-
 // Minimum go version, which its output does not require masking passwords in URLs.
 const minGoVersionForMasking = "go1.13"
 
@@ -237,7 +234,7 @@ func getGoVersion() (string, error) {
 func prepareGlobalRegExp() error {
 	var err error
 	if protocolRegExp == nil {
-		protocolRegExp, err = initRegExp(credentialsInUrlRegexp, removeCredentials)
+		protocolRegExp, err = initRegExp(CredentialsInUrlRegexp, RemoveCredentials)
 		if err != nil {
 			return err
 		}
@@ -260,12 +257,6 @@ func initRegExp(regex string, execFunc func(pattern *gofrogcmd.CmdOutputPattern)
 	return outputPattern, nil
 }
 
-// Remove the credentials information from the line.
-func removeCredentials(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
-	splitResult := strings.Split(pattern.MatchedResults[0], "//")
-	return strings.Replace(pattern.Line, pattern.MatchedResults[0], splitResult[0]+"//", 1), nil
-}
-
 // GetCachePath returns the location of downloads dir inside the GOMODCACHE
 func GetCachePath() (string, error) {
 	goModCachePath, err := GetGoModCachePath()

utils/masking.go

@@ -0,0 +1,15 @@
+package utils
+
+import (
+	gofrogcmd "github.com/jfrog/gofrog/io"
+	"strings"
+)
+
+// #nosec G101 -- False positive - no hardcoded credentials.
+const CredentialsInUrlRegexp = `(?:http|https|git)://.+@`
+
+// Remove the credentials information from the line.
+func RemoveCredentials(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
+	splitResult := strings.Split(pattern.MatchedResults[0], "//")
+	return strings.ReplaceAll(pattern.Line, pattern.MatchedResults[0], splitResult[0]+"//***@"), nil
+}

utils/pythonutils/utils.go

@@ -28,6 +28,11 @@ const (
 
 type PythonTool string
 
+var (
+	credentialsInUrlRegexp = regexp.MustCompile(utils.CredentialsInUrlRegexp)
+	catchAllRegexp         = regexp.MustCompile(".*")
+)
+
 // Parse pythonDependencyPackage list to dependencies map. (mapping dependency to his child deps)
 // Also returns a list of project's root dependencies
 func parseDependenciesToGraph(packages []pythonDependencyPackage) (map[string][]string, []string, error) {
@@ -177,7 +182,7 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup,
 	// Create a parser for multi line pattern matches.
 	lineBuffer := ""
 	collectingMultiLineValue := false
-	parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: regexp.MustCompile(".*"), ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
+	parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: catchAllRegexp, ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
 		// Check if the line matches the startCollectingPattern.
 		if !collectingMultiLineValue && startCollectionRegexp.MatchString(pattern.Line) {
 			// Start collecting lines.
@@ -207,6 +212,53 @@ func getMultilineSplitCaptureOutputPattern(startCollectingPattern, captureGroup,
 	return
 }
 
+// Mask the pre-known credentials that are provided as command arguments from logs.
+// This function creates a log parser for each credentials argument.
+func maskPreKnownCredentials(args []string) (parsers []*gofrogcmd.CmdOutputPattern) {
+	for _, arg := range args {
+		// If this argument is a credentials argument, create a log parser that masks it.
+		if credentialsInUrlRegexp.MatchString(arg) {
+			parsers = append(parsers, maskCredentialsArgument(arg, credentialsInUrlRegexp)...)
+		}
+	}
+	return
+}
+
+// Creates a log parser that masks a pre-known credentials argument from logs.
+// Support both multiline (using the line buffer) and single line credentials.
+func maskCredentialsArgument(credentialsArgument string, credentialsRegex *regexp.Regexp) (parsers []*gofrogcmd.CmdOutputPattern) {
+	lineBuffer := ""
+	parsers = append(parsers, &gofrogcmd.CmdOutputPattern{RegExp: catchAllRegexp, ExecFunc: func(pattern *gofrogcmd.CmdOutputPattern) (string, error) {
+		return handlePotentialCredentialsInLogLine(pattern.Line, credentialsArgument, &lineBuffer, credentialsRegex)
+	}})
+
+	return
+}
+
+func handlePotentialCredentialsInLogLine(patternLine, credentialsArgument string, lineBuffer *string, credentialsRegex *regexp.Regexp) (string, error) {
+	patternLine = strings.TrimSpace(patternLine)
+	if patternLine == "" {
+		return patternLine, nil
+	}
+
+	*lineBuffer += patternLine
+	// If the accumulated line buffer is not a prefix of the credentials argument, reset the buffer and return the line unchanged.
+	if !strings.HasPrefix(credentialsArgument, *lineBuffer) {
+		*lineBuffer = ""
+		return patternLine, nil
+	}
+
+	// When the whole credential was found (aggregated multiline or single line), return it filtered.
+	if credentialsRegex.MatchString(*lineBuffer) {
+		filteredLine, err := utils.RemoveCredentials(&gofrogcmd.CmdOutputPattern{Line: *lineBuffer, MatchedResults: credentialsRegex.FindStringSubmatch(*lineBuffer)})
+		*lineBuffer = ""
+		return filteredLine, err
+	}
+
+	// Avoid logging parts of the credentials till they are fully found.
+	return "", nil
+}
+
 func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log, srcPath string) (map[string]entities.Dependency, error) {
 	if tool == Pipenv {
 		// Add verbosity flag to pipenv commands to collect necessary data
@@ -272,6 +324,8 @@ func InstallWithLogParsing(tool PythonTool, commandArgs []string, log utils.Log,
 	// Extract cached file, stored in Artifactory. (value at log may be split into multiple lines)
 	parsers = append(parsers, getMultilineSplitCaptureOutputPattern(startUsingCachedPattern, usingCacheCaptureGroup, endPattern, saveCaptureGroupAsDependencyInfo)...)
 
+	parsers = append(parsers, maskPreKnownCredentials(commandArgs)...)
+
 	// Extract already installed packages names.
 	parsers = append(parsers, &gofrogcmd.CmdOutputPattern{
 		RegExp: regexp.MustCompile(`^Requirement\salready\ssatisfied:\s(\w[\w-.]+)`),

utils/pythonutils/utils_test.go

@@ -2,6 +2,8 @@ package pythonutils
 
 import (
 	"fmt"
+	"github.com/jfrog/build-info-go/utils"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -139,3 +141,79 @@ func runDummyTextStream(t *testing.T, txt string, parsers []*gofrogcmd.CmdOutput
 		}
 	}
 }
+
+func TestMaskPreKnownCredentials(t *testing.T) {
+	tests := []struct {
+		name                string
+		inputText           string
+		credentialsArgument string
+	}{
+		{
+			name: "Single line credentials",
+			inputText: `
+Preparing Installation of "toml==0.10.2; python_version >= '2.6' and 
+python_version not in '3.0, 3.1, 3.2' 
+--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b 
+--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+$ 
+/usr/local/Cellar/pipenv/2023.12.1/libexec/lib/python3.12/site-packages/pipenv/p
+atched/pip/__pip-runner__.py install -i 
+https://user:[email protected]/artifactory/api/pypi/cli-pipenv-pypi-virtual-1715766379/simple 
+--no-input --upgrade --no-deps -r 
+/var/folders/2c/cdvww2550p90b0sdbz6w6jqc0000gn/T/pipenv-bs956chg-requirements/pi
+penv-hejkfcsj-hashed-reqs.txt`,
+			credentialsArgument: "https://user:[email protected]/artifactory/api/pypi/cli-pipenv-pypi-virtual-1715766379/simple",
+		},
+		{
+			name: "Multiline credentials",
+			inputText: `
+Preparing Installation of "toml==0.10.2; python_version >= '2.6' and 
+python_version not in '3.0, 3.1, 3.2' 
+--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b 
+--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+$ 
+/usr/local/Cellar/pipenv/2023.12.1/libexec/lib/python3.12/site-packages/pipenv/p
+atched/pip/__pip-runner__.py install -i 
+https://user:not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an.
+actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an.
+actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an.
+[email protected]/artifactory/api/pypi/cli-pipenv-pypi-virtual-17
+15766379/simple 
+--no-input --upgrade --no-deps -r 
+/var/folders/2c/cdvww2550p90b0sdbz6w6jqc0000gn/T/pipenv-bs956chg-requirements/pi
+penv-hejkfcsj-hashed-reqs.txt`,
+			credentialsArgument: "https://user:not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." +
+				"actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." +
+				"actual.token.not.an.actual.token.not.an.actual.token.not.an.actual.token.not.an." +
+				"[email protected]/artifactory/api/pypi/cli-pipenv-pypi-virtual-17" +
+				"15766379/simple",
+		},
+	}
+
+	for _, testCase := range tests {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Contains(t, getOnelinerText(testCase.inputText), testCase.credentialsArgument)
+			outputText := maskCredentialsInText(t, testCase.inputText, testCase.credentialsArgument)
+			assert.NotContains(t, getOnelinerText(outputText), testCase.credentialsArgument)
+		})
+	}
+}
+
+// This method mimics RunCmdWithOutputParser, in which the masking parsers will be used.
+func maskCredentialsInText(t *testing.T, text, credentialsArgument string) string {
+	lines := strings.Split(text, "\n")
+	credentialsRegex := regexp.MustCompile(utils.CredentialsInUrlRegexp)
+	lineBuffer := ""
+	outputText := ""
+
+	for _, line := range lines {
+		outputLine, err := handlePotentialCredentialsInLogLine(line, credentialsArgument, &lineBuffer, credentialsRegex)
+		assert.NoError(t, err)
+		outputText += outputLine + "\n"
+	}
+	return outputText
+}
+
+func getOnelinerText(inputText string) string {
+	return strings.ReplaceAll(inputText, "\n", "")
+}