fix(piperenv): add JSON data cleaning to handle invalid UTF-8 and control characters (#5449)

Author: phgermanov

cmd/writePipelineEnv.go

@@ -107,7 +107,11 @@ func handleEncryption(password string, data []byte) ([]byte, error) {
 
 func parseInput(data []byte) (piperenv.CPEMap, error) {
 	commonPipelineEnv := piperenv.CPEMap{}
-	decoder := json.NewDecoder(bytes.NewReader(data))
+
+	// Clean invalid UTF-8 sequences that can cause JSON parsing errors
+	cleanData := piperenv.CleanJSONData(data)
+
+	decoder := json.NewDecoder(bytes.NewReader(cleanData))
 	decoder.UseNumber()
 	if err := decoder.Decode(&commonPipelineEnv); err != nil {
 		return nil, err

cmd/writePipelineEnv_test.go

@@ -0,0 +1,85 @@
+//go:build unit
+// +build unit
+
+package cmd
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/SAP/jenkins-library/pkg/piperenv"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCleanJSONDataWritePipeline(t *testing.T) {
+	t.Parallel()
+
+	// Test valid UTF-8 (should be unchanged)
+	validData := []byte(`{"git":{"commitMessage":"This is a valid commit message"}}`)
+	result := piperenv.CleanJSONData(validData)
+	require.Equal(t, validData, result)
+
+	// Test emoji with valid UTF-8 (should be unchanged)
+	emojiData := []byte(`{"git":{"commitMessage":"🚀 feat: add new feature"}}`)
+	result = piperenv.CleanJSONData(emojiData)
+	require.Equal(t, emojiData, result)
+
+	// Test data with JSON control character (like \x16)
+	invalidData := []byte("{\"git\":{\"commitMessage\":\"Test \x16 control char\"}}")
+	result = piperenv.CleanJSONData(invalidData)
+	require.NotEqual(t, invalidData, result)
+	require.True(t, json.Valid(result), "Result should be valid JSON")
+
+	// Verify we can parse the cleaned data as JSON
+	var parsed map[string]interface{}
+	err := json.Unmarshal(result, &parsed)
+	require.NoError(t, err)
+	require.Contains(t, parsed, "git")
+
+	// The control character should be escaped as unicode
+	require.Contains(t, string(result), "\\u0016")
+}
+
+func TestParseInputWithInvalidUTF8(t *testing.T) {
+	t.Parallel()
+
+	// Test parsing JSON with control character
+	invalidJSON := []byte("{\"git\":{\"commitMessage\":\"Test \x16 control char commit\"}}")
+
+	// Should not fail due to control characters
+	cpeMap, err := parseInput(invalidJSON)
+	require.NoError(t, err)
+	require.NotNil(t, cpeMap)
+
+	// Verify we can access the git section
+	if git, ok := cpeMap["git"]; ok {
+		require.NotNil(t, git)
+		if gitMap, ok := git.(map[string]interface{}); ok {
+			commitMsg, exists := gitMap["commitMessage"]
+			require.True(t, exists)
+			require.IsType(t, "", commitMsg)
+			// The message should be present and non-empty
+			require.NotEmpty(t, commitMsg)
+		}
+	}
+}
+
+func TestParseInputWithValidEmoji(t *testing.T) {
+	t.Parallel()
+
+	// Test parsing JSON with valid emoji
+	validJSON := []byte(`{"git":{"commitMessage":"🚀 feat: add new feature"}}`)
+
+	cpeMap, err := parseInput(validJSON)
+	require.NoError(t, err)
+	require.NotNil(t, cpeMap)
+
+	// Verify emoji is preserved
+	if git, ok := cpeMap["git"]; ok {
+		if gitMap, ok := git.(map[string]interface{}); ok {
+			commitMsg, exists := gitMap["commitMessage"]
+			require.True(t, exists)
+			require.Equal(t, "🚀 feat: add new feature", commitMsg)
+		}
+	}
+}

pkg/piperenv/CPEMap.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/SAP/jenkins-library/pkg/log"
 )
@@ -122,7 +123,11 @@ func readFileContent(fullPath string) (string, interface{}, bool, error) {
 	if strings.HasSuffix(fullPath, ".json") {
 		// value is json encoded
 		var value interface{}
-		decoder := json.NewDecoder(bytes.NewReader(fileContent))
+
+		// Clean invalid UTF-8 sequences that can cause JSON parsing errors
+		cleanContent := CleanJSONData(fileContent)
+
+		decoder := json.NewDecoder(bytes.NewReader(cleanContent))
 		decoder.UseNumber()
 		err = decoder.Decode(&value)
 		if err != nil {
@@ -135,3 +140,73 @@ func readFileContent(fullPath string) (string, interface{}, bool, error) {
 	}
 	return fileName, string(fileContent), toBeEmptied, nil
 }
+
+// CleanJSONData handles both invalid UTF-8 sequences and JSON control characters that can cause parsing errors
+func CleanJSONData(data []byte) []byte {
+	// First ensure valid UTF-8
+	if !utf8.Valid(data) {
+		data = []byte(strings.ToValidUTF8(string(data), "\uFFFD"))
+	}
+
+	// Check if it's already valid JSON - if so, return as-is
+	if json.Valid(data) {
+		return data
+	}
+
+	// If not valid JSON, try to escape control characters in string literals
+	s := string(data)
+	result := strings.Builder{}
+	inString := false
+	escaped := false
+
+	for _, r := range s {
+		if !inString {
+			result.WriteRune(r)
+			if r == '"' {
+				inString = true
+			}
+			continue
+		}
+
+		if escaped {
+			result.WriteRune(r)
+			escaped = false
+			continue
+		}
+
+		if r == '\\' {
+			escaped = true
+			result.WriteRune(r)
+			continue
+		}
+
+		if r == '"' {
+			inString = false
+			result.WriteRune(r)
+			continue
+		}
+
+		// Handle control characters (0x00-0x1F) that are invalid in JSON strings
+		if r < 0x20 {
+			switch r {
+			case '\b':
+				result.WriteString("\\b")
+			case '\f':
+				result.WriteString("\\f")
+			case '\n':
+				result.WriteString("\\n")
+			case '\r':
+				result.WriteString("\\r")
+			case '\t':
+				result.WriteString("\\t")
+			default:
+				// Use unicode escape for other control characters
+				result.WriteString(fmt.Sprintf("\\u%04x", int(r)))
+			}
+		} else {
+			result.WriteRune(r)
+		}
+	}
+
+	return []byte(result.String())
+}

pkg/piperenv/CPEMap_test.go

@@ -107,3 +107,59 @@ func TestCommonPipelineEnvDirNotPresent(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, cpe, 0)
 }
+
+func TestCleanJSONData(t *testing.T) {
+	t.Parallel()
+
+	// Test valid UTF-8 (should be unchanged)
+	validData := []byte(`{"commitMessage":"This is a valid commit message"}`)
+	result := CleanJSONData(validData)
+	require.Equal(t, validData, result)
+
+	// Test emoji with valid UTF-8 (should be unchanged)
+	emojiData := []byte(`{"commitMessage":"🚀 feat: add new feature"}`)
+	result = CleanJSONData(emojiData)
+	require.Equal(t, emojiData, result)
+
+	// Test data with JSON control character (like \x16)
+	invalidData := []byte("{\"commitMessage\":\"Test \x16 invalid char\"}")
+	result = CleanJSONData(invalidData)
+	require.NotEqual(t, invalidData, result)
+	require.True(t, json.Valid(result), "Result should be valid JSON")
+
+	// Verify we can parse the cleaned data as JSON
+	var parsed map[string]interface{}
+	err := json.Unmarshal(result, &parsed)
+	require.NoError(t, err)
+	require.Contains(t, parsed, "commitMessage")
+
+	// The control character should be escaped as unicode
+	require.Contains(t, string(result), "\\u0016")
+}
+
+func TestReadFileContentWithInvalidUTF8(t *testing.T) {
+	t.Parallel()
+
+	// Create a temporary JSON file with control characters
+	tmpDir := t.TempDir()
+	jsonFile := path.Join(tmpDir, "test.json")
+
+	// Write JSON with control character that causes parsing errors
+	invalidJSON := []byte("{\"commitMessage\":\"Test \x16 control char commit\"}")
+	err := os.WriteFile(jsonFile, invalidJSON, 0644)
+	require.NoError(t, err)
+
+	// Try to read the file - should not fail due to control characters
+	_, value, _, err := readFileContent(jsonFile)
+	require.NoError(t, err)
+	require.NotNil(t, value)
+
+	// Verify we can extract the commit message
+	if valueMap, ok := value.(map[string]interface{}); ok {
+		commitMsg, exists := valueMap["commitMessage"]
+		require.True(t, exists)
+		require.IsType(t, "", commitMsg)
+		// The control character should be properly handled in the message
+		require.NotEmpty(t, commitMsg)
+	}
+}