Skip to content

Commit

Permalink
feat: filter uploaded Snyk Code files by .gitignore, fix some minor w…
Browse files Browse the repository at this point in the history
…orkspace scan bugs [ROAD-794] (#34)

Co-authored-by: Aayush Attri <[email protected]>
  • Loading branch information
bastiandoetsch and attriaayush authored Apr 13, 2022
1 parent 7b24a18 commit 983f082
Show file tree
Hide file tree
Showing 18 changed files with 321 additions and 81 deletions.
16 changes: 16 additions & 0 deletions .run/go test github.com_snyk_snyk-ls [INTEG].run.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="go test github.com/snyk/snyk-ls/server [INTEG]" type="GoTestRunConfiguration" factoryName="Go Test">
<module name="snyk-lsp" />
<working_directory value="$PROJECT_DIR$/" />
<envs>
<env name="INTEG_TESTS" value="true" />
</envs>
<root_directory value="$PROJECT_DIR$" />
<kind value="PACKAGE" />
<package value="github.com/snyk/snyk-ls/server" />
<directory value="$PROJECT_DIR$/server" />
<filePath value="$PROJECT_DIR$" />
<framework value="gotest" />
<method v="2" />
</configuration>
</component>
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ LDFLAGS_DEV := "-X 'github.com/snyk/snyk-ls/config.Development=true' -X 'github.
.PHONY: tools
tools:
ifeq (,$(wildcard ./.bin/golangci-lint*))
@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b .bin/ v1.44.2
@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b .bin/ v1.45.2
else
@echo "==> Required tooling is already installed"
endif
Expand Down Expand Up @@ -47,10 +47,10 @@ build:
# workaround for missing .exe extension on Windows
ifeq ($(OS),Windows_NT)
@go build -o $(BUILD_DIR)/$(PROJECT_NAME).$(DEV_GOOS).$(DEV_GOARCH).exe \
-ldflags=-ldflags=$(LDFLAGS_DEV)
-ldflags=$(LDFLAGS_DEV)
else
@go build -o $(BUILD_DIR)/$(PROJECT_NAME).$(DEV_GOOS).$(DEV_GOARCH) \
-ldflags=-ldflags=$(LDFLAGS_DEV)
-ldflags=$(LDFLAGS_DEV)
endif

## run: Compile and run LSP server.
Expand Down
8 changes: 0 additions & 8 deletions code/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ const (
jsonOverheadPerFile = jsonUriOverhead + jsonContentOverhead
)

var bundleMutex = &sync.Mutex{}

func getTotalDocPayloadSize(uri sglsp.DocumentURI, file File) int {
return len(jsonHashSizePerFile) + len(jsonOverheadPerFile) + len([]byte(uri)) + len([]byte(file.Content))
}
Expand Down Expand Up @@ -97,9 +95,6 @@ func (b *BundleImpl) createBundleFromSource() error {
}

func (b *BundleImpl) AddToBundleDocuments(files map[sglsp.DocumentURI]sglsp.TextDocumentItem) FilesNotAdded {
bundleMutex.Lock()
defer bundleMutex.Unlock()

if b.BundleDocuments == nil {
b.BundleDocuments = make(map[sglsp.DocumentURI]File)
}
Expand Down Expand Up @@ -155,9 +150,6 @@ func (b *BundleImpl) FetchDiagnosticsData(
dChan chan lsp.DiagnosticResult,
clChan chan lsp.CodeLensResult,
) {
bundleMutex.Lock()
defer bundleMutex.Unlock()

defer wg.Done()
defer log.Debug().Str("method", "FetchDiagnosticsData").Msg("done.")

Expand Down
32 changes: 20 additions & 12 deletions diagnostics/diagnostics.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ func ClearDiagnosticsCache(uri sglsp.DocumentURI) {
delete(documentDiagnosticCache, uri)
}

func ClearEntireDiagnosticsCache() {
documentDiagnosticCache = map[sglsp.DocumentURI][]lsp.Diagnostic{}
}

func ClearRegisteredDocuments() {
registeredDocuments = map[sglsp.DocumentURI]sglsp.TextDocumentItem{}
}

func UpdateDocument(uri sglsp.DocumentURI, changes []sglsp.TextDocumentContentChangeEvent) {
file := registeredDocuments[uri]
for i := range changes {
Expand All @@ -39,26 +47,26 @@ func UnRegisterDocument(file sglsp.DocumentURI) {
delete(registeredDocuments, file)
}

func GetDiagnostics(rootUri sglsp.DocumentURI) []lsp.Diagnostic {
func GetDiagnostics(uri sglsp.DocumentURI) []lsp.Diagnostic {
// serve from cache
diagnosticSlice := documentDiagnosticCache[rootUri]
diagnosticSlice := documentDiagnosticCache[uri]
if len(diagnosticSlice) > 0 {
log.Info().Str("method", "GetDiagnostics").
Msgf("Cached: Diagnostics for %s", rootUri)
Msgf("Cached: Diagnostics for %s", uri)

return diagnosticSlice
}

var diagnostics map[sglsp.DocumentURI][]lsp.Diagnostic
var codeLenses map[sglsp.DocumentURI][]sglsp.CodeLens

diagnostics, codeLenses = fetchAllRegisteredDocumentDiagnostics(rootUri, ScanLevelFile)
diagnostics, codeLenses = fetchAllRegisteredDocumentDiagnostics(uri, lsp.ScanLevelFile)
addToCache(diagnostics, codeLenses)

return documentDiagnosticCache[rootUri]
return documentDiagnosticCache[uri]
}

func fetchAllRegisteredDocumentDiagnostics(rootUri sglsp.DocumentURI, level ScanLevel) (map[sglsp.DocumentURI][]lsp.Diagnostic, map[sglsp.DocumentURI][]sglsp.CodeLens) {
func fetchAllRegisteredDocumentDiagnostics(uri sglsp.DocumentURI, level lsp.ScanLevel) (map[sglsp.DocumentURI][]lsp.Diagnostic, map[sglsp.DocumentURI][]sglsp.CodeLens) {
log.Info().
Str("method", "fetchAllRegisteredDocumentDiagnostics").
Msg("started.")
Expand All @@ -82,15 +90,15 @@ func fetchAllRegisteredDocumentDiagnostics(rootUri sglsp.DocumentURI, level Scan
clChan := make(chan lsp.CodeLensResult, len(registeredDocuments))

for _, myBundle := range bundles {
go myBundle.FetchDiagnosticsData(string(rootUri), &wg, dChan, clChan)
go myBundle.FetchDiagnosticsData(string(uri), &wg, dChan, clChan)
}

if level == ScanLevelWorkspace {
go iac.ScanWorkspace(rootUri, &wg, dChan, clChan)
go oss.ScanWorkspace(rootUri, &wg, dChan, clChan)
if level == lsp.ScanLevelWorkspace {
go iac.ScanWorkspace(uri, &wg, dChan, clChan)
go oss.ScanWorkspace(uri, &wg, dChan, clChan)
} else {
go iac.ScanFile(rootUri, &wg, dChan, clChan)
go oss.ScanFile(registeredDocuments[rootUri], &wg, dChan, clChan)
go iac.ScanFile(uri, &wg, dChan, clChan)
go oss.ScanFile(registeredDocuments[uri], &wg, dChan, clChan)
}

wg.Wait()
Expand Down
8 changes: 0 additions & 8 deletions diagnostics/types.go

This file was deleted.

79 changes: 68 additions & 11 deletions diagnostics/workspace.go
Original file line number Diff line number Diff line change
@@ -1,36 +1,47 @@
package diagnostics

import (
"io/fs"
"os"
"path/filepath"
"strings"
"sync"

"github.com/snyk/snyk-ls/lsp"

"github.com/rs/zerolog/log"
ignore "github.com/sabhiram/go-gitignore"
sglsp "github.com/sourcegraph/go-lsp"

"github.com/snyk/snyk-ls/lsp"
)

var registeredDocsMutex = &sync.Mutex{}

func registerAllFilesFromWorkspace(workspaceUri sglsp.DocumentURI) error {
workspace, err := filepath.Abs(strings.ReplaceAll(
string(workspaceUri),
"file://", ""),
)
// this is not a mistake - eclipse reports workspace folders with `file:` pre-prended
workspace, err :=
filepath.Abs(
strings.ReplaceAll(strings.ReplaceAll(string(workspaceUri), "file://", ""), "file:", ""),
)

if err != nil {
return err
}

var patterns []string
patterns, err = loadIgnorePatterns(workspace)
if err != nil {
return err
}

return filepath.Walk(workspace, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
return filepath.WalkDir(workspace, func(path string, dirEntry os.DirEntry, _ error) error {
if dirEntry == nil || dirEntry.IsDir() {
return nil
}

content, _ := os.ReadFile(path)
if ignored(patterns, path) {
return nil
}

content, err := os.ReadFile(path)
file := sglsp.TextDocumentItem{
URI: sglsp.DocumentURI("file://" + path),
Text: string(content),
Expand All @@ -44,6 +55,52 @@ func registerAllFilesFromWorkspace(workspaceUri sglsp.DocumentURI) error {
})
}

func loadIgnorePatterns(workspace string) ([]string, error) {
var ignores = ""
log.Debug().
Str("method", "loadIgnorePatterns").
Str("workspace", workspace).
Msg("searching for ignore files")
err := filepath.WalkDir(workspace, func(path string, dirEntry os.DirEntry, _ error) error {
if dirEntry == nil || dirEntry.IsDir() {
return nil
}

if !(strings.HasSuffix(path, ".gitignore") || strings.HasSuffix(path, ".dcignore")) {
return nil
}
log.Debug().Str("method", "loadIgnorePatterns").Str("file", path).Msg("found ignore file")
content, err := os.ReadFile(path)
if err != nil {
log.Err(err).Msg("Can't read" + path)
}
ignores += string(content)
return err
})

if err != nil {
return nil, err
}

patterns := strings.Split(ignores, "\n")
log.Debug().Interface("ignorePatterns", patterns).Msg("Loaded ignore patterns")
return patterns, nil
}

func ignored(patterns []string, path string) bool {
ignored := false
gitIgnore := ignore.CompileIgnoreLines(patterns...)
for _, pattern := range patterns {
ignored = gitIgnore.MatchesPath(path)
if ignored {
log.Trace().Str("method", "ignored").Str("pattern", pattern).Str("path", path).Msg("matched")
return true
}
log.Trace().Str("method", "ignored").Str("pattern", pattern).Str("path", path).Msg("not matched")
}
return false
}

func workspaceDiagnostics(workspaceUri sglsp.DocumentURI, wg *sync.WaitGroup) {
defer wg.Done()

Expand All @@ -57,7 +114,7 @@ func workspaceDiagnostics(workspaceUri sglsp.DocumentURI, wg *sync.WaitGroup) {
Msg("Error occurred while registering files from workspace")
}

diagnostics, codeLenses = fetchAllRegisteredDocumentDiagnostics(workspaceUri, ScanLevelWorkspace)
diagnostics, codeLenses = fetchAllRegisteredDocumentDiagnostics(workspaceUri, lsp.ScanLevelWorkspace)
addToCache(diagnostics, codeLenses)
}

Expand Down
121 changes: 121 additions & 0 deletions diagnostics/workspace_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package diagnostics

import (
"os"
"strings"
"testing"

"github.com/rs/zerolog/log"
sglsp "github.com/sourcegraph/go-lsp"
"github.com/stretchr/testify/assert"
)

func Test_ignored_ignoredGlob(t *testing.T) {
ignoredPath := "test.xml"

err := os.WriteFile(ignoredPath, []byte("test"), 0600)
defer os.RemoveAll(ignoredPath)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't create file " + ignoredPath)
}
patterns := []string{"**/ignored.txt", "*.xml"}

assert.True(t, ignored(patterns, ignoredPath))
}

func Test_ignored_notIgnored(t *testing.T) {
notIgnoredPath := "not-ignored.txt"
err := os.WriteFile(notIgnoredPath, []byte("test"), 0600)
defer os.RemoveAll(notIgnoredPath)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't create file " + notIgnoredPath)
}
patterns := []string{"**/ignored.txt", "*.xml"}

assert.False(t, ignored(patterns, notIgnoredPath))
}

func Test_ignored_doubleAsterisk(t *testing.T) {
ignoredDoubleAsteriskPath := "test-ignore/ignored.txt"
testIgnoreDir := "test-ignore"
err := os.Mkdir(testIgnoreDir, 0700)
defer os.RemoveAll(testIgnoreDir)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't create testIgnoreDir" + testIgnoreDir)
}
err = os.WriteFile(ignoredDoubleAsteriskPath, []byte("test"), 0600)
defer os.RemoveAll(ignoredDoubleAsteriskPath)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't create file " + ignoredDoubleAsteriskPath)
}
patterns := []string{"**/ignored.txt", "*.xml"}
assert.True(t, ignored(patterns, ignoredDoubleAsteriskPath))
}

func Test_LoadIgnorePatternsWithIgnoreFilePresent(t *testing.T) {
expectedPatterns, tempDir, _, _ := setupIgnoreWorkspace()
defer os.RemoveAll(tempDir)

actualPatterns, err := loadIgnorePatterns(tempDir)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't load .gitignore from workspace " + tempDir)
}
assert.Equal(t, strings.Split(expectedPatterns, "\n"), actualPatterns)
}

func Test_LoadIgnorePatternsWithoutIgnoreFilePresent(t *testing.T) {
temp, err := os.MkdirTemp(os.TempDir(), "loadIgnoreTest")
defer os.RemoveAll(temp)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't set up test directory")
}
var actualPatterns []string
actualPatterns, err = loadIgnorePatterns(temp)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't load .gitignore from workspace")
}
assert.Equal(t, []string{""}, actualPatterns)
}

func Test_RegisterAllFilesFromWorkspace_Without_Ignored(t *testing.T) {
registeredDocuments = map[sglsp.DocumentURI]sglsp.TextDocumentItem{}
_, workspace, ignoredFilePath, notIgnoredFilePath := setupIgnoreWorkspace()
defer os.RemoveAll(workspace)

err := registerAllFilesFromWorkspace(sglsp.DocumentURI("file://" + workspace))
if err != nil {
log.Fatal().Err(err).Msg("Error while registering " + workspace)
}
assert.Equal(t, 2, len(registeredDocuments)) //.gitignore & notIgnoredFilePath
assert.NotEqual(t, sglsp.TextDocumentItem{}, registeredDocuments[sglsp.DocumentURI("file://"+notIgnoredFilePath)])
assert.Equal(t, sglsp.TextDocumentItem{}, registeredDocuments[sglsp.DocumentURI(ignoredFilePath)])
}

func writeTestGitIgnore(ignorePatterns string) string {
temp, err := os.MkdirTemp(os.TempDir(), "loadIgnorePatterns")
if err != nil {
log.Fatal().Err(err).Msg("Couldn't create temp dir")
}
err = os.WriteFile(temp+string(os.PathSeparator)+".gitignore", []byte(ignorePatterns), 0600)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't write .gitignore")
}
return temp
}

func setupIgnoreWorkspace() (string, string, string, string) {
expectedPatterns := "*.xml\n**/*.txt\nbin"
tempDir := writeTestGitIgnore(expectedPatterns)

ignoredFilePath := tempDir + string(os.PathSeparator) + "ignored.xml"
err := os.WriteFile(ignoredFilePath, []byte("test"), 0600)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't write ignored file ignored.xml")
}
notIgnoredFilePath := tempDir + string(os.PathSeparator) + "not-ignored.java"
err = os.WriteFile(notIgnoredFilePath, []byte("test"), 0600)
if err != nil {
log.Fatal().Err(err).Msg("Couldn't write ignored file not-ignored.java")
}
return expectedPatterns, tempDir, ignoredFilePath, notIgnoredFilePath
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/creachadair/jrpc2 v0.35.5
github.com/gomarkdown/markdown v0.0.0-20220114203417-14399d5448c4
github.com/rs/zerolog v1.26.1
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/sourcegraph/go-lsp v0.0.0-20200429204803-219e11d77f5d
github.com/stretchr/testify v1.7.0
github.com/subosito/gotenv v1.2.0
Expand Down
Loading

0 comments on commit 983f082

Please sign in to comment.