Add support for nested ignore files (#117)

**Please check if the PR fulfills these requirements** - [x] The commit message follows our [guidelines](https://github.com/get-woke/woke/blob/main/CONTRIBUTING.md) - [x] Tests for the changes have been added (for bug fixes / features) - [x] Docs have been added / updated (for bug fixes / features) **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...) Feature **What is the current behavior?** (You can also link to an open issue here) Woke currently only looks at .gitignore and .wokeignore files in the current directory. **What is the new behavior (if this is a feature change)?** Woke will now traverse a repo and build a priority ranked list of ignore rules in exactly the same way that git does for .gitignore files at different directory levels. The tool will look for the root git directory (where the .git folder is) and use that for the root. If no root git directory is found then woke will just run with the current directory as root (still traversing children folders). **Does this PR introduce a breaking change?** (What changes might users need to make due to this PR?) No **Other information**: Addresses #98
get-woke · Mar 18, 2022 · 6531f50 · 6531f50
1 parent 1b32c2b
commit 6531f50
Show file tree

Hide file tree

Showing 19 changed files with 554 additions and 132 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -37,6 +37,15 @@
 		"go.lintFlags": [
 			"--fast"
 		],
+		"go.coverOnSave": true,
+		"go.coverageDecorator": {
+			"type": "gutter",
+			"coveredHighlightColor": "rgba(64,128,128,0.5)",
+			"uncoveredHighlightColor": "rgba(128,64,64,0.25)",
+			"coveredGutterStyle": "blockgreen",
+			"uncoveredGutterStyle": "blockred"
+		},
+		"go.coverOnSingleTest": true,
 		"editor.formatOnSave": true,
 		"files.insertFinalNewline": true,
 		"files.trimFinalNewlines": true,

diff --git a/.devcontainer/scripts/docker-debian.sh b/.devcontainer/scripts/docker-debian.sh
@@ -7,13 +7,17 @@
 # Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker.md
 # Maintainer: The VS Code and Codespaces Teams
 #
-# Syntax: ./docker-debian.sh [enable non-root docker socket access flag] [source socket] [target socket] [non-root user] [use moby]
+# Syntax: ./docker-debian.sh [enable non-root docker socket access flag] [source socket] [target socket] [non-root user] [use moby] [CLI version]
+# shellcheck disable=SC2068,SC2086,SC2155,SC2012,SC2236,SC1091
 
 ENABLE_NONROOT_DOCKER=${1:-"true"}
 SOURCE_SOCKET=${2:-"/var/run/docker-host.sock"}
 TARGET_SOCKET=${3:-"/var/run/docker.sock"}
 USERNAME=${4:-"automatic"}
 USE_MOBY=${5:-"true"}
+DOCKER_VERSION=${6:-"latest"}
+MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc"
+DOCKER_DASH_COMPOSE_VERSION="1"
 
 set -e
 
@@ -39,8 +43,23 @@ elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then
     USERNAME=root
 fi
 
+# Get central common setting
+get_common_setting() {
+    if [ "${common_settings_file_loaded}" != "true" ]; then
+        curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping."
+        common_settings_file_loaded=true
+    fi
+    if [ -f "/tmp/vsdc-settings.env" ]; then
+        local multi_line=""
+        if [ "$2" = "true" ]; then multi_line="-z"; fi
+        local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')"
+        if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi
+    fi
+    echo "$1=${!1}"
+}
+
 # Function to run apt-get if needed
-apt-get-update-if-needed()
+apt_get_update_if_needed()
 {
     if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then
         echo "Running apt-get update..."
@@ -50,33 +69,114 @@ apt-get-update-if-needed()
     fi
 }
 
+# Checks if packages are installed and installs them if not
+check_packages() {
+    if ! dpkg -s "$@" > /dev/null 2>&1; then
+        apt_get_update_if_needed
+        apt-get -y install --no-install-recommends "$@"
+    fi
+}
+
+# Figure out correct version of a three part version number is not passed
+find_version_from_git_tags() {
+    local variable_name=$1
+    local requested_version=${!variable_name}
+    if [ "${requested_version}" = "none" ]; then return; fi
+    local repository=$2
+    local prefix=${3:-"tags/v"}
+    local separator=${4:-"."}
+    local last_part_optional=${5:-"false"}
+    if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then
+        local escaped_separator=${separator//./\\.}
+        local last_part
+        if [ "${last_part_optional}" = "true" ]; then
+            last_part="(${escaped_separator}[0-9]+)?"
+        else
+            last_part="${escaped_separator}[0-9]+"
+        fi
+        local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$"
+        local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)"
+        if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then
+            declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)"
+        else
+            set +e
+            declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")"
+            set -e
+        fi
+    fi
+    if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then
+        echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2
+        exit 1
+    fi
+    echo "${variable_name}=${!variable_name}"
+}
+
 # Ensure apt is in non-interactive to avoid prompts
 export DEBIAN_FRONTEND=noninteractive
 
-# Install apt-transport-https, curl, gpg if missing
-if ! dpkg -s apt-transport-https curl ca-certificates > /dev/null 2>&1 || ! type gpg > /dev/null 2>&1; then
-    apt-get-update-if-needed
-    apt-get -y install --no-install-recommends apt-transport-https curl ca-certificates gnupg2
+# Install dependencies
+check_packages apt-transport-https curl ca-certificates gnupg2 dirmngr
+if ! type git > /dev/null 2>&1; then
+    apt_get_update_if_needed
+    apt-get -y install git
+fi
+
+# Source /etc/os-release to get OS info
+. /etc/os-release
+# Fetch host/container arch.
+architecture="$(dpkg --print-architecture)"
+
+# Set up the necessary apt repos (either Microsoft's or Docker's)
+if [ "${USE_MOBY}" = "true" ]; then
+
+    cli_package_name="moby-cli"
+
+    # Import key safely and import Microsoft apt repo
+    get_common_setting MICROSOFT_GPG_KEYS_URI
+    curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg
+    echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list
+else
+    # Name of proprietary engine package
+    cli_package_name="docker-ce-cli"
+
+    # Import key safely and import Docker apt repo
+    curl -fsSL https://download.docker.com/linux/${ID}/gpg | gpg --dearmor > /usr/share/keyrings/docker-archive-keyring.gpg
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list
+fi
+
+# Refresh apt lists
+apt-get update
+
+# Soft version matching for CLI
+if [ "${DOCKER_VERSION}" = "latest" ] || [ "${DOCKER_VERSION}" = "lts" ] || [ "${DOCKER_VERSION}" = "stable" ]; then
+    # Empty, meaning grab whatever "latest" is in apt repo
+    cli_version_suffix=""
+else
+    # Fetch a valid version from the apt-cache (eg: the Microsoft repo appends +azure, breakfix, etc...)
+    docker_version_dot_escaped="${DOCKER_VERSION//./\\.}"
+    docker_version_dot_plus_escaped="${docker_version_dot_escaped//+/\\+}"
+    # Regex needs to handle debian package version number format: https://www.systutorials.com/docs/linux/man/5-deb-version/
+    docker_version_regex="^(.+:)?${docker_version_dot_plus_escaped}([\\.\\+ ~:-]|$)"
+    set +e # Don't exit if finding version fails - will handle gracefully
+    cli_version_suffix="=$(apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")"
+    set -e
+    if [ -z "${cli_version_suffix}" ] || [ "${cli_version_suffix}" = "=" ]; then
+        echo "(!) No full or partial Docker / Moby version match found for \"${DOCKER_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:"
+        apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+'
+        exit 1
+    fi
+    echo "cli_version_suffix ${cli_version_suffix}"
 fi
 
 # Install Docker / Moby CLI if not already installed
 if type docker > /dev/null 2>&1; then
     echo "Docker / Moby CLI already installed."
 else
-    # Source /etc/os-release to get OS info
-    . /etc/os-release
     if [ "${USE_MOBY}" = "true" ]; then
-        # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install
-        curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg
-        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list
-        apt-get update
-        apt-get -y install --no-install-recommends moby-cli moby-buildx
+        apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx
+        apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping."
     else
-        # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install
-        curl -fsSL https://download.docker.com/linux/${ID}/gpg | gpg --dearmor > /usr/share/keyrings/docker-archive-keyring.gpg
-        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list
-        apt-get update
-        apt-get -y install --no-install-recommends docker-ce-cli
+        apt-get -y install --no-install-recommends docker-ce-cli${cli_version_suffix}
     fi
 fi
 
@@ -90,19 +190,26 @@ else
     fi
     if [ "${TARGET_COMPOSE_ARCH}" != "x86_64" ]; then
         # Use pip to get a version that runns on this architecture
-        if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv pipx > /dev/null 2>&1; then
-            apt-get-update-if-needed
-            apt-get -y install python3-minimal python3-pip libffi-dev python3-venv pipx
+        if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv > /dev/null 2>&1; then
+            apt_get_update_if_needed
+            apt-get -y install python3-minimal python3-pip libffi-dev python3-venv
         fi
         export PIPX_HOME=/usr/local/pipx
         mkdir -p ${PIPX_HOME}
         export PIPX_BIN_DIR=/usr/local/bin
+        export PYTHONUSERBASE=/tmp/pip-tmp
         export PIP_CACHE_DIR=/tmp/pip-tmp/cache
-        pipx install --system-site-packages --pip-args '--no-cache-dir --force-reinstall' docker-compose
+        pipx_bin=pipx
+        if ! type pipx > /dev/null 2>&1; then
+            pip3 install --disable-pip-version-check --no-warn-script-location  --no-cache-dir --user pipx
+            pipx_bin=/tmp/pip-tmp/bin/pipx
+        fi
+        ${pipx_bin} install --system-site-packages --pip-args '--no-cache-dir --force-reinstall' docker-compose
         rm -rf /tmp/pip-tmp
     else
-        LATEST_COMPOSE_VERSION=$(basename "$(curl -fsSL -o /dev/null -w "%{url_effective}" https://github.com/docker/compose/releases/latest)")
-        curl -fsSL "https://github.com/docker/compose/releases/download/${LATEST_COMPOSE_VERSION}/docker-compose-$(uname -s)-${TARGET_COMPOSE_ARCH}" -o /usr/local/bin/docker-compose
+        find_version_from_git_tags DOCKER_DASH_COMPOSE_VERSION "https://github.com/docker/compose" "tags/"
+        echo "(*) Installing docker-compose ${DOCKER_DASH_COMPOSE_VERSION}..."
+        curl -fsSL "https://github.com/docker/compose/releases/download/${DOCKER_DASH_COMPOSE_VERSION}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose
         chmod +x /usr/local/bin/docker-compose
     fi
 fi
@@ -111,6 +218,7 @@ fi
 if [ -f "/usr/local/share/docker-init.sh" ]; then
     exit 0
 fi
+echo "docker-init doesnt exist, adding..."
 
 # By default, make the source and target sockets the same
 if [ "${SOURCE_SOCKET}" != "${TARGET_SOCKET}" ]; then
@@ -128,7 +236,7 @@ fi
 # If enabling non-root access and specified user is found, setup socat and add script
 chown -h "${USERNAME}":root "${TARGET_SOCKET}"
 if ! dpkg -s socat > /dev/null 2>&1; then
-    apt-get-update-if-needed
+    apt_get_update_if_needed
     apt-get -y install socat
 fi
 tee /usr/local/share/docker-init.sh > /dev/null \

diff --git a/cmd/root.go b/cmd/root.go
@@ -101,9 +101,19 @@ func rootRunE(cmd *cobra.Command, args []string) error {
 
 	var ignorer *ignore.Ignore
 	if !noIgnore {
-		ignorer = ignore.NewIgnore(cfg.IgnoreFiles)
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		fs, err := ignore.GetRootGitDir(cwd)
+		if err != nil {
+			return err
+		}
+		ignorer, err = ignore.NewIgnore(fs, cfg.IgnoreFiles)
+		if err != nil {
+			return err
+		}
 	}
-
 	p := parser.NewParser(cfg.Rules, ignorer)
 
 	print, err := printer.NewPrinter(outputName, output.Stdout)

diff --git a/cmd/root_test.go b/cmd/root_test.go
@@ -111,6 +111,9 @@ func TestRunE(t *testing.T) {
 
 	t.Run("findings w error", func(t *testing.T) {
 		exitOneOnFailure = true
+		// don't ignore testdata folder
+		noIgnore = true
+
 		t.Cleanup(func() {
 			exitOneOnFailure = false
 		})

diff --git a/docs/ignore.md b/docs/ignore.md
@@ -74,3 +74,27 @@ func main() {
   fmt.Println("and here is the blacklist")
 }
 ```
+
+## Nested Ignore Files
+
+`woke` will apply ignore rules from nested ignore files to any child files/folders, similar to a nested `.gitignore` file. Nested ignore files work for any ignore file type listed above.
+
+```txt
+project
+│   README.md
+│   .wokeignore (applies to whole project)
+│
+└───folder1
+│   │   file011.txt
+│   │   file012.txt
+│   │   .wokeignore (applies to file011.txt, file012.txt, and subfolder1)
+│   │
+│   └───subfolder1
+│       │   file111.txt
+│       │   file112.txt
+│       │   ...
+│
+└───folder2
+    │   file021.txt
+    │   file022.txt
+```
diff --git a/docs/snippets/woke.md b/docs/snippets/woke.md
@@ -31,4 +31,4 @@ woke [globs ...] [flags]
       --stdin                   Read from stdin
 ```
 
-###### Auto generated by spf13/cobra on 22-Nov-2021
+###### Auto generated by spf13/cobra on 6-Dec-2021
diff --git a/go.mod b/go.mod
@@ -6,7 +6,8 @@ require (
 	github.com/caitlinelfring/go-env-default v1.0.0
 	github.com/fatih/color v1.13.0
 	github.com/get-woke/fastwalk v1.0.0
-	github.com/get-woke/go-gitignore v1.1.2
+	github.com/go-git/go-billy/v5 v5.3.1
+	github.com/go-git/go-git/v5 v5.4.2
 	github.com/mattn/go-colorable v0.1.12
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/rs/zerolog v1.26.0
@@ -17,11 +18,14 @@ require (
 )
 
 require (
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/mapstructure v1.4.2 // indirect
@@ -34,8 +38,12 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
 	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
+
+replace github.com/go-git/go-git/v5 => github.com/inclusive-dev-tools/go-git/v5 v5.4.4