fix #7 & prepare simplifying sed-patterns (#36)

Andrwe · Jan 1, 2024 · 73e8781 · 73e8781
1 parent eb13129
commit 73e8781
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 8 deletions.
diff --git a/privoxy-blocklist.sh b/privoxy-blocklist.sh
@@ -117,16 +117,18 @@ TMPDIR="/tmp/\${TMPNAME}"
 #    3 = incredibly loud (function debugging)
 DBG=0
 EOF
-        exit 1
+        exit 2
     fi
 
     if [[ ! -r "${SCRIPTCONF}" ]]; then
         debug "Can't read ${SCRIPTCONF}. Permission denied." -1
     fi
 
     # load script config
+    _dbg="${DBG:-0}"
     # shellcheck disable=SC1090
     source "${SCRIPTCONF}"
+    DBG="${_dbg}"
     # load privoxy config
     # shellcheck disable=SC1090
     if [[ -r "${INIT_CONF}" ]]; then
@@ -183,6 +185,14 @@ function main() {
     for url in "${URLS[@]}"; do
         debug "Processing ${url} ...\n" 0
         file="${TMPDIR}/$(basename "${url}")"
+        address_file="${TMPDIR}/$(basename "${url}").address"
+        address_except_file="${TMPDIR}/$(basename "${url}").address_except"
+        url_file="${TMPDIR}/$(basename "${url}").url"
+        url_except_file="${TMPDIR}/$(basename "${url}").url_except"
+        domain_name_file="${TMPDIR}/$(basename "${url}").domain"
+        domain_name_except_file="${TMPDIR}/$(basename "${url}").domain_except"
+        regex_file="${TMPDIR}/$(basename "${url}").regex"
+        regex_except_file="${TMPDIR}/$(basename "${url}").regex_except"
         actionfile=${file%\.*}.script.action
         filterfile=${file%\.*}.script.filter
         list="$(basename "${file%\.*}")"
@@ -197,11 +207,29 @@ function main() {
             continue
         fi
 
+        # remove comments
+        sed -i '/^!.*/d;1,1 d' "${file}"
+        set +e
+        # generate rule based files
+        ## domain-name block
+        grep -E '^\|\|.*' "${file}" > "${domain_name_file}"
+        grep -E '^@@\|\|.*' "${file}" > "${domain_name_except_file}"
+        ## exact address block
+        grep -E '^\|[^|].*\|' "${file}" > "${address_file}"
+        grep -E '^@@\|[^|].*\|' "${file}" > "${address_except_file}"
+        ## url block
+        grep '^/[^^]' "${file}" > "${url_file}"
+        grep '^@@/[^^]' "${file}" > "${url_except_file}"
+        ## regex block
+        grep '^/^' "${file}" > "${regex_file}"
+        grep '^@@/^' "${file}" > "${regex_except_file}"
+        set -e
+
         # convert AdblockPlus list to Privoxy list
         # blacklist of urls
         debug "Creating actionfile for ${list} ..." 1
         echo -e "{ +block{${list}} }" > "${actionfile}"
-        sed '/^!.*/d;1,1 d;/^@@.*/d;/\$.*/d;/#/d;s/\./\\./g;s/\?/\\?/g;s/\*/.*/g;s/(/\\(/g;s/)/\\)/g;s/\[/\\[/g;s/\]/\\]/g;s/\^/[\/\&:\?=_]/g;s/^||/\./g;s/^|/^/g;s/|$/\$/g;/|/d' "${file}" >> "${actionfile}"
+        sed '/\$.*/d;/#/d;s/\?/\\?/g;s/\*/.*/g;s/(/\\(/g;s/)/\\)/g;s/\[/\\[/g;s/\]/\\]/g;s/\^$//g;s/^||/\./g;s/^|/^/g;s/|$/\$/g;/|/d' "${domain_name_file}" >> "${actionfile}"
 
         debug "... creating filterfile for ${list} ..." 1
         echo "FILTER: ${list} Tag filter of ${list}" > "${filterfile}"
@@ -238,7 +266,7 @@ function main() {
         debug "... creating and adding whitlist for urls ..." 1
         # whitelist of urls
         echo "{ -block }" >> "${actionfile}"
-        sed '/^@@.*/!d;s/^@@//g;/\$.*/d;/#/d;s/\./\\./g;s/\?/\\?/g;s/\*/.*/g;s/(/\\(/g;s/)/\\)/g;s/\[/\\[/g;s/\]/\\]/g;s/\^/[\/\&:\?=_]/g;s/^||/\./g;s/^|/^/g;s/|$/\$/g;/|/d' "${file}" >> "${actionfile}"
+        sed 's/^@@//g;/\$.*/d;/#/d;s/\./\\./g;s/\?/\\?/g;s/\*/.*/g;s/(/\\(/g;s/)/\\)/g;s/\[/\\[/g;s/\]/\\]/g;s/\^/[\/\&:\?=_]/g;s/^||/\./g;s/^|/^/g;s/|$/\$/g;/|/d' "${domain_name_except_file}" >> "${actionfile}"
         debug "... created and added whitelist - creating and adding image handler ..." 1
         # whitelist of image urls
         echo "{ -block +handle-as-image }" >> "${actionfile}"
@@ -340,9 +368,10 @@ while getopts ":c:hrqv:" opt; do
     esac
 done
 
+prepare
+
 trap 'rm -fr "${TMPDIR}";exit' INT TERM EXIT
 
-prepare
 lock
 debug "URL-List: ${URLS}\nPrivoxy-Configdir: ${PRIVOXY_DIR}\nTemporary directory: ${TMPDIR}" 2
 "${method}"

diff --git a/tests/test_01_root_execute.py b/tests/test_01_root_execute.py
@@ -13,7 +13,7 @@ def test_config_generator(shell, privoxy_blocklist) -> None:
     if config.exists():
         config.unlink()
     ret = shell.run(privoxy_blocklist)
-    assert ret.returncode == 1
+    assert ret.returncode == 2
     assert "Creating default one and exiting" in ret.stdout
     assert config.exists()
 
@@ -24,7 +24,7 @@ def test_custom_config_generator(shell, tmp_path, privoxy_blocklist) -> None:
     if config.exists():
         config.unlink()
     ret = shell.run(privoxy_blocklist, "-c", str(config))
-    assert ret.returncode == 1
+    assert ret.returncode == 2
     assert "Creating default one and exiting" in ret.stdout
     assert config.exists()
 
@@ -49,9 +49,13 @@ def test_request_success(start_privoxy, supported_schemes) -> None:
 
 def test_request_block_url(start_privoxy, supported_schemes) -> None:
     """Test URLs blocked by privoxy due to easylist."""
-    # FIXME: see https://github.com/Andrwe/privoxy-blocklist/issues/7
+    urls = [
+        "andrwe.org/ads/",
+        "andrwe.jp/ads/",
+        "pubfeed.linkby.com",
+        f"s3.{'a'*6}.amazonaws.com/{'0123abcd'*6}/{'ab,12'*2}/",
+    ]
     urls = ["andrwe.org/ads/", "andrwe.jp/ads/", "pubfeed.linkby.com"]
-    urls = ["andrwe.org/ads/", "andrwe.jp/ads/"]
     run_requests(start_privoxy, supported_schemes, urls, [403])