Merge pull request #61 from sdr-enthusiasts/dev

Fix for logging issues

docker-compose.yml

@@ -20,7 +20,7 @@ services:
       # locations.conf file. Then, before restarting, please make sure to set AUTOGENERATE=OFF below.
       - AUTOGENERATE=ON
       # ----------------------------------------------------------------------------------------
-      # VERBOSELOGS indicates if the web server logs are written to the Docker Container logs
+      # VERBOSELOG indicates if the web server logs are written to the Docker Container logs
       # (accessible with `docker logs webproxy`) or that they are discarded. Values are ON (default) or OFF
       - VERBOSELOG=ON
       # ----------------------------------------------------------------------------------------

rootfs/etc/s6-overlay/s6-rc.d/nginx-log-to-console/run

@@ -0,0 +1,3 @@
+#!/bin/sh
+exec /etc/s6-overlay/scripts/nginx-log-to-console
+

rootfs/etc/s6-overlay/s6-rc.d/nginx-log-to-console/type

@@ -0,0 +1 @@
+longrun

rootfs/etc/s6-overlay/scripts/20-webproxy

@@ -160,16 +160,10 @@ fi
 ln -sf /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default
 
 # set logging to what VERBOSELOG says. Default is off, "ON" means to stdout, "FILE" means to access.log/error.log file in external volume
-if chk_enabled "${VERBOSELOG}"
-then
-  "${s6wrap[@]}" echo "Enabled HTTP logging to container logs"
-  sed -i 's|\(^\s*access_log \).*|\1/dev/stdout ;|' /etc/nginx/nginx.conf
-  sed -i 's|\(^\s*error_log \).*|\1/dev/stdout ;|' /etc/nginx/nginx.conf
-elif [[ "${VERBOSELOG,,}" == "file" ]]
-then
-  "${s6wrap[@]}" echo "Enabled HTTP logging to files in shared volume"
-  sed -i 's|\(^\s*access_log \).*|\1/run/nginx/access.log ;|' /etc/nginx/nginx.conf
-  sed -i 's|\(^\s*error_log \).*|\1/run/nginx/error.log ;|' /etc/nginx/nginx.conf
+if chk_enabled "${VERBOSELOG}" || [[ "${VERBOSELOG,,}" == "file" ]] || chk_enabled "$IPMAPS" || chk_enabled "$IPTABLES_BLOCK"; then
+  "${s6wrap[@]}" echo "Enabled HTTP logging"
+  sed -i 's|\(^\s*access_log\).*|\1 /var/log/nginx/access.log ;|' /etc/nginx/nginx.conf
+  sed -i 's|\(^\s*error_log\).*|\1 /var/log/nginx/error.log ;|' /etc/nginx/nginx.conf
 else
   "${s6wrap[@]}" echo "HTTP logging is disabled"
 fi

rootfs/etc/s6-overlay/scripts/60-block-ips

@@ -8,21 +8,11 @@
 # shellcheck disable=SC1091
 source /scripts/common
 
-IPTABLES_BLOCK="${IPTABLES_BLOCK,,}"
-IPTABLES_BLOCK="${IPTABLES_BLOCK:0:3}"
-
 cp -f /root/manage_ipblock /usr/local/bin >/dev/null 2>&1
 
-if [[ "$IPTABLES_BLOCK" != "ena" ]] && [[ "$IPTABLES_BLOCK" != "on" ]]
-then
+if ! chk_enabled "$IPTABLES_BLOCK" || [[ -z "$BLOCKBOT" ]]; then
   # shellcheck disable=SC2154
-  "${s6wrap[@]}" echo "Bot blocking is disabled. Continuing..."
-  exit 0
-fi
-
-if [[ -z "$BLOCKBOT" ]]
-then
-  "${s6wrap[@]}" echo "Bot blocking is disabled. Continuing..."
+  "${s6wrap[@]}" echo "Bot blocking is not enabled. Continuing..."
   exit 0
 fi
 

rootfs/etc/s6-overlay/scripts/block-ips

@@ -6,17 +6,8 @@ source /scripts/common
 
 SLEEPTIME=60
 
-# check if the IPTABLES_BLOCK is disabled and if so, stop execution of this service:
-IPTABLES_BLOCK="${IPTABLES_BLOCK,,}"
-IPTABLES_BLOCK="${IPTABLES_BLOCK:0:3}"
-if [[ "$IPTABLES_BLOCK" != "ena" ]] && [[ "$IPTABLES_BLOCK" != "on" ]]
-then
-  "${s6wrap[@]}" echo "Bot blocking is disabled"
-  sleep infinity
-fi
-
-if [[ -z "$BLOCKBOT" ]]
-then
+# check if the IPTABLES_BLOCK is not enabled and if so, stop execution of this service:
+if ! chk_enabled "$IPTABLES_BLOCK" || [[ -z "$BLOCKBOT" ]]; then
   "${s6wrap[@]}" echo "Bot blocking is disabled"
   sleep infinity
 fi
@@ -25,65 +16,67 @@ fi
 
 while true
 do
-  # Read access log and process any GEOIP or BLOCKBOT response codes. Note - also naturally occurring responses with these codes are picked up:
-  awk -v "g=$GEOIP_RESPONSECODE" -v "b=$BLOCKBOT_RESPONSECODE" -v "d=$(date +%s)" '{if ($9 == b || $9 == g) print $1 " " $9 " " d}' /var/log/nginx/access.log >> /run/nginx/ip-blocklist
+  # Do stuff if the access log file exists. Sometimes, it doesn't exist if there isn't much traffic
+  if [[ -f /var/log/nginx/access.log ]]; then
+    # Read access log and process any GEOIP or BLOCKBOT response codes. Note - also naturally occurring responses with these codes are picked up:
+    awk -v "g=$GEOIP_RESPONSECODE" -v "b=$BLOCKBOT_RESPONSECODE" -v "d=$(date +%s)" '{if ($9 == b || $9 == g) print $1 " " $9 " " d}' /var/log/nginx/access.log >> /run/nginx/ip-blocklist
 
-  # Read $BLOCKBOT into an array and remove all leading/trailing spaces:
-  readarray -d "," -t BOT_ARRAY <<< "$BLOCKBOT"
-  for ((n=0; n<${#BOT_ARRAY[*]}; n++))
-  do
-    # this is slightly inefficient as it iterates through the entire array.
-    # However, all-at-once solutions like "y=($(printf '%s\n' "${x[@]}"|xargs))" appear to split each element at any internal whitespace :(
-    BOT_ARRAY[n]="$(xargs <<< "${BOT_ARRAY[n]}")"
-  done
-  # Parse through the log files and to the blocklist.
-  # It's OK to add duplicates; `sort -u` will only keep the first (oldest) occurrence
-  grep -i -h -w -f <(printf '%s\n' "${BOT_ARRAY[@]}") /var/log/nginx/access.log 2>/dev/null | awk -v "d=$(date +%s)" '{print $1 " logs_crawler " d}' >> /run/nginx/ip-blocklist
+    # Read $BLOCKBOT into an array and remove all leading/trailing spaces:
+    readarray -d "," -t BOT_ARRAY <<< "$BLOCKBOT"
+    for ((n=0; n<${#BOT_ARRAY[*]}; n++))
+    do
+      # this is slightly inefficient as it iterates through the entire array.
+      # However, all-at-once solutions like "y=($(printf '%s\n' "${x[@]}"|xargs))" appear to split each element at any internal whitespace :(
+      BOT_ARRAY[n]="$(xargs <<< "${BOT_ARRAY[n]}")"
+    done
+    # Parse through the log files and to the blocklist.
+    # It's OK to add duplicates; `sort -u` will only keep the first (oldest) occurrence
+    grep -i -h -w -f <(printf '%s\n' "${BOT_ARRAY[@]}") /var/log/nginx/access.log 2>/dev/null | awk -v "d=$(date +%s)" '{print $1 " logs_crawler " d}' >> /run/nginx/ip-blocklist
 
-  sort -u -k 1,1 /run/nginx/ip-blocklist | sort -n > /tmp/ip-blocklist
-  mv -f /tmp/ip-blocklist /run/nginx/ip-blocklist
+    sort -u -k 1,1 /run/nginx/ip-blocklist | sort -n > /tmp/ip-blocklist
+    mv -f /tmp/ip-blocklist /run/nginx/ip-blocklist
 
-  # Re-process the blocklist and add them to the IP tables for DROPping:
-  blocktable=()
-  if [[ -f /run/nginx/ip-blocklist ]]
-  then
-    while read -ra line
-    do
-      if [[ -n "${line[0]}" ]] && [[ "${line[0]:0:1}" != "#" ]] && ! grep -f /run/nginx/ip-allowlist >/dev/null 2>&1 <<< "${line[0]}" && ! iptables -C INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
-      then
-        iptables -I INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
-        blocktable+=("${line[0]}")
-      fi
-    done < /run/nginx/ip-blocklist
-  fi
+    # Re-process the blocklist and add them to the IP tables for DROPping:
+    blocktable=()
+    if [[ -f /run/nginx/ip-blocklist ]]
+    then
+      while read -ra line
+      do
+        if [[ -n "${line[0]}" ]] && [[ "${line[0]:0:1}" != "#" ]] && ! grep -f /run/nginx/ip-allowlist >/dev/null 2>&1 <<< "${line[0]}" && ! iptables -C INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
+        then
+          iptables -I INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
+          blocktable+=("${line[0]}")
+        fi
+      done < /run/nginx/ip-blocklist
+    fi
 
-  # Notify the logs of any additions to the blocklist:
-  if (( ${#blocktable[@]} > 0 ))
-  then
-    "${s6wrap[@]}" echo "These ${#blocktable[@]} IP(s) have been added to the iptables Block List: ${blocktable[@]}. Currently, there are $(sed -e '/^$/d ; /^#/d' /run/nginx/ip-blocklist |wc -l) blocked IP addresses."
-  fi
+    # Notify the logs of any additions to the blocklist:
+    if (( ${#blocktable[@]} > 0 ))
+    then
+      "${s6wrap[@]}" echo "These ${#blocktable[@]} IP(s) have been added to the iptables Block List: ${blocktable[@]}. Currently, there are $(sed -e '/^$/d ; /^#/d' /run/nginx/ip-blocklist |wc -l) blocked IP addresses."
+    fi
 
-  # Go through the blocklist and determine if any entries have timed out:
-  if (( IPJAILTIME > 0 ))
-  then
-    deletes=()
-    cp /run/nginx/ip-blocklist /tmp
-    while read -ra line
-    do
-      if [[ -n "${line[2]}" ]] && [[ "${line[0]:0:1}" != "#" ]] && (( line[2] > 0 )) && (( line[2] + IPJAILTIME < $(date +%s) )) || grep -f /run/nginx/ip-allowlist >/dev/null 2>&1 <<< "${line[0]}"
+    # Go through the blocklist and determine if any entries have timed out:
+    if (( IPJAILTIME > 0 ))
+    then
+      deletes=()
+      cp /run/nginx/ip-blocklist /tmp
+      while read -ra line
+      do
+        if [[ -n "${line[2]}" ]] && [[ "${line[0]:0:1}" != "#" ]] && (( line[2] > 0 )) && (( line[2] + IPJAILTIME < $(date +%s) )) || grep -f /run/nginx/ip-allowlist >/dev/null 2>&1 <<< "${line[0]}"
+        then
+          sed -i '/^'"${line[0]}"'/d' /run/nginx/ip-blocklist
+          iptables -D INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
+          deletes+=("${line[0]}")
+        fi
+      done < /tmp/ip-blocklist
+      rm -f /tmp/ip-blocklist
+      if [[ "${#deletes[*]}" -gt 0 ]]
       then
-        sed -i '/^'"${line[0]}"'/d' /run/nginx/ip-blocklist
-        iptables -D INPUT -s "${line[0]}" -j DROP >/dev/null 2>&1
-        deletes+=("${line[0]}")
+        "${s6wrap[@]}" echo "These ${#deletes[@]} IP(s) have timed out and have been removed from the Block List: ${deletes[@]}. Currently, there are $(sed -e '/^$/d ; /^#/d' /run/nginx/ip-blocklist |wc -l) blocked IP addresses."
       fi
-    done < /tmp/ip-blocklist
-    rm -f /tmp/ip-blocklist
-    if [[ "${#deletes[*]}" -gt 0 ]]
-    then
-      "${s6wrap[@]}" echo "These ${#deletes[@]} IP(s) have timed out and have been removed from the Block List: ${deletes[@]}. Currently, there are $(sed -e '/^$/d ; /^#/d' /run/nginx/ip-blocklist |wc -l) blocked IP addresses."
     fi
   fi
-
   # Sleep a while before repeating everything:
   sleep $SLEEPTIME
 

rootfs/etc/s6-overlay/scripts/nginx-log-to-console

@@ -0,0 +1,24 @@
+#!/command/with-contenv bash
+#shellcheck shell=bash disable=SC2154,SC1091
+
+source /scripts/common
+
+if ! chk_enabled "${VERBOSELOG}"; then
+  "${s6wrap[@]}" echo "Logging to console is disabled. Set \"VERBOSELOG=on\" if you want to see the nginx access and error logs"
+  sleep infinity
+fi
+
+# tail the log files to console:
+
+{ "${s6wrap[@]}" tail -F /var/log/nginx/access.log; } &
+accesspid="$!"
+
+{ "${s6wrap[@]}" tail -F /var/log/nginx/error.log; } &
+errorpid="$!"
+
+# Wait until any of them exits for any reason
+wait -n
+
+# kill both processes before exiting, so the restart of the service is done cleanly
+kill $accesspid >/dev/null 2>&1 || true
+kill $errorpid >/dev/null 2>&1 || true

rootfs/etc/s6-overlay/scripts/nginx-logrotate

@@ -3,15 +3,8 @@
 
 source /scripts/common
 
-if [[ -z "$LOGROTATE_INTERVAL" ]]
-then
-  LOGROTATE_INTERVAL=3600
-fi
-
-if [[ -z "$LOGROTATE_MAXBACKUPS" ]]
-then
-  LOGROTATE_MAXBACKUPS=24
-fi
+LOGROTATE_INTERVAL="${LOGROTATE_INTERVAL:-3600}"
+LOGROTATE_MAXBACKUPS="${LOGROTATE_MAXBACKUPS:-24}"
 
 "${s6wrap[@]}" echo "Started as an s6 service; nginx logs are rotated every $LOGROTATE_INTERVAL seconds; up to $LOGROTATE_MAXBACKUPS are kept."
 
@@ -29,9 +22,9 @@ do
     count=$LOGROTATE_MAXBACKUPS
     while (( --count > 0 ))
     do
-      mv -f /var/log/nginx/access.log.$((count-1)) /var/log/nginx/access.log.$count >/dev/null 2>&1
+      mv -f /var/log/nginx/access.log.$((count-1)) /var/log/nginx/access.log.$count >/dev/null 2>&1 || true
     done
-    mv -f /var/log/nginx/access.log /var/log/nginx/access.log.0
+    mv -f /var/log/nginx/access.log /var/log/nginx/access.log.0 || true
     "${s6wrap[@]}" echo "nginx access logs have been rotated. Next rotation at $(date -d "+$LOGROTATE_INTERVAL seconds")."
   fi
 
@@ -40,9 +33,9 @@ do
     count=$LOGROTATE_MAXBACKUPS
     while (( --count > 0 ))
     do
-      mv -f /var/log/nginx/error.log.$((count-1)) /var/log/nginx/error.log.$count >/dev/null 2>&1
+      mv -f /var/log/nginx/error.log.$((count-1)) /var/log/nginx/error.log.$count >/dev/null 2>&1 || true
     done
-    mv -f /var/log/nginx/error.log /var/log/nginx/error.log.0
+    mv -f /var/log/nginx/error.log /var/log/nginx/error.log.0 || true
     "${s6wrap[@]}" echo "nginx error logs have been rotated. Next rotation at $(date -d "+$LOGROTATE_INTERVAL seconds")."
   fi