Skip to content
💾 Fetch || ⏫ Update Wordlists 📙

💾 Fetch || ⏫ Update Wordlists 📙 #36

Sign in to view logs

💾 Fetch || ⏫ Update Wordlists 📙

💾 Fetch || ⏫ Update Wordlists 📙 #36

Workflow file for this run

.github/workflows/generate_update_wordlists.yaml at 2802d48
name: 💾 Fetch || ⏫ Update Wordlists 📙
#MAX_RUNTIME: ~ 05-10 mins
on:
workflow_dispatch:
schedule:
- cron: "15 23 * * *" # ( 11:15 PM UTC --> 05:00 AM Morning )
jobs:
generate:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
path: main
- name: Install Coreutils
run: |
# Presets
set -x ; set +e
#--------------#
sudo apt update -y
sudo apt install coreutils curl dos2unix git jq moreutils wget -y
# Do again, sometimes fails
sudo apt install coreutils curl dos2unix git jq moreutils wget -y
pip install ansi2txt
continue-on-error: true
- name: Install eget
run: |
# Presets
set -x ; set +e
#--------------#
# eget for bins
sudo wget "https://raw.githubusercontent.com/Azathothas/Toolpacks/main/x86_64/eget" -O "/usr/local/bin/eget"
sudo chmod +xwr "/usr/local/bin/eget"
- name: Install Deps for Wordium
run: |
# Presets
set -x ; set +e
#--------------#
#anew
sudo eget "https://raw.githubusercontent.com/Azathothas/Toolpacks/main/x86_64/anew" --to "/usr/local/bin/anew"
anew -h
continue-on-error: true
- name: Clean Cloned Repos (I)
run: |
# Presets
set -x ; set +e
#--------------#
# Del all Dirs
find "$GITHUB_WORKSPACE/main" -maxdepth 1 -type d ! -name '.git*' ! -name 'main' ! -name '*Logs*' ! -name '*Misc*' -exec rm -rf {} \; 2>/dev/null
continue-on-error: true
- name: Setup Env
run: |
# Presets
set -x ; set +e
#--------------#
# Create Output Dir
mkdir -p "$GITHUB_WORKSPACE/main/Logs"
continue-on-error: true
- name: Run Wordium
run: |
# Presets
set -x ; set +e
#--------------#
#Install
sudo curl -qflSL "https://raw.githubusercontent.com/Azathothas/Arsenal/main/wordium/wordium.sh" -o "/usr/local/bin/wordium"
sudo chmod +xwr "/usr/local/bin/wordium"
#Run , (DO NOT SPECIFY /Wordlists)
wordium --wordlist-dir $GITHUB_WORKSPACE/main/ | tee "/tmp/wordium.log"
cat "/tmp/wordium.log" | ansi2txt > "$GITHUB_WORKSPACE/main/Logs/wordium.log"
cat "/tmp/wordium.log" | ansi2txt | grep -A999999 "➼Updated Wordlists:" > "/tmp/wordium_stat.txt"
continue-on-error: true
- name: Clean Cloned Repos (II)
run: |
# Presets
set -x ; set +e
#--------------#
# Del all Dirs
find "$GITHUB_WORKSPACE/main" -maxdepth 1 -type d ! -name '.git*' ! -name 'main' ! -name '*Logs*' ! -name '*Misc*' -exec rm -rf {} \; 2>/dev/null
continue-on-error: true
- name: Enrichments (Anew || Sort Misc)
run: |
# Presets
set -x ; set +e
#--------------#
#anew things
#APIs
cat "$GITHUB_WORKSPACE/main/Misc/api_docs.txt" | anew -q "$GITHUB_WORKSPACE/main/x_api_tiny.txt"
cat "$GITHUB_WORKSPACE/main/Misc/graphql_paths.txt" | anew -q "$GITHUB_WORKSPACE/main/x_api_tiny.txt"
#LHFs
cat "$GITHUB_WORKSPACE/main/Misc/api_docs.txt" | anew -q "$GITHUB_WORKSPACE/main/x_lhf_mini.txt"
cat "$GITHUB_WORKSPACE/main/Misc/graphql_paths.txt" | anew -q "$GITHUB_WORKSPACE/main/x_lhf_mini.txt"
#Ports
cat "$GITHUB_WORKSPACE/main/Misc/ports_sussy.txt" | anew -q "$GITHUB_WORKSPACE/main/Misc/ports_top_20k.txt"
#Sort
cd "$GITHUB_WORKSPACE/main"
#Main
find . -maxdepth 1 -type f -name '*.txt' -exec sort -u {} -o {} \;
#Misc
find ./Misc -type f -name '*.txt' -exec sort -u {} -o {} \;
- name: Dos2Unix Everything
run: |
# Presets
set -x ; set +e
#--------------#
cd "$GITHUB_WORKSPACE/main"
find . -type f -name '*.txt' -exec dos2unix {} \;
continue-on-error: true
- name: Update README.md
run: |
# Presets
set -x ; set +e
#--------------#
cd "$GITHUB_WORKSPACE/main"
cat ./INFO.md > ./README.md
echo -e "" >> ./README.md
echo '---' >> ./README.md
echo '```console' >> ./README.md
echo -e "" >> ./README.md
echo -e "--> METADATA" >> ./README.md
/bin/bash -c 'PS4="$ "; cat /tmp/wordium_stat.txt '&>> ./README.md
echo -e "" >> ./README.md
echo -e '```\n' >> ./README.md
echo -e "" >> ./README.md
echo '---' >> ./README.md
echo -e "" >> ./README.md
working-directory: main
continue-on-error: true
- name: Get DateTime
run: |
# Date Time
NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)')
echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV
- name: Git Pull
run: |
cd "$GITHUB_WORKSPACE/main" && git pull origin main
continue-on-error: true
- uses: stefanzweifel/git-auto-commit-action@v4
with:
repository: ./main
commit_user_name: Azathothas # defaults to "github-actions[bot]"
commit_user_email: [email protected] # defaults to "41898282+github-actions[bot]@users.noreply.github.com"
commit_message: "✅ Fetch latest Wordlists 📙 <-- ${{ env.NEPALI_TIME }} ⌚"
#push_options: '--force'