From a696f344acdae0cb902a76459e7bc9e27e9fcfc8 Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 25 Apr 2024 14:29:44 -0700 Subject: [PATCH 1/6] Redid Debian setup scripts for k3s - Redid Debian setup scripts for k3s --- debian_k3s_prereqs_only.sh | 37 +++++++++++++ debian_k3s_setup.sh | 98 +++++++++++++++++++++++++++++++++ debian_minikube_prereqs_only.sh | 60 -------------------- debian_minikube_setup.sh | 81 --------------------------- 4 files changed, 135 insertions(+), 141 deletions(-) create mode 100755 debian_k3s_prereqs_only.sh create mode 100755 debian_k3s_setup.sh delete mode 100644 debian_minikube_prereqs_only.sh delete mode 100644 debian_minikube_setup.sh diff --git a/debian_k3s_prereqs_only.sh b/debian_k3s_prereqs_only.sh new file mode 100755 index 0000000..1e494eb --- /dev/null +++ b/debian_k3s_prereqs_only.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +sudo pwd &> /dev/null + +if ! command -v curl &> /dev/null; then + echo -e "[*] curl could not be found, attempting to install...\n" + sudo apt-get update + sudo apt-get install curl -y +fi + +clear -x + +# Step 1: Install k3s +echo -e "\n[*] Installing k3s...\n" +curl -sfL https://get.k3s.io | sh - +mkdir -p ~/.kube && sudo k3s kubectl config view --raw > ~/.kube/config +chmod 600 ~/.kube/config +export KUBECONFIG=~/.kube/config + +# Step 2: Install Helm +echo -e "\n[*] Installing Helm...\n" +curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null +sudo apt-get install apt-transport-https --yes +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list +sudo apt-get update +sleep 10 +sudo apt-get install helm + +# Step 3: Install Dependencies with Helm +echo -e "\n[*] Installing Dependencies with Helm...\n" +helm install elastic-operator eck-operator --repo https://helm.elastic.co --namespace elastic-system --create-namespace --set managedNamespaces='{default}' + +clear -x +echo -e "\n[*] Nemesis k3s prereqs installed." +echo -e "[*] Install Nemesis with:\n" +echo -e "\thelm install --repo https://specterops.github.io/Nemesis/ nemesis-quickstart quickstart" +echo -e "\thelm install --repo https://specterops.github.io/Nemesis/ nemesis nemesis --timeout '45m' --set operation.nemesisHttpServer="https://\:443/"\n" diff --git a/debian_k3s_setup.sh b/debian_k3s_setup.sh new file mode 100755 index 0000000..14c80f1 --- /dev/null +++ b/debian_k3s_setup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +sudo pwd &> /dev/null + +if ! command -v curl &> /dev/null; then + echo -e "[*] curl could not be found, attempting to install...\n" + sudo apt-get update + sudo apt-get install curl -y +fi + +validate_ip() { + if [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + IFS='.' read -r -a octets <<< "$1" + for octet in "${octets[@]}"; do + if (( octet > 255 )); then + return 1 + fi + done + return 0 + else + return 1 + fi +} + +clear -x + +if [ -z "$1" ]; then + while true; do + read -p "No IP address provided. Would you like to use the default IP 127.0.0.1 or enter another? (y/n): " answer + answer=$(echo "$answer" | tr '[:upper:]' '[:lower:]') + + if [[ "$answer" == "y" ]]; then + read -p "Enter an IP address or press enter to use 127.0.0.1: " user_ip + if [ -z "$user_ip" ]; then + IP="127.0.0.1" + elif validate_ip "$user_ip"; then + IP="$user_ip" + else + echo "Invalid IP address format. Please enter a valid IPv4 address." + continue + fi + break + elif [[ "$answer" == "n" ]]; then + echo "No IP set, exiting script." + exit 1 + else + echo "Please answer 'y' or 'n'." + fi + done +else + # IP was provided as argument, validate it + if validate_ip "$1"; then + IP="$1" + else + echo "Invalid IP address provided." + exit 1 + fi +fi + +echo "[*] Using IP: $IP" + +# Step 1: Install k3s +echo -e "\n[*] Installing k3s...\n" +curl -sfL https://get.k3s.io | sh - +mkdir -p ~/.kube && sudo k3s kubectl config view --raw > ~/.kube/config +chmod 600 ~/.kube/config +export KUBECONFIG=~/.kube/config + +# Step 2: Install Helm +echo -e "\n[*] Installing Helm...\n" +curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null +sudo apt-get install apt-transport-https --yes +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list +sudo apt-get update +sleep 10 +sudo apt-get install helm + +# Step 3: Install Dependencies with Helm +echo -e "\n[*] Installing Dependencies with Helm...\n" +helm install elastic-operator eck-operator --repo https://helm.elastic.co --namespace elastic-system --create-namespace --set managedNamespaces='{default}' + +# Step 4: Install the quickstart Chart +sleep 30 +echo -e "\n[*] Installing Quickstart chart...\n" +helm install --repo https://specterops.github.io/Nemesis/ nemesis-quickstart quickstart + +# Step 5: Install Nemesis chart +sleep 30 +echo -e "\n[*] Installing Nemesis chart...\n" +helm install --repo https://specterops.github.io/Nemesis/ nemesis nemesis --timeout '45m' --set operation.nemesisHttpServer="https://$IP:443/" + +export NEMESIS_BASIC_AUTH_USER=$(sudo kubectl get secret --namespace "default" basic-auth -o jsonpath="{.data.username}" | base64 -d) +export NEMESIS_BASIC_AUTH_PASSWORD=$(sudo kubectl get secret --namespace "default" basic-auth -o jsonpath="{.data.password}" | base64 -d) + +clear -x +echo -e "\n[*] Nemesis installed, but some pods may still be standing up for the next 5-10 minutes." +echo -e "[*] You can check pod deployment status with 'sudo kubectl get pods -A'" +echo -e "[*] Once all pods are up, browse to https://$IP:443/ and log in with '$NEMESIS_BASIC_AUTH_USER:$NEMESIS_BASIC_AUTH_PASSWORD'\n" diff --git a/debian_minikube_prereqs_only.sh b/debian_minikube_prereqs_only.sh deleted file mode 100644 index 5bd8fe7..0000000 --- a/debian_minikube_prereqs_only.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Installs the prereqs needed to install Nemesis - -# Install Docker -sudo apt-get update -sudo apt-get install -y curl -sudo mkdir /etc/apt/keyrings/ 2>/dev/null -curl -fsSL https://get.docker.com -o /tmp/get-docker.sh -sudo sh /tmp/get-docker.sh -sudo apt install -y docker-compose -sudo usermod -aG docker $USER - - -# echo the next part of the script to a temp location so we can run with newgrp -cat << EOF > /tmp/setup_part_2.sh - -# Install Kubectl -curl -LO "https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - -# Install Minikube -curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 -sudo install minikube-linux-amd64 /usr/local/bin/minikube -minikube config set memory 12288 -minikube config set cpus 3 -minikube start - -# Install Helm -curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null -echo "deb [arch=\$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list -sudo apt-get update -sudo apt-get install -y helm - -# Install k8s prereqs -# Add Elastic repository -helm repo add elastic https://helm.elastic.co -# Add Bitnami repository -helm repo add bitnami https://charts.bitnami.com/bitnami -# Add NGINX repository -helm repo add nginx https://kubernetes.github.io/ingress-nginx -# Install NGINX ingress -helm install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --set prometheus.create=true --set prometheus.port=9113 --set tcp.5044="default/nemesis-ls-beats:5044" --set controller.config."proxy-body-size"="5000m" -# Install ElasticSearch operator to manage "default" namespace. The managedNamespaces field will need to be configured if you desire to install Nemesis in a different namespace -helm install elastic-operator elastic/eck-operator --namespace elastic-system --create-namespace --set managedNamespaces='{default}' - -curl https://raw.githubusercontent.com/SpecterOps/Nemesis/helm/helm/nemesis/values.yaml -o nemesis-values.yaml -curl https://raw.githubusercontent.com/SpecterOps/Nemesis/helm/helm/quickstart/values.yaml -o quickstart-values.yaml -EOF - - -# kick off the script with new group privileges using "newgrp" -newgrp docker </dev/null -curl -fsSL https://get.docker.com -o /tmp/get-docker.sh -sudo sh /tmp/get-docker.sh -sudo apt install -y docker-compose openssh-server -sudo service ssh start -sudo usermod -aG docker $USER - - -# echo the next part of the script to a temp location so we can run with newgrp -cat << EOF > /tmp/setup_part_2.sh - -# Install Kubectl -curl -LO "https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - -# Install Minikube -curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 -sudo install minikube-linux-amd64 /usr/local/bin/minikube -minikube config set memory 12288 -minikube config set cpus 3 -minikube start - -# Install Helm -curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null -echo "deb [arch=\$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list -sudo apt-get update -sudo apt-get install -y helm - -# Install k8s prereqs -# Add Elastic repository -helm repo add elastic https://helm.elastic.co -# Add Bitnami repository -helm repo add bitnami https://charts.bitnami.com/bitnami -# Add NGINX repository -helm repo add nginx https://kubernetes.github.io/ingress-nginx -# Install NGINX ingress -helm install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --set prometheus.create=true --set prometheus.port=9113 --set tcp.5044="default/nemesis-ls-beats:5044" --set controller.config."proxy-body-size"="5000m" -# Install ElasticSearch operator to manage "default" namespace. The managedNamespaces field will need to be configured if you desire to install Nemesis in a different namespace -helm install elastic-operator elastic/eck-operator --namespace elastic-system --create-namespace --set managedNamespaces='{default}' - -# run the Nemesis quickstart -echo -e "\nInstalling Nemesis quickstart\n" -helm install --repo https://specterops.github.io/Nemesis/ nemesis-quickstart quickstart -# If using the local repo: -# helm install quickstart ./helm/quickstart - -# install Nemesis -echo -e "\n\n\nInstalling Nemesis itself (this may take some time to pull down all containers)...\n" -helm install --repo https://specterops.github.io/Nemesis/ nemesis nemesis --timeout '45m' -# If using the local repo: -# helm install nemesis ./helm/nemesis --timeout '45m' -EOF - - -# kick off the script with new Docker group privileges using "newgrp" -newgrp docker < Date: Thu, 25 Apr 2024 15:01:17 -0700 Subject: [PATCH 2/6] Documentation tweaks Bumped version, in README, added more thanks, added blog posts --- README.md | 6 ++++-- docs/quickstart.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c6b1b5..0849060 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

-version 0.1.0a +version 1.0.0 Slack @@ -48,6 +48,8 @@ See [development.md](./docs/development.md) | Post Name | Publication Date | Link | |---------------------------------------------|------------------|------------------------------------------------------------------------------------| +| *Nemesis 1.0.0* | Apr 25, 2024 | https://posts.specterops.io/nemesis-1-0-0-8c6b745dc7c5 | +| *Summoning RAGnarok With Your Nemesis* | Mar 13, 2024 | https://posts.specterops.io/summoning-ragnarok-with-your-nemesis-7c4f0577c93b | | *Shadow Wizard Registry Gang: Structured Registry Querying* | Sep 5, 2023 | https://posts.specterops.io/shadow-wizard-registry-gang-structured-registry-querying-9a2fab62a26f | | *Hacking With Your Nemesis* | Aug 9, 2023 | https://posts.specterops.io/hacking-with-your-nemesis-7861f75fcab4 | | *Challenges In Post-Exploitation Workflows* | Aug 2, 2023 | https://posts.specterops.io/challenges-in-post-exploitation-workflows-2b3469810fe9 | @@ -58,4 +60,4 @@ See [development.md](./docs/development.md) Nemesis is built on large chunk of other people's work. Throughout the codebase we've provided citations, references, and applicable licenses for anything used or adapted from public sources. If we're forgotten proper credit anywhere, please let us know or submit a pull request! -We also want to acknowledge Evan McBroom, Hope Walker, and Carlo Alcantara from SpecterOps for their help with the initial Nemesis concept and amazing feedback throughout the development process. +We also want to acknowledge Evan McBroom, Hope Walker, and Carlo Alcantara from [SpecterOps](https://specterops.io/) for their help with the initial Nemesis concept and amazing feedback throughout the development process. Also thanks to [Matt Ehrnschwender](https://twitter.com/M_alphaaa) for tons of k3s and GitHub workflow help! diff --git a/docs/quickstart.md b/docs/quickstart.md index f134b12..d17c4f9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,6 +1,6 @@ # Quickstart Guide -Here's a quickstart guide to setting up the Nemesis platform using k3s and Helm on Debian 11. This guide will start a Nemesis server listening on the server "192.168.1.10". Change this IP to the host that you will be accessing Nemesis from. If this does not fit your installation need, see the full [setup guide](setup.md). +Here's a quickstart guide to setting up the Nemesis platform using k3s and Helm on Debian 11. This guide will start a Nemesis server listening on the server "192.168.1.10". Change this IP to the host that you will be accessing Nemesis from. If this does not fit your installation need, see the full [setup guide](setup.md). This process is automated in the [debian_k3s_setup.sh](../debian_k3s_setup.sh) script, which accepts an IP as an argument. ### Prerequisites From 782b391ebfdf44111433f2e046f286fd3305eb89 Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 25 Apr 2024 15:01:27 -0700 Subject: [PATCH 3/6] Added CHANGELOG.md - Added CHANGELOG.md --- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..59f43cb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [1.0.0] - 2024-04-25 + +### Added + +- Proper host and temporal modeling +- Helm charts for deployment and publishing of images to [Dockerhub](https://hub.docker.com/u/specterops) +- Hasura API endpoint +- Additional documentation, including (finally) a usage guide +- Hosting of documentation on [GitHub Pages](https://specterops.github.io/Nemesis/) +- `monitor` command to submit_to_nemesis.sh for continual file submission +- Jupyter notebooks back into the stack +- Processing for Chromium JSON cookie dumps +- Automatic expunging of expired data via the `data_expunge` task + +### Changed + +- Dropped Docker/Minikube support, replaced with k3s +- Drastically simplified/streamlined setup process +- Any compatible file is now handled by Apache Tika instead of a subset +- Removed the Tensorflow model hosting and DeepPass as the model just wasn't accurate enough to be useful +- Streamlined NLP indexing to prevent choking and exposed a /nlp/ route for search +- Streamlined hash cracking and added in deduplication so hashes aren’t cracked twice +- Revamped text search to use fuzzy search fused with semantic search +- Countless Dashboard changes + +### Fixed + +- Too many bugs to count \ No newline at end of file From 09d3d0f0e8995a5596b1d66135d4a8b6d05de5ad Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 25 Apr 2024 15:14:22 -0700 Subject: [PATCH 4/6] Update index.md --- docs/index.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 43298b7..7c97bc6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@

-version 0.1.0a +version 1.0.0 Slack @@ -34,7 +34,9 @@ Built on Kubernetes with scale in mind, our goal with Nemesis was to create a ce Nemesis aims to automate a number of repetitive tasks operators encounter on engagements, empower operators’ analytic capabilities and collective knowledge, and create structured and unstructured data stores of as much operational data as possible to help guide future research and facilitate offensive data analysis. ## Setup / Installation -See the [setup instructions](setup.md). +Follow the [quickstart guide](quickstart.md) + +Or see the full [setup instructions](setup.md) ## Usage See the [Nemesis Usage Guide](usage_guide.md). @@ -46,6 +48,8 @@ See [development.md](development.md) | Post Name | Publication Date | Link | |---------------------------------------------|------------------|------------------------------------------------------------------------------------| +| *Nemesis 1.0.0* | Apr 25, 2024 | https://posts.specterops.io/nemesis-1-0-0-8c6b745dc7c5 | +| *Summoning RAGnarok With Your Nemesis* | Mar 13, 2024 | https://posts.specterops.io/summoning-ragnarok-with-your-nemesis-7c4f0577c93b | | *Shadow Wizard Registry Gang: Structured Registry Querying* | Sep 5, 2023 | https://posts.specterops.io/shadow-wizard-registry-gang-structured-registry-querying-9a2fab62a26f | | *Hacking With Your Nemesis* | Aug 9, 2023 | https://posts.specterops.io/hacking-with-your-nemesis-7861f75fcab4 | | *Challenges In Post-Exploitation Workflows* | Aug 2, 2023 | https://posts.specterops.io/challenges-in-post-exploitation-workflows-2b3469810fe9 | @@ -56,4 +60,4 @@ See [development.md](development.md) Nemesis is built on large chunk of other people's work. Throughout the codebase we've provided citations, references, and applicable licenses for anything used or adapted from public sources. If we're forgotten proper credit anywhere, please let us know or submit a pull request! -We also want to acknowledge Evan McBroom, Hope Walker, and Carlo Alcantara from SpecterOps for their help with the initial Nemesis concept and amazing feedback throughout the development process. +We also want to acknowledge Evan McBroom, Hope Walker, and Carlo Alcantara from [SpecterOps](https://specterops.io/) for their help with the initial Nemesis concept and amazing feedback throughout the development process. Also thanks to [Matt Ehrnschwender](https://twitter.com/M_alphaaa) for tons of k3s and GitHub workflow help! From f1665e75ab26b21c09fec76ffca4126840983263 Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 25 Apr 2024 16:29:45 -0700 Subject: [PATCH 5/6] Update quickstart.md --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index d17c4f9..bdbb179 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,6 +1,6 @@ # Quickstart Guide -Here's a quickstart guide to setting up the Nemesis platform using k3s and Helm on Debian 11. This guide will start a Nemesis server listening on the server "192.168.1.10". Change this IP to the host that you will be accessing Nemesis from. If this does not fit your installation need, see the full [setup guide](setup.md). This process is automated in the [debian_k3s_setup.sh](../debian_k3s_setup.sh) script, which accepts an IP as an argument. +Here's a quickstart guide to setting up the Nemesis platform using k3s and Helm on Debian 11. This guide will start a Nemesis server listening on the server "192.168.1.10". Change this IP to the host that you will be accessing Nemesis from. If this does not fit your installation need, see the full [setup guide](setup.md). This process is automated in the [debian_k3s_setup.sh](https://github.com/SpecterOps/Nemesis/blob/main/debian_k3s_setup.sh) script, which accepts an IP as an argument. ### Prerequisites From 0563ef6b4d27c66216030a14a1f0c98ab6d1a2b7 Mon Sep 17 00:00:00 2001 From: Will Date: Thu, 25 Apr 2024 17:00:18 -0700 Subject: [PATCH 6/6] Update new_connector.md --- docs/new_connector.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/new_connector.md b/docs/new_connector.md index 827db14..57f172a 100644 --- a/docs/new_connector.md +++ b/docs/new_connector.md @@ -64,7 +64,7 @@ r = requests.request("POST", f"{NEMESIS_URL}/data", auth=basic, data=data, heade ## Other Structured Data -For other types of structured data, only a single message needs to be posted to the `http:///api/data` API route, e.g. Step 2 in the downloading processing example. The `metadata["data_type"]` field should be one of the types defined in the [ODR](odr/references/). The appropriate ODR document will also define the fields and structure needed for the datatype. +For other types of structured data, only a single message needs to be posted to the `http:///api/data` API route, e.g. Step 2 in the downloading processing example. The `metadata["data_type"]` field should be one of the types defined in the [ODR](https://github.com/SpecterOps/Nemesis/tree/main/docs/odr). The appropriate ODR document will also define the fields and structure needed for the datatype. Note that the "data" section of the message is an array of dictionaries, i.e., multiple instances of a datatype can be posted in a single message. For example, multiple process messages can exist in the single post.