diff --git a/.gitignore b/.gitignore index 5ee6c08..890b676 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ tests/snapshots/output tests/snapshots/tmp +# Prometheus node_exporter +node_exporter + # Editors .vscode/ .idea/ diff --git a/Dockerfile b/Dockerfile index b098a39..01e21c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,6 @@ -FROM ubuntu:22.04 +FROM ipal-ids-base:v1 -RUN apt-get update \ - && apt-get -y install software-properties-common sudo g++ \ - && apt-get -y install pip vim -RUN apt-get -y install libgsl-dev git - -# create a user and add them to sudoers -RUN useradd -ms /bin/bash ipal && echo "ipal ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ipal -USER ipal - -WORKDIR /home/ipal/ipal_ids_framework/ -COPY --chown=ipal . . - -# Install IIDS framework -WORKDIR /home/ipal/ipal_ids_framework/ -RUN sudo pip install numpy # for ar to install correctly +COPY . . RUN sudo pip install . RUN sudo pip install -r requirements-dev.txt diff --git a/Dockerfile-base b/Dockerfile-base new file mode 100644 index 0000000..fc7312b --- /dev/null +++ b/Dockerfile-base @@ -0,0 +1,23 @@ +FROM ubuntu:22.04 + +RUN apt-get update +RUN apt-get -y install linux-headers-$(uname -r) +RUN apt-get -y install software-properties-common sudo g++ +RUN apt-get -y install pip vim +RUN apt-get -y install libgsl-dev git +RUN apt-get -y install iproute2 iputils-ping ncat + +# create a user and add them to sudoers +# RUN useradd -ms /bin/bash ipal && echo "ipal ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ipal +# USER ipal + +WORKDIR /home/ipal/ipal_ids_framework/ +# COPY --chown=ipal . . +COPY requirements.txt . + +# Install IIDS framework +WORKDIR /home/ipal/ipal_ids_framework/ +RUN sudo pip install numpy # for ar to install correctly +RUN sudo pip install -r requirements.txt + +CMD /bin/bash diff --git a/ids/interarrivaltime/Mean.py b/ids/interarrivaltime/Mean.py index adf5199..5c2512f 100644 --- a/ids/interarrivaltime/Mean.py +++ b/ids/interarrivaltime/Mean.py @@ -45,7 +45,10 @@ def train(self, ipal=None, state=None): # Load timestamps for each identifier with self._open_file(ipal) as f: for line in f.readlines(): - ipal_msg = json.loads(line) + try: + ipal_msg = json.loads(line) + except: + continue # dump broken data - e.g. broken pipe recovery timestamp = ipal_msg["timestamp"] identifier = self._get_identifier(ipal_msg) diff --git a/ipal_iids/iids.py b/ipal_iids/iids.py index af13d90..e8bdc7d 100755 --- a/ipal_iids/iids.py +++ b/ipal_iids/iids.py @@ -13,6 +13,12 @@ from ids.utils import get_all_iidss +# Prometheus Client +from ipal_iids.tools.prometheus import PrometheusClient +from prometheus_client import Enum, Summary, Counter +IDS_STATE = Enum('ids_state', 'Current IDS state', states=['starting','training','operating']) +PACKET_TIME = Summary('ids_packet_inspection_seconds', 'Time spent inspecting packet', ['method','ids']) +IDS_ALERT = Counter('ids_alert', 'Counts the alert that occur while inspecting', ["ids"]) # Wrapper for hiding .gz files def open_file(filename, mode): @@ -80,6 +86,12 @@ def prepare_arg_parser(parser): help="input file of IPAL state messages to train the IDS on ('-' stdin, '*.gz' compressed).", required=False, ) + parser.add_argument( + "--train.time", + dest="train_time", + metavar="INT", + help="[UNUSED] initial training time for the IDS - only effective when train.ipal==live.ipal or train.state==live.ipal. When both are a simple file, IDS will re-inspect trained data" + ) parser.add_argument( "--live.ipal", dest="live_ipal", @@ -192,7 +204,7 @@ def load_settings(args): # noqa: C901 settings.logger.error("no IDS configuration provided, exiting") exit(1) - # Parse training input + # Parse training input (TODO: its probably better if these also were file handles like live input for consistency. file handles can also be reset (e.g. fd.seek(0,0))) if args.train_ipal: settings.train_ipal = args.train_ipal if args.train_state: @@ -277,7 +289,7 @@ def train_idss(idss): continue settings.logger.error( - "Required arguement: {} for IDS {}".format(ids._requires, ids._name) + "Required argument: {} for IDS {}".format(ids._requires, ids._name) ) exit(1) @@ -320,13 +332,19 @@ def live_idss(idss): if ipal_msg is None and settings.live_ipal: line = settings.live_ipalfd.readline() if line: - ipal_msg = json.loads(line) + try: + ipal_msg = json.loads(line) + except: + continue # dump broken data - e.g. broken pipe recovery # load a new state if state_msg is None and settings.live_state: line = settings.live_statefd.readline() if line: - state_msg = json.loads(line) + try: + state_msg = json.loads(line) + except: + continue # dump broken data - e.g. broken pipe recovery # Determine smallest timestamp ipal or state? if ipal_msg and state_msg: @@ -339,51 +357,60 @@ def live_idss(idss): break # Process next message - if is_ipal_smaller: - ipal_msg["metrics"] = {} - ipal_msg["ids"] = False - - for ids in idss: - if ids.requires("live.ipal"): - alert, metric = ids.new_ipal_msg(ipal_msg) - ipal_msg["ids"] = ( - ipal_msg["ids"] or alert - ) # combine alerts with or (TODO config ?) - ipal_msg["metrics"][ids._name] = metric - - if settings.output: - - if _first_ipal_msg: - ipal_msg["_iids-config"] = settings.iids_settings_to_dict() - _first_ipal_msg = False - - settings.outputfd.write(json.dumps(ipal_msg) + "\n") - settings.outputfd.flush() - ipal_msg = None - else: - state_msg["metrics"] = {} - state_msg["ids"] = False - - for ids in idss: - if ids.requires("live.state"): - alert, metric = ids.new_state_msg(state_msg) - state_msg["ids"] = ( - state_msg["ids"] or alert - ) # combine alerts with or (TODO config ?) - state_msg["metrics"][ids._name] = metric - - if settings.output: - - if _first_state_msg: - state_msg["_iids-config"] = settings.iids_settings_to_dict() - _first_state_msg = False - - settings.outputfd.write(json.dumps(state_msg) + "\n") - settings.outputfd.flush() - state_msg = None + with PACKET_TIME.labels(method="combined", ids="all").time(): + if is_ipal_smaller: + ipal_msg["metrics"] = {} + ipal_msg["ids"] = False + + for ids in idss: + with PACKET_TIME.labels(method="ipal", ids=ids._name).time(): + if ids.requires("live.ipal"): + alert, metric = ids.new_ipal_msg(ipal_msg) + ipal_msg["ids"] = ( + ipal_msg["ids"] or alert + ) # combine alerts with or (TODO config ?) + ipal_msg["metrics"][ids._name] = metric + if alert: + IDS_ALERT.labels(ids=ids._name).inc() + + if settings.output: + + if _first_ipal_msg: + ipal_msg["_iids-config"] = settings.iids_settings_to_dict() + _first_ipal_msg = False + + settings.outputfd.write(json.dumps(ipal_msg) + "\n") + settings.outputfd.flush() + ipal_msg = None + else: + state_msg["metrics"] = {} + state_msg["ids"] = False + + for ids in idss: + with PACKET_TIME.labels(method="state", ids=ids._name).time(): + if ids.requires("live.state"): + alert, metric = ids.new_state_msg(state_msg) + state_msg["ids"] = ( + state_msg["ids"] or alert + ) # combine alerts with or (TODO config ?) + state_msg["metrics"][ids._name] = metric + if alert: + IDS_ALERT.labels(ids=ids._name).inc() + + if settings.output: + + if _first_state_msg: + state_msg["_iids-config"] = settings.iids_settings_to_dict() + _first_state_msg = False + + settings.outputfd.write(json.dumps(state_msg) + "\n") + settings.outputfd.flush() + state_msg = None def main(): + _prom = PrometheusClient() + IDS_STATE.state('starting') # Argument parser and settings parser = argparse.ArgumentParser() prepare_arg_parser(parser) @@ -395,10 +422,12 @@ def main(): try: # Train IDSs settings.logger.info("Start IDS training...") + IDS_STATE.state('training') train_idss(idss) # Live IDS settings.logger.info("Start IDS live...") + IDS_STATE.state('operating') live_idss(idss) except BrokenPipeError: devnull = os.open(os.devnull, os.O_WRONLY) diff --git a/ipal_iids/tools/prometheus.py b/ipal_iids/tools/prometheus.py new file mode 100644 index 0000000..7497f36 --- /dev/null +++ b/ipal_iids/tools/prometheus.py @@ -0,0 +1,6 @@ +from prometheus_client import start_http_server + +class PrometheusClient(): + + def __init__(self, port=9103) -> None: + start_http_server(port) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ddff104..baac112 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ torch git+https://github.com/RhysU/ar.git matplotlib pomegranate +prometheus-client \ No newline at end of file diff --git a/setup.py b/setup.py index 1a228ab..b2382e0 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ "ar @ git+https://github.com/RhysU/ar.git", "matplotlib", "pomegranate", + "prometheus-client" ], tests_require=["pytest"], url="https://github.com/fkie-cad/ipal_ids_framework", diff --git a/start-ids.sh b/start-ids.sh new file mode 100755 index 0000000..904077d --- /dev/null +++ b/start-ids.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -x +set -E +SERVERPORT_IPAL="$1" ; shift +SERVERPORT_STATE="$1" ; shift +TRAINFILE_NAME="$1" ; shift + +### Server +# mkfifo opens a stream, that will not close - IIDS will continue reading input +mkfifo IPAL_INPUT +mkfifo STATE_INPUT + +# start ncat server +echo "Starting server on port $SERVERPORT_IPAL and $SERVERPORT_STATE for $(hostname -i)" + +ncat -lk --append -p $SERVERPORT_IPAL | tee $TRAINFILE_NAME.ipal IPAL_INPUT & +ncat -lk --append -p $SERVERPORT_STATE | tee $TRAINFILE_NAME.state STATE_INPUT & + +### Node exporter +./node_exporter --web.listen-address=":9102" & + + +### IDS +./ipal-iids $@ & + + +wait -n +exit $?