Skip to content

Commit

Permalink
Merge pull request #109 from rocknsm/dcode/pull-102/fsf
Browse files Browse the repository at this point in the history
Updates towards merging in FSF
LGTM
  • Loading branch information
jeffgeiger authored Feb 20, 2017
2 parents 8e989f4 + 0e892eb commit 3b1093e
Show file tree
Hide file tree
Showing 12 changed files with 304 additions and 66 deletions.
7 changes: 7 additions & 0 deletions ansible/files/logrotate-fsf.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/data/fsf/*.log
{
rotate 3
missingok
compress
create 0644 fsf fsf
}
27 changes: 27 additions & 0 deletions ansible/files/logstash-fsf-es.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
input {
file {
codec => "json"
path => "/data/fsf/scan.log"
add_field => { "[@metadata][stage]" => "fsf" }
}
}

filter {
if "_jsonparsefailure" in [tags] {
drop { }
}
if [@metadata][stage] == "fsf" {
mutate { remove_field => ["path"] }
}
}

output {
if [@metadata][stage] == "fsf" {
#stdout { codec => rubydebug }
elasticsearch {
hosts => ["127.0.0.1"]
index => "fsf-%{+YYYY.MM.dd}"
document_type => "fsf"
}
}
}
106 changes: 67 additions & 39 deletions ansible/files/rock_start
Original file line number Diff line number Diff line change
@@ -1,43 +1,71 @@
#!/bin/bash

echo "Starting Zookeeper..."
systemctl start zookeeper
sleep 5
systemctl status zookeeper | egrep "^\s*Active"

echo "Starting Elasticsearch..."
systemctl start elasticsearch
sleep 5
systemctl status elasticsearch | egrep "^\s*Active"

echo "Starting Kafka..."
systemctl start kafka
sleep 5
systemctl status kafka | egrep "^\s*Active"

echo "Starting Logstash..."
systemctl start logstash
sleep 5
systemctl status logstash | egrep "^\s*Active"

echo "Starting Kibana..."
systemctl start kibana
sleep 5
systemctl status kibana | egrep "^\s*Active"

echo "Starting Suricata..."
systemctl start suricata
sleep 5
systemctl status suricata | egrep "^\s*Active"

echo "Starting Bro..."
systemctl start broctl
sleep 5
systemctl status broctl | egrep "^\s*Active"

#echo "Starting Stenographer..."
#systemctl start stenographer
#sleep 5
#systemctl status stenographer | egrep "^\s*Active"
function feature_enabled() {
grep -qiE "^enable_$1: (true|yes)" /etc/rocknsm/config.yml;
return $?
}

if feature_enabled zookeeper; then
echo "Starting Zookeeper..."
systemctl start zookeeper
sleep 5
systemctl status zookeeper | egrep "^\s*Active"
fi

if feature_enabled elasticsearch; then
echo "Starting Elasticsearch..."
systemctl start elasticsearch
sleep 5
systemctl status elasticsearch | egrep "^\s*Active"
fi

if feature_enabled kafka; then
echo "Starting Kafka..."
systemctl start kafka
sleep 5
systemctl status kafka | egrep "^\s*Active"
fi

if feature_enabled logstash; then
echo "Starting Logstash..."
systemctl start logstash
sleep 5
systemctl status logstash | egrep "^\s*Active"
fi

if feature_enabled kibana; then
echo "Starting Kibana..."
systemctl start kibana
sleep 5
systemctl status kibana | egrep "^\s*Active"
fi

if feature_enabled suricata; then
echo "Starting Suricata..."
systemctl start suricata
sleep 5
systemctl status suricata | egrep "^\s*Active"
fi

if feature_enabled bro; then
echo "Starting Bro..."
systemctl start broctl
sleep 5
systemctl status broctl | egrep "^\s*Active"
fi

if feature_enabled stenographer; then
echo "Starting Stenographer..."
systemctl start stenographer
sleep 5
systemctl status stenographer | egrep "^\s*Active"
fi

if feature_enabled fsf; then
echo "Starting FSF..."
systemctl start fsf
sleep 5
systemctl status fsf | egrep "^\s*Active"
fi

exit 0
7 changes: 7 additions & 0 deletions ansible/files/rock_status
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ export MON_IFS=$(cat /opt/bro/etc/node.cfg | grep interface | \
systemctl status suricata
}

#----------------------------------------------------------------------------
## FSF
##----------------------------------------------------------------------------
@test "Check that FSF is running" {
systemctl status fsf
}

#----------------------------------------------------------------------------
# ZOOKEEPER
#----------------------------------------------------------------------------
Expand Down
76 changes: 54 additions & 22 deletions ansible/files/rock_stop
Original file line number Diff line number Diff line change
@@ -1,27 +1,59 @@
#!/bin/bash

#echo "Stopping Stenographer..."
#systemctl stop stenographer

echo "Stopping Suricata..."
systemctl stop suricata

echo "Stopping Bro..."
systemctl stop broctl

echo "Stopping Logstash..."
systemctl stop logstash

echo "Stopping Kibana..."
systemctl stop kibana

echo "Stopping Elasticsearch..."
systemctl stop elasticsearch

echo "Stopping Kafka..."
systemctl stop kafka

echo "Stopping Zookeeper..."
systemctl stop zookeeper
function feature_enabled() {
grep -qiE "^enable_$1: (true|yes)" /etc/rocknsm/config.yml;
return $?
}

if feature_enabled fsf; then
echo "Stopping FSF..."
systemctl stop fsf
fi

if feature_enabled suricata; then
echo "Stopping Stenographer..."
systemctl stop stenographer
fi

if feature_enabled suricata; then
echo "Stopping Suricata..."
systemctl stop suricata
fi

if feature_enabled snort; then
echo "Stopping Snort..."
systemctl stop snortd
fi

if feature_enabled bro; then
echo "Stopping Bro..."
systemctl stop broctl
fi

if feature_enabled logstash; then
echo "Stopping Logstash..."
systemctl stop logstash
fi

if feature_enabled kibana; then
echo "Stopping Kibana..."
systemctl stop kibana
fi

if feature_enabled elasticsearch; then
echo "Stopping Elasticsearch..."
systemctl stop elasticsearch
fi

if feature_enabled kafka; then
echo "Stopping Kafka..."
systemctl stop kafka
fi

if feature_enabled zookeeper; then
echo "Stopping Zookeeper..."
systemctl stop zookeeper
fi

exit 0
20 changes: 20 additions & 0 deletions ansible/files/rocknsm_config.dist.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,21 @@ with_kibana: true
with_zookeeper: true
with_kafka: true
with_nginx: true
with_fsf: true

# Feature options - Don't flip these unless you know what you're doing
enable_stenographer: false
enable_bro: true
enable_suricata: true
enable_snort: false
enable_pulledpork: true
enable_logstash: true
enable_elasticsearch: true
enable_kibana: true
enable_zookeeper: true
enable_kafka: true
enable_nginx: true
enable_fsf: false

rocknsm_package_list:
- java-1.8.0-openjdk-headless
Expand Down Expand Up @@ -84,6 +99,11 @@ suricata_group: suricata
suricata_data_dir: "{{ rock_data_dir }}/suricata"
pulled_pork_repo: https://github.com/shirkdog/pulledpork.git
pulled_pork_oinkcode: 796f26a2188c4c953ced38ff3ec899d8ae543350
fsf_user: fsf
fsf_group: fsf
fsf_data_dir: "{{ rock_data_dir }}/fsf"
fsf_archive_dir: "{{ fsf_data_dir }}/archive"
fsf_client_logfile: "{{ fsf_data_dir }}/client.log"
kafka_user: kafka
kafka_group: kafka
kafka_data_dir: "{{ rock_data_dir }}/kafka"
Expand Down
75 changes: 74 additions & 1 deletion ansible/rock.yml
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
- { pkg: zookeeper, test: "{{with_zookeeper}}", state: installed }
- { pkg: kafka, test: "{{with_kafka}}", state: installed }
- { pkg: kafkacat, test: "{{with_kafka}}", state: installed }
- { pkg: fsf, test: "{{with_fsf}}", state: installed }
- { pkg: chrony, state: installed }
- { pkg: firewalld, state: installed }
- { pkg: postfix, state: absent }
Expand Down Expand Up @@ -381,7 +382,6 @@
git:
repo: "{{ bro_rockscripts_repo }}"
dest: /opt/bro/share/bro/site/scripts/rock
depth: 1
version: "{{ bro_rockscripts_branch }}"
when: rock_online_install

Expand All @@ -403,6 +403,17 @@
force: yes
when: not rock_online_install

- name: Update owner for ROCK NSM Bro scripts
file:
path: /opt/bro/share/bro/site/scripts/rock
owner: "{{ bro_user }}"
group: "{{ bro_group }}"
state: directory
recurse: yes
follow: yes
tags:
- bro_scripts

- name: Add ROCK scripts to local.bro
lineinfile:
dest: /opt/bro/share/bro/site/local.bro
Expand Down Expand Up @@ -665,6 +676,59 @@
{{ "/usr/bin/systemctl restart snortd;" if with_snort else None }}
when: with_pulledpork

#######################################################
######################## FSF ##########################
#######################################################

- name: Create FSF data dir
file:
path: "{{ fsf_data_dir }}"
mode: 0755
owner: "{{ fsf_user }}"
group: "{{ fsf_group }}"
state: directory
when: with_fsf

- name: Create FSF archive dir
file:
path: "{{ fsf_archive_dir }}"
mode: 0755
owner: "{{ fsf_user }}"
group: "{{ fsf_group }}"
state: directory
when: with_fsf

- name: Configure logrotate for FSF logs
copy:
src: files/logrotate-fsf.conf
dest: /etc/logrotate.d/fsf.conf
mode: 0644
owner: root
group: root
when: with_fsf

- name: Configure fsf-server
template:
src: templates/fsf-server-config.j2
dest: /opt/fsf/fsf-server/conf/config.py
owner: "{{ fsf_user }}"
group: "{{ fsf_group }}"
mode: 0644
when: with_fsf

- name: Configure fsf-client
template:
src: templates/fsf-client-config.j2
dest: /opt/fsf/fsf-client/conf/config.py
owner: "{{ fsf_user }}"
group: "{{ fsf_group }}"
mode: 0644
when: with_fsf

- name: Enable and start FSF
service: name=fsf state=started enabled=yes
when: with_fsf and enable_fsf

######################################################
################### Setup Zookeeper ##################
######################################################
Expand Down Expand Up @@ -831,6 +895,15 @@
group: root
when: with_logstash and with_suricata

- name: Configure Logstash to pickup the FSF scan.log
copy:
src: "files/logstash-fsf-es.conf"
dest: "/etc/logstash/conf.d/fsf-es.conf"
mode: 0644
owner: root
group: root
when: with_logstash and with_fsf

- name: Enable and start Logstash
service: name=logstash state=started enabled=yes
when: with_logstash
Expand Down
Loading

0 comments on commit 3b1093e

Please sign in to comment.