README

# Install Elastic Search in Ubuntu 20.04

sudo apt update
sudo apt install openjdk-8-jdk
java -version
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee –a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt update
sudo apt install elasticsearch
sudo cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
sudo nano /etc/elasticsearch/elasticsearch.yml
     network.host: localhost
     http.port: 9200
sudo systemctl start elasticsearch.service
sudo systemctl enable elasticsearch.service
curl -X GET "localhost:9200"

ps aux | grep java

sudo systemctl restart elasticsearch.service
curl -X GET "localhost:9200"

# GET INDEX 
curl -X GET "localhost:9200/_cat/indices?v"
# CREATE INDEX 
curl -X PUT "localhost:9200/NAME"

curl -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
"user" : "JAI",
"message" : "Hello to Elasticsearch"
}' "http://localhost:9200/jai/_doc/"

curl -X GET "localhost:9200/jai/"


# Check logs 
less /var/log/elasticsearch/FILENAME.log

# Check node configuration 
curl localhost:9200/_cat/nodes?v

# SETUP ESLAB CLUSTER 
sudo apt update
sudo apt install openjdk-8-jdk
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee –a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt update
sudo apt install elasticsearch
wget https://raw.githubusercontent.com/Jaibw/elk/main/eslab-data.yml
cat  eslab-data.yml
sed 's/NODENAME/USER0#/g' eslab-data.yml > elasticsearch.yml
cat elasticsearch.yml
sudo cp elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
sudo systemctl start elasticsearch.service
sudo systemctl enable elasticsearch.service
curl -X GET "localhost:9200"

# INSTALL KIBANA 
sudo apt update
sudo apt install openjdk-8-jdk
java -version
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee –a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt update
sudo apt-get install kibana
wget https://raw.githubusercontent.com/Jaibw/elk/main/kibana.yml
cat kibana.yml
sudo cp kibana.yml /etc/kibana/kibana.yml
sudo systemctl start kibana
sudo systemctl enable kibana

# DEV Tools 
GET /
GET _license
GET _cat/health
GET _cat/health?v
GET _cat/nodes
GET _cat/nodes?v
PUT easy_index_by_NAME                          # create an index 
GET _cat/indices?help
GET _cat/indices
GET _cat/indices?v
GET _cat/indices?format=json
GET _cat/indices?format=text
GET _cat/indices?format=yaml
GET _cat/indices?format=smile
GET _cat/indices?format=cbor
GET _cat/indices?format=json&pretty                
GET _cat/indices?v&h=health,index&format=text         # select a header 
GET _cat/indices?s=index:asc                          # sort 
PUT demo_settings_index_by_jai
{
  "settings": {
    "number_of_replicas": 2,
    "number_of_shards": 1
  }
}
GET _cat/indices?v
PUT subjects_by_jai
{
  "mappings": {
    "properties": {
      "name": {
        "type": "text"
      },
      "total_marks": {
        "type": "float"
      },
      "passing_marks": {
        "type": "float"
      },
      "description": {
        "type": "text"
      }
    }
  }
}
GET subjects_by_jai
GET _cat/indices?v
PUT purchase_jan_2022_by_jai
{
  "aliases": {
    "purchase_last_2022_by_jai": {}
  }
}
GET purchase_jan_2022_by_jai
GET purchase_last_2022_by_jai
GET _cat/aliases

# LOAD BULK DATA IN ELASTIC SEARCH 
cd ~
curl -X PUT "localhost:9200/logs_by_NAME"
sed 's/logs/logs_by_NAME/g' logs.json > logs_by_NAME.json
curl -H 'Content-type: application/x-ndjson' http://localhost:9200/logs_by_NAME/_bulk --data-binary @logs_by_NAME.json
curl -X GET "localhost:9200/logs_by_NAME/_refresh"
curl -X GET "localhost:9200/_cat/indices?v" | grep logs_by_NAME
curl -X GET "localhost:9200/logs_by_NAME/_search?pretty=true&q=*:*&size=1"

# CURD IN ELASTIC SEARCH 
PUT data_by_NAME
GET data_by_NAME
POST data_by_NAME/_doc/1
{
  "Name" : "NAME",
  "User" : "user--",
  "Password" : "pass123"
}
GET data_by_NAME/_doc/1
PUT data_by_NAME/_doc/1 
{
  "Name" : "NAME",
  "User" : "user--",
  "Password" : "pass123456"
}
GET data_by_NAME/_doc/1
DELETE data_by_NAME/_doc/1
DELETE data_by_NAME


# INSTALL NGINX AND FILEBEATS 
sudo apt update
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee –a /etc/apt/sources.list.d/elastic-7.x.list
sudo apt update
sudo apt install filebeat
sudo apt install nginx 
sudo systemctl start nginx 
sudo systemctl enable nginx
Open Chrome and access nginx on http://PUBLICIP
sudo filebeat modules enable nginx
sudo filebeat modules enable system
wget https://raw.githubusercontent.com/Jaibw/elk/main/filebeat-template.yml
sed 's/USERNAME/yourname/g' filebeat-template.yml > filebeat.yml
sudo cp /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.bak
sudo cp filebeat.yml /etc/filebeat/filebeat.yml
sudo filebeat modules list
sudo systemctl restart filebeat
sudo systemctl status filebeat

Open DevTools : http://54.176.154.120:5601/app/dev_tools#/console 
Try below requests - 
   GET /_cat/indices?v                                  # GET YOUR INDEX NAME 
   GET /INDEXNAME/_search?pretty=true&q=*:*


# METRICBEAT INSTALL
sudo apt install metricbeat
sudo metricbeat modules enable nginx
wget https://raw.githubusercontent.com/Jaibw/elk/main/metricbeat-template.yml
sed 's/USERNAME/yourname/g' metricbeat-template.yml > metricbeat.yml
sudo cp /etc/metricbeat/metricbeat.yml /etc/metricbeat/metricbeat.yml.bak
sudo cp metricbeat.yml /etc/metricbeat/metricbeat.yml
sudo systemctl enable metricbeat
sudo systemctl restart metricbeat
sudo systemctl status metricbeat

Open DevTools : http://54.176.154.120:5601/app/dev_tools#/console 
Try below requests - 
   GET /_cat/indices?v                                  # GET YOUR INDEX NAME WITH mb-yourname
   GET /INDEXNAME/_search?pretty=true&q=*:*


# INSTALL HEARTBEAT IN CENTOS 
sudo -s -H
yum -y install java-11-openjdk java-11-openjdk-devel
java -version
cat > /etc/yum.repos.d/elasticsearch.repo <<EOF
[elasticsearch-7.x]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

yum install heartbeat-elastic
wget https://raw.githubusercontent.com/Jaibw/elk/main/heartbeat-template.yml
wget https://raw.githubusercontent.com/Jaibw/elk/main/icmp.yml
sed 's/USERNAME/yourname/g' heartbeat-template.yml > heartbeat.yml
cp /etc/heartbeat/heartbeat.yml /etc/heartbeat/heartbeat.yml.bak
cp heartbeat.yml /etc/heartbeat/heartbeat.yml
cp icmp.yml /etc/heartbeat/monitors.d/icmp.yml
systemctl enable heartbeat-elastic
chkconfig --add heartbeat-elastic
systemctl start heartbeat-elastic
systemctl status heartbeat-elastic

# INSTALL PACKET BEAT 
sudo apt install packetbeat
wget https://raw.githubusercontent.com/Jaibw/elk/main/packetbeat-template.yml
sed 's/USERNAME/yourname/g' packetbeat-template.yml > packetbeat.yml
sudo cp /etc/packetbeat/packetbeat.yml /etc/packetbeat/packetbeat.yml.bak
sudo cp packetbeat.yml /etc/packetbeat/packetbeat.yml
sudo systemctl enable packetbeat
sudo systemctl restart packetbeat
sudo systemctl status packetbeat

# INSTALL LOGSTASH WITH FILEBEATS 
sudo apt install logstash
wget https://raw.githubusercontent.com/Jaibw/elk/main/easy-logstash-template.yml
sed 's/USERNAME/yourname/g' easy-logstash-template.yml > logstash.yml
sudo cp /etc/logstash/logstash.yml /etc/logstash/logstash.yml.bak
sudo cp logstash.yml /etc/logstash/logstash.yml
wget https://raw.githubusercontent.com/Jaibw/elk/main/logstash-filebeat-template.yml
sudo cp logstash-filebeat-template.yml /etc/filebeat/filebeat.yml
sudo systemctl restart filebeat
sudo systemctl start logstash


GET _cat/nodes?v
GET _cat/indices?v

GET mb-jaik-7.17.0-2022.02.24

GET mb-jaik-7.17.0-2022.02.24/_search

GET mb-jaik-7.17.0-2022.02.24/_search
{
  "query": {
    "term": {
      "agent.hostname": {
        "value": "ip-172-31-15-192"
      }
    }
  }
}

GET mb-*/_search
{
  "query": {
    "term": {
      "host.ip": {
        "value": "172.31.13.188"
      }
    }
  }
}

GET mb-*/_search
{
  "query": {
    "term": {
      "host.name": {
        "value": "172.31.13.188"
      }
    }
  },
  "_source": ["system", "cloud", "service"]
}




GET mb-*/_search
{
  "size": 5000,
  "from": 5000
  , 
  "query": {
    "terms": {
      "agent.hostname" : [
        "ip-172-31-19-40",
        "ip-172-31-1-61"
        ]
    }
  }, 
  "_source": ["agent"]
}


GET mb-adhitya-7.17.0-2022.02.24/_search
GET mb-jaik-7.17.0-2022.02.23/_search


GET mb-adhitya-7.17.0-2022.02.24,mb-jaik-7.17.0-2022.02.23/_search
{
  "size": 10000,
  "_source": ["index"]
}

## check with Distict 
GET mb-*/_search
{
  "from": 8000,
  "size": 100,
  "query": {
    "term": {
      "host.ip": {
        "value": "172.31.0.0/16"
      }
    }
  },
  "_source": ["host.ip"]
}



GET mb-adhitya-7.17.0-2022.02.24,mb-jaik-7.17.0-2022.02.23/_search
{
   "_source": ["system.cpu.cores", "cloud.machine.type", "service.type"]
}




GET _analyze 
{
  "analyzer": "english",
  "text": "this is a simple demo for elastic search"
}

GET mb-jaik-7.17.0-2022.02.23

GET jaik-7.17.0-2022.02.23/_search

GET */_search
{
  "size": 100, 
 "_source": ["message"]
}

GET */_search
{
 "query": {
   "match": {
      "message": "Win32_PnPEntity"
   }
 },
 "_source": ["message"]
}


GET */_search
{
 "query": {
   "match": {
      "message": "administrator"
   }
 },
 "_source": ["message"]
}

GET */_search
{
 "query": {
   "match": {
      "message": "Administrator"
   }
 },
 "_source": ["message"]
}

GET */_search
{
 "query": {
   "match": {
      "message": "Admin"
   }
 },
 "_source": ["message"]
}

GET */_search
{
  "size": 100, 
 "query": {
   "match": {
      "message": "Network GUID"
   }
 },
 "_source": ["message"]
}


GET */_search
{
  "size": 100, 
 "query": {
   "match_phrase": {
      "message": "Password Last Set"
   }
 },
 "_source": ["message"]
}


GET */_search
{
  "size": 100, 
 "query": {
   "match_phrase_prefix": {
      "message": "SAM"
   }
 },
 "_source": ["message"]
}


GET */_search
{
  "size": 100, 
 "query": {
   "match_phrase_prefix": {
      "message": "ADMIN"
   }
 },
 "_source": ["message"]
}


GET .ds-winlog-8.0.0-2022.02.10-2022.02.23-000001/_search
{
 "size": 10, 
 "query": {
   "match_phrase_prefix": {
      "message": "Password Policy"
   }
 },
 "_source": ["message"]
}

POST _sql?format=txt
{
  "query": """
  SELECT * FROM ".tasks"
  """
}

POST _sql?format=txt
{
  "query": """
  SELECT ecs.version, host.hostname, host.os.version, system.load.cores  FROM "mb-jaik-7.17.0-2022.02.24" limit 10
 
  """
}

POST _sql?format=txt
{
  "query": """
  SELECT ecs.version, host.hostname, host.os.version, system.load.cores  FROM "mb-jaik-7.17.0-2022.02.24" where ecs.version = '1.12.0' limit 1
  """
}


GET pb-jai-7.17.0-2022.02.24/_search
GET pb-jai-7.17.0-2022.02.24/_search
{
  "aggs": {
    "max_bytes": {
      "max": { "field": "source.bytes" }
    }
  }, 
  "_source": ["source.bytes", "source.ip"]
}

GET pb-jai-7.17.0-2022.02.24/_search
{
  "size": 0, 
  "aggs": {
    "min_bytes": {
      "min": { "field": "source.bytes" }
    }
  }, 
  "_source": ["source.bytes", "source.ip"]
}

GET pb-jai-7.17.0-2022.02.24/_search
{
  "size": 0, 
  "aggs": {
    "total_bytes": {
      "sum": { "field": "source.bytes" }
    }
  }, 
  "_source": ["source.bytes", "source.ip"]
}

GET pb-jai-7.17.0-2022.02.24/_search
{
  "size": 0, 
  "aggs": {
    "avg_bytes": {
      "avg": { "field": "source.bytes" }
    }
  }, 
  "_source": ["source.bytes", "source.ip"]
}

GET pb-*/_search
{
  "size": 0, 
  "aggs": {
    "avg_bytes": {
      "avg": { "field": "source.bytes" }
    }
  }, 
  "_source": ["source.bytes", "source.ip"]
}

GET jaik-7.17.0-2022.02.23/_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "day"
      }
    }
  }
}

GET jaik-7.17.0-2022.02.23/_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "hour"
      }
    }
  }
}

GET jaik-7.17.0-2022.02.23/_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "month"
      }
    }
  }
}

GET */_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "day"
      }
    }
  }
}

GET */_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "hour"
      }
    }
  }
}

GET */_search
{
  "size": 0, 
  "aggs": {
    "events_per_hour": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "month"
      }
    }
  }
}


GET mb-*/_search
{
  "size": 1,
  "aggs": {
    "distinct_ips_count": {
      "cardinality": {
        "field": "host.ip"
      }
    }
  }, 
  "query": {
    "term": {
      "host.ip": {
        "value": "172.31.0.0/16"
      }
    }
  },
  "_source": ["host.ip"]
}


GET mb-*/_search
{
  "size": 1,
  "aggs": {
    "distinct_ips": {
      "terms": {
        "field": "host.ip"
      }
    }
  }, 
  "query": {
    "term": {
      "host.ip": {
        "value": "172.31.0.0/16"
      }
    }
  },
  "_source": ["host.ip"]
}

GET mb-*/_search
{
  "size": 0,
  "aggs": {
    "distinct_ips": {
      "terms": {
        "field": "agent.hostname"
      }
    }
  }, 
  "query": {
    "terms": {
      "agent.hostname" : [
        "ip-172-31-19-40",
        "ip-172-31-1-61"
        ]
    }
  }, 
  "_source": ["agent"]
}


# download Logstash binay and test your first config
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.17.0-linux-x86_64.tar.gz
tar xvfz logstash-7.17.0-linux-x86_64.tar.gz
cd logstash-7.17.0/
wget https://raw.githubusercontent.com/Jaibw/elk/main/logstash/hello-world.conf
bin/logstash -f hello-world.conf        
hello       # wait for sometime - 2 m 

Ctrl + C    # to exit from interactive session 


# Logstash  Plugins Ref: 
https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html
https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html
https://www.elastic.co/guide/en/logstash/6.8/plugins-inputs-imap.html
https://www.elastic.co/guide/en/logstash/current/plugins-filters-csv.html
https://www.elastic.co/guide/en/logstash/current/plugins-outputs-kafka.html

# CSV UPLOAD TO ES - Logstash
cd ~
cd logstash-7.17.0/
bin/logstash-plugin list
wget https://raw.githubusercontent.com/Jaibw/elk/main/cm23FEB2022bhav.csv
wget https://raw.githubusercontent.com/Jaibw/elk/main/logstash/csv-upload.conf
sed 's/USERNAME/name/g' csv-upload.conf > csvupload.conf
cat csvupload.conf
bin/logstash -f csvupload.conf 


# CUSTOM LOGS WITH LOGSTASH 
cd ~
cd logstash-7.17.0/
wget https://raw.githubusercontent.com/Jaibw/elk/main/logstash/samplelogs.txt
cat samplelogs.txt
wget https://raw.githubusercontent.com/Jaibw/elk/main/logstash/custom-logs.conf
sed 's/USERNAME/name/g' custom-logs.conf > customlogs.conf
cat customlogs.conf
bin/logstash -f customlogs.conf