Django wrapper

.gitignore

@@ -1,6 +1,6 @@
-*secret*
+**secret**
 .venv
 .env
-db.sqlite3
-sqlite3.db
 migrations/
+.db
+.sqlite3

commands.txt

@@ -1,2 +1,18 @@
-# Start ssh-agent -> add ssh-key -> ssh into iou2 with localtunnel
-eval `ssh-agent`; ssh-add /c/Users/twide/.ssh/uninett_rsa; ssh -L 8899:127.0.0.1:8899 -J [email protected] [email protected] -A
+# 
+
+# Start ssh-agent -> add ssh-key -> ssh into iou2 with localtunnel (jupyter lab)
+eval `ssh-agent`; ssh-add ~/.ssh/uninett_rsa; ssh -L 8899:127.0.0.1:8899 -J [email protected] [email protected] -A
+
+# Start ssh-agent -> add ssh-key -> ssh into iou2 with localtunnel (jupyter notebook)
+eval `ssh-agent`; ssh-add ~/.ssh/uninett_rsa; ssh -L 9988:127.0.0.1:9988 -J [email protected] [email protected] -A
+
+# ncdu (disk usage)
+# -x: do not cross filesystem bounderies
+# -r: read only / prevent delete options
+ncdu -xr
+
+# du
+# -h: human readable M, G
+# -s: summarize
+# -d, --max-depth=N
+du -hs -d 0

django_wrapper/apps/microdep/management/commands/conn.py

@@ -0,0 +1,57 @@
+# imports
+import gzip
+import glob
+import os
+import pathlib
+
+from django.db import connections
+from django.utils import timezone
+from django.core import management
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+
+
+from apps.microdep import models as microdep_models
+# End: imports -----------------------------------------------------------------
+
+# Settings:
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        print(f"\n== COMMAND: {__file__} ==")
+        # https://stackoverflow.com/questions/29840382/single-django-app-to-use-multiple-sqlite3-files-for-database
+        # https://stackoverflow.com/questions/56733112/how-to-create-new-database-connection-in-django
+        # https://stackoverflow.com/questions/14254315/django-dynamic-database-file
+        # print(connections.databases)
+
+        """
+        Create a database for each host in /dynga/data/
+        Analyze crude streams and create Gap in db
+        """
+
+        crude_pattern = '**/*crude*.gz'
+
+        data_root = pathlib.Path(os.environ['DATA_ROOT']).resolve()
+
+        # finds all crude files
+        for path in data_root.rglob(crude_pattern): # /**/<data_root>/<domain>/<host>/<yyyy.mm.dd>/*crude*.gz
+            path_tokens = path.as_posix().split('/')
+            domain = path_tokens[-4]
+            host = path_tokens[-3]
+            db_key = f"{domain}_{host}"
+
+            # get or create db
+            # if not connections.databases.get(db_key, None):
+            #     print(db_key)
+            #     connections.databases[db_key] = dict(connections.databases['default'])
+            #     connections.databases[db_key]['NAME'] = path.parent.parent / f"{db_key}.db"
+                # print(connections.databases[db_key])
+
+
+            # analyze file
+            # add gaps to db in data_root/domain/host
+
+

django_wrapper/apps/microdep/management/commands/crude_parser.py

@@ -0,0 +1,73 @@
+# imports
+import gzip
+
+from django.utils import timezone
+from django.core import management
+from django.core.management.base import BaseCommand
+
+from microdep import models as microdep_models
+from microdep import utils as microdep_utils
+# End: imports -----------------------------------------------------------------
+
+# Settings:
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        # Positional arguments
+        parser.add_argument('path', nargs='+', type=int)
+
+        # Named (optional) arguments
+        parser.add_argument(
+            '--delete',
+            action='store_true',
+            help='Delete poll instead of closing it',
+        )
+
+    def handle(self, *args, **options):
+        print(f"\n== COMMAND: {__file__} ==")
+
+
+        # crude = 'C:/Users/twide/my_projects/git/ml-microdep/secret/fredrikstad-mp.hiof.no/2021-02-07/crude-00_00_01.gz'
+        # traceroute = 'C:/Users/twide/my_projects/git/ml-microdep/secret/fredrikstad-mp.hiof.no/2021-02-07/traceroute_128.39.19.150.gz'
+        # tcptraceroute = 'C:/Users/twide/my_projects/git/ml-microdep/secret/fredrikstad-mp.hiof.no/2021-02-07/tcptraceroute_128.39.19.150.gz'
+        # vmstat = 'C:/Users/twide/my_projects/git/ml-microdep/secret/fredrikstad-mp.hiof.no/2021-02-07/vmstat.gz'
+        tekno_crude = 'C:/Users/twide/my_projects/git/ml-microdep/secret/teknobyen-mp.uninett.no/2021-02-07/crude-00_00_02.gz'
+        # traceroute2 = 'C:/Users/twide/my_projects/git/ml-microdep/secret/teknobyen-mp.uninett.no/2021-02-07/traceroute_128.39.19.150.gz'
+        # tcptraceroute2 = 'C:/Users/twide/my_projects/git/ml-microdep/secret/teknobyen-mp.uninett.no/2021-02-07/tcptraceroute_128.39.19.150.gz'
+        # vmstat2 = 'C:/Users/twide/my_projects/git/ml-microdep/secret/teknobyen-mp.uninett.no/2021-02-07/vmstat.gz'
+
+
+        ca = microdep_utils.CrudeAnalyzer(window_size=1000)
+
+        with gzip.open(filename=tekno_crude, mode='rt') as file:
+
+            for i, line in enumerate(file):
+                key_values = line.split() # list of key-value pairs # ['ID=1', 'RX=15434', ...]
+
+                if i % 1000000 == 0:
+                    print(f"{i}: {line}")
+
+                try:
+                    record = microdep_models.CrudeRecord.objects.create(
+                        stream_id = key_values[0].split('=')[1], # ID=1
+                        seq = key_values[1].split('=')[1],
+                        src = key_values[2].split('=')[1].split(':')[0], # SRC=ip:port
+                        dst = key_values[3].split('=')[1].split(':')[0], # DST=ip:port
+                        tx = key_values[4].split('=')[1],
+                        rx = key_values[5].split('=')[1],
+                        size = key_values[6].split('=')[1],
+                        hoplimit = key_values[7].split('=')[1],
+                    )
+                    ca.add_record(record)
+                    # print(record.__dict__)
+
+                except Exception as e:
+                    print(e)
+                    print(i)
+                    print(line)
+
+        print(ca)
+        # print(ca.__dict__)
+        # End of handle