Portscan (#133)

* Add portscan integration

* Remove internal deps from unit testing

* Fix ioloop issues

* Update default configuration

* Update py_clib and add logging configuration

.ci/security.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PYLINT_FILES=$(find . -name "*.py" -not -path "./tests/*" -not -path "./venv/*")
+PYLINT_FILES=$(find . -name "*.py" -not -path "./tests/*" -not -path "./venv/*" -not -path "./internal_deps*")
 
 bandit -ll ${PYLINT_FILES} | tee bandit.txt
 exit ${PIPESTATUS[0]}

.ci/unit_tests.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-FILES=$(find . -maxdepth 1 -type d)
-nosetests --with-xunit --with-coverage --cover-erase --cover-xml --cover-package=. ${FILES} || exit 1
+FILES=$(find . -maxdepth 1 -type d -not -path "./internal_deps*")
+nosetests --with-xunit --with-coverage --cover-erase --cover-xml --cover-package=. ${FILES} || exit 1

.gitmodules

@@ -3,4 +3,4 @@
 	url = [email protected]:FCG-LLC/ci-utils.git
 [submodule "internal_deps/py_cslib"]
 	path = internal_deps/py_cslib
-	url = [email protected]:FCG-LLC/py_cslib
+	url = [email protected]:FCG-LLC/py_cslib.git

aucote.py

@@ -156,8 +156,11 @@ async def run_scan(self, as_service=True):
             self._storage.init_schema()
             self.ioloop.add_callback(self.web_server.run)
 
+            tcp_host = cfg['tcpportscan.host']
+            tcp_port = int(cfg['tcpportscan.port'])
+
             self.scanners = [
-                TCPScanner(aucote=self, as_service=as_service),
+                TCPScanner(host=tcp_host, port=tcp_port, aucote=self, as_service=as_service),
                 UDPScanner(aucote=self, as_service=as_service)
             ]
 

aucote_cfg.py

@@ -12,18 +12,28 @@
 # default values
 from utils.toucan import Toucan
 
+LOG_DIR = path.join(path.dirname(__file__), 'logs')
+
 _DEFAULT = {
     'logging': {
         'root': {
-            'file': lambda: path.join(path.dirname(__file__), 'aucote.log'),
+            'file': path.join(LOG_DIR, 'aucote.log'),
             'level': 'info',
             'max_file_size': 10 * 1024 * 1024,
             'max_files': 5,
             'format': '%(levelname)s %(asctime)s %(funcName)s: %(message)s',
             'propagate': True
         },
         'storage': {
-            'file': lambda: path.join(path.dirname(__file__), 'storage.log'),
+            'file': path.join(LOG_DIR, 'storage.log'),
+            'level': 'info',
+            'max_file_size': 10 * 1024 * 1024,
+            'max_files': 10,
+            'format': '%(levelname)s %(asctime)s %(message)s',
+            'propagate': False
+        },
+        'pycslib': {
+            'file': path.join(LOG_DIR, 'pycslib.log'),
             'level': 'info',
             'max_file_size': 10 * 1024 * 1024,
             'max_files': 10,

aucote_cfg.yaml.example

@@ -31,6 +31,13 @@ logging:
         max_files: 10
         format: "%(levelname)s %(asctime)s %(message)s"
         propagate: False
+    pycslib:
+        file: logs/pycslib.log
+        level: debug
+        max_file_size: 10485760
+        max_files: 10
+        format: "%(levelname)s %(asctime)s %(message)s"
+        propagate: False
 
 # storage: Storage configuration
 #    path: <str> - Path to the storage
@@ -101,4 +108,13 @@ rabbit:
     host: localhost
     port: 5672
     username: guest
-    password: guest
+    password: guest
+
+# TCP portscan is a remote scanning service used for TCP portdetection
+# tcpportscan:
+#   host: <str>
+#   port: <int>
+
+tcpportscan:
+    host: portscan
+    port: 1339

package.sh

@@ -4,7 +4,7 @@ set -e
 
 VERSION=$1
 PACKAGE_NAME=aucote
-PYTHON_FILES=`find . -not -path './venv*' -and -not -path './package*' -and -name '*.py'`
+PYTHON_FILES=`find . -not -path './venv*' -and -not -path './package*' -not -path './internal_deps*' -and -name '*.py'`
 PACKAGE_FILES="venv ${PYTHON_FILES} fixtures/exploits/* static/* aucote_cfg.yaml.example aucote_cfg_default.yaml"
 PACKAGE_PATH=package/${PACKAGE_NAME}_${VERSION}
 PACKAGE_TARGET_PATH=${PACKAGE_PATH}/opt/${PACKAGE_NAME}

scans/tcp_scanner.py

@@ -1,16 +1,23 @@
+from asyncio import get_event_loop
+
 from scans.scanner import Scanner
 from structs import TransportProtocol
-from tools.masscan import MasscanPorts
-from tools.nmap.ports import PortsScan
+from utils.portscan import PortscanScanner
 
 
 class TCPScanner(Scanner):
     PROTOCOL = TransportProtocol.TCP
     NAME = 'tcp'
 
+    def __init__(self, host, port, *args, **kwargs):
+        super(TCPScanner, self).__init__(*args, **kwargs)
+        self.host = host
+        self.port = port
+        self._tcp_scanner = PortscanScanner(self.host, self.port, get_event_loop())
+
     @property
     def scanners(self):
         return {
-            self.IPV4: [MasscanPorts(udp=False)],
-            self.IPV6: [PortsScan(ipv6=True, tcp=True, udp=False)]
+            self.IPV4: [self._tcp_scanner],
+            self.IPV6: [self._tcp_scanner]
         }

structs.py

@@ -494,7 +494,7 @@ class Port(object):
         'http-proxy': 'http'
     }
 
-    def __init__(self, node, number, transport_protocol):
+    def __init__(self, node, number, transport_protocol, scan=None):
         """
         Args:
             node (Node):
@@ -511,7 +511,7 @@ def __init__(self, node, number, transport_protocol):
         self.apps = []
         self.protocol = None
         self.banner = None
-        self.scan = None
+        self.scan = scan
         self.interface = None
 
     def __eq__(self, other):

tests/test_api/test_api.py

@@ -76,7 +76,7 @@ def get_app(self):
         for vulnerability in (self.vulnerability_1, self.vulnerability_2):
             self.storage.save_vulnerability(vulnerability)
 
-        self.scanner = TCPScanner(aucote=self.aucote)
+        self.scanner = TCPScanner(aucote=self.aucote, host='localhost', port=1339)
         self.scanner.NAME = 'test_name'
         self.scanner.scan_start = 1290
         self.scanner.nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]

tests/test_api/test_scanners_handler.py

@@ -18,7 +18,7 @@ def setUp(self):
 
     def get_app(self):
         self.aucote = MagicMock()
-        self.scanner = TCPScanner(aucote=self.aucote)
+        self.scanner = TCPScanner(aucote=self.aucote, host='localhost', port=1339)
         self.scanner.NAME = 'test_name'
         self.scanner.scan_start = 1290
         self.scanner.nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]
@@ -61,7 +61,7 @@ def test_scanner(self, cfg):
             'previous_scan': 1260,
             'previous_scan_human': '1970-01-01T00:21:00+00:00',
             'scan': 'test_name',
-            'scanners': {'IPv4': ['masscan'], 'IPv6': ['nmap']},
+            'scanners': {'IPv4': ['portscan'], 'IPv6': ['portscan']},
             'status': 'IDLE'
         }
         response = self.fetch('/api/v1/scanner/test_name', method='GET')

tests/test_root/test_aucote.py

@@ -63,6 +63,10 @@ def setUp(self, cfg, mock_storage):
                     'host': 'localhost',
                     'port': '1234'
                 }
+            },
+            'tcpportscan': {
+                'host': 'localhost',
+                'port': '1239'
             }
         }
         self.aucote = Aucote(exploits=MagicMock(), kudu_queue=MagicMock(), tools_config=MagicMock())
@@ -209,7 +213,7 @@ async def test_scan(self, cfg, tcp_scanner, udp_scanner, tools_scanner, mock_sto
 
         await self.aucote.run_scan(as_service=False)
 
-        tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False)
+        tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False, host='localhost', port=1239)
         udp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False)
         self.assertFalse(tools_scanner.called)
 
@@ -242,7 +246,7 @@ async def test_service(self, cfg, tcp_scanner, udp_scanner, tools_scanner, mock_
 
         await self.aucote.run_scan(as_service=True)
 
-        tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True)
+        tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True, host='localhost', port=1239)
         udp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True)
         tools_scanner.assert_called_once_with(aucote=self.aucote, name='tools')
         self.aucote.async_task_manager.add_crontab_task.assert_has_calls((

tests/test_scans/test_tcp_scanner.py

@@ -7,17 +7,13 @@
 class TCPScannerTest(TestCase):
     def setUp(self):
         self.aucote = MagicMock()
-        self.scanner = TCPScanner(aucote=self.aucote, as_service=False)
+        self.scanner = TCPScanner(aucote=self.aucote, as_service=False, host='localhost', port=1339)
 
-    @patch('scans.tcp_scanner.MasscanPorts')
-    @patch('scans.tcp_scanner.PortsScan')
-    def test_scanners(self, scan, masscan):
+    def test_scanners(self):
         result = self.scanner.scanners
         expected = {
-            self.scanner.IPV4: [masscan.return_value],
-            self.scanner.IPV6: [scan.return_value]
+            self.scanner.IPV4: [self.scanner._tcp_scanner],
+            self.scanner.IPV6: [self.scanner._tcp_scanner]
         }
 
         self.assertEqual(result, expected)
-        scan.assert_called_once_with(ipv6=True, tcp=True, udp=False)
-        masscan.assert_called_once_with(udp=False)

utils/portscan.py

@@ -0,0 +1,32 @@
+from asyncio import ensure_future
+from collections import namedtuple
+
+from pycslib.scan_engines import Portscan
+from pycslib.utils.nmap import ports_to_string
+
+from aucote_cfg import cfg
+from structs import TransportProtocol, Port, Scan
+from tools.nmap.tool import NmapTool
+
+
+class PortscanScanner(object):
+    def __init__(self, host, port, io_loop):
+        self.portscan = Portscan(host, port, io_loop)
+        self.command = namedtuple('command', 'NAME')('portscan')
+        ensure_future(self.portscan.connect(), loop=io_loop)
+
+    async def scan_ports(self, nodes):
+        include_ports = NmapTool.ports_from_list(tcp=cfg['portdetection.tcp.ports.include']).get(TransportProtocol.TCP)
+        exclude_ports = NmapTool.ports_from_list(tcp=cfg['portdetection.tcp.ports.exclude']).get(TransportProtocol.TCP)
+
+        ports = list(include_ports - exclude_ports)
+
+        task = {str(node.ip): ports_to_string(set(ports)) for node in nodes}
+
+        found_ports = await self.portscan.send(task)
+
+        return list({
+            Port(number=port, node=node, transport_protocol=TransportProtocol.TCP, scan=Scan(start=node.scan.start))
+            for node in nodes
+            for port in found_ports.get(str(node.ip))
+        })