diff --git a/.ci/security.sh b/.ci/security.sh index 22912844..792ee181 100644 --- a/.ci/security.sh +++ b/.ci/security.sh @@ -1,6 +1,6 @@ #!/bin/bash -PYLINT_FILES=$(find . -name "*.py" -not -path "./tests/*" -not -path "./venv/*") +PYLINT_FILES=$(find . -name "*.py" -not -path "./tests/*" -not -path "./venv/*" -not -path "./internal_deps*") bandit -ll ${PYLINT_FILES} | tee bandit.txt exit ${PIPESTATUS[0]} \ No newline at end of file diff --git a/.ci/unit_tests.sh b/.ci/unit_tests.sh index ab552487..6c73e4ca 100644 --- a/.ci/unit_tests.sh +++ b/.ci/unit_tests.sh @@ -1,4 +1,4 @@ #!/bin/bash -FILES=$(find . -maxdepth 1 -type d) -nosetests --with-xunit --with-coverage --cover-erase --cover-xml --cover-package=. ${FILES} || exit 1 \ No newline at end of file +FILES=$(find . -maxdepth 1 -type d -not -path "./internal_deps*") +nosetests --with-xunit --with-coverage --cover-erase --cover-xml --cover-package=. ${FILES} || exit 1 diff --git a/.gitmodules b/.gitmodules index c9dd491c..7ddc740e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = git@github.com:FCG-LLC/ci-utils.git [submodule "internal_deps/py_cslib"] path = internal_deps/py_cslib - url = git@github.com:FCG-LLC/py_cslib + url = git@github.com:FCG-LLC/py_cslib.git diff --git a/aucote.py b/aucote.py index d21f9a18..fe29959f 100644 --- a/aucote.py +++ b/aucote.py @@ -156,8 +156,11 @@ async def run_scan(self, as_service=True): self._storage.init_schema() self.ioloop.add_callback(self.web_server.run) + tcp_host = cfg['tcpportscan.host'] + tcp_port = int(cfg['tcpportscan.port']) + self.scanners = [ - TCPScanner(aucote=self, as_service=as_service), + TCPScanner(host=tcp_host, port=tcp_port, aucote=self, as_service=as_service), UDPScanner(aucote=self, as_service=as_service) ] diff --git a/aucote_cfg.py b/aucote_cfg.py index 50f394be..a757d788 100644 --- a/aucote_cfg.py +++ b/aucote_cfg.py @@ -12,10 +12,12 @@ # default values from utils.toucan import Toucan +LOG_DIR = path.join(path.dirname(__file__), 'logs') + _DEFAULT = { 'logging': { 'root': { - 'file': lambda: path.join(path.dirname(__file__), 'aucote.log'), + 'file': path.join(LOG_DIR, 'aucote.log'), 'level': 'info', 'max_file_size': 10 * 1024 * 1024, 'max_files': 5, @@ -23,7 +25,15 @@ 'propagate': True }, 'storage': { - 'file': lambda: path.join(path.dirname(__file__), 'storage.log'), + 'file': path.join(LOG_DIR, 'storage.log'), + 'level': 'info', + 'max_file_size': 10 * 1024 * 1024, + 'max_files': 10, + 'format': '%(levelname)s %(asctime)s %(message)s', + 'propagate': False + }, + 'pycslib': { + 'file': path.join(LOG_DIR, 'pycslib.log'), 'level': 'info', 'max_file_size': 10 * 1024 * 1024, 'max_files': 10, diff --git a/aucote_cfg.yaml.example b/aucote_cfg.yaml.example index 2bfbbc1c..e0d18fd8 100644 --- a/aucote_cfg.yaml.example +++ b/aucote_cfg.yaml.example @@ -31,6 +31,13 @@ logging: max_files: 10 format: "%(levelname)s %(asctime)s %(message)s" propagate: False + pycslib: + file: logs/pycslib.log + level: debug + max_file_size: 10485760 + max_files: 10 + format: "%(levelname)s %(asctime)s %(message)s" + propagate: False # storage: Storage configuration # path: - Path to the storage @@ -101,4 +108,13 @@ rabbit: host: localhost port: 5672 username: guest - password: guest \ No newline at end of file + password: guest + +# TCP portscan is a remote scanning service used for TCP portdetection +# tcpportscan: +# host: +# port: + +tcpportscan: + host: portscan + port: 1339 diff --git a/internal_deps/py_cslib b/internal_deps/py_cslib index 65294dda..cf164431 160000 --- a/internal_deps/py_cslib +++ b/internal_deps/py_cslib @@ -1 +1 @@ -Subproject commit 65294ddaa243aa57c63cc2c2b97bc38be6941c70 +Subproject commit cf1644319994e2ad91b6e03d6e30c856f27e2270 diff --git a/package.sh b/package.sh index 76d3de9d..e927db9a 100755 --- a/package.sh +++ b/package.sh @@ -4,7 +4,7 @@ set -e VERSION=$1 PACKAGE_NAME=aucote -PYTHON_FILES=`find . -not -path './venv*' -and -not -path './package*' -and -name '*.py'` +PYTHON_FILES=`find . -not -path './venv*' -and -not -path './package*' -not -path './internal_deps*' -and -name '*.py'` PACKAGE_FILES="venv ${PYTHON_FILES} fixtures/exploits/* static/* aucote_cfg.yaml.example aucote_cfg_default.yaml" PACKAGE_PATH=package/${PACKAGE_NAME}_${VERSION} PACKAGE_TARGET_PATH=${PACKAGE_PATH}/opt/${PACKAGE_NAME} diff --git a/scans/tcp_scanner.py b/scans/tcp_scanner.py index 673be12d..602ecac0 100644 --- a/scans/tcp_scanner.py +++ b/scans/tcp_scanner.py @@ -1,16 +1,23 @@ +from asyncio import get_event_loop + from scans.scanner import Scanner from structs import TransportProtocol -from tools.masscan import MasscanPorts -from tools.nmap.ports import PortsScan +from utils.portscan import PortscanScanner class TCPScanner(Scanner): PROTOCOL = TransportProtocol.TCP NAME = 'tcp' + def __init__(self, host, port, *args, **kwargs): + super(TCPScanner, self).__init__(*args, **kwargs) + self.host = host + self.port = port + self._tcp_scanner = PortscanScanner(self.host, self.port, get_event_loop()) + @property def scanners(self): return { - self.IPV4: [MasscanPorts(udp=False)], - self.IPV6: [PortsScan(ipv6=True, tcp=True, udp=False)] + self.IPV4: [self._tcp_scanner], + self.IPV6: [self._tcp_scanner] } diff --git a/structs.py b/structs.py index 9e0f1243..0278aae1 100644 --- a/structs.py +++ b/structs.py @@ -494,7 +494,7 @@ class Port(object): 'http-proxy': 'http' } - def __init__(self, node, number, transport_protocol): + def __init__(self, node, number, transport_protocol, scan=None): """ Args: node (Node): @@ -511,7 +511,7 @@ def __init__(self, node, number, transport_protocol): self.apps = [] self.protocol = None self.banner = None - self.scan = None + self.scan = scan self.interface = None def __eq__(self, other): diff --git a/tests/test_api/test_api.py b/tests/test_api/test_api.py index 44989a1d..908527f6 100644 --- a/tests/test_api/test_api.py +++ b/tests/test_api/test_api.py @@ -76,7 +76,7 @@ def get_app(self): for vulnerability in (self.vulnerability_1, self.vulnerability_2): self.storage.save_vulnerability(vulnerability) - self.scanner = TCPScanner(aucote=self.aucote) + self.scanner = TCPScanner(aucote=self.aucote, host='localhost', port=1339) self.scanner.NAME = 'test_name' self.scanner.scan_start = 1290 self.scanner.nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))] diff --git a/tests/test_api/test_scanners_handler.py b/tests/test_api/test_scanners_handler.py index 67522d02..28963701 100644 --- a/tests/test_api/test_scanners_handler.py +++ b/tests/test_api/test_scanners_handler.py @@ -18,7 +18,7 @@ def setUp(self): def get_app(self): self.aucote = MagicMock() - self.scanner = TCPScanner(aucote=self.aucote) + self.scanner = TCPScanner(aucote=self.aucote, host='localhost', port=1339) self.scanner.NAME = 'test_name' self.scanner.scan_start = 1290 self.scanner.nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))] @@ -61,7 +61,7 @@ def test_scanner(self, cfg): 'previous_scan': 1260, 'previous_scan_human': '1970-01-01T00:21:00+00:00', 'scan': 'test_name', - 'scanners': {'IPv4': ['masscan'], 'IPv6': ['nmap']}, + 'scanners': {'IPv4': ['portscan'], 'IPv6': ['portscan']}, 'status': 'IDLE' } response = self.fetch('/api/v1/scanner/test_name', method='GET') diff --git a/tests/test_root/test_aucote.py b/tests/test_root/test_aucote.py index 114cc7c3..061886e7 100644 --- a/tests/test_root/test_aucote.py +++ b/tests/test_root/test_aucote.py @@ -63,6 +63,10 @@ def setUp(self, cfg, mock_storage): 'host': 'localhost', 'port': '1234' } + }, + 'tcpportscan': { + 'host': 'localhost', + 'port': '1239' } } self.aucote = Aucote(exploits=MagicMock(), kudu_queue=MagicMock(), tools_config=MagicMock()) @@ -209,7 +213,7 @@ async def test_scan(self, cfg, tcp_scanner, udp_scanner, tools_scanner, mock_sto await self.aucote.run_scan(as_service=False) - tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False) + tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False, host='localhost', port=1239) udp_scanner.assert_called_once_with(aucote=self.aucote, as_service=False) self.assertFalse(tools_scanner.called) @@ -242,7 +246,7 @@ async def test_service(self, cfg, tcp_scanner, udp_scanner, tools_scanner, mock_ await self.aucote.run_scan(as_service=True) - tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True) + tcp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True, host='localhost', port=1239) udp_scanner.assert_called_once_with(aucote=self.aucote, as_service=True) tools_scanner.assert_called_once_with(aucote=self.aucote, name='tools') self.aucote.async_task_manager.add_crontab_task.assert_has_calls(( diff --git a/tests/test_scans/test_tcp_scanner.py b/tests/test_scans/test_tcp_scanner.py index af62e56c..746da70d 100644 --- a/tests/test_scans/test_tcp_scanner.py +++ b/tests/test_scans/test_tcp_scanner.py @@ -7,17 +7,13 @@ class TCPScannerTest(TestCase): def setUp(self): self.aucote = MagicMock() - self.scanner = TCPScanner(aucote=self.aucote, as_service=False) + self.scanner = TCPScanner(aucote=self.aucote, as_service=False, host='localhost', port=1339) - @patch('scans.tcp_scanner.MasscanPorts') - @patch('scans.tcp_scanner.PortsScan') - def test_scanners(self, scan, masscan): + def test_scanners(self): result = self.scanner.scanners expected = { - self.scanner.IPV4: [masscan.return_value], - self.scanner.IPV6: [scan.return_value] + self.scanner.IPV4: [self.scanner._tcp_scanner], + self.scanner.IPV6: [self.scanner._tcp_scanner] } self.assertEqual(result, expected) - scan.assert_called_once_with(ipv6=True, tcp=True, udp=False) - masscan.assert_called_once_with(udp=False) diff --git a/utils/portscan.py b/utils/portscan.py new file mode 100644 index 00000000..d04945be --- /dev/null +++ b/utils/portscan.py @@ -0,0 +1,32 @@ +from asyncio import ensure_future +from collections import namedtuple + +from pycslib.scan_engines import Portscan +from pycslib.utils.nmap import ports_to_string + +from aucote_cfg import cfg +from structs import TransportProtocol, Port, Scan +from tools.nmap.tool import NmapTool + + +class PortscanScanner(object): + def __init__(self, host, port, io_loop): + self.portscan = Portscan(host, port, io_loop) + self.command = namedtuple('command', 'NAME')('portscan') + ensure_future(self.portscan.connect(), loop=io_loop) + + async def scan_ports(self, nodes): + include_ports = NmapTool.ports_from_list(tcp=cfg['portdetection.tcp.ports.include']).get(TransportProtocol.TCP) + exclude_ports = NmapTool.ports_from_list(tcp=cfg['portdetection.tcp.ports.exclude']).get(TransportProtocol.TCP) + + ports = list(include_ports - exclude_ports) + + task = {str(node.ip): ports_to_string(set(ports)) for node in nodes} + + found_ports = await self.portscan.send(task) + + return list({ + Port(number=port, node=node, transport_protocol=TransportProtocol.TCP, scan=Scan(start=node.scan.start)) + for node in nodes + for port in found_ports.get(str(node.ip)) + })