Add several fixes (#137)

* Fix timeout

* Fix for Topdis api v2

* Remove logging scripts output and log only kudu message size

aucote.py

@@ -124,7 +124,7 @@ def __init__(self, exploits, kudu_queue, tools_config):
         self._storage = Storage(filename=cfg['storage.path'], nodes_limit=cfg['storage.max_nodes_query'])
 
         self.ioloop = IOLoop.current()
-        self.topdis = Topdis(cfg['topdis.api.host'], cfg['topdis.api.port'])
+        self.topdis = Topdis(cfg['topdis.api.host'], cfg['topdis.api.port'], cfg['topdis.api.base'])
 
         self.async_task_manager = AsyncTaskManager.instance(parallel_tasks=cfg['service.scans.parallel_tasks'])
         self._throttling_consumer = ThrottlingConsumer(manager=self.async_task_manager)

aucote_cfg.yaml.example

@@ -72,11 +72,13 @@ kuduworker:
 #   api:
 #     host: <str|ipaddress> - host of api
 #     port: <int> - port of api
+#     base: <str> - base address for topdis
 
 topdis:
     api:
         host: topdis
         port: 1234
+        base: /api/v1
 
 # Toucan is centralized node manager. It provides configuration for Aucote
 # toucan:

structs.py

@@ -5,6 +5,7 @@
 import ipaddress
 import re
 from abc import ABCMeta, abstractmethod
+from ctypes import c_uint32
 from enum import Enum
 import time
 
@@ -84,8 +85,7 @@ def __hash__(self):
 
 class Node:
     """
-    Node object consist of name, id and ip
-
+    Represents node in the network. Every pair ip:node_id is considered as separated node
     """
 
     def __init__(self, node_id, ip):
@@ -98,10 +98,24 @@ def __init__(self, node_id, ip):
         """
         self.name = None
         self.ip = ip
+        self._id = None
         self.id = node_id
         self.scan = None
         self.os = Service()
 
+    @property
+    def id(self):
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        """
+        Set node id. If it's negative, convert to unsigned int
+        """
+        if value is not None:
+            value = c_uint32(value).value
+        self._id = value
+
     def __eq__(self, other):
         return isinstance(other, Node) and self.ip == other.ip and self.id == other.id
 

tests/test_root/test_aucote.py

@@ -61,7 +61,8 @@ def setUp(self, cfg, mock_storage):
             'topdis': {
                 'api': {
                     'host': 'localhost',
-                    'port': '1234'
+                    'port': '1234',
+                    'base': '/api/v1'
                 }
             },
             'tcpportscan': {

tests/test_root/test_structs.py

@@ -62,9 +62,9 @@ def test_node_comparison_none(self):
 
     def test_hash(self):
         ip = MagicMock()
-        id = MagicMock()
+        id = -1
 
-        expected = hash((id, ip))
+        expected = hash((4294967295, ip))
         result = hash(Node(ip=ip, node_id=id))
 
         self.assertEqual(result, expected)

tests/test_utils/test_topdis.py

@@ -194,7 +194,7 @@ class TopdisTest(AsyncTestCase):
 
     def setUp(self):
         super(TopdisTest, self).setUp()
-        self.topdis = Topdis('localhost', 1234)
+        self.topdis = Topdis('localhost', 1234, '/api/v1')
         self.req_future = Future()
         self.http_client_response = MagicMock()
         self.http_client_response.body = self.TODIS_RESPONSE

tools/common/command.py

@@ -4,6 +4,7 @@
 """
 import logging as log
 import subprocess
+from datetime import timedelta
 
 from tornado import gen
 from tornado import process
@@ -75,9 +76,9 @@ async def async_call(self, args=None, timeout=None):
             return_code, stdout, stderr = await coroutine
         else:
             try:
-                return_code, stdout, stderr = await gen.with_timeout(timeout, coroutine)
+                return_code, stdout, stderr = await gen.with_timeout(timedelta(seconds=timeout), coroutine)
             except gen.TimeoutError as exception:
-                log.exception("Command %s timed out while executing %s", self.NAME, cmd)
+                log.exception("Command %s timed out after %s while executing %s", self.NAME, timeout, cmd)
                 task.proc.kill()
                 raise exception
 

utils/kudu_queue.py

@@ -139,7 +139,7 @@ def send_msg(self, msg, dont_wait=False):
         """
 
         assert isinstance(msg, KuduMsg)
-        log.debug('sending bytes to kuduworker: %s', bytes_str(msg.data))
+        log.debug('sending bytes to kuduworker: %s', len(msg.data))
         flags = DONTWAIT if dont_wait else 0
         try:
             self._socket.send(msg.data, flags)

utils/task.py

@@ -71,7 +71,7 @@ def store_vulnerability(self, vuln):
             None
 
         """
-        log.debug('Found vulnerability: port=%s exploit=%s output=%s', vuln.port, vuln.exploit.id, vuln.output)
+        log.debug('Found vulnerability %s for %s', vuln.exploit.id, vuln.port)
         msg = Serializer.serialize_port_vuln(vuln.port, vuln)
         self.kudu_queue.send_msg(msg)
 

utils/topdis.py

@@ -23,8 +23,8 @@ class Topdis(object):
     max_retry_time = 30
     max_retry_count = 20
 
-    def __init__(self, hostname, port):
-        self.api = 'http://{0}:{1}/api/v1'.format(hostname, port)
+    def __init__(self, hostname, port, api):
+        self.api = 'http://{0}:{1}{2}'.format(hostname, port, api)
 
     @retry_if_fail(min_retry_time, max_retry_time, max_retry_count, HTTPError)
     async def get_nodes(self):