diff --git a/analyzers/Qintel/qintel_helper.py b/analyzers/Qintel/qintel_helper.py new file mode 100644 index 000000000..47106f78a --- /dev/null +++ b/analyzers/Qintel/qintel_helper.py @@ -0,0 +1,263 @@ +# Copyright (c) 2009-2021 Qintel, LLC +# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) + +from urllib.request import Request, urlopen +from urllib.parse import urlencode +from urllib.error import HTTPError +from time import sleep +from json import loads +import os +from copy import deepcopy +from datetime import datetime, timedelta +from gzip import GzipFile + +VERSION = '1.0.1' +USER_AGENT = 'integrations-helper' +MAX_RETRY_ATTEMPTS = 5 + +DEFAULT_HEADERS = { + 'User-Agent': f'{USER_AGENT}/{VERSION}' +} + +REMOTE_MAP = { + 'pmi': 'https://api.pmi.qintel.com', + 'qwatch': 'https://api.qwatch.qintel.com', + 'qauth': 'https://api.qauth.qintel.com', + 'qsentry_feed': 'https://qsentry.qintel.com', + 'qsentry': 'https://api.qsentry.qintel.com' +} + +ENDPOINT_MAP = { + 'pmi': { + 'ping': '/users/me', + 'cve': 'cves' + }, + 'qsentry_feed': { + 'anon': '/files/anonymization', + 'mal_hosting': '/files/malicious_hosting' + }, + 'qsentry': {}, + 'qwatch': { + 'ping': '/users/me', + 'exposures': 'exposures' + }, + 'qauth': {} +} + + +def _get_request_wait_time(attempts): + """ Use Fibonacci numbers for determining the time to wait when rate limits + have been encountered. + """ + + n = attempts + 3 + a, b = 1, 0 + for _ in range(n): + a, b = a + b, a + + return a + + +def _search(**kwargs): + remote = kwargs.get('remote') + max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS)) + params = kwargs.get('params', {}) + headers = _set_headers(**kwargs) + + logger = kwargs.get('logger') + + params = urlencode(params) + url = remote + "?" + params + req = Request(url, headers=headers) + + request_attempts = 1 + while request_attempts < max_retries: + try: + return urlopen(req) + + except HTTPError as e: + response = e + + except Exception as e: + raise Exception('API connection error') from e + + if response.code not in [429, 504]: + raise Exception(f'API connection error: {response}') + + if request_attempts < max_retries: + wait_time = _get_request_wait_time(request_attempts) + + if response.code == 429: + msg = 'rate limit reached on attempt {request_attempts}, ' \ + 'waiting {wait_time} seconds' + + if logger: + logger(msg) + + else: + msg = f'connection timed out, retrying in {wait_time} seconds' + if logger: + logger(msg) + + sleep(wait_time) + + else: + raise Exception('Max API retries exceeded') + + request_attempts += 1 + + +def _set_headers(**kwargs): + headers = deepcopy(DEFAULT_HEADERS) + + if kwargs.get('user_agent'): + headers['User-Agent'] = \ + f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}" + + # TODO: deprecate + if kwargs.get('client_id') or kwargs.get('client_secret'): + try: + headers['Cf-Access-Client-Id'] = kwargs['client_id'] + headers['Cf-Access-Client-Secret'] = kwargs['client_secret'] + except KeyError: + raise Exception('missing client_id or client_secret') + + if kwargs.get('token'): + headers['x-api-key'] = kwargs['token'] + + return headers + + +def _set_remote(product, query_type, **kwargs): + remote = kwargs.get('remote') + endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type)) + + if not remote: + remote = REMOTE_MAP[product] + + if not endpoint: + raise Exception('invalid search type') + + remote = remote.rstrip('/') + endpoint = endpoint.lstrip('/') + + return f'{remote}/{endpoint}' + + +def _process_qsentry(resp): + if resp.getheader('Content-Encoding', '') == 'gzip': + with GzipFile(fileobj=resp) as file: + for line in file.readlines(): + yield loads(line) + + +def search_pmi(search_term, query_type, **kwargs): + """ + Search PMI + + :param str search_term: Search term + :param str query_type: Query type [cve|ping] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('pmi', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'identifier': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qwatch(search_term, search_type, query_type, **kwargs): + """ + Search QWatch for exposed credentials + + :param str search_term: Search term + :param str search_type: Search term type [domain|email] + :param str query_type: Query type [exposures] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN')) + + params = kwargs.get('params', {}) + if search_type: + params.update({search_type: search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qauth(search_term, **kwargs): + """ + Search QAuth + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qauth', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qsentry(search_term, **kwargs): + """ + Search QSentry + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qsentry', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs): + """ + Fetch the most recent QSentry Feed + + :param str query_type: Feed type [anon|mal_hosting] + :param dict kwargs: extra client args [remote|token|params] + :param datetime feed_date: feed date to fetch + :return: API JSON response object + :rtype: Iterator[dict] + """ + + remote = _set_remote('qsentry_feed', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d') + kwargs['remote'] = f'{remote}/{feed_date}' + + resp = _search(**kwargs) + for r in _process_qsentry(resp): + yield r diff --git a/analyzers/Qintel/qintel_qsentry.json b/analyzers/Qintel/qintel_qsentry.json new file mode 100644 index 000000000..edd6e740c --- /dev/null +++ b/analyzers/Qintel/qintel_qsentry.json @@ -0,0 +1,27 @@ +{ + "name": "Qintel_QSentry", + "version": "1.0", + "url": "None", + "author": "Qintel, LLC", + "license": "Apache 2.0", + "description": "Queries Qintel QSentry for ip reputation data", + "dataTypeList": ["ip"], + "command": "Qintel/qintel_qsentry.py", + "baseConfig": "Qintel_QSentry", + "configurationItems": [ + { + "name": "token", + "description": "API token for Qintel", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "remote", + "description": "API URL (optional)", + "type": "string", + "multi": false, + "required": false + } + ] +} \ No newline at end of file diff --git a/analyzers/Qintel/qintel_qsentry.py b/analyzers/Qintel/qintel_qsentry.py new file mode 100755 index 000000000..221bd5ad5 --- /dev/null +++ b/analyzers/Qintel/qintel_qsentry.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from cortexutils.analyzer import Analyzer + +from qintel_helper import search_qsentry + + +class Qintel(Analyzer): + + VERSION = '1.0' + + def __init__(self): + Analyzer.__init__(self) + self.token = self.get_param('config.token', None, 'Missing API Key') + self.remote = self.get_param('config.remote', None) + + def _enrich(self, data): + + kwargs = { + 'token': self.token, + 'user_agent': f'cortex/{self.VERSION}' + } + + try: + return search_qsentry(data, **kwargs) + except Exception as e: + self.error(f'Qintel API request failed: {str(e)}') + + def summary(self, raw): + taxonomies = [] + ns = 'Qintel' + + for tag in self.res.get('tags', []): + level = 'suspicious' + + if tag == 'criminal': + level = 'malicious' + + tax = self.build_taxonomy(level, ns, 'tag', tag) + taxonomies.append(tax) + + return {'taxonomies': taxonomies} + + def run(self): + if self.data_type not in ['ip']: + self.error(f'Unsupported data type: {self.data_type}') + + data = self.getData() + self.res = self._enrich(data) + + self.report({ + 'Qintel': self.res + }) + + +if __name__ == '__main__': + Qintel().run() diff --git a/thehive-templates/Qintel_QSentry/long.html b/thehive-templates/Qintel_QSentry/long.html new file mode 100644 index 000000000..03275e29c --- /dev/null +++ b/thehive-templates/Qintel_QSentry/long.html @@ -0,0 +1,47 @@ + +
+
+ {{name}} +
+
+
+ {{artifact.data | fang}} +

+ + + + + + + + + + + + + + + + + +
ASN:{{ content.Qintel.asn }}
Tags:{{ content.Qintel.tags.join(', ').toString() }}
Last Seen:{{ content.Qintel.last_seen }}
Descriptions: +
    +
  • {{ i }}
    • +
+
+
+
+ + + +
+
+ {{artifact.data | fang}} +
+
+
+
{{name}}
+
{{content.errorMessage}}
+
+
+
\ No newline at end of file diff --git a/thehive-templates/Qintel_QSentry/short.html b/thehive-templates/Qintel_QSentry/short.html new file mode 100644 index 000000000..9fd48f9fa --- /dev/null +++ b/thehive-templates/Qintel_QSentry/short.html @@ -0,0 +1,3 @@ + + {{t.namespace}}:{{t.predicate}}="{{t.value}}" + \ No newline at end of file