From b644e9558a4871584bc6c7ab7b2ffff2b8bc5e3a Mon Sep 17 00:00:00 2001 From: Kashyap Date: Wed, 17 Apr 2019 19:57:43 +0000 Subject: [PATCH 1/4] fix for issue/217-Specifying-asterisk-as-query-causes-error --- awslogs/bin.py | 55 ++++++++++++++++++++++++++++++++++++----- awslogs/core.py | 47 ++++++++++------------------------- awslogs/exceptions.py | 10 +++++++- tests/test_it.py | 57 ++++++++++++++++++++++++++++++++++--------- 4 files changed, 116 insertions(+), 53 deletions(-) diff --git a/awslogs/bin.py b/awslogs/bin.py index 87d2286..183f9dc 100644 --- a/awslogs/bin.py +++ b/awslogs/bin.py @@ -1,11 +1,16 @@ import os +import re import sys import locale import codecs import argparse +from datetime import datetime, timedelta +from dateutil.parser import parse +from dateutil.tz import tzutc import boto3 from botocore.client import ClientError +from botocore.compat import total_seconds from termcolor import colored from . import exceptions @@ -13,6 +18,45 @@ from ._version import __version__ +def regex_str(s): + """Verifies that the s is a valid python regex + if s is not a valid regex then an exception is raised""" + try: + re.compile(s) + except Exception as e: + raise exceptions.InvalidPythonRegularExpressionError('Log stream name pattern', s) + + return s + + +def seconds_since_epoch(datetime_text): + """Parse ``datetime_text`` into a seconds since epoch.""" + + if not datetime_text: + return None + + ago_regexp = r'(\d+)\s?(m|minute|minutes|h|hour|hours|d|day|days|w|weeks|weeks)(?: ago)?' + ago_match = re.match(ago_regexp, datetime_text) + + if ago_match: + amount, unit = ago_match.groups() + amount = int(amount) + unit = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800}[unit[0]] + date = datetime.utcnow() + timedelta(seconds=unit * amount * -1) + else: + try: + date = parse(datetime_text) + except ValueError: + raise exceptions.UnknownDateError(datetime_text) + + if date.tzinfo: + if date.utcoffset != 0: + date = date.astimezone(tzutc()) + date = date.replace(tzinfo=None) + + return int(total_seconds(date - datetime(1970, 1, 1))) * 1000 + + def main(argv=None): if sys.version_info < (3, 0): @@ -57,13 +101,13 @@ def add_common_arguments(parser): def add_date_range_arguments(parser, default_start='5m'): parser.add_argument("-s", "--start", - type=str, + type=seconds_since_epoch, dest='start', default=default_start, help="Start time (default %(default)s)") parser.add_argument("-e", "--end", - type=str, + type=seconds_since_epoch, dest='end', help="End time") @@ -81,7 +125,7 @@ def add_date_range_arguments(parser, default_start='5m'): help="log group name") get_parser.add_argument("log_stream_name", - type=str, + type=regex_str, default="ALL", nargs='?', help="log stream name") @@ -167,10 +211,9 @@ def add_date_range_arguments(parser, default_start='5m'): type=str, help="log group name") - # Parse input - options, args = parser.parse_known_args(argv) - try: + # Parse input + options, args = parser.parse_known_args(argv) logs = AWSLogs(**vars(options)) if not hasattr(options, 'func'): parser.print_help() diff --git a/awslogs/core.py b/awslogs/core.py index 8af6a0e..db5cb88 100644 --- a/awslogs/core.py +++ b/awslogs/core.py @@ -3,21 +3,26 @@ import os import time import errno -from datetime import datetime, timedelta +import logging +from datetime import datetime from collections import deque import boto3 import botocore -from botocore.compat import json, six, total_seconds +from botocore.compat import json, six import jmespath from termcolor import colored -from dateutil.parser import parse -from dateutil.tz import tzutc from . import exceptions +logger = logging.getLogger('awslogs') +FORMAT = '%(asctime)-15s %(message)s' +logging.basicConfig(format=FORMAT, filename="/tmp/awslogs.log") +# setLevel to logging.DEBUG to enable logging. +logger.setLevel(logging.CRITICAL) + COLOR_ENABLED = { 'always': True, @@ -59,6 +64,7 @@ class AWSLogs(object): ALL_WILDCARD = 'ALL' def __init__(self, **kwargs): + logger.debug('AWSLogs(): kwargs: %s', kwargs) self.aws_region = kwargs.get('aws_region') self.aws_access_key_id = kwargs.get('aws_access_key_id') self.aws_secret_access_key = kwargs.get('aws_secret_access_key') @@ -75,8 +81,8 @@ def __init__(self, **kwargs): self.output_timestamp_enabled = kwargs.get('output_timestamp_enabled') self.output_ingestion_time_enabled = kwargs.get( 'output_ingestion_time_enabled') - self.start = self.parse_datetime(kwargs.get('start')) - self.end = self.parse_datetime(kwargs.get('end')) + self.start = kwargs.get('start') + self.end = kwargs.get('end') self.query = kwargs.get('query') if self.query is not None: self.query_expression = jmespath.compile(self.query) @@ -92,7 +98,7 @@ def __init__(self, **kwargs): def _get_streams_from_pattern(self, group, pattern): """Returns streams in ``group`` matching ``pattern``.""" pattern = '.*' if pattern == self.ALL_WILDCARD else pattern - reg = re.compile('^{0}'.format(pattern)) + reg = re.compile('{0}'.format(pattern)) for stream in self.get_streams(group): if re.match(reg, stream): yield stream @@ -250,7 +256,6 @@ def get_streams(self, log_group_name=None): kwargs = {'logGroupName': log_group_name or self.log_group_name} window_start = self.start or 0 window_end = self.end or sys.float_info.max - paginator = self.client.get_paginator('describe_log_streams') for page in paginator.paginate(**kwargs): for stream in page.get('logStreams', []): @@ -269,29 +274,3 @@ def color(self, text, color): return colored(text, color) return text - def parse_datetime(self, datetime_text): - """Parse ``datetime_text`` into a ``datetime``.""" - - if not datetime_text: - return None - - ago_regexp = r'(\d+)\s?(m|minute|minutes|h|hour|hours|d|day|days|w|weeks|weeks)(?: ago)?' - ago_match = re.match(ago_regexp, datetime_text) - - if ago_match: - amount, unit = ago_match.groups() - amount = int(amount) - unit = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800}[unit[0]] - date = datetime.utcnow() + timedelta(seconds=unit * amount * -1) - else: - try: - date = parse(datetime_text) - except ValueError: - raise exceptions.UnknownDateError(datetime_text) - - if date.tzinfo: - if date.utcoffset != 0: - date = date.astimezone(tzutc()) - date = date.replace(tzinfo=None) - - return int(total_seconds(date - datetime(1970, 1, 1))) * 1000 diff --git a/awslogs/exceptions.py b/awslogs/exceptions.py index c1dcf67..9409639 100644 --- a/awslogs/exceptions.py +++ b/awslogs/exceptions.py @@ -30,4 +30,12 @@ class NoStreamsFilteredError(BaseAWSLogsException): code = 7 def hint(self): - return ("No streams match your pattern '{0}' for the given time period.").format(*self.args) + return "No streams match your pattern '{0}' for the given time period.".format(*self.args) + + +class InvalidPythonRegularExpressionError(BaseAWSLogsException): + + code = 8 + + def hint(self): + return "{0} '{1}' is not a valid Python regular expression.".format(*self.args) diff --git a/tests/test_it.py b/tests/test_it.py index b54f30d..74747b8 100644 --- a/tests/test_it.py +++ b/tests/test_it.py @@ -16,7 +16,14 @@ from awslogs import AWSLogs from awslogs.exceptions import UnknownDateError -from awslogs.bin import main +from awslogs.bin import main, seconds_since_epoch + +import logging +logger = logging.getLogger('awslogs') +FORMAT = '%(asctime)-15s %(message)s' +logging.basicConfig(format=FORMAT, filename="/tmp/awslogs.log") +# setLevel to logging.DEBUG to enable logging. +logger.setLevel(logging.CRITICAL) def mapkeys(keys, rec_lst): @@ -30,9 +37,10 @@ def mapkeys(keys, rec_lst): class TestAWSLogsDatetimeParse(unittest.TestCase): + + @patch('awslogs.bin.datetime') @patch('awslogs.core.boto3_client') - @patch('awslogs.core.datetime') - def test_parse_datetime(self, datetime_mock, botoclient): + def test_seconds_since_epoch(self, botoclient, datetime_mock): awslogs = AWSLogs() datetime_mock.utcnow.return_value = datetime(2015, 1, 1, 3, 0, 0, 0) @@ -42,8 +50,8 @@ def iso2epoch(iso_str): dt = datetime.strptime(iso_str, "%Y-%m-%d %H:%M:%S") return int(total_seconds(dt - datetime(1970, 1, 1)) * 1000) - self.assertEqual(awslogs.parse_datetime(''), None) - self.assertEqual(awslogs.parse_datetime(None), None) + self.assertEqual(seconds_since_epoch(''), None) + self.assertEqual(seconds_since_epoch(None), None) plan = (('2015-01-01 02:59:00', '1m'), ('2015-01-01 02:59:00', '1m ago'), ('2015-01-01 02:59:00', '1minute'), @@ -81,10 +89,11 @@ def iso2epoch(iso_str): ) for expected_iso, dateutil_time in plan: - self.assertEqual(awslogs.parse_datetime(dateutil_time), - iso2epoch(expected_iso)) + self.assertEqual(seconds_since_epoch(dateutil_time), + iso2epoch(expected_iso), + msg='expected_iso: {}, dateutil_time: {}'.format(expected_iso, dateutil_time)) - self.assertRaises(UnknownDateError, awslogs.parse_datetime, '???') + self.assertRaises(UnknownDateError, seconds_since_epoch, '???') class TestAWSLogs(unittest.TestCase): @@ -230,8 +239,8 @@ def test_get_streams(self, botoclient): ['A', 'B', 'C', 'D', 'E', 'F', 'G']) @patch('awslogs.core.boto3_client') - @patch('awslogs.core.AWSLogs.parse_datetime') - def test_get_streams_filtered_by_date(self, parse_datetime, botoclient): + @patch('awslogs.bin.seconds_since_epoch') + def test_get_streams_filtered_by_date(self, seconds_since_epoch_mock, botoclient): client = Mock() botoclient.return_value = client client.get_paginator.return_value.paginate.return_value = [ @@ -243,8 +252,8 @@ def test_get_streams_filtered_by_date(self, parse_datetime, botoclient): ], } ] - parse_datetime.side_effect = [5, 7] - awslogs = AWSLogs(log_group_name='group', start='5', end='7') + seconds_since_epoch_mock.side_effect = [5, 7] + awslogs = AWSLogs(log_group_name='group', start=5, end=7) self.assertEqual([g for g in awslogs.get_streams()], ['B', 'C', 'E']) @patch('awslogs.core.boto3_client') @@ -630,3 +639,27 @@ def test_boto3_client_creation(self, mock_core_session): awslogs = AWSLogs() self.assertEqual(client, awslogs.client) + + @patch('awslogs.core.boto3_client') + @patch('sys.stderr', new_callable=StringIO) + def test_invalid_stream_regex(self, mock_stderr, botoclient): + self.maxDiff = None + botoclient.return_value = None + exit_code = main("awslogs get LG_NAME *".split()) + self.assertEqual(mock_stderr.getvalue(), + colored("Log stream name pattern '*' is not a valid Python regular expression.\n", + "red")) + assert exit_code == 8 + + @patch('awslogs.core.boto3_client') + @patch('sys.stdout', new_callable=StringIO) + def test_valid_stream_regex(self, mock_stdout, botoclient): + logger.debug('botoclient: %s', botoclient) + self.maxDiff = None + self.set_ABCDE_logs(botoclient) + exit_code = main("awslogs streams LG_NAME .*".split()) + output = mock_stdout.getvalue() + expected = ("DDD\n" + "EEE\n") + assert output == expected + assert exit_code == 0 From c7f3da3a6057da04627af4177ace43becf4bbd12 Mon Sep 17 00:00:00 2001 From: Kashyap Date: Wed, 17 Apr 2019 21:57:44 +0000 Subject: [PATCH 2/4] fix for issue 180 --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 99ccb2e..020d47b 100644 --- a/README.rst +++ b/README.rst @@ -79,9 +79,9 @@ Options * ``awslogs groups``: List existing groups * ``awslogs streams GROUP``: List existing streams withing ``GROUP`` -* ``awslogs get GROUP [STREAM_EXPRESSION]``: Get logs matching ``STREAM_EXPRESSION`` in ``GROUP``. +* ``awslogs get GROUP [STREAM_EXPRESSION]``: Get logs from streams with names matching ``STREAM_EXPRESSION`` in log group ``GROUP``. - - Expressions can be regular expressions or the wildcard ``ALL`` if you want any and don't want to type ``.*``. + - STREAM_EXPRESSION is a python regular expression accepted by ``re.compile()`` `described here `_. Expression ``ALL`` is reserved and is same as ``'.*'``. Remember to quote/escape shell special characters to ensure they are not gobbled up by shell variable expansion. E.g. ``'2014-04.*'`` instead of ``2014-04.*`` **Note:** You need to provide to all these options a valid AWS region using ``--aws-region`` or ``AWS_REGION`` env variable. From c780f245701989335d9bd33802cb3d0386dbf635 Mon Sep 17 00:00:00 2001 From: Kashyap Date: Wed, 17 Apr 2019 22:01:48 +0000 Subject: [PATCH 3/4] remove extra new line at end --- awslogs/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awslogs/core.py b/awslogs/core.py index db5cb88..83f673d 100644 --- a/awslogs/core.py +++ b/awslogs/core.py @@ -273,4 +273,3 @@ def color(self, text, color): if self.color_enabled: return colored(text, color) return text - From c3a74128a876ba5690c362551c759ef1ea4e195a Mon Sep 17 00:00:00 2001 From: Kashyap Date: Wed, 17 Apr 2019 22:03:18 +0000 Subject: [PATCH 4/4] remove assertion message --- tests/test_it.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_it.py b/tests/test_it.py index 74747b8..86a9774 100644 --- a/tests/test_it.py +++ b/tests/test_it.py @@ -90,8 +90,7 @@ def iso2epoch(iso_str): for expected_iso, dateutil_time in plan: self.assertEqual(seconds_since_epoch(dateutil_time), - iso2epoch(expected_iso), - msg='expected_iso: {}, dateutil_time: {}'.format(expected_iso, dateutil_time)) + iso2epoch(expected_iso)) self.assertRaises(UnknownDateError, seconds_since_epoch, '???')