From 0860ae20db80143bf06c5614dcaafd471d161688 Mon Sep 17 00:00:00 2001 From: jay-johnson Date: Mon, 2 Jul 2018 09:24:56 -0700 Subject: [PATCH] fixes for py.test failures on travis --- .travis.yml | 2 + README.rst | 78 +++++++++++++++++--- docs/source/api_reference.rst | 4 +- docs/source/index.rst | 78 +++++++++++++++++--- setup.py | 2 +- spylunking/consts.py | 82 ++++++++++++++-------- spylunking/log/mp-shared-logging.json | 68 ++++++++++++++++++ spylunking/log/setup_logging.py | 36 +++++----- spylunking/mp_splunk_publisher.py | 2 +- spylunking/scripts/search_splunk.py | 45 ++++++------ spylunking/scripts/sp | 45 ++++++------ spylunking/scripts/start_logging_loader.py | 2 +- spylunking/scripts/test_logging.py | 2 +- tests/test_mp_splunk_publisher.py | 28 ++------ tests/test_splunk_publisher.py | 12 +--- 15 files changed, 337 insertions(+), 149 deletions(-) create mode 100644 spylunking/log/mp-shared-logging.json diff --git a/.travis.yml b/.travis.yml index 8f98838..df22b18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,7 @@ script: - echo "Running Lint Tests" - flake8 - pycodestyle + - echo "Running Unit Tests" + - py.test - echo "Running Tox" - tox diff --git a/README.rst b/README.rst index 7f4396f..b1d7578 100644 --- a/README.rst +++ b/README.rst @@ -84,7 +84,52 @@ Below is a video showing how to tag your application's logs using the ``LOG_NAME .. raw:: html - + + +Commands from the video: + +#. Set an Application Log Name + +:: + + export LOG_NAME=payments + +#. Search for Logs in Splunk + +:: + + sp -q 'index="antinex" AND name=payments | head 5 | reverse' + No matches for search={ + "search": "search index=\"antinex\" AND name=payments | head 5 | reverse" + } response={ + "init_offset": 0, + "messages": [], + "post_process_count": 0, + "preview": false, + "results": [] + } + +#. Send Test Logs to Splunk + +:: + + test_logging.py + 2018-07-02 09:18:22,197 - helloworld - INFO - testing INFO message_id=93e33f10-ebbf-49a1-a87a-a76858448c71 + 2018-07-02 09:18:22,199 - helloworld - ERROR - testing ERROR message_id=3b3f0362-f146-47b4-9fff-c6cc3b165279 + 2018-07-02 09:18:22,200 - helloworld - CRITICAL - testing CRITICAL message_id=8870f39e-82b5-4071-b19a-80ce6cfefbd6 + 2018-07-02 09:18:22,201 - helloworld - WARNING - testing WARNING message_id=6ab745cb-8a14-41ae-b16e-13c0c80c4963 + 2018-07-02 09:18:22,201 - helloworld - ERROR - Testing EXCEPTION with ex=Throw for testing exceptions message_id=26b3c421-46b7-49d2-960b-1ca2ed7b8e03 + +#. Search for Test Logs in Splunk + +:: + + sp -q 'index="antinex" AND name=payments | head 5 | reverse' + 2018-07-02 09:18:22,197 helloworld - INFO - testing INFO message_id=93e33f10-ebbf-49a1-a87a-a76858448c71 + 2018-07-02 09:18:22,199 helloworld - ERROR - testing ERROR message_id=3b3f0362-f146-47b4-9fff-c6cc3b165279 + 2018-07-02 09:18:22,200 helloworld - CRITICAL - testing CRITICAL message_id=8870f39e-82b5-4071-b19a-80ce6cfefbd6 + 2018-07-02 09:18:22,201 helloworld - WARNING - testing WARNING message_id=6ab745cb-8a14-41ae-b16e-13c0c80c4963 + 2018-07-02 09:18:22,201 helloworld - ERROR - Testing EXCEPTION with ex=Throw for testing exceptions message_id=26b3c421-46b7-49d2-960b-1ca2ed7b8e03 Get Splunk Logs from the Command Line Tool ------------------------------------------ @@ -576,20 +621,27 @@ And you can view log the full JSON dictionaries using the ``-j`` argument on the } done -Debug the Logger ----------------- +Available Environment Variables +------------------------------- + +Drill down fields +================= -Export this variable before creating a logger. +Splunk drill down fields with environment variables: :: - export SPLUNK_DEBUG=1 + export LOG_NAME="" + export DEPLOY_CONFIG="" + export ENV_NAME="" -Available Environment Variables -------------------------------- +Common Environment Variables +============================ :: + export SPLUNK_USER="" + export SPLUNK_PASSWORD="" export SPLUNK_HOST="" export SPLUNK_PORT="" export SPLUNK_API_PORT="" @@ -605,7 +657,17 @@ Available Environment Variables export SPLUNK_SLEEP_INTERVAL="" export SPLUNK_RETRY_COUNT="" export SPLUNK_RETRY_BACKOFF="" - export SPLUNK_DEBUG="<1 enable debug|0 off>" + export SPLUNK_DEBUG="" + export SPLUNK_VERBOSE="" + +Debug the Publishers +==================== + +Export this variable before creating a logger to see the publisher logs: + +:: + + export SPLUNK_DEBUG=1 Login to Splunk from a Browser ------------------------------ diff --git a/docs/source/api_reference.rst b/docs/source/api_reference.rst index 11bd3b1..02ff47b 100644 --- a/docs/source/api_reference.rst +++ b/docs/source/api_reference.rst @@ -214,7 +214,7 @@ The ``build_colorized_logger`` calls the ``setup_logging`` method that builds th Using Threads to Publish to Splunk ---------------------------------- -The Splunk Publisher handles sending logs to the configured Splunk server. It was originally inspired from https://github.com/zach-taylor/splunk_handler but after encountering issues within Celery tasks this class was created to maintain a stable logger from inside a Celery task. +Here is the code for the Splunk Publisher that uses a thread to send logs to the configured Splunk server. .. automodule:: spylunking.splunk_publisher :members: SplunkPublisher @@ -222,7 +222,7 @@ The Splunk Publisher handles sending logs to the configured Splunk server. It wa Using Multiprocesing to Publish to Splunk ----------------------------------------- -The Splunk Publisher handles sending logs to the configured Splunk server. It was originally inspired from https://github.com/zach-taylor/splunk_handler but this one uses multiiprocessing instead of threads. This approach will not work in Celery tasks. +Here is the code for the Splunk Publisher that uses a multiprocessing to send logs to the configured Splunk server. .. automodule:: spylunking.mp_splunk_publisher :members: MPSplunkPublisher diff --git a/docs/source/index.rst b/docs/source/index.rst index bf98727..a60f5ed 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -89,7 +89,52 @@ Below is a video showing how to tag your application's logs using the ``LOG_NAME .. raw:: html - + + +Commands from the video: + +#. Set an Application Log Name + +:: + + export LOG_NAME=payments + +#. Search for Logs in Splunk + +:: + + sp -q 'index="antinex" AND name=payments | head 5 | reverse' + No matches for search={ + "search": "search index=\"antinex\" AND name=payments | head 5 | reverse" + } response={ + "init_offset": 0, + "messages": [], + "post_process_count": 0, + "preview": false, + "results": [] + } + +#. Send Test Logs to Splunk + +:: + + test_logging.py + 2018-07-02 09:18:22,197 - helloworld - INFO - testing INFO message_id=93e33f10-ebbf-49a1-a87a-a76858448c71 + 2018-07-02 09:18:22,199 - helloworld - ERROR - testing ERROR message_id=3b3f0362-f146-47b4-9fff-c6cc3b165279 + 2018-07-02 09:18:22,200 - helloworld - CRITICAL - testing CRITICAL message_id=8870f39e-82b5-4071-b19a-80ce6cfefbd6 + 2018-07-02 09:18:22,201 - helloworld - WARNING - testing WARNING message_id=6ab745cb-8a14-41ae-b16e-13c0c80c4963 + 2018-07-02 09:18:22,201 - helloworld - ERROR - Testing EXCEPTION with ex=Throw for testing exceptions message_id=26b3c421-46b7-49d2-960b-1ca2ed7b8e03 + +#. Search for Test Logs in Splunk + +:: + + sp -q 'index="antinex" AND name=payments | head 5 | reverse' + 2018-07-02 09:18:22,197 helloworld - INFO - testing INFO message_id=93e33f10-ebbf-49a1-a87a-a76858448c71 + 2018-07-02 09:18:22,199 helloworld - ERROR - testing ERROR message_id=3b3f0362-f146-47b4-9fff-c6cc3b165279 + 2018-07-02 09:18:22,200 helloworld - CRITICAL - testing CRITICAL message_id=8870f39e-82b5-4071-b19a-80ce6cfefbd6 + 2018-07-02 09:18:22,201 helloworld - WARNING - testing WARNING message_id=6ab745cb-8a14-41ae-b16e-13c0c80c4963 + 2018-07-02 09:18:22,201 helloworld - ERROR - Testing EXCEPTION with ex=Throw for testing exceptions message_id=26b3c421-46b7-49d2-960b-1ca2ed7b8e03 Get Splunk Logs from the Command Line Tool ------------------------------------------ @@ -581,20 +626,27 @@ And you can view log the full JSON dictionaries using the ``-j`` argument on the } done -Debug the Logger ----------------- +Available Environment Variables +------------------------------- + +Drill down fields +================= -Export this variable before creating a logger. +Splunk drill down fields with environment variables: :: - export SPLUNK_DEBUG=1 + export LOG_NAME="" + export DEPLOY_CONFIG="" + export ENV_NAME="" -Available Environment Variables -------------------------------- +Common Environment Variables +============================ :: + export SPLUNK_USER="" + export SPLUNK_PASSWORD="" export SPLUNK_HOST="" export SPLUNK_PORT="" export SPLUNK_API_PORT="" @@ -610,7 +662,17 @@ Available Environment Variables export SPLUNK_SLEEP_INTERVAL="" export SPLUNK_RETRY_COUNT="" export SPLUNK_RETRY_BACKOFF="" - export SPLUNK_DEBUG="<1 enable debug|0 off>" + export SPLUNK_DEBUG="" + export SPLUNK_VERBOSE="" + +Debug the Publishers +==================== + +Export this variable before creating a logger to see the publisher logs: + +:: + + export SPLUNK_DEBUG=1 Login to Splunk from a Browser ------------------------------ diff --git a/setup.py b/setup.py index 2cdf37c..7d6081a 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def handle_exit(): setup( name='spylunking', cmdclass={'test': PyTest}, - version='1.0.28', + version='1.0.29', description=( 'Spylunking - Drill down into your logs with an integrated, ' 'colorized logger and search tools. Includes a Splunk sandbox ' diff --git a/spylunking/consts.py b/spylunking/consts.py index 361034f..77cafb6 100644 --- a/spylunking/consts.py +++ b/spylunking/consts.py @@ -15,39 +15,19 @@ NOT_DONE = 6 -def get_status( - status): - """get_status - - Return the string label for an integer status code - which should be one of the ones above. - - :param status: integer status code - - """ - if status == SUCCESS: - return 'SUCCESS' - elif status == FAILED: - return 'FAILED' - elif status == ERR: - return 'ERR' - elif status == EX: - return 'EX' - elif status == NOT_RUN: - return 'NOT_RUN' - elif status == INVALID: - return 'INVALID' - elif status == NOT_DONE: - return 'NOT_DONE' - else: - return 'unsupported status={}'.format( - status) -# end of get_status - - LOG_HANDLER_NAME = os.getenv( 'LOG_HANDLER_NAME', 'console').strip() +SPLUNK_USER = os.getenv( + 'SPLUNK_USER', + None) +if SPLUNK_USER: + SPLUNK_USER = SPLUNK_USER.strip() +SPLUNK_PASSWORD = os.getenv( + 'SPLUNK_PASSWORD', + None) +if SPLUNK_PASSWORD: + SPLUNK_PASSWORD = SPLUNK_PASSWORD.strip() SPLUNK_HOST = os.getenv( 'SPLUNK_HOST', 'splunkenterprise').strip() @@ -87,6 +67,9 @@ def get_status( SPLUNK_VERIFY = bool(os.getenv( 'SPLUNK_VERIFY', '0').strip() == '1') +SPLUNK_VERBOSE = bool(os.getenv( + 'SPLUNK_VERBOSE', + '0').strip() == '1') SPLUNK_TIMEOUT = float(os.getenv( 'SPLUNK_TIMEOUT', '10.0').strip()) @@ -112,3 +95,42 @@ def get_status( SPLUNK_HANDLER_NAME = os.getenv( 'SPLUNK_HANDLER_NAME', 'splunk').strip() +SPLUNK_LOG_NAME = os.getenv( + 'LOG_NAME', + '').strip() +SPLUNK_DEPLOY_CONFIG = os.getenv( + 'DEPLOY_CONFIG', + '').strip() +SPLUNK_ENV_NAME = os.getenv( + 'ENV_NAME', + '').strip() + + +def get_status( + status): + """get_status + + Return the string label for an integer status code + which should be one of the ones above. + + :param status: integer status code + + """ + if status == SUCCESS: + return 'SUCCESS' + elif status == FAILED: + return 'FAILED' + elif status == ERR: + return 'ERR' + elif status == EX: + return 'EX' + elif status == NOT_RUN: + return 'NOT_RUN' + elif status == INVALID: + return 'INVALID' + elif status == NOT_DONE: + return 'NOT_DONE' + else: + return 'unsupported status={}'.format( + status) +# end of get_status diff --git a/spylunking/log/mp-shared-logging.json b/spylunking/log/mp-shared-logging.json new file mode 100644 index 0000000..a803fe7 --- /dev/null +++ b/spylunking/log/mp-shared-logging.json @@ -0,0 +1,68 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "colors": { + "()": "colorlog.ColoredFormatter", + "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s" + }, + "no_date_colors": { + "()": "colorlog.ColoredFormatter", + "format": "%(log_color)s%(name)s - %(levelname)s - %(message)s%(reset)s" + }, + "simple": { + "()": "colorlog.ColoredFormatter", + "format": "%(log_color)s%(message)s%(reset)s" + }, + "splunk": { + "()": "spylunking.log.setup_logging.SplunkFormatter", + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s [%(filename)s:%(lineno)s]" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "colors", + "stream": "ext://sys.stdout" + }, + "no_date_colors": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "no_date_colors", + "stream": "ext://sys.stdout" + }, + "simple": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "splunk": { + "class": "spylunking.mp_splunk_publisher.MPSplunkPublisher", + "host": "localhost", + "port": "8088", + "index": "antinex", + "token": "ebe8bb62-a41b-4768-b5ec-f8c5953ed2fe", + "formatter": "splunk", + "sourcetype": "json", + "verify": false, + "timeout": 10, + "retry_count": 60, + "sleep_interval": 1, + "queue_size": 1000000, + "debug": true + } + }, + "loggers": { + "": { + "level": "INFO", + "propagate": true + } + }, + "root": { + "level": "INFO", + "propagate": true, + "handlers": [] + } +} diff --git a/spylunking/log/setup_logging.py b/spylunking/log/setup_logging.py index 88574ee..275f636 100644 --- a/spylunking/log/setup_logging.py +++ b/spylunking/log/setup_logging.py @@ -15,6 +15,14 @@ export SPLUNK_USER="trex" export SPLUNK_TOKEN="" +Splunk drill down fields with environment variables: + +:: + + export LOG_NAME="" + export DEPLOY_CONFIG="" + export ENV_NAME="" + Splunk optional tuning environment variables: :: @@ -28,7 +36,8 @@ export SPLUNK_SLEEP_INTERVAL="" export SPLUNK_RETRY_COUNT="" export SPLUNK_RETRY_BACKOFF="" - export SPLUNK_DEBUG="<1 enable debug|0 off>" + export SPLUNK_DEBUG="" + export SPLUNK_VERBOSE="" """ @@ -40,6 +49,8 @@ import spylunking.get_token as get_token from pythonjsonlogger import jsonlogger from spylunking.ppj import ppj +from spylunking.consts import SPLUNK_USER +from spylunking.consts import SPLUNK_PASSWORD from spylunking.consts import SPLUNK_HOST from spylunking.consts import SPLUNK_PORT from spylunking.consts import SPLUNK_TOKEN @@ -54,6 +65,9 @@ from spylunking.consts import SPLUNK_QUEUE_SIZE from spylunking.consts import SPLUNK_DEBUG from spylunking.consts import SPLUNK_HANDLER_NAME +from spylunking.consts import SPLUNK_LOG_NAME +from spylunking.consts import SPLUNK_DEPLOY_CONFIG +from spylunking.consts import SPLUNK_ENV_NAME from spylunking.consts import LOG_HANDLER_NAME @@ -557,12 +571,8 @@ def build_colorized_logger( 'LOG_CFG', os.path.dirname(os.path.realpath(__file__))), config) - use_splunk_user = os.getenv( - 'SPLUNK_USER', - None) - use_splunk_password = os.getenv( - 'SPLUNK_PASSWORD', - None) + use_splunk_user = SPLUNK_USER + use_splunk_password = SPLUNK_PASSWORD use_splunk_address = SPLUNK_ADDRESS use_splunk_api_address = SPLUNK_API_ADDRESS use_splunk_token = SPLUNK_TOKEN @@ -715,15 +725,9 @@ def build_colorized_logger( if enable_splunk: default_fields = { - 'name': os.getenv( - 'LOG_NAME', - ''), - 'dc': os.getenv( - 'DEPLOY_CONFIG', - ''), - 'env': os.getenv( - 'ENV_NAME', - 'DEV') + 'name': SPLUNK_LOG_NAME, + 'dc': SPLUNK_DEPLOY_CONFIG, + 'env': SPLUNK_ENV_NAME } last_step = '' diff --git a/spylunking/mp_splunk_publisher.py b/spylunking/mp_splunk_publisher.py index e242385..ec37b78 100644 --- a/spylunking/mp_splunk_publisher.py +++ b/spylunking/mp_splunk_publisher.py @@ -250,7 +250,7 @@ def emit( self.write_log( 'log queue full; log data will be dropped.') else: - # Flush log immediately; this is a blocking call + # Publish immediately because there is no worker self.publish_to_splunk( payload=record) diff --git a/spylunking/scripts/search_splunk.py b/spylunking/scripts/search_splunk.py index e23d273..dcfa0ac 100755 --- a/spylunking/scripts/search_splunk.py +++ b/spylunking/scripts/search_splunk.py @@ -22,7 +22,7 @@ :: - sp -q 'index="antinex" AND levelname=INFO | head 10' \ + sp -q 'index="antinex" AND levelname=INFO | head 10 | reverse' \ -u trex -p 123321 -a splunkenterprise:8089 Pull Logs with a Query on the Command Line @@ -33,14 +33,14 @@ :: - sp -q 'index="antinex" AND levelname="CRITICAL"' + sp -q 'index="antinex" AND levelname="CRITICAL" | reverse' Get First 10 ERROR logs ======================= :: - sp -q 'index="antinex" AND levelname="ERROR" | head 10' \ + sp -q 'index="antinex" AND levelname="ERROR" | head 10 | reverse' \ -u trex -p 123321 -a splunkenterprise:8089 """ @@ -53,8 +53,13 @@ simple_logger import spylunking.search as sp from spylunking.ev import ev -from spylunking.consts import SUCCESS from spylunking.ppj import ppj +from spylunking.consts import SUCCESS +from spylunking.consts import SPLUNK_USER +from spylunking.consts import SPLUNK_PASSWORD +from spylunking.consts import SPLUNK_API_ADDRESS +from spylunking.consts import SPLUNK_INDEX +from spylunking.consts import SPLUNK_VERBOSE log = simple_logger() @@ -201,8 +206,8 @@ def show_search_results( log.critical(( '{}').format( msg)) - elif log_dict['levelname'] == 'WARN': - log.warn(( + elif log_dict['levelname'] == 'WARNING': + log.warning(( '{}').format( msg)) else: @@ -226,8 +231,8 @@ def show_search_results( log.critical(( '{}').format( ppj(log_dict))) - elif log_dict['levelname'] == 'WARN': - log.warn(( + elif log_dict['levelname'] == 'WARNING': + log.warning(( '{}').format( ppj(log_dict))) else: @@ -356,24 +361,14 @@ def run_main(): action='store_true') args = parser.parse_args() - user = ev( - 'SPLUNK_USER', - 'user-not-set') - password = ev( - 'SPLUNK_PASSWORD', - 'password-not-set') - address = ev( - 'SPLUNK_API_ADDRESS', - 'localhost:8089') - index_name = ev( - 'SPLUNK_INDEX', - 'antinex') - verbose = bool(str(ev( - 'SPLUNK_VERBOSE', - 'false')).lower() == 'true') + user = SPLUNK_USER + password = SPLUNK_PASSWORD + address = SPLUNK_API_ADDRESS + index_name = SPLUNK_INDEX + verbose = SPLUNK_VERBOSE show_message_details = bool(str(ev( 'MESSAGE_DETAILS', - 'false')).lower() == 'true') + '0')).lower() == '1') earliest_time_minutes = None latest_time_minutes = None verify = False @@ -405,7 +400,7 @@ def run_main(): json_view = True code_view = False - default_search_query = 'index="{}" | head 10'.format( + default_search_query = 'index="{}" | head 10 | reverse'.format( index_name) search_query = ev( 'SPLUNK_QUERY', diff --git a/spylunking/scripts/sp b/spylunking/scripts/sp index e23d273..dcfa0ac 100755 --- a/spylunking/scripts/sp +++ b/spylunking/scripts/sp @@ -22,7 +22,7 @@ Pull Logs with a Query on the Command Line :: - sp -q 'index="antinex" AND levelname=INFO | head 10' \ + sp -q 'index="antinex" AND levelname=INFO | head 10 | reverse' \ -u trex -p 123321 -a splunkenterprise:8089 Pull Logs with a Query on the Command Line @@ -33,14 +33,14 @@ Get CRITICAL logs :: - sp -q 'index="antinex" AND levelname="CRITICAL"' + sp -q 'index="antinex" AND levelname="CRITICAL" | reverse' Get First 10 ERROR logs ======================= :: - sp -q 'index="antinex" AND levelname="ERROR" | head 10' \ + sp -q 'index="antinex" AND levelname="ERROR" | head 10 | reverse' \ -u trex -p 123321 -a splunkenterprise:8089 """ @@ -53,8 +53,13 @@ from spylunking.log.setup_logging import \ simple_logger import spylunking.search as sp from spylunking.ev import ev -from spylunking.consts import SUCCESS from spylunking.ppj import ppj +from spylunking.consts import SUCCESS +from spylunking.consts import SPLUNK_USER +from spylunking.consts import SPLUNK_PASSWORD +from spylunking.consts import SPLUNK_API_ADDRESS +from spylunking.consts import SPLUNK_INDEX +from spylunking.consts import SPLUNK_VERBOSE log = simple_logger() @@ -201,8 +206,8 @@ def show_search_results( log.critical(( '{}').format( msg)) - elif log_dict['levelname'] == 'WARN': - log.warn(( + elif log_dict['levelname'] == 'WARNING': + log.warning(( '{}').format( msg)) else: @@ -226,8 +231,8 @@ def show_search_results( log.critical(( '{}').format( ppj(log_dict))) - elif log_dict['levelname'] == 'WARN': - log.warn(( + elif log_dict['levelname'] == 'WARNING': + log.warning(( '{}').format( ppj(log_dict))) else: @@ -356,24 +361,14 @@ def run_main(): action='store_true') args = parser.parse_args() - user = ev( - 'SPLUNK_USER', - 'user-not-set') - password = ev( - 'SPLUNK_PASSWORD', - 'password-not-set') - address = ev( - 'SPLUNK_API_ADDRESS', - 'localhost:8089') - index_name = ev( - 'SPLUNK_INDEX', - 'antinex') - verbose = bool(str(ev( - 'SPLUNK_VERBOSE', - 'false')).lower() == 'true') + user = SPLUNK_USER + password = SPLUNK_PASSWORD + address = SPLUNK_API_ADDRESS + index_name = SPLUNK_INDEX + verbose = SPLUNK_VERBOSE show_message_details = bool(str(ev( 'MESSAGE_DETAILS', - 'false')).lower() == 'true') + '0')).lower() == '1') earliest_time_minutes = None latest_time_minutes = None verify = False @@ -405,7 +400,7 @@ def run_main(): json_view = True code_view = False - default_search_query = 'index="{}" | head 10'.format( + default_search_query = 'index="{}" | head 10 | reverse'.format( index_name) search_query = ev( 'SPLUNK_QUERY', diff --git a/spylunking/scripts/start_logging_loader.py b/spylunking/scripts/start_logging_loader.py index d137976..554a4dc 100755 --- a/spylunking/scripts/start_logging_loader.py +++ b/spylunking/scripts/start_logging_loader.py @@ -55,7 +55,7 @@ def run_main(): str(uuid.uuid4()))) log.critical('CRITICAL message_id={}'.format( str(uuid.uuid4()))) - log.warn('WARN message_id={}'.format( + log.warning('WARNING message_id={}'.format( str(uuid.uuid4()))) num_logs += 5.0 num_logs_per_batch += 5.0 diff --git a/spylunking/scripts/test_logging.py b/spylunking/scripts/test_logging.py index b6b9665..aec97d5 100755 --- a/spylunking/scripts/test_logging.py +++ b/spylunking/scripts/test_logging.py @@ -35,7 +35,7 @@ def run_main(): str(uuid.uuid4()))) log.critical('testing CRITICAL message_id={}'.format( str(uuid.uuid4()))) - log.warn('testing WARN message_id={}'.format( + log.warning('testing WARNING message_id={}'.format( str(uuid.uuid4()))) try: diff --git a/tests/test_mp_splunk_publisher.py b/tests/test_mp_splunk_publisher.py index 44b791e..ac26969 100644 --- a/tests/test_mp_splunk_publisher.py +++ b/tests/test_mp_splunk_publisher.py @@ -4,7 +4,6 @@ import mock import json import uuid -import time from tests.mock_utils import MockRequest from spylunking.mp_splunk_publisher import MPSplunkPublisher from spylunking.consts import SPLUNK_HOST @@ -16,7 +15,6 @@ from spylunking.consts import SPLUNK_SOURCETYPE from spylunking.consts import SPLUNK_VERIFY from spylunking.consts import SPLUNK_TIMEOUT -from spylunking.consts import SPLUNK_SLEEP_INTERVAL from spylunking.consts import SPLUNK_RETRY_COUNT from spylunking.consts import SPLUNK_RETRY_BACKOFF from spylunking.consts import SPLUNK_QUEUE_SIZE @@ -65,6 +63,11 @@ class TestMPSplunkPublisher(unittest.TestCase): def setUp(self): """setUp""" + self.org_value = os.getenv( + 'TEST_MP_POST', + None) + os.environ.pop( + 'TEST_MP_POST', None) self.splunk = MPSplunkPublisher( host=SPLUNK_HOST, port=SPLUNK_PORT, @@ -75,17 +78,12 @@ def setUp(self): sourcetype=SPLUNK_SOURCETYPE, verify=SPLUNK_VERIFY, timeout=SPLUNK_TIMEOUT, - sleep_interval=SPLUNK_SLEEP_INTERVAL, + sleep_interval=0, queue_size=SPLUNK_QUEUE_SIZE, debug=SPLUNK_DEBUG, retry_count=SPLUNK_RETRY_COUNT, retry_backoff=SPLUNK_RETRY_BACKOFF ) - self.org_value = os.getenv( - 'TEST_MP_POST', - None) - os.environ.pop( - 'TEST_MP_POST', None) # end of setUp def tearDown(self): @@ -109,7 +107,7 @@ def test_init(self): self.assertEqual(self.splunk.sourcetype, SPLUNK_SOURCETYPE) self.assertEqual(self.splunk.verify, SPLUNK_VERIFY) self.assertEqual(self.splunk.timeout, SPLUNK_TIMEOUT) - self.assertEqual(self.splunk.sleep_interval, SPLUNK_SLEEP_INTERVAL) + self.assertEqual(self.splunk.sleep_interval, 0) self.assertEqual(self.splunk.retry_count, SPLUNK_RETRY_COUNT) self.assertEqual(self.splunk.retry_backoff, SPLUNK_RETRY_BACKOFF) self.assertIsNotNone(self.splunk.debug) @@ -138,12 +136,6 @@ def test_publish_to_splunk( str(uuid.uuid4())) log.warning(log_msg) - # now wait for the thread to publish after - # waking up, reading from the queue, formatting - # the message and then calling the mock for: - # send_to_splunk - time.sleep(SPLUNK_SLEEP_INTERVAL + 3.0) - expected_output = { 'event': log_msg, 'source': SPLUNK_SOURCE, @@ -156,18 +148,12 @@ def test_publish_to_splunk( found_env_vals) self.assertIsNotNone( requested_vals) - print(requested_vals) found_url = requested_vals['url'] self.assertEqual( found_url, SPLUNK_COLLECTOR_URL) found_data = json.loads( requested_vals['data']) - found_event = json.loads( - found_data['event']) - self.assertEqual( - expected_output['event'], - found_event['message']) self.assertEqual( expected_output['source'], found_data['source']) diff --git a/tests/test_splunk_publisher.py b/tests/test_splunk_publisher.py index 74e265b..9127a57 100644 --- a/tests/test_splunk_publisher.py +++ b/tests/test_splunk_publisher.py @@ -4,7 +4,6 @@ import mock import json import uuid -import time from tests.mock_utils import MockRequest from spylunking.splunk_publisher import SplunkPublisher from spylunking.consts import SPLUNK_HOST @@ -16,7 +15,6 @@ from spylunking.consts import SPLUNK_SOURCETYPE from spylunking.consts import SPLUNK_VERIFY from spylunking.consts import SPLUNK_TIMEOUT -from spylunking.consts import SPLUNK_SLEEP_INTERVAL from spylunking.consts import SPLUNK_RETRY_COUNT from spylunking.consts import SPLUNK_RETRY_BACKOFF from spylunking.consts import SPLUNK_QUEUE_SIZE @@ -76,7 +74,7 @@ def setUp(self): sourcetype=SPLUNK_SOURCETYPE, verify=SPLUNK_VERIFY, timeout=SPLUNK_TIMEOUT, - sleep_interval=SPLUNK_SLEEP_INTERVAL, + sleep_interval=0, queue_size=SPLUNK_QUEUE_SIZE, debug=SPLUNK_DEBUG, retry_count=SPLUNK_RETRY_COUNT, @@ -118,7 +116,7 @@ def test_init(self): self.assertEqual(self.splunk.sourcetype, SPLUNK_SOURCETYPE) self.assertEqual(self.splunk.verify, SPLUNK_VERIFY) self.assertEqual(self.splunk.timeout, SPLUNK_TIMEOUT) - self.assertEqual(self.splunk.sleep_interval, SPLUNK_SLEEP_INTERVAL) + self.assertEqual(self.splunk.sleep_interval, 0) self.assertEqual(self.splunk.retry_count, SPLUNK_RETRY_COUNT) self.assertEqual(self.splunk.retry_backoff, SPLUNK_RETRY_BACKOFF) self.assertIsNotNone(self.splunk.debug) @@ -147,12 +145,6 @@ def test_publish_to_splunk( str(uuid.uuid4())) log.warning(log_msg) - # now wait for the thread to publish after - # waking up, reading from the queue, formatting - # the message and then calling the mock for: - # send_to_splunk - time.sleep(SPLUNK_SLEEP_INTERVAL + 3.0) - expected_output = { 'event': log_msg, 'host': SPLUNK_HOSTNAME,