diff --git a/.gitignore b/.gitignore index b5efc896..f12c2d06 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ Documentation/ .pytest_cache/ coverage.xml htmlcov/ +.hypothesis/ diff --git a/.travis.yml b/.travis.yml index 0c30ee58..ae5333dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,10 @@ language: python sudo: required dist: xenial -cache: pip +cache: + pip: true + directories: + - $TRAVIS_BUILD_DIR/.hypothesis python: - '2.7' - '3.4' @@ -27,50 +30,61 @@ matrix: include: - python: 2.7 env: TOXENV=flake8 + cache: pip stage: lint - python: 2.7 env: TOXENV=flakeplus + cache: pip stage: lint - python: 2.7 env: TOXENV=pydocstyle + cache: pip stage: lint - python: 2.7 env: TOXENV=apicheck + cache: pip stage: lint - python: 2.7 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: 3.4 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: 3.5 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: 3.6 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: 3.7 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: pypy2.7-6.0 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration - python: pypy3.5-6.0 env: MATRIX_TOXENV=integration-rabbitmq services: - docker + cache: pip stage: integration before_install: diff --git a/amqp/serialization.py b/amqp/serialization.py index 758a3c03..5296e37f 100644 --- a/amqp/serialization.py +++ b/amqp/serialization.py @@ -7,10 +7,13 @@ from __future__ import absolute_import, unicode_literals import calendar +import struct import sys from datetime import datetime from decimal import Decimal from io import BytesIO +import logging +from pprint import pformat from .exceptions import FrameSyntaxError from .five import int_types, items, long_t, string, string_t @@ -19,10 +22,12 @@ from .utils import bytes_to_str as pstr_t from .utils import str_to_bytes +AMQP_LOGGER = logging.getLogger('amqp') + ftype_t = chr if sys.version_info[0] == 3 else None ILLEGAL_TABLE_TYPE = """\ - Table type {0!r} not handled by amqp. +Table type {0!r} not handled by amqp. """ ILLEGAL_TABLE_TYPE_WITH_KEY = """\ @@ -30,7 +35,14 @@ """ ILLEGAL_TABLE_TYPE_WITH_VALUE = """\ - Table type {0!r} not handled by amqp. [value: {1!r}] +Table type {0!r} not handled by amqp. [value: {1!r}] +""" + +ILLEGAL_TABLE_VALUE_WITH_TYPE = """\ +Table type {0!r} cannot store {1!r} as it's value is below or above +the storage limits. +Details: +{2!r} """ @@ -58,11 +70,11 @@ def _read_item(buf, offset=0, unpack_from=unpack_from, ftype_t=ftype_t): offset += blen # 'b': short-short int elif ftype == 'b': - val, = unpack_from('>B', buf, offset) + val, = unpack_from('>b', buf, offset) offset += 1 # 'B': short-short unsigned int elif ftype == 'B': - val, = unpack_from('>b', buf, offset) + val, = unpack_from('>B', buf, offset) offset += 1 # 'U': short int elif ftype == 'U': @@ -242,6 +254,9 @@ def loads(format, buf, offset=0, else: raise FrameSyntaxError(ILLEGAL_TABLE_TYPE.format(p)) append(val) + if __debug__: + AMQP_LOGGER.debug("Deserialized the following stream:\n%s\nInto these values:\n%s", + buf, pformat(values)) return values, offset @@ -322,7 +337,12 @@ def dumps(format, values): write(pack('>Q', long_t(calendar.timegm(val.utctimetuple())))) _flushbits(bits, write) - return out.getvalue() + result = out.getvalue() + if __debug__: + AMQP_LOGGER.debug("Serialized the following values:\n%s\nInto:\n%s", + pformat(values), result) + + return result def _write_table(d, write, bits, pack=pack): @@ -338,6 +358,10 @@ def _write_table(d, write, bits, pack=pack): except ValueError: raise FrameSyntaxError( ILLEGAL_TABLE_TYPE_WITH_KEY.format(type(v), k, v)) + except struct.error as e: + raise FrameSyntaxError( + ILLEGAL_TABLE_VALUE_WITH_TYPE.format(type(v), v, str(e)) + ) table_data = out.getvalue() write(pack('>I', len(table_data))) write(table_data) @@ -348,10 +372,14 @@ def _write_array(l, write, bits, pack=pack): awrite = out.write for v in l: try: - _write_item(v, awrite, bits) + _write_item(v, awrite, bits, array_types=True) except ValueError: raise FrameSyntaxError( ILLEGAL_TABLE_TYPE_WITH_VALUE.format(type(v), v)) + except struct.error as e: + raise FrameSyntaxError( + ILLEGAL_TABLE_VALUE_WITH_TYPE.format(type(v), v, str(e)) + ) array_data = out.getvalue() write(pack('>I', len(array_data))) write(array_data) @@ -361,21 +389,50 @@ def _write_item(v, write, bits, pack=pack, string_t=string_t, bytes=bytes, string=string, bool=bool, float=float, int_types=int_types, Decimal=Decimal, datetime=datetime, dict=dict, list=list, tuple=tuple, - None_t=None): - if isinstance(v, (string_t, bytes)): - if isinstance(v, string): - v = v.encode('utf-8', 'surrogatepass') - write(pack('>cI', b'S', len(v))) + None_t=None, array_types=False): + if isinstance(v, string_t): + v = v.encode('utf-8', 'surrogatepass') + string_length = len(v) + + if array_types: + # Only arrays support short strings + if string_length > 255: + write(pack('>cI', b'S', string_length)) + else: + write(pack('>cB', b's', string_length)) + else: + write(pack('>cI', b'S', string_length)) + write(v) + elif isinstance(v, bytes): + write(pack('>cI', b'x', len(v))) write(v) elif isinstance(v, bool): write(pack('>cB', b't', int(v))) elif isinstance(v, float): write(pack('>cd', b'd', v)) elif isinstance(v, int_types): - if v > 2147483647 or v < -2147483647: + if 127 >= v >= -128: + # signed short short int + write(pack('>cb', b'b', v)) + elif 255 >= v >= 0: + # unsigned short short int + write(pack('>cB', b'B', v)) + elif 32767 >= v >= -32768: + # short int + write(pack('>ch', b'U', v)) + elif 65535 >= v >= 0: + # unsigned short int + write(pack('>cH', b'u', v)) + elif 2147483647 >= v >= -2147483648: + # long int + write(pack('>cl', b'I', v)) + elif 4294967295 >= v >= 0: + # unsigned long int + write(pack('>cL', b'i', v)) + elif 9223372036854775807 >= v >= -9223372036854775807: write(pack('>cq', b'L', v)) - else: - write(pack('>ci', b'I', v)) + elif 18446744073709551615 >= v >= 0: + write(pack('>cQ', b'l', v)) elif isinstance(v, Decimal): sign, digits, exponent = v.as_tuple() v = 0 diff --git a/requirements/test.txt b/requirements/test.txt index d523f9e3..98821c0d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,3 +1,4 @@ case>=1.3.1 pytest>=3.0 pytest-sugar>=0.9.1 +hypothesis>=4.8.0 diff --git a/t/integration/test_rmq.py b/t/integration/test_rmq.py index e5f04962..d7c4c3a6 100644 --- a/t/integration/test_rmq.py +++ b/t/integration/test_rmq.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +import logging import os import pytest @@ -16,7 +17,8 @@ def connection(request): @pytest.mark.env('rabbitmq') -def test_connect(connection): +def test_connect(connection, caplog): + caplog.set_level(logging.DEBUG) connection.connect() connection.close() @@ -46,6 +48,8 @@ def setup_conn(self, connection): ) ) def test_publish_consume(self, publish_method, mandatory, immediate): + message = amqp.Message('Unittest') + method = getattr(self.channel, publish_method) callback = Mock() self.channel.queue_declare( queue='py-amqp-unittest', durable=False, exclusive=True @@ -56,8 +60,8 @@ def test_publish_consume(self, publish_method, mandatory, immediate): # http://www.rabbitmq.com/blog/2012/11/19/breaking-things-with-rabbitmq-3-0/ if immediate and publish_method == "basic_publish_confirm": with pytest.raises(amqp.exceptions.AMQPNotImplementedError) as exc: - getattr(self.channel, publish_method)( - amqp.Message('Unittest'), + method( + message, routing_key='py-amqp-unittest', mandatory=mandatory, immediate=immediate @@ -69,8 +73,8 @@ def test_publish_consume(self, publish_method, mandatory, immediate): return else: - getattr(self.channel, publish_method)( - amqp.Message('Unittest'), + method( + message, routing_key='py-amqp-unittest', mandatory=mandatory, immediate=immediate diff --git a/t/unit/test_serialization.py b/t/unit/test_serialization.py index 9a3a0bf1..3a1ec674 100644 --- a/t/unit/test_serialization.py +++ b/t/unit/test_serialization.py @@ -1,16 +1,34 @@ from __future__ import absolute_import, unicode_literals +import os +import sys from datetime import datetime from decimal import Decimal from math import ceil +from string import printable import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st from amqp.basic_message import Message from amqp.exceptions import FrameSyntaxError from amqp.platform import pack from amqp.serialization import GenericContent, _read_item, dumps, loads +CI = os.environ.get("CI") +SUPPRESSED_HEALTH_CHECKS = (HealthCheck.too_slow,) if CI else () + + +def _filter_large_decimals(v): + sign, digits, exponent = v.as_tuple() + v = 0 + for d in digits: + v = (v * 10) + d + if v > 2147483647 or v < -2147483647: + return False + return True + class _ANY(object): @@ -21,6 +39,39 @@ def __ne__(self, other): return other is None +base_types_strategy = ( + st.integers(min_value=-9223372036854775807, + max_value=18446744073709551615) | # noqa: W503 + st.booleans() | # noqa: W503 + st.text() | # noqa: W503 + st.builds( + lambda d: d.replace(microsecond=0), + st.datetimes( + min_value=datetime(2000, 1, 1), + max_value=datetime(2300, 1, 1) + ) + ) | # noqa: W503 + st.decimals( + min_value=Decimal(-2147483648), + max_value=Decimal(2147483647), + allow_nan=False + ).filter(_filter_large_decimals) | # noqa: W503 + st.floats(allow_nan=False) | # noqa: W503 + st.binary() | # noqa: W503 + st.none() +) + +arrays_strategy = st.recursive( + st.deferred(lambda: base_types_strategy), + st.lists +).filter(lambda x: isinstance(x, list)) + +tables_strategy = st.recursive( + st.deferred(lambda: base_types_strategy | arrays_strategy), + lambda y: st.dictionaries(st.text(printable, max_size=255), y) +).filter(lambda x: isinstance(x, dict)) + + class test_serialization: @pytest.mark.parametrize('descr,frame,expected,cast', [ @@ -73,31 +124,22 @@ def test_loads_unknown_type(self): loads('y', 'asdsad') def test_float(self): - assert (int(loads(b'fb', dumps(b'fb', [32.31, False]))[0][0] * 100) == - 3231) - - def test_table(self): - table = { - 'foo': 32, - 'bar': 'baz', - 'nil': None, - 'array': [ - 1, True, 'bar' - ] - } + actual = loads(b'fb', dumps(b'fb', [32.31, False]))[0][0] * 100 + assert int(actual) == 3231 + + @given(tables_strategy) + @settings(suppress_health_check=SUPPRESSED_HEALTH_CHECKS) + @pytest.mark.xfail(sys.version_info <= (3, 0), + reason="Unicode Problems on Python 2.x") + def test_table(self, table): assert loads(b'F', dumps(b'F', [table]))[0][0] == table - def test_array(self): - array = [ - 'A', 1, True, 33.3, - Decimal('55.5'), Decimal('-3.4'), - datetime(2015, 3, 13, 10, 23), - {'quick': 'fox', 'amount': 1}, - [3, 'hens'], - None, - ] + @given(arrays_strategy) + @settings(suppress_health_check=SUPPRESSED_HEALTH_CHECKS) + @pytest.mark.xfail(sys.version_info <= (3, 0), + reason="Unicode Problems on Python 2.x") + def test_array(self, array): expected = list(array) - expected[6] = _ANY() assert expected == loads('A', dumps('A', [array]))[0][0] diff --git a/tox.ini b/tox.ini index 1fb953ca..e6b6580f 100644 --- a/tox.ini +++ b/tox.ini @@ -30,9 +30,12 @@ basepython = install_command = python -m pip --disable-pip-version-check install {opts} {packages} commands_pre = integration-rabbitmq: ./wait_for_rabbitmq.sh +commands_post = + integration-rabbitmq: ./rabbitmq_logs.sh docker = integration-rabbitmq: rabbitmq:alpine dockerenv = PYAMQP_INTEGRATION_INSTANCE=1 +passenv = CI [testenv:apicheck] commands =