diff --git a/ChangeLog b/ChangeLog index c8a4112c..18f6b5ec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,18 @@ +Version 0.7.5 +https://github.com/edgewall/genshi/releases/tag/0.7.5 +(Nov 18 2020, from branches/stable/0.7.x) + + * Fix handling of slices containing function call, variable name and attribute + lookup AST nodes in Python 3.9 in template scripts (template expressions + already correctly handled these cases). Thank you to Roger Leigh for + finding this issue and contributing the fix for it. + * C speedup module now available for Python >= 3.3. Support was added for + PEP 393 (flexible string representation). Thank you to Inada Naoki for + contributing this major enhancement. + * Remove the custom 2to3 fixers (no longer used since the removal of 2to3 + in 0.7.4). + + Version 0.7.4 https://github.com/edgewall/genshi/releases/tag/0.7.4 (Nov 3 2020, from branches/stable/0.7.x) diff --git a/MANIFEST.in b/MANIFEST.in index 68a0e9b6..1e7de1fb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,9 @@ include ChangeLog include COPYING recursive-include genshi *.py *.c *.txt *.html -recursive-include scripts *.py -recursive-include fixes *.py recursive-include examples * recursive-include doc *.html *.css *.txt *.png *.gif *.py *.ini COPYING recursive-exclude doc/logo.lineform * exclude doc/2000ft.graffle global-exclude *.pyc -include fixes/*.* recursive-include genshi/template/tests/templates *.html *.txt diff --git a/fixes/__init__.py b/fixes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fixes/fix_unicode_in_strings.py b/fixes/fix_unicode_in_strings.py deleted file mode 100644 index 696c6023..00000000 --- a/fixes/fix_unicode_in_strings.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Fixer that changes expressions inside strings literals from u"..." to "...". - -""" - -import re -from lib2to3 import fixer_base - -_literal_re = re.compile(r"(.+?)\b[uU]([rR]?[\'\"])") - -class FixUnicodeInStrings(fixer_base.BaseFix): - - PATTERN = "STRING" - - def transform(self, node, results): - new = node.clone() - new.value = _literal_re.sub(r"\1\2", new.value) - return new diff --git a/genshi/_speedups.c b/genshi/_speedups.c index 600c7aee..d913a807 100644 --- a/genshi/_speedups.c +++ b/genshi/_speedups.c @@ -56,6 +56,8 @@ PyDoc_STRVAR(Markup__doc__, "Marks a string as being safe for inclusion in HTML/XML output without\n\ needing to be escaped."); +#if PY_VERSION_HEX < 0x03030000 + static PyObject * escape(PyObject *text, int quotes) { @@ -169,6 +171,132 @@ escape(PyObject *text, int quotes) return ret; } +#else /* if PY_VERSION_HEX < 0x03030000 */ + +static PyObject * +escape(PyObject *text, int quotes) +{ + PyObject *args, *ret; + + if (PyObject_TypeCheck(text, &MarkupType)) { + Py_INCREF(text); + return text; + } + if (PyObject_HasAttrString(text, "__html__")) { + ret = PyObject_CallMethod(text, "__html__", NULL); + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(ret); + return NULL; + } + PyTuple_SET_ITEM(args, 0, ret); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + + PyObject *in = PyObject_Str(text); + if (in == NULL) { + return NULL; + } + Py_ssize_t utf8_len; + const char *utf8 = PyUnicode_AsUTF8AndSize(in, &utf8_len); + if (utf8 == NULL) { + Py_DECREF(in); + return NULL; + } + + /* First we need to figure out how long the escaped string will be */ + Py_ssize_t len = 0, inn = 0; + for (Py_ssize_t i = 0; i < utf8_len; i++) { + switch (utf8[i]) { + case '&': len += 5; inn++; break; + case '"': len += quotes ? 5 : 1; inn += quotes ? 1 : 0; break; + case '<': + case '>': len += 4; inn++; break; + default: len++; + } + } + + /* Do we need to escape anything at all? */ + if (!inn) { + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF((PyObject *) in); + return NULL; + } + PyTuple_SET_ITEM(args, 0, in); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; + } + + char *buffer = (char*)PyMem_Malloc(len); + if (buffer == NULL) { + Py_DECREF(in); + return NULL; + } + + Py_ssize_t outn = 0; + const char *inp = utf8; + char *outp = buffer; + while (utf8_len > inp - utf8) { + if (outn == inn) { + /* copy rest of string if we have already replaced everything */ + memcpy(outp, inp, utf8_len - (inp - utf8)); + break; + } + switch (*inp) { + case '&': + memcpy(outp, "&", 5); + outp += 5; + outn++; + break; + case '"': + if (quotes) { + memcpy(outp, """, 5); + outp += 5; + outn++; + } else { + *outp++ = *inp; + } + break; + case '<': + memcpy(outp, "<", 4); + outp += 4; + outn++; + break; + case '>': + memcpy(outp, ">", 4); + outp += 4; + outn++; + break; + default: + *outp++ = *inp; + } + inp++; + } + + Py_DECREF(in); + + PyObject *out = PyUnicode_FromStringAndSize(buffer, len); + PyMem_Free(buffer); + if (out == NULL) { + return NULL; + } + args = PyTuple_New(1); + if (args == NULL) { + Py_DECREF(out); + return NULL; + } + PyTuple_SET_ITEM(args, 0, out); + ret = MarkupType.tp_new(&MarkupType, args, NULL); + Py_DECREF(args); + return ret; +} + +#endif /* if PY_VERSION_HEX < 0x03030000 */ + PyDoc_STRVAR(escape__doc__, "Create a Markup instance from a string and escape special characters\n\ it may contain (<, >, & and \").\n\ diff --git a/genshi/filters/i18n.py b/genshi/filters/i18n.py index 43c91713..65e7eaba 100644 --- a/genshi/filters/i18n.py +++ b/genshi/filters/i18n.py @@ -346,7 +346,7 @@ def __init__(self, value, template=None, namespaces=None, lineno=-1, def attach(cls, template, stream, value, namespaces, pos): if type(value) is dict: numeral = value.get('numeral', '').strip() - assert numeral is not '', "at least pass the numeral param" + assert numeral != '', "at least pass the numeral param" params = [v.strip() for v in value.get('params', '').split(',')] value = '%s; ' % numeral + ', '.join(params) return super(ChooseDirective, cls).attach(template, stream, value, diff --git a/genshi/template/astutil.py b/genshi/template/astutil.py index 9c6ff29f..d5b33bdf 100644 --- a/genshi/template/astutil.py +++ b/genshi/template/astutil.py @@ -734,17 +734,13 @@ def _process_slice(node): self.visit(node.step) elif isinstance(node, _ast.Index): self.visit(node.value) - elif isinstance(node, _ast_Constant): - self.visit_Constant(node) - elif isinstance(node, _ast.UnaryOp): - self.visit_UnaryOp(node) elif isinstance(node, _ast.ExtSlice): self.visit(node.dims[0]) for dim in node.dims[1:]: self._write(', ') self.visit(dim) else: - raise NotImplementedError('Slice type not implemented') + self.visit(node) _process_slice(node.slice) self._write(']') diff --git a/genshi/template/tests/eval.py b/genshi/template/tests/eval.py index e8910189..a8dcd844 100644 --- a/genshi/template/tests/eval.py +++ b/genshi/template/tests/eval.py @@ -360,6 +360,31 @@ def test_slice_negative_end(self): res = expr.evaluate({'numbers': list(range(5))}) self.assertEqual([0, 1, 2, 3], res) + def test_slice_constant(self): + expr = Expression("numbers[1]") + res = expr.evaluate({"numbers": list(range(5))}) + self.assertEqual(res, 1) + + def test_slice_call(self): + def f(): + return 2 + expr = Expression("numbers[f()]") + res = expr.evaluate({"numbers": list(range(5)), "f": f}) + self.assertEqual(res, 2) + + def test_slice_name(self): + expr = Expression("numbers[v]") + res = expr.evaluate({"numbers": list(range(5)), "v": 2}) + self.assertEqual(res, 2) + + def test_slice_attribute(self): + class ValueHolder: + def __init__(self): + self.value = 3 + expr = Expression("numbers[obj.value]") + res = expr.evaluate({"numbers": list(range(5)), "obj": ValueHolder()}) + self.assertEqual(res, 3) + def test_access_undefined(self): expr = Expression("nothing", filename='index.html', lineno=50, lookup='lenient') @@ -941,6 +966,71 @@ def test_with_statement_with_multiple_items(self): finally: os.remove(path) + def test_slice(self): + suite = Suite("x = numbers[0:2]") + data = {"numbers": [0, 1, 2, 3]} + suite.execute(data) + self.assertEqual([0, 1], data["x"]) + + def test_slice_with_vars(self): + suite = Suite("x = numbers[start:end]") + data = {"numbers": [0, 1, 2, 3], "start": 0, "end": 2} + suite.execute(data) + self.assertEqual([0, 1], data["x"]) + + def test_slice_copy(self): + suite = Suite("x = numbers[:]") + data = {"numbers": [0, 1, 2, 3]} + suite.execute(data) + self.assertEqual([0, 1, 2, 3], data["x"]) + + def test_slice_stride(self): + suite = Suite("x = numbers[::stride]") + data = {"numbers": [0, 1, 2, 3, 4], "stride": 2} + suite.execute(data) + self.assertEqual([0, 2, 4], data["x"]) + + def test_slice_negative_start(self): + suite = Suite("x = numbers[-1:]") + data = {"numbers": [0, 1, 2, 3, 4], "stride": 2} + suite.execute(data) + self.assertEqual([4], data["x"]) + + def test_slice_negative_end(self): + suite = Suite("x = numbers[:-1]") + data = {"numbers": [0, 1, 2, 3, 4], "stride": 2} + suite.execute(data) + self.assertEqual([0, 1, 2, 3], data["x"]) + + def test_slice_constant(self): + suite = Suite("x = numbers[1]") + data = {"numbers": [0, 1, 2, 3, 4]} + suite.execute(data) + self.assertEqual(1, data["x"]) + + def test_slice_call(self): + def f(): + return 2 + suite = Suite("x = numbers[f()]") + data = {"numbers": [0, 1, 2, 3, 4], "f": f} + suite.execute(data) + self.assertEqual(2, data["x"]) + + def test_slice_name(self): + suite = Suite("x = numbers[v]") + data = {"numbers": [0, 1, 2, 3, 4], "v": 2} + suite.execute(data) + self.assertEqual(2, data["x"]) + + def test_slice_attribute(self): + class ValueHolder: + def __init__(self): + self.value = 3 + suite = Suite("x = numbers[obj.value]") + data = {"numbers": [0, 1, 2, 3, 4], "obj": ValueHolder()} + suite.execute(data) + self.assertEqual(3, data["x"]) + def suite(): suite = unittest.TestSuite() diff --git a/setup.py b/setup.py index 35f2c15f..bc01e570 100755 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def _unavailable(self, exc): # - CPython >= 3.3 (the new Unicode C API is not supported yet) speedups = Feature( "optional C speed-enhancements", - standard = not is_pypy and sys.version_info < (3, 3), + standard = not is_pypy, ext_modules = [ Extension('genshi._speedups', ['genshi/_speedups.c']), ],