Merge pull request #72 from camptocamp/simplify_in

Better simplification of IN SQL clauses

c2cwsgiutils/stats_pyramid.py

@@ -1,6 +1,7 @@
 """
 Generate statsd metrics for pyramid and SQLAlchemy events.
 """
+import pyparsing
 import pyramid.config
 import pyramid.events
 import pyramid.request
@@ -67,6 +68,12 @@ def _before_rendered_callback(event: pyramid.events.BeforeRender) -> None:  # pr
         request.add_finished_callback(_create_finished_cb("render", measure))
 
 
+# parser to take a whole parenthesis expression until the matching end parenthesis and get the remaining.
+EAT_PARENTESIS = \
+    pyparsing.nestedExpr('(', ')', content=pyparsing.Word(pyparsing.printables, excludeChars="()")) + \
+    pyparsing.ZeroOrMore(pyparsing.Word(pyparsing.printables + ' \t'))
+
+
 def _simplify_sql(sql: str) -> str:
     """
     Simplify SQL statements to make them easier on the eye and shorter for the stats.
@@ -76,7 +83,11 @@ def _simplify_sql(sql: str) -> str:
     sql = re.sub(r"SELECT .*? FROM", "SELECT FROM", sql)
     sql = re.sub(r"INSERT INTO (.*?) \(.*", r"INSERT INTO \1", sql)
     sql = re.sub(r"SET .*? WHERE", "SET WHERE", sql)
-    sql = re.sub(r"IN \((?:%\(\w+\)\w(?:, *)?)+\)", "IN (?)", sql)
+    for in_ in reversed(list(re.compile(r" IN \(").finditer(sql))):
+        before = sql[0:in_.start()]
+        content = sql[in_.end() - 1:]
+        parsed = EAT_PARENTESIS.parseString(content)
+        sql = before + ' IN (?)' + (' ' + parsed[1] if len(parsed) > 1 else '')
     return re.sub(r"%\(\w+\)\w", "?", sql)
 
 

requirements.txt

@@ -11,6 +11,7 @@ mypy==0.560
 netifaces==0.10.6
 objgraph==3.3.0
 psycopg2==2.7.3.2
+pyparsing==2.2.0
 pyramid==1.9.1
 pyramid_tm==2.2
 pytest==3.3.2

tests/test_stats_pyramid.py

@@ -0,0 +1,16 @@
+from c2cwsgiutils.stats_pyramid import _simplify_sql as simplify_sql   # pylint: disable=W0212
+
+
+def test_simplify_sql():
+
+    assert simplify_sql("SELECT a FROM xxx WHERE b IN (1, 2, 3)") == \
+        "SELECT FROM xxx WHERE b IN (?)"
+
+    assert simplify_sql("SELECT a FROM xxx WHERE b IN (1, 2, 3) AND c=2") == \
+        "SELECT FROM xxx WHERE b IN (?) AND c=2"
+
+    assert simplify_sql("SELECT a FROM xxx WHERE b IN (1, 2, 3) AND c IN (d, e)") == \
+        "SELECT FROM xxx WHERE b IN (?) AND c IN (?)"
+
+    assert simplify_sql("SELECT a FROM xxx WHERE b IN (1, 2) AND c IN ((d, 1), (e, 2))") == \
+        "SELECT FROM xxx WHERE b IN (?) AND c IN (?)"