Conformance 2016

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
 build*
 *.egg*
-dist
+dist
+/.cache

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "twitter-text-conformance"]
-	path = twitter-text-conformance
-	url = https://github.com/dryan/twitter-text-conformance.git
+[submodule "twitter-text"]
+	path = twitter-text
+	url = https://github.com/twitter/twitter-text.git

.travis.yml

@@ -1,11 +1,11 @@
+sudo: false
 language: python
 python:
-  - "2.6"
   - "2.7"
 
 install: 
-  - "pip install . --use-mirrors"
-  - "pip install -r requirements.txt --use-mirrors"
-script: "python ./tests.py"
+  - "pip install ."
+  - "pip install -r requirements.txt"
+script: "py.test"
 notifications:
   email: false

README.md

@@ -1,6 +1,6 @@
 A port of the Ruby gem [twitter-text-rb](https://github.com/twitter/twitter-text-rb) to Python.
 
-[![Build Status](https://travis-ci.org/dryan/twitter-text-py.png?branch=master)](https://travis-ci.org/dryan/twitter-text-py)
+[![Build Status](https://travis-ci.org/muckrack/twitter-text-py.svg?branch=master)](https://travis-ci.org/muckrack/twitter-text-py)
 
 # Changes in 2.0
 

conftest.py

@@ -0,0 +1,177 @@
+# encoding=utf-8
+
+from __future__ import unicode_literals
+
+import json
+import os
+
+import pytest
+import yaml
+
+import twitter_text
+from twitter_text.encoding import force_text, smart_bytes
+
+# from http://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects
+#from yaml import Loader, SafeLoader
+
+
+narrow_build = True
+try:
+    unichr(0x20000)
+    narrow_build = False
+except:
+    pass
+
+
+#def construct_yaml_str(self, node):
+#    return self.construct_scalar(node)
+#Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
+#SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
+
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    raise Exception('You need to install BeautifulSoup4 to run the tests')
+
+
+def pytest_collect_file(parent, path):
+    if path.ext == '.yml':
+        return YamlFile(path, parent)
+
+
+class YamlException(Exception):
+    """ custom exception for error reporting. """
+
+
+class YamlFile(pytest.File):
+    def collect(self):
+        filename = os.path.splitext(os.path.basename(self.fspath.strpath))[0]
+        if filename not in TEST_MAP:
+            print "Skipping {}; not supported".format(filename)
+            return
+        if TEST_MAP[filename].get('requires_wide_build') and narrow_build:
+            print "Skipping {} due to narrow build".format(filename)
+            return
+        raw = yaml.safe_load(force_text(self.fspath.open().read()))
+        if 'tests' not in raw:
+            return
+        for section, specs in raw['tests'].items():
+            for spec in specs:
+                yield YamlItem(self, filename, section, spec)
+
+
+TEST_MAP = {
+    'autolink': {
+        'cls': twitter_text.autolink.Autolink,
+        'options': {'suppress_no_follow': True},
+        'methods': {
+            'usernames': 'auto_link_usernames_or_lists',
+            'cashtags': 'auto_link_cashtags',
+            'urls': 'auto_link_urls',
+            'hashtags': 'auto_link_hashtags',
+            'all': 'auto_link',
+            'lists': 'auto_link_usernames_or_lists',
+            'json': 'auto_link_with_json',
+        },
+        'ignore_attribute_order': set([
+            'usernames',
+            'cashtags',
+            'urls',
+            'hashtags',
+            'all',
+            'lists',
+            'json',
+        ])
+    },
+    'extract': {
+        'cls': twitter_text.extractor.Extractor,
+        'methods': {
+            'mentions': 'extract_mentioned_screen_names',
+            'mentions_with_indices': 'extract_mentioned_screen_names_with_indices',
+            'mentions_or_lists_with_indices': 'extract_mentions_or_lists_with_indices',
+            'replies': 'extract_reply_screen_name',
+            'urls': 'extract_urls',
+            'urls_with_indices': 'extract_urls_with_indices',
+            'hashtags': 'extract_hashtags',
+            'cashtags': 'extract_cashtags',
+            'hashtags_with_indices': 'extract_hashtags_with_indices',
+            'cashtags_with_indices': 'extract_cashtags_with_indices',
+        },
+    },
+    'hit_highlighting': {
+        'cls': twitter_text.highlighter.HitHighlighter,
+        'methods': {
+            'plain_text': 'hit_highlight',
+            'with_links': 'hit_highlight',
+        },
+        'ignore_attribute_order': set([
+            'with_links',
+        ])
+    },
+    'validate': {
+        'cls': twitter_text.validation.Validation,
+        'requires_wide_build': True,
+        'methods': {
+            'tweets': 'valid_tweet_text',
+            'usernames': 'valid_username',
+            'lists': 'valid_list',
+            'hashtags': 'valid_hashtag',
+            'urls': 'valid_url',
+            'urls_without_protocol': ('valid_url', {'require_protocol': False}),
+            'lengths': 'tweet_length',
+        },
+    }
+}
+
+
+class YamlItem(pytest.Item):
+    def __init__(self, parent, filename, section, spec):
+        self.section = section
+        self.filename = filename
+        self.spec = spec
+        name = "{}:{}:{}".format(filename, section, spec['description'])
+        super(YamlItem, self).__init__(name, parent)
+
+    def _equal_without_attribute_order(self, result, expected):
+        # Beautiful Soup sorts the attributes for us so we can skip all the hoops the ruby version jumps through
+        return BeautifulSoup(result, "lxml") == BeautifulSoup(expected, "lxml")
+
+    def runtest(self):
+        if self.filename not in TEST_MAP:
+            raise YamlException("{} file not supported".format(self.section))
+        if self.section not in TEST_MAP[self.filename]['methods']:
+            raise YamlException("{}:{} section not supported".format(self.section))
+        cls = TEST_MAP[self.filename]['cls']
+        instance = cls(self.spec['text'])
+        args = []
+        try:
+            method_name, kwargs = TEST_MAP[self.filename]['methods'][self.section]
+            kwargs = kwargs.copy()
+        except ValueError:
+            kwargs = {}
+            method_name = TEST_MAP[self.filename]['methods'][self.section]
+        if 'json' in self.spec:
+            args.append(json.loads(self.spec['json']))
+        if 'options' in TEST_MAP[self.filename]:
+            kwargs['options'] = TEST_MAP[self.filename]['options']
+        if 'hits' in self.spec:
+            kwargs['hits'] = self.spec['hits']
+        result = getattr(instance, method_name)(*args, **kwargs)
+        if self.section in TEST_MAP[self.filename].get('ignore_attribute_order', ()):
+            equal = self._equal_without_attribute_order(result, self.spec['expected'])
+        else:
+            equal = result == self.spec['expected']
+        if not equal:
+            raise YamlException("{} != {}".format(result, self.spec['expected']))
+
+    def repr_failure(self, excinfo):
+        """ called when self.runtest() raises an exception. """
+        if isinstance(excinfo.value, YamlException):
+            return smart_bytes("\n".join([
+                "usecase execution failed",
+                "   {}".format(*excinfo.value.args)
+            ]))
+
+    def reportinfo(self):
+        return self.fspath, 0, smart_bytes("usecase: %s" % self.name)

requirements.txt

@@ -1,3 +1,8 @@
 argparse==1.2.1
 PyYAML==3.10
-beautifulsoup4==4.2.0
+beautifulsoup4==4.4.1
+Django==1.9.6
+lxml==3.4.4
+pytest==2.9.1
+py==1.4.29
+regex==2016.04.25

setup.py

@@ -1,5 +1,5 @@
 from setuptools import setup, find_packages
- 
+
 setup(
     name='twitter-text-py',
     version='2.0.2',
@@ -19,5 +19,5 @@
     ],
     include_package_data=True,
     install_requires=['setuptools'],
-    license = "BSD"
+    license="BSD"
 )