bouncer: implement /collector and /test-helpers (#118)

* bouncer: implement /collector and /test-helpers

This allows us to deploy a bouncer with a much simpler API. So, we can
start using such better API in apps. This makes us ready to switch to
use the OONI orchestra rather than legacy services, since the API added
here is the same API of some OONI orchestra endpoints.

* Make code compliant with bouncer v2.0.0 spec

* bouncer: factor common code used by /collector and /test-helpers

See https://github.com/TheTorProject/ooni-backend/pull/118/files#r267408775

oonib/bouncer/api.py

@@ -1,7 +1,8 @@
 from oonib.bouncer import handlers
 # XXX: if bouncer is configured
 bouncerAPI = [
-    # return a collector and helper of the requested type
+    (r"/api/v1/collectors", handlers.APIv1Collectors),
+    (r"/api/v1/test-helpers", handlers.APIv1TestHelpers),
     (r"/bouncer", handlers.BouncerQueryHandler),
     (r"/bouncer/test-helpers", handlers.BouncerTestHelpers),
     (r"/bouncer/net-tests", handlers.BouncerNetTests),

oonib/bouncer/handlers.py

@@ -190,6 +190,77 @@ def filterByNetTests(self, requested_nettests):
             nettests.append(nettest)
         return {'net-tests': nettests}
 
+    def formatCollectorsWithoutPolicy(self):
+        ''' Formats the collectors without policy for the new
+            /api/v1/collectors endpoint. '''
+        results = []
+        for collector in self.knownCollectorsWithoutPolicy:
+            results.append({'type': 'onion', 'address': collector})
+            section = self.bouncerFile.get('collector')
+            if not section: continue
+            info = section.get(collector)
+            if not info: continue
+            for alt in info['collector-alternate']:
+                r = self.format_alternate_address(alt)
+                if r:
+                    results.append(r)
+        return results
+
+    def formatTestHelpersWithoutPolicy(self):
+        ''' Formats the test helpers without policy for the new
+            /api/v1/test-helpers endpoint. '''
+        results = {}
+        for collector in self.knownCollectorsWithoutPolicy:
+            section = self.bouncerFile.get('collector')
+            if not section: continue
+            info = section.get(collector)
+            if not info: continue
+            helper = info.get('test-helper')
+            if helper:
+                for k, v in helper.items():
+                    results.setdefault(k, []).append({
+                        'type': 'legacy',
+                        'address': v,
+                    })
+            alt = info.get('test-helper-alternate')
+            if alt:
+                for k, v in alt.items():
+                    results.setdefault(k, [])
+                    for e in v:
+                        r = self.format_alternate_address(e)
+                        if r:
+                            results[k].append(r)
+        return results
+
+    @staticmethod
+    def format_alternate_address(entry):
+        res = {
+            'type': entry.get('type'),
+            'address': entry.get('address'),
+            'front': entry.get('front'),
+        }
+        if not res['type'] or not res['address']:
+            return None  # reject invalid input with missing mandatory fields
+        if not res['front']:
+            del res['front']  # make sure we do not emit a None optional field
+        return res
+
+
+class APIv1Collectors(OONIBHandler):
+    def initialize(self):
+        self.bouncer = Bouncer(config.main.bouncer_file)
+
+    def get(self):
+        self.write(self.bouncer.formatCollectorsWithoutPolicy())
+
+
+class APIv1TestHelpers(OONIBHandler):
+    def initialize(self):
+        self.bouncer = Bouncer(config.main.bouncer_file)
+
+    def get(self):
+        self.write(self.bouncer.formatTestHelpersWithoutPolicy())
+
 
 class BouncerHandlerBase(OONIBHandler):
     def initialize(self):

oonib/tests/test_bouncer.py

@@ -6,6 +6,7 @@
 from cyclone import web
 
 from oonib.bouncer.api import bouncerAPI
+from oonib.bouncer.handlers import Bouncer
 from oonib.tests.handler_helpers import HandlerTestCase
 
 fake_bouncer_file = """
@@ -107,6 +108,23 @@
 """ % (reports_dir, archive_dir, input_dir, decks_dir, bouncer_filename)
 
 
+class FormatHelperAddressTest(HandlerTestCase):
+    def test_behaviour(self):
+        vector = (
+            ({}, None),
+            ({'type': 'https'}, None),
+            ({'address': 'https://a.org'}, None),
+            ({'address': 'https://a.org', 'type': 'https'},
+             {'address': 'https://a.org', 'type': 'https'}),
+            ({'address': 'https://a.org',
+              'type': 'cloudfront', 'front': 'aaa'},
+             {'address': 'https://a.org',
+              'type': 'cloudfront', 'front': 'aaa'}),
+        )
+        for inputs, expects in vector:
+            outputs = Bouncer.format_alternate_address(inputs)
+        self.assertEqual(expects, outputs)
+
 class BaseTestBouncer(HandlerTestCase):
     def setUp(self, *args, **kw):
         self.make_dir(reports_dir)
@@ -571,4 +589,54 @@ def test_traceroute(self):
         response_body = json.loads(response.body)
         print(response_body['net-tests'])
 
+    @defer.inlineCallbacks
+    def test_collectors(self):
+        response = yield self.request('/api/v1/collectors', 'GET', None)
+        response_body = json.loads(response.body)
+        self.assertEqual(response_body, [{
+            "address": "httpo://ihiderha53f36lsd.onion",
+            "type": "onion",
+        }, {
+            "address": "https://a.collector.ooni.io:4441",
+            "type": "https"
+        }, {
+            "address": "https://das0y2z2ribx3.cloudfront.net",
+            "front": "a0.awsstatic.com",
+            "type": "cloudfront"
+        }])
+
+    @defer.inlineCallbacks
+    def test_test_helpers(self):
+        response = yield self.request('/api/v1/test-helpers', 'GET', None)
+        response_body = json.loads(response.body)
+        self.assertEqual(response_body, {
+            "dns": [{"type": "legacy", "address": "213.138.109.232:57004"}],
+            "http-return-json-headers": [{
+                "type": "legacy",
+                "address": "http://38.107.216.10:80"
+            }],
+            "ssl": [{
+                "type": "legacy",
+                "address": "https://213.138.109.232"
+            }],
+            "tcp-echo": [{
+                "type": "legacy",
+                "address": "213.138.109.232"
+            }],
+            "traceroute": [{
+                "type": "legacy",
+                "address": "213.138.109.232"
+            }],
+            "web-connectivity": [{
+                "type": "legacy",
+                "address": "httpo://7jne2rpg5lsaqs6b.onion"
+            }, {
+                "address": "https://a.web-connectivity.th.ooni.io:4442",
+                "type": "https",
+            }, {
+                "address": "https://d2vt18apel48hw.cloudfront.net",
+                "front": "a0.awsstatic.com",
+                "type": "cloudfront",
+            }]
+        })