diff --git a/admin.py b/admin.py index 90694464..217dbb71 100644 --- a/admin.py +++ b/admin.py @@ -101,8 +101,19 @@ def UpdateManufacturers(self): manufacturers_to_delete = [] # invalidate the cache now memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY) - memcache.delete(memcache_keys.MANUFACTURER_MODEL_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY_2) + # This is linked to manufacturer PIDs, so invalidate memcache.delete(memcache_keys.MANUFACTURER_PID_COUNT_KEY) + # These need invalidating as they display manufacturer names + memcache.delete(memcache_keys.MANUFACTURER_MODEL_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_CONTROLLER_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_NODE_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_SOFTWARE_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_SPLITTER_COUNTS) + memcache.delete(memcache_keys.MANUFACTURER_CONTROLLER_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_NODE_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_SOFTWARE_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_SPLITTER_COUNTS_2) added = removed = updated = errors = 0 for manufacturer in Manufacturer.all(): @@ -147,6 +158,7 @@ def UpdateManufacturers(self): def UpdateManufacturerLinks(self): # TODO(Peter): Add ability to remove links if not present in data? + # TODO(Peter): Only clear caches if we've done something new_data = {} for id, name in MANUFACTURER_LINKS: new_data[id] = name @@ -154,6 +166,12 @@ def UpdateManufacturerLinks(self): present_manufacturers = set() # invalidate the cache now memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY) + memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY_2) + # These need invalidating as they display manufacturer links + memcache.delete(memcache_keys.MANUFACTURER_CONTROLLER_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_NODE_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_SOFTWARE_COUNTS_2) + memcache.delete(memcache_keys.MANUFACTURER_SPLITTER_COUNTS_2) added = updated = missing = errors = 0 for manufacturer in Manufacturer.all(): @@ -164,8 +182,6 @@ def UpdateManufacturerLinks(self): if not(manufacturer.link) or (new_link != manufacturer.link): try: # add/update if required - manufacturer.link = new_link - manufacturer.put() if manufacturer.link: logging.info('Updating link for %d (%s) %s -> %s' % (id, manufacturer.name, manufacturer.link, new_link)) @@ -174,6 +190,8 @@ def UpdateManufacturerLinks(self): logging.info('Adding link for %d (%s) - %s' % (id, manufacturer.name, new_link)) added += 1 + manufacturer.link = new_link + manufacturer.put() except BadValueError as e: logging.error('Failed to add link for 0x%hx (%s) - %s: %s' % (id, manufacturer.name, new_link, e)) @@ -455,6 +473,7 @@ def UpdateControllers(self): CONTROLLER_DATA, Controller, [memcache_keys.MANUFACTURER_CONTROLLER_COUNTS, + memcache_keys.MANUFACTURER_CONTROLLER_COUNTS_2, memcache_keys.TAG_CONTROLLER_COUNTS], timestamp_keys.CONTROLLERS) @@ -466,6 +485,7 @@ def UpdateNodes(self): NODE_DATA, Node, [memcache_keys.MANUFACTURER_NODE_COUNTS, + memcache_keys.MANUFACTURER_NODE_COUNTS_2, memcache_keys.TAG_NODE_COUNTS], timestamp_keys.NODES) @@ -477,6 +497,7 @@ def UpdateSplitters(self): SPLITTER_DATA, Splitter, [memcache_keys.MANUFACTURER_SPLITTER_COUNTS, + memcache_keys.MANUFACTURER_SPLITTER_COUNTS_2, memcache_keys.TAG_SPLITTER_COUNTS], timestamp_keys.SPLITTERS) @@ -488,6 +509,7 @@ def UpdateSoftware(self): SOFTWARE_DATA, Software, [memcache_keys.MANUFACTURER_SOFTWARE_COUNTS, + memcache_keys.MANUFACTURER_SOFTWARE_COUNTS_2, memcache_keys.TAG_SOFTWARE_COUNTS], timestamp_keys.SOFTWARE) diff --git a/api/json_v1.py b/api/json_api.py similarity index 63% rename from api/json_v1.py rename to api/json_api.py index 10aa2c82..de068c52 100644 --- a/api/json_v1.py +++ b/api/json_api.py @@ -12,9 +12,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -# json.py +# json_api.py # Copyright (C) 2012 Simon Newton -# Version 1 of the JSON API. +# Version 1 and 2 of the JSON API. from model import Controller, LastUpdateTime, Manufacturer, Node, Pid, ProductTag, Responder, Software, Splitter import common @@ -29,13 +29,17 @@ class ManufacturerList(webapp.RequestHandler): """Return the list of all manufacturers.""" + API_VERSION = 1 CACHE_KEY = memcache_keys.MANUFACTURER_CACHE_KEY def get(self): - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' self.response.headers['Cache-Control'] = 'public; max-age=300;' - response = memcache.get(memcache_keys.MANUFACTURER_CACHE_KEY) + response = memcache.get(self.CACHE_KEY) if response is None: response = self.BuildResponse() if not memcache.add(self.CACHE_KEY, response): @@ -44,16 +48,29 @@ def get(self): def BuildResponse(self): manufacturers = [] - for manufacturer in Manufacturer.all(): - manufacturers.append({ + query = Manufacturer.all() + if self.API_VERSION > 1: + query.order('name') + for manufacturer in query: + manufacturer_entry = { 'name': manufacturer.name, 'id': manufacturer.esta_id - }) + } + if self.API_VERSION > 1 and manufacturer.link: + manufacturer_entry['link'] = manufacturer.link + manufacturers.append(manufacturer_entry) return json.dumps({'manufacturers': manufacturers}) +class ManufacturerList2(ManufacturerList): + API_VERSION = 2 + CACHE_KEY = memcache_keys.MANUFACTURER_CACHE_KEY_2 + + class ManufacturerLookup(webapp.RequestHandler): """Query on manufacturer ID.""" + API_VERSION = 1 + def get(self): manufacturer = common.GetManufacturer(self.request.get('manufacturer')) if manufacturer is None: @@ -64,16 +81,26 @@ def get(self): 'name': manufacturer.name, 'esta_id': manufacturer.esta_id, } - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1 and manufacturer.link: + output['link'] = manufacturer.link + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' self.response.headers['Cache-Control'] = 'public; max-age=300;' self.response.out.write(json.dumps(output)) +class ManufacturerLookup2(ManufacturerLookup): + API_VERSION = 2 + + class ResponderFirmware(webapp.RequestHandler): """Return the latest firmware for a responder.""" + API_VERSION = 1 + def get(self): - responder = common.LookupModel(self.request.get('manufacturer'), - self.request.get('model')) + responder = common.LookupModelFromRequest(self.request) if responder is None: self.error(404) return @@ -85,18 +112,38 @@ def get(self): output = { 'version': version.version_id, 'label': version.label, - 'URL': '', } - self.response.headers['Content-Type'] = 'text/json' + # Standardise on link in API v2 upwards + if self.API_VERSION > 1: + output['link'] = None + else: + output['URL'] = '' + # Add other useful info in API v2 upwards + if self.API_VERSION > 1: + output['manufacturer_name'] = responder.manufacturer.name + output['manufacturer_id'] = responder.manufacturer.esta_id + output['device_model_id'] = responder.device_model_id + output['model_description'] = responder.model_description + if responder.manufacturer.link: + output['manufacturer_link'] = responder.manufacturer.link + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/json' self.response.headers['Cache-Control'] = 'public; max-age=300;' self.response.out.write(json.dumps(output)) +class ResponderFirmware2(ResponderFirmware): + API_VERSION = 2 + + class ResponderPersonalities(webapp.RequestHandler): """Returns the personalities for a responder.""" + API_VERSION = 1 + def get(self): - responder = common.LookupModel(self.request.get('manufacturer'), - self.request.get('model')) + responder = common.LookupModelFromRequest(self.request) if responder is None: self.error(404) return @@ -109,6 +156,8 @@ def get(self): 'description': personality.description, 'index': personality.index, } + if self.API_VERSION > 1 and personality.slot_count: + personality_info['slot_count'] = personality.slot_count personalities.append(personality_info) version_output = { @@ -125,15 +174,29 @@ def get(self): 'model_description': responder.model_description, 'versions': versions, } - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1 and responder.manufacturer.link: + output['manufacturer_link'] = responder.manufacturer.link + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' self.response.headers['Cache-Control'] = 'public; max-age=300;' self.response.out.write(json.dumps(output)) +class ResponderPersonalities2(ResponderPersonalities): + API_VERSION = 2 + + class UpdateTimeHandler(webapp.RequestHandler): """Return the last update time for various parts of the index.""" + API_VERSION = 1 + def get(self): - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' # timestamp name : json key timestamp_pairs = { @@ -154,10 +217,19 @@ def get(self): self.response.out.write(json.dumps(output)) +class UpdateTimeHandler2(UpdateTimeHandler): + API_VERSION = 2 + + class ProductTags(webapp.RequestHandler): """Return the tags and number of products for each.""" + API_VERSION = 1 + def get(self): - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' tag_list = memcache.get(self.MemcacheKey()) if not tag_list: tag_list = [] @@ -177,8 +249,13 @@ def get(self): class ProductManufacturers(webapp.RequestHandler): """Return the manufactures and number of products for each.""" + API_VERSION = 1 + def get(self): - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' manufacturer_list = memcache.get(self.MemcacheKey()) if not manufacturer_list: query = self.ProductType().all() @@ -191,6 +268,8 @@ def get(self): 'name': manufacturer.name, 'count': 0, } + if self.API_VERSION > 1 and manufacturer.link: + manufacturer_by_id[manufacturer.esta_id]['link'] = manufacturer.link manufacturer_by_id[manufacturer.esta_id]['count'] += 1 manufacturer_list = manufacturer_by_id.values() manufacturer_list.sort(key=lambda x: x['name']) @@ -208,6 +287,13 @@ def MemcacheKey(self): return memcache_keys.MANUFACTURER_CONTROLLER_COUNTS +class ControllerManufacturers2(ControllerManufacturers): + API_VERSION = 2 + + def MemcacheKey(self): + return memcache_keys.MANUFACTURER_CONTROLLER_COUNTS_2 + + class ControllerTags(ProductTags): def ProductType(self): return Controller @@ -216,6 +302,10 @@ def MemcacheKey(self): return memcache_keys.TAG_CONTROLLER_COUNTS +class ControllerTags2(ControllerTags): + API_VERSION = 2 + + # Nodes class NodeManufacturers(ProductManufacturers): def ProductType(self): @@ -225,6 +315,13 @@ def MemcacheKey(self): return memcache_keys.MANUFACTURER_NODE_COUNTS +class NodeManufacturers2(NodeManufacturers): + API_VERSION = 2 + + def MemcacheKey(self): + return memcache_keys.MANUFACTURER_NODE_COUNTS_2 + + class NodeTags(ProductTags): def ProductType(self): return Node @@ -233,6 +330,10 @@ def MemcacheKey(self): return memcache_keys.TAG_NODE_COUNTS +class NodeTags2(NodeTags): + API_VERSION = 2 + + # Software class SoftwareManufacturers(ProductManufacturers): def ProductType(self): @@ -242,6 +343,13 @@ def MemcacheKey(self): return memcache_keys.MANUFACTURER_SOFTWARE_COUNTS +class SoftwareManufacturers2(SoftwareManufacturers): + API_VERSION = 2 + + def MemcacheKey(self): + return memcache_keys.MANUFACTURER_SOFTWARE_COUNTS_2 + + class SoftwareTags(ProductTags): def ProductType(self): return Software @@ -250,6 +358,10 @@ def MemcacheKey(self): return memcache_keys.TAG_SOFTWARE_COUNTS +class SoftwareTags2(SoftwareTags): + API_VERSION = 2 + + # Splitters class SplitterManufacturers(ProductManufacturers): def ProductType(self): @@ -259,6 +371,13 @@ def MemcacheKey(self): return memcache_keys.MANUFACTURER_SPLITTER_COUNTS +class SplitterManufacturers2(SplitterManufacturers): + API_VERSION = 2 + + def MemcacheKey(self): + return memcache_keys.MANUFACTURER_SPLITTER_COUNTS_2 + + class SplitterTags(ProductTags): def ProductType(self): return Splitter @@ -267,10 +386,19 @@ def MemcacheKey(self): return memcache_keys.TAG_SPLITTER_COUNTS +class SplitterTags2(SplitterTags): + API_VERSION = 2 + + class PidCounts(webapp.RequestHandler): """Return the count of PID usage.""" + API_VERSION = 1 + def get(self): - self.response.headers['Content-Type'] = 'text/plain' + if self.API_VERSION > 1: + self.response.headers['Content-Type'] = 'application/json' + else: + self.response.headers['Content-Type'] = 'text/plain' self.response.headers['Cache-Control'] = 'public; max-age=300;' self.response.out.write(self.BuildResponse()) @@ -278,9 +406,15 @@ def BuildResponse(self): param_counts = {} responders = 0 max_manufacturer_pids = 0 - max_manufacturer_responder = '' + if self.API_VERSION > 1: + max_manufacturer_responder = None + else: + max_manufacturer_responder = '' max_pids = 0 - max_pids_responder = '' + if self.API_VERSION > 1: + max_pids_responder = None + else: + max_pids_responder = '' for responder in Responder.all(): params = [] version = common.GetLatestSoftware(responder) @@ -296,10 +430,16 @@ def BuildResponse(self): if params: responders += 1 if manufacturer_pids > max_manufacturer_pids: - max_manufacturer_responder = responder.model_description + if responder.model_description: + max_manufacturer_responder = responder.model_description + else: + max_manufacturer_responder = responder.device_model_id max_manufacturer_pids = manufacturer_pids if len(params) > max_pids: - max_pids_responder = responder.model_description + if responder.model_description: + max_pids_responder = responder.model_description + else: + max_pids_responder = responder.device_model_id max_pids = len(params) pids = [] @@ -326,6 +466,10 @@ def BuildResponse(self): return json.dumps(output) +class PidCounts2(PidCounts): + API_VERSION = 2 + + app = webapp.WSGIApplication( [ ('/api/json/1/manufacturers', ManufacturerList), @@ -342,5 +486,19 @@ def BuildResponse(self): ('/api/json/1/splitter_tags', SplitterTags), ('/api/json/1/splitter_manufacturers', SplitterManufacturers), ('/api/json/1/pid_counts', PidCounts), + ('/api/json/2/manufacturers', ManufacturerList2), + ('/api/json/2/manufacturer', ManufacturerLookup2), + ('/api/json/2/latest_responder_firmware', ResponderFirmware2), + ('/api/json/2/responder_personalities', ResponderPersonalities2), + ('/api/json/2/update_times', UpdateTimeHandler2), + ('/api/json/2/controller_tags', ControllerTags2), + ('/api/json/2/controller_manufacturers', ControllerManufacturers2), + ('/api/json/2/node_tags', NodeTags2), + ('/api/json/2/node_manufacturers', NodeManufacturers2), + ('/api/json/2/software_tags', SoftwareTags2), + ('/api/json/2/software_manufacturers', SoftwareManufacturers2), + ('/api/json/2/splitter_tags', SplitterTags2), + ('/api/json/2/splitter_manufacturers', SplitterManufacturers2), + ('/api/json/2/pid_counts', PidCounts2), ], debug=True) diff --git a/api/proto_v1.py b/api/proto_api.py similarity index 77% rename from api/proto_v1.py rename to api/proto_api.py index f92873f5..4fbc243c 100644 --- a/api/proto_v1.py +++ b/api/proto_api.py @@ -12,32 +12,44 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -# proto_v1.py +# proto_api.py # Copyright (C) 2012 Simon Newton -# Version 1 of the Proto API +# Version 1 and 2 of the Proto API from model import Manufacturer from google.appengine.ext import webapp class ManufacturerList(webapp.RequestHandler): + API_VERSION = 1 + """Return the list of all manufacturers.""" def get(self): self.response.headers['Content-Type'] = 'text/plain' self.response.headers['Cache-Control'] = 'public; max-age=300;' output = [] - for manufacturer in Manufacturer.all(): + query = Manufacturer.all() + if self.API_VERSION > 1: + query.order('name') + for manufacturer in query: output.append("manufacturer {") output.append(" name: \"%s\"" % manufacturer.name) output.append(" id: %d" % manufacturer.esta_id) + if self.API_VERSION > 1 and manufacturer.link: + output.append(" link: \"%s\"" % manufacturer.link) output.append("}") self.response.out.write('\n'.join(output)) +class ManufacturerList2(ManufacturerList): + API_VERSION = 2 + + app = webapp.WSGIApplication( [ ('/api/proto/1/manufacturers', ManufacturerList), + ('/api/proto/2/manufacturers', ManufacturerList2), ], debug=True) diff --git a/app.yaml b/app.yaml index 81179a72..7f5f18c8 100644 --- a/app.yaml +++ b/app.yaml @@ -66,8 +66,8 @@ handlers: script: export.export_application # API handlers -- url: /api/(json|proto)/1/.* - script: api.\1_v1.app +- url: /api/(json|proto)/(1|2)/.* + script: api.\1_api.app # Admin page - url: /admin(/.*)? diff --git a/memcache_keys.py b/memcache_keys.py index e7870979..93575393 100644 --- a/memcache_keys.py +++ b/memcache_keys.py @@ -22,6 +22,9 @@ # Number of manufacturer pids MANUFACTURER_CACHE_KEY = 'manufacturers' +# Number of manufacturer pids API v2 +MANUFACTURER_CACHE_KEY_2 = 'manufacturers_2' + # Number of manufacturer pids MANUFACTURER_PID_COUNT_KEY = 'manufacturer_pid_count' @@ -43,6 +46,18 @@ # Manufacturer splitter counts MANUFACTURER_SPLITTER_COUNTS = 'manufacturer_splitters' +# Manufacturer controller counts API v2 +MANUFACTURER_CONTROLLER_COUNTS_2 = 'manufacturer_controllers_2' + +# Manufacturer node counts API v2 +MANUFACTURER_NODE_COUNTS_2 = 'manufacturer_nodes_2' + +# Manufacturer software counts API v2 +MANUFACTURER_SOFTWARE_COUNTS_2 = 'manufacturer_software_2' + +# Manufacturer splitter counts API v2 +MANUFACTURER_SPLITTER_COUNTS_2 = 'manufacturer_splitters_2' + # Category model counts CATEGORY_MODEL_COUNTS = 'category_models' diff --git a/pid_loader.py b/pid_loader.py index 5a4707ef..f23464ca 100644 --- a/pid_loader.py +++ b/pid_loader.py @@ -129,6 +129,13 @@ def UpdateIfRequired(self, new_pid_data, manufacturer_id=0): pid = Pid(manufacturer=manufacturer, pid_id=new_pid_data['value'], name=new_pid_data['name']) + # Always save a PID, even if it doesn't have commands, then we can store + # the name of a PID we don't know the full details on + save = True + + if pid.name != new_pid_data.get('name'): + pid.name = new_pid_data.get('name') + save = True if pid.link != new_pid_data.get('link'): pid.link = new_pid_data.get('link')