Merge pull request #6 from DaMouse404/auto-retry

Retry support for requests

README.md

@@ -3,6 +3,8 @@ iBL Fixture Generator
 
 Script fixtures to be generated from live data. Modify only the data you care about and keep fixtures up to date.
 
+Caches the results for 1 hour after a successful request and retries 3 times to allow for API downtime.
+
 Quick start
 -----------
 Setup your fixture file, all fixtures share a common structure so copy paste is your friend:
@@ -59,7 +61,7 @@ module.exports = function (creator, fixtureName) {
 ## Fixture
 
 The base object you operate on has the following methods:
-    
+
 ```javascript
 getEpisode(n) / getProgramme(n) / getElement(n)
 ```
@@ -108,8 +110,8 @@ group.set({
 });
 ```
 
-Elements should also provide helper methods which need to modify multiple attributes 
-to be consistent such as availability (which should modify `status` and 
+Elements should also provide helper methods which need to modify multiple attributes
+to be consistent such as availability (which should modify `status` and
 `availability` information at the same time to be useful).
 
 ### All elements inherit some base fuctions

lib/Fetcher.js

@@ -2,7 +2,8 @@ var requestLib = require('request'),
     qs = require('querystring'),
     u = require('underscore'),
     Q = require('q'),
-    fs = require('fs')
+    fs = require('fs'),
+    retry = require('retry')
 
 function Fetcher(config) {
     this.config = config;
@@ -13,7 +14,14 @@ function Fetcher(config) {
         api_key: config.apiKey
     };
     requestOptions = {
-        timeout: 20000
+        timeout: 1000
+    };
+    retryOptions = {
+        retries: 3,
+        factor: 1.5,
+        minTimeout: 2000,
+        maxTimeout: 5000,
+        randomize: false
     };
 
     if (config.proxy)
@@ -40,28 +48,37 @@ function Fetcher(config) {
 Fetcher.prototype._request = function (feedName, params) {
     var defer = Q.defer(),
         that = this,
+        operation = retry.operation(retryOptions),
         params = u.extend({}, this.defaultRequestParams, params),
         url = this.config.iblUrl + feedName + '.json?' + qs.stringify(params),
         requestOptions = u.extend({}, this.requestOptions, {url:url});
 
     cached = that.getFromCache(feedName, params);
 
     if (cached === false) {
-        this.request(requestOptions, function (err, response, body) {
-            if (err || response.statusCode !== 200) {
-                console.log('[WARN] Failed getting feed', feedName, err, body);
-                defer.reject(err);
-            } else {
-                that.addToCache(feedName, params, body);
-                try {
-                    feedOb = JSON.parse(body);
-                } catch (e) {
-                    defer.reject('Could not parse feed response as JSON')
+        operation.attempt(function(currentAttempt) {
+            that.request(requestOptions, function (err, response, body) {
+                if (operation.retry(err)) {
+                    console.log("[INFO] Retrying ", feedName);
+                    return;
                 }
 
-                if (feedOb)
-                    defer.resolve(feedOb);
-            }
+                if (err || response.statusCode !== 200) {
+                    err = operation.mainError();
+                    console.log('[WARN] Failed getting feed', feedName, err, body);
+                    defer.reject(err);
+                } else {
+                    that.addToCache(feedName, params, body);
+                    try {
+                        feedOb = JSON.parse(body);
+                    } catch (e) {
+                        defer.reject('Could not parse feed response as JSON')
+                    }
+
+                    if (feedOb)
+                        defer.resolve(feedOb);
+                }
+            });
         });
     } else {
         defer.resolve(cached);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fixturator",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "description": "",
   "main": "index.js",
   "repository": {
@@ -11,7 +11,8 @@
     "request": "*",
     "querystring": "0.2.0",
     "q": "~1",
-    "underscore": "*"
+    "underscore": "*",
+    "retry": "~0.6.1"
   },
   "author": "Josh Priestley",
   "license": "BSD"