diff --git a/README.md b/README.md index 393c6a2..a5667b6 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,24 @@ To preseed a request, issue a JSON `POST` request to chameleon at the `_seed` en Field | Description ----- | ----------- -Method | Method is the HTTP method used to match the incoming request. Case insensitive, supports arbitrary methods -URL | URL is the absolute or relative URL to match in requests. Only the path and querystring are used -Body | Body is the raw content -StatusCode | StatusCode is the [HTTP status code](http://en.wikipedia.org/wiki/List_of_HTTP_status_codes) -Headers | Headers is a map of headers in the format of string key to string value +`Request` | Request is the request payload including a URL, Method and Body +`Response` | Response is the response to be cached and sent back for a given request + +**Request** + +Field | Description +----- | ----------- +`Body` | Body is the content for the request. May be empty where body doesn't make sense (e.g. `GET` requests) +`Method` | Method is the HTTP method used to match the incoming request. Case insensitive, supports arbitrary methods +`URL` | URL is the absolute or relative URL to match in requests. Only the path and querystring are used + +**Response** + +Field | Description +----- | ----------- +`Body` | Body is the content for the request. May be empty where body doesn't make sense (e.g. `GET` requests) +`Headers` | Headers is a map of headers in the format of string key to string value +`StatusCode` | StatusCode is the [HTTP status code](http://en.wikipedia.org/wiki/List_of_HTTP_status_codes) of the response Repeated, duplicate requests to preseed the cache will be discarded and the cache unaffected. @@ -67,14 +80,19 @@ Here is an example of preseeding the cache with a JSON response for a `GET` requ import requests preseed = json.dumps({ - 'URL': '/foobar', - 'Method': 'GET, - 'Body': '{"key": "value"}', - 'StatusCode': 200, - 'Headers': { - 'Content-Type': 'application/json', - 'Other-Header': 'something-else', - } + 'Request': { + 'Body': '', + 'URL': '/foobar', + 'Method': 'GET', + }, + 'Response': { + 'Body': '{"key": "value"}', + 'Headers': { + 'Content-Type': 'application/json', + 'Other-Header': 'something-else', + }, + 'StatusCode': 200, + }, }) response = requests.post('http://localhost:6005/_seed', data=preseed) @@ -93,14 +111,12 @@ Check out the [example](./example) directory to see preseeding in action. ### How chameleon caches responses -chameleon makes a hash for a given request URI and method and uses that to cache content. What that means: +chameleon makes a hash for a given request URI, request method and request body and uses that to cache content. What that means: * a request of `GET /foo/` will be cached differently than `GET /bar/` * a request of `GET /foo/5` will be cached differently than `GET /foo/6` * a request of `DELETE /foo/5` will be cached differently than `DELETE /foo/6` -* a request of `POST /foo` with a body of `{"hi":"hello}` will be cached the same as a - request of `POST /foo` with a body of `{"spam":"eggs"}`. To get around this, set a header of `chameleon-hash-body` - to any value. This will instruct chameleon to use the entire body as part of the hash. +* a request of `POST /foo` with a body of `{"hi":"hello}` will be cached differently than a request of `POST /foo` with a body of `{"spam":"eggs"}`. To ignore the request body, set a header of `chameleon-no-hash-body` to any value. This will instruct chameleon to ignore the body as part of the hash. ### Writing custom hasher diff --git a/example/app.py b/example/app.py index 034c2eb..e48c7ec 100644 --- a/example/app.py +++ b/example/app.py @@ -72,8 +72,8 @@ def do_GET(self): self.wfile.write(resp.msg.upper()) def do_HASHED(self): - # Custom method that hashes a post with body - self._do_patch_post_put(POST_SERVICE_URL, 'POST', {'chameleon-hash-body': 'true'}) + # Custom method that doesn't hash a post with body + self._do_patch_post_put(POST_SERVICE_URL, 'POST', {'chameleon-no-hash-body': 'true'}) def do_SEEDED(self): url = urlparse.urljoin(SERVICE_URL, self.path[1:]) diff --git a/example/testing_data/7806c1446ee40bed04d1d3f1e5c4a206 b/example/testing_data/7806c1446ee40bed04d1d3f1e5c4a206 deleted file mode 100644 index ff386b8..0000000 --- a/example/testing_data/7806c1446ee40bed04d1d3f1e5c4a206 +++ /dev/null @@ -1,24 +0,0 @@ -{ - "args": {}, - "data": "{\"post\": \"body\"}", - "files": {}, - "form": {}, - "headers": { - "Accept-Encoding": "identity", - "Chameleon-Hash-Body": "true", - "Connect-Time": "1", - "Connection": "close", - "Content-Length": "16", - "Content-Type": "application/json", - "Host": "httpbin.org", - "Total-Route-Time": "0", - "User-Agent": "Python-urllib/2.7", - "Via": "1.1 vegur", - "X-Request-Id": "7610816a-5047-4cbe-9476-ecb479d07144" - }, - "json": { - "post": "body" - }, - "origin": "99.245.54.15", - "url": "https://httpbin.org/post" -} diff --git a/example/testing_data/9835adf25e3ecc09431cdf3079bb822a b/example/testing_data/9835adf25e3ecc09431cdf3079bb822a index 1565a8f..c3c881e 100644 --- a/example/testing_data/9835adf25e3ecc09431cdf3079bb822a +++ b/example/testing_data/9835adf25e3ecc09431cdf3079bb822a @@ -13,7 +13,7 @@ "Total-Route-Time": "0", "User-Agent": "Python-urllib/2.7", "Via": "1.1 vegur", - "X-Request-Id": "26b789b8-b3a3-451e-8f3f-5b237464c15e" + "X-Request-Id": "4baca827-0052-462a-9995-c65680a1f10d" }, "json": { "spam": "eggs" diff --git a/example/testing_data/c884f9c06bdfd2dac66e6af8e2e3c4c1 b/example/testing_data/c884f9c06bdfd2dac66e6af8e2e3c4c1 index 3bccf42..4f7f873 100644 --- a/example/testing_data/c884f9c06bdfd2dac66e6af8e2e3c4c1 +++ b/example/testing_data/c884f9c06bdfd2dac66e6af8e2e3c4c1 @@ -5,15 +5,15 @@ "form": {}, "headers": { "Accept-Encoding": "identity", - "Connect-Time": "6", + "Connect-Time": "2", "Connection": "close", "Content-Length": "14", "Content-Type": "application/json", "Host": "httpbin.org", - "Total-Route-Time": "1", + "Total-Route-Time": "0", "User-Agent": "Python-urllib/2.7", "Via": "1.1 vegur", - "X-Request-Id": "bea001b0-88c2-49f7-8875-c332a36402ef" + "X-Request-Id": "62f3b7d8-260c-4d6f-934e-40ecac8f9c6a" }, "json": { "foo": "bar" diff --git a/example/testing_data/dbc78ad575723d20eb5469356ac19562 b/example/testing_data/dbc78ad575723d20eb5469356ac19562 index 52a3abf..a08c71a 100644 --- a/example/testing_data/dbc78ad575723d20eb5469356ac19562 +++ b/example/testing_data/dbc78ad575723d20eb5469356ac19562 @@ -12,7 +12,7 @@ "Total-Route-Time": "0", "User-Agent": "Python-urllib/2.7", "Via": "1.1 vegur", - "X-Request-Id": "e8299e3b-866f-49cc-8227-41e1e51ae67e" + "X-Request-Id": "11347282-b40d-47a3-aeb0-e19ba6cc1817" }, "json": null, "origin": "99.245.54.15", diff --git a/example/testing_data/f131cff22faf4cb6acd94098b42a9452 b/example/testing_data/f131cff22faf4cb6acd94098b42a9452 index 038ff78..2e36d4d 100644 --- a/example/testing_data/f131cff22faf4cb6acd94098b42a9452 +++ b/example/testing_data/f131cff22faf4cb6acd94098b42a9452 @@ -13,7 +13,7 @@ "Total-Route-Time": "0", "User-Agent": "Python-urllib/2.7", "Via": "1.1 vegur", - "X-Request-Id": "c99008c6-efcb-4a08-bb6d-47d24fb0bd2e" + "X-Request-Id": "c400e705-56ce-400b-aac0-71a39dd309b0" }, "json": { "hi": "hello" diff --git a/example/testing_data/spec.json b/example/testing_data/spec.json index a2efcd3..927611d 100644 --- a/example/testing_data/spec.json +++ b/example/testing_data/spec.json @@ -8,7 +8,7 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "0", "Content-Type": "text/html; charset=utf-8", - "Date": "Sun, 04 Jan 2015 22:29:56 GMT", + "Date": "Mon, 12 Jan 2015 00:42:41 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } @@ -23,7 +23,7 @@ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "*", "Content-Length": "135", - "Date": "Sun, 04 Jan 2015 22:29:57 GMT", + "Date": "Mon, 12 Jan 2015 00:42:41 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur", "X-More-Info": "http://tools.ietf.org/html/rfc2324" @@ -40,7 +40,7 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "0", "Content-Type": "text/html; charset=utf-8", - "Date": "Sun, 04 Jan 2015 22:29:57 GMT", + "Date": "Mon, 12 Jan 2015 00:42:41 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } @@ -56,7 +56,7 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "470", "Content-Type": "application/json", - "Date": "Sun, 04 Jan 2015 22:29:57 GMT", + "Date": "Mon, 12 Jan 2015 00:42:42 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } @@ -72,7 +72,7 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "549", "Content-Type": "application/json", - "Date": "Sun, 04 Jan 2015 22:29:57 GMT", + "Date": "Mon, 12 Jan 2015 00:42:42 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } @@ -88,29 +88,13 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "546", "Content-Type": "application/json", - "Date": "Sun, 04 Jan 2015 22:29:58 GMT", + "Date": "Mon, 12 Jan 2015 00:42:42 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } }, "key": "c884f9c06bdfd2dac66e6af8e2e3c4c1" }, - { - "response": { - "status_code": 200, - "content": "7806c1446ee40bed04d1d3f1e5c4a206", - "headers": { - "Access-Control-Allow-Credentials": "true", - "Access-Control-Allow-Origin": "*", - "Content-Length": "586", - "Content-Type": "application/json", - "Date": "Sun, 04 Jan 2015 22:29:58 GMT", - "Server": "gunicorn/18.0", - "Via": "1.1 vegur" - } - }, - "key": "7806c1446ee40bed04d1d3f1e5c4a206" - }, { "response": { "status_code": 200, @@ -120,11 +104,11 @@ "Access-Control-Allow-Origin": "*", "Content-Length": "549", "Content-Type": "application/json", - "Date": "Sun, 04 Jan 2015 22:29:58 GMT", + "Date": "Mon, 12 Jan 2015 00:42:43 GMT", "Server": "gunicorn/18.0", "Via": "1.1 vegur" } }, "key": "9835adf25e3ecc09431cdf3079bb822a" } -] +] \ No newline at end of file diff --git a/example/tests.py b/example/tests.py index 0a27b0b..126a08d 100644 --- a/example/tests.py +++ b/example/tests.py @@ -20,13 +20,18 @@ def get_name(code): def preseed(url, method): payload = json.dumps({ - 'URL': url, - 'Method': method, - 'StatusCode': 942, - 'Body': '{"key": "value"}', - 'Headers': { - 'Content-Type': 'application/json', - } + 'Request': { + 'URL': url, + 'Method': method, + 'Body': '', + }, + 'Response': { + 'StatusCode': 942, + 'Body': '{"key": "value"}', + 'Headers': { + 'Content-Type': 'application/json', + } + }, }) req = urllib2.Request('http://localhost:{}/_seed'.format(TEST_CHAMELEON_PORT), payload, {'Content-type': 'application/json'}) req.get_method = lambda: 'POST' @@ -65,7 +70,7 @@ def test_post_returns_post_body(self): req.get_method = lambda: 'HASHED' resp = urllib2.urlopen(req) parsed = json.loads(resp.read()) - self.assertEqual({'post': 'body'}, parsed['json']) + self.assertEqual({'foo': 'bar'}, parsed['json']) def test_patch_returns_body(self): url = 'http://localhost:{}/patch'.format(TEST_APP_PORT) @@ -91,7 +96,8 @@ def test_delete_returns_200(self): self.assertEqual(200, resp.getcode()) def test_preseed(self): - preseed('/encoding/utf8', 'GET') # Preseed this URL and Method with some data + resp = preseed('/encoding/utf8', 'GET') # Preseed this URL and Method with some data + self.assertIn(resp.getcode(), (200, 201)) url = 'http://localhost:{}/encoding/utf8'.format(TEST_APP_PORT) req = urllib2.Request(url) req.get_method = lambda: 'SEEDED' diff --git a/handlers.go b/handlers.go index 02115c5..0d1c511 100644 --- a/handlers.go +++ b/handlers.go @@ -13,11 +13,16 @@ import ( ) type preseedResponse struct { - URL string - Method string - Body string - StatusCode int - Headers map[string]string + Request struct { + Body string + URL string + Method string + } + Response struct { + Body string + StatusCode int + Headers map[string]string + } } // PreseedHandler preseeds a Cacher, according to a Hasher @@ -32,7 +37,11 @@ func PreseedHandler(cacher Cacher, hasher Hasher) http.HandlerFunc { return } - fakeReq, err := http.NewRequest(preseedResp.Method, preseedResp.URL, strings.NewReader(preseedResp.Body)) + fakeReq, err := http.NewRequest( + preseedResp.Request.Method, + preseedResp.Request.URL, + strings.NewReader(preseedResp.Request.Body), + ) if err != nil { w.WriteHeader(500) fmt.Fprint(w, err) @@ -42,17 +51,17 @@ func PreseedHandler(cacher Cacher, hasher Hasher) http.HandlerFunc { response := cacher.Get(hash) if response != nil { - log.Printf("-> Proxying [preseeding;cached: %v] to %v\n", hash, preseedResp.URL) + log.Printf("-> Proxying [preseeding;cached: %v] to %v\n", hash, preseedResp.Request.URL) w.WriteHeader(200) return } - log.Printf("-> Proxying [preseeding;not cached: %v] to %v\n", hash, preseedResp.URL) + log.Printf("-> Proxying [preseeding;not cached: %v] to %v\n", hash, preseedResp.Request.URL) rec := httptest.NewRecorder() - rec.Body = bytes.NewBufferString(preseedResp.Body) - rec.Code = preseedResp.StatusCode - for name, value := range preseedResp.Headers { + rec.Body = bytes.NewBufferString(preseedResp.Response.Body) + rec.Code = preseedResp.Response.StatusCode + for name, value := range preseedResp.Response.Headers { rec.Header().Set(name, value) } diff --git a/handlers_test.go b/handlers_test.go index e83ee9d..e491e37 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -105,12 +105,17 @@ func TestPreseedHandler(t *testing.T) { serverURL.Path = "/_seed" req, _ := http.NewRequest("POST", serverURL.String(), strings.NewReader( `{ - "URL": "/foobar", - "Method": "GET", - "Body": "FOOBAR BODY", - "StatusCode": 942, - "Headers": { - "Content-Type": "application/json" + "Request": { + "URL": "/foobar", + "Method": "GET", + "Body": "" + }, + "Response": { + "Body": "FOOBAR BODY", + "StatusCode": 942, + "Headers": { + "Content-Type": "application/json" + } } }`, )) @@ -118,7 +123,7 @@ func TestPreseedHandler(t *testing.T) { preseedHandler.ServeHTTP(w, req) if w.Code != 201 { - t.Errorf("Got: `%v`; Expected: `201`", w.Code) + t.Errorf("Got: `%v`; Expected: `201`; Error was `%v`", w.Code, w.Body.String()) } serverURL.Path = "/foobar" @@ -138,6 +143,65 @@ func TestPreseedHandler(t *testing.T) { } } +func TestPreseedHandlerWithRequestBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + if string(body) != `{"post":"body"}` { + t.Errorf("Got: `%v`; Expected `{\"post\":\"body\"}`", string(body)) + w.WriteHeader(500) + return + } + w.WriteHeader(200) + })) + defer server.Close() + + serverURL, _ := url.Parse(server.URL) + cache := mockCacher{data: make(map[string]*CachedResponse)} + cachedProxyHandler := CachedProxyHandler( + serverURL, + cache, + DefaultHasher{}, + ) + preseedHandler := PreseedHandler( + cache, + DefaultHasher{}, + ) + + // Seed /foobar + serverURL.Path = "/_seed" + req, _ := http.NewRequest("POST", serverURL.String(), strings.NewReader( + `{ + "Request": { + "URL": "/foobar", + "Method": "POST", + "Body": "{\"foo\":\"bar\"}" + }, + "Response": { + "Body": "FOOBAR BODY", + "StatusCode": 942, + "Headers": { + "Content-Type": "application/json" + } + } + }`, + )) + w := httptest.NewRecorder() + preseedHandler.ServeHTTP(w, req) + + serverURL.Path = "/foobar" + req, _ = http.NewRequest("POST", serverURL.String(), strings.NewReader(`{"foo":"bar"}`)) + w = httptest.NewRecorder() + cachedProxyHandler.ServeHTTP(w, req) + + req, _ = http.NewRequest("POST", serverURL.String(), strings.NewReader(`{"post":"body"}`)) + w = httptest.NewRecorder() + cachedProxyHandler.ServeHTTP(w, req) + + if w.Code != 200 { + t.Errorf("Server wasn't hit with the correct body") + } +} + func TestPreseedHandlerBadJSON(t *testing.T) { preseedHandler := PreseedHandler( mockCacher{}, @@ -160,12 +224,17 @@ func TestPreseedHandlerCachesDuplicateRequest(t *testing.T) { ) payload := `{ - "URL": "/foobar", - "Method": "GET", - "Body": "FOOBAR BODY", - "StatusCode": 942, - "Headers": { - "Content-Type": "application/json" + "Request": { + "URL": "/foobar", + "Method": "GET", + "Body": "" + }, + "Response": { + "Body": "FOOBAR BODY", + "StatusCode": 942, + "Headers": { + "Content-Type": "application/json" + } } }` @@ -193,12 +262,17 @@ func TestPreseedHandlerBadURL(t *testing.T) { ) payload := `{ - "URL": "%&%", - "Method": "GET", - "Body": "FOOBAR BODY", - "StatusCode": 942, - "Headers": { - "Content-Type": "application/json" + "Request": { + "URL": "%&%", + "Method": "GET", + "Body": "" + }, + "Response": { + "Body": "FOOBAR BODY", + "StatusCode": 942, + "Headers": { + "Content-Type": "application/json" + } } }` diff --git a/hash.go b/hash.go index a01d512..da8d0b5 100644 --- a/hash.go +++ b/hash.go @@ -43,16 +43,16 @@ type DefaultHasher struct { } // Hash returns a hash for a given request. -// The default behavior is to hash the URL and method -// but if the header 'chameleon-hash-body' exists, the body -// will be used to hash as well. +// The default behavior is to hash the URL, request method and body +// but if the header 'chameleon-no-hash-body' exists, the body +// will not be included in the hash. func (k DefaultHasher) Hash(r *http.Request) string { hasher := md5.New() hash := r.URL.RequestURI() + r.Method // This method always succeeds _, _ = hasher.Write([]byte(hash)) - if r.Header.Get("chameleon-hash-body") != "" { + if r.Body != nil && r.Header.Get("chameleon-no-hash-body") == "" { var buf bytes.Buffer _, err := buf.ReadFrom(r.Body) if err != nil { diff --git a/hash_test.go b/hash_test.go index d851c54..93ea1fc 100644 --- a/hash_test.go +++ b/hash_test.go @@ -23,22 +23,37 @@ func (c testCommander) NewCmd(command string, stderr io.Writer, stdin io.Reader) return cmd } -func TestDefaultHasherWithBody(t *testing.T) { +func TestDefaultHasherExcludesBody(t *testing.T) { hasher := DefaultHasher{} body := "HASH THIS BODY" req, _ := http.NewRequest("POST", "/foobar", strings.NewReader(body)) - req.Header.Set("chameleon-hash-body", "true") + req.Header.Set("chameleon-no-hash-body", "true") hash := hasher.Hash(req) md5Hasher := md5.New() - md5Hasher.Write([]byte(req.URL.RequestURI() + req.Method + body)) + md5Hasher.Write([]byte(req.URL.RequestURI() + req.Method)) expected := hex.EncodeToString(md5Hasher.Sum(nil)) if hash != expected { t.Errorf("Got: `%v`; Expected: `%v`", hash, expected) } } +func TestDefaultHasherIncludesBody(t *testing.T) { + hasher := DefaultHasher{} + + body := "HASH THIS BODY" + reqWithHeader, _ := http.NewRequest("POST", "/foobar", strings.NewReader(body)) + reqWithHeader.Header.Set("chameleon-hash-body", "true") + reqWithoutHeader, _ := http.NewRequest("POST", "/foobar", strings.NewReader(body)) + withHeader := hasher.Hash(reqWithHeader) + withoutHeader := hasher.Hash(reqWithoutHeader) + + if withoutHeader != withHeader { + t.Errorf("Request hashes do not match: `%v` != `%v`", withoutHeader, withHeader) + } +} + func TestCmdHasher(t *testing.T) { var stdin bytes.Buffer hasher := CmdHasher{Command: "/bin/cat", Commander: testCommander{stdin: &stdin}}