WIP Tests

Author: oldpatricka

Makefile

@@ -15,10 +15,10 @@ test-client: docker
 	docker run --rm --entrypoint tox $(IMAGE) -c /opt/client/tox.ini
 
 test-ingester: docker
-	docker run --rm --entrypoint py.test $(IMAGE) ingester
+	docker run --rm --entrypoint py.test -vv $(IMAGE) ingester
 
 test-api: docker
-	docker run --rm --entrypoint py.test $(IMAGE) api
+	docker run --rm --entrypoint py.test -vv $(IMAGE) api
 
 .PHONY: test  # Run the tests
 test:

ingester/tests/data/multi-notification-copy-s3-file.json

@@ -31,7 +31,7 @@
     "expected_datalake_records": [
       {
         "url": "s3://foo/existing",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "version": 0,
         "time_index_key": "16552:syslog",
@@ -83,7 +83,7 @@
     "expected_datalake_records": [
       {
         "url": "s3://foo/new_name",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "version": 0,
         "time_index_key": "16552:syslog",

ingester/tests/data/s3-notification-multipart-event.json

@@ -30,7 +30,7 @@
       "expected_datalake_records": [{
       "version": 0,
       "url": "s3://datalake-test/2-california/syslog/1612876178000/9fd061c46d004031b2ceafbb729a0ea3-syslog-284.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "18667:syslog",
       "work_id_index_key": "null9fd061c46d004031b2ceafbb729a0ea3:syslog",
@@ -57,7 +57,7 @@
     "records": [
       {
         "url": "s3://datalake-test/2-california/syslog/1612876178000/9fd061c46d004031b2ceafbb729a0ea3-syslog-284.txt",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "metadata": {
           "version": 0,

ingester/tests/data/s3-notification-null-end-time.json

@@ -29,7 +29,7 @@
     "expected_datalake_records": [{
       "version": 0,
       "url": "s3://datalake-test/3-0a16/commands/1429084800000/34182375d96f460789e9ab4fcb6b4475-commands-655.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "16552:commands",
       "work_id_index_key": "null34182375d96f460789e9ab4fcb6b4475:commands",
@@ -55,7 +55,7 @@
     "records": [
       {
         "url": "s3://datalake-test/3-0a16/commands/1429084800000/34182375d96f460789e9ab4fcb6b4475-commands-655.txt",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "metadata": {
           "version": 0,

ingester/tests/data/s3-notification-one-record.json

@@ -29,7 +29,7 @@
     "expected_datalake_records": [{
       "version": 0,
       "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "16552:syslog",
       "work_id_index_key": "null2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b:syslog",
@@ -55,7 +55,7 @@
     "records": [
       {
         "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "metadata": {
           "version": 0,

ingester/tests/data/s3-notification-two-records.json

@@ -29,7 +29,7 @@
     "expected_datalake_records": [{
       "version": 0,
       "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "16552:syslog",
       "work_id_index_key": "null2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b:syslog",
@@ -49,7 +49,7 @@
     {
       "version": 0,
       "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "16553:syslog",
       "work_id_index_key": "null2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b:syslog",
@@ -69,7 +69,7 @@
     {
       "version": 0,
       "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "time_index_key": "16554:syslog",
       "work_id_index_key": "null2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b:syslog",
@@ -95,7 +95,7 @@
     "records": [
       {
         "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
-        "size": 0,
+        "size": 36,
         "create_time": 236574060000,
         "metadata": {
           "version": 0,

ingester/tests/data/s3-notification-two-s3-records.json

@@ -42,7 +42,7 @@
     },
     "expected_datalake_records": [{
       "version": 0,
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
       "time_index_key": "16552:syslog",
@@ -62,7 +62,7 @@
     },
     {
       "version": 0,
-      "size": 0,
+      "size": 36,
       "create_time": 236574060000,
       "url": "s3://datalake-test/9-kentucky/nginx/1430092800000/f9213e49accd52432b3cfea94239c0eaa90f4431-nginx.gz",
       "time_index_key": "16552:nginx",
@@ -90,7 +90,7 @@
       {
         "url": "s3://datalake-test/2-california/syslog/1430092800000/2544375f44efb2ef5568d8a9ccf8ea9e0c9f122b-syslog-284.txt",
         "create_time": 236574060000,
-        "size": 0,
+        "size": 36,
         "metadata": {
           "version": 0,
           "work_id": null,
@@ -106,7 +106,7 @@
       {
         "url": "s3://datalake-test/9-kentucky/nginx/1430092800000/f9213e49accd52432b3cfea94239c0eaa90f4431-nginx.gz",
         "create_time": 236574060000,
-        "size": 0,
+        "size": 36,
         "metadata": {
           "version": 0,
           "work_id": null,