From 22af953e0d6825e21616c0feb0b1748e97983af4 Mon Sep 17 00:00:00 2001 From: Nikhil Sinha <131262146+nikhilsinhaparseable@users.noreply.github.com> Date: Sun, 27 Oct 2024 14:03:36 +0530 Subject: [PATCH] test added for schema detection and updated data type for datetime fields (#78) --- model.go | 646 +++++++++++++++++++++++++++++++++++--------------- quest_test.go | 10 +- test_utils.go | 92 +------ 3 files changed, 479 insertions(+), 269 deletions(-) diff --git a/model.go b/model.go index f207737..551b740 100644 --- a/model.go +++ b/model.go @@ -51,6 +51,282 @@ const AlertBody string = `{ ] }` +const SchemaPayload string = `{ + "fields":[ + { + "name": "source_time", + "data_type": "string" + }, + { + "name": "level", + "data_type": "string" + }, + { + "name": "message", + "data_type": "string" + }, + { + "name": "version", + "data_type": "string" + }, + { + "name": "user_id", + "data_type": "int" + }, + { + "name": "device_id", + "data_type": "int" + }, + { + "name": "session_id", + "data_type": "string" + }, + { + "name": "os", + "data_type": "string" + }, + { + "name": "host", + "data_type": "string" + }, + { + "name": "uuid", + "data_type": "string" + }, + { + "name": "location", + "data_type": "string" + }, + { + "name": "timezone", + "data_type": "string" + }, + { + "name": "user_agent", + "data_type": "string" + }, + { + "name": "runtime", + "data_type": "string" + }, + { + "name": "request_body", + "data_type": "string" + }, + { + "name": "status_code", + "data_type": "int" + }, + { + "name": "response_time", + "data_type": "int" + }, + { + "name": "process_id", + "data_type": "int" + }, + { + "name": "app_meta", + "data_type": "string" + } + ] + }` + +const SampleJson string = ` +[ + { + "device_id": 1855, + "host": "192.168.1.100", + "level": "error", + "message": "Application started", + "os": "macOS", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 279, + "response_time": 34, + "session_id": "xyz", + "source_time": "2024-10-27T05:13:26.743", + "status_code": 400, + "timezone": "qtv", + "user_id": 46008, + "uuid": "8d01b3cb-825a-4988-92db-96b5493db772", + "version": "1.2.0" + }, + { + "device_id": 4499, + "host": "192.168.1.100", + "level": "info", + "message": "Application started", + "os": "Linux", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "request_body": "npcftnmxcdrydlsvcwotlzbokmqwacnoitaezoddrvmtzeszjdpliukklzoxdkewimglolofpdedyutvsaobebojjokzflcdmlvu", + "response_time": 34, + "runtime": "bfa", + "session_id": "abc", + "source_time": "2024-10-27T05:13:26.743", + "timezone": "hzr", + "user_id": 76351, + "uuid": "cf33d641-ffab-4523-989c-58dd02a011b7", + "version": "1.0.0" + }, + { + "device_id": 2733, + "host": "192.168.1.100", + "level": "info", + "message": "Application started", + "os": "Linux", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 975, + "session_id": "pqr", + "source_time": "2024-10-27T05:13:26.743", + "status_code": 500, + "timezone": "cut", + "user_agent": "PineApple", + "user_id": 69023, + "uuid": "2daf065d-ecff-4a3e-a5e1-7b4a9e404bc1", + "version": "1.1.0" + }, + { + "device_id": 150, + "host": "112.168.1.110", + "level": "error", + "location": "wukxvqjlqdxjjpvy", + "message": "Application is failing", + "os": "Windows", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "request_body": "jrdjsnssrmemqdphdabrmukpsguddyifqthxockkaqxccainkyywqohuefynnxlofpgvdsoijnqavipzbkcoxegfurbxehsbnftc", + "response_time": 22, + "session_id": "pqr", + "source_time": "2024-10-27T05:13:26.742", + "timezone": "wxu", + "user_id": 88506, + "uuid": "050a043a-83f3-4f2a-81be-8d7b68f081e9", + "version": "1.2.0" + }, + { + "device_id": 4489, + "host": "112.168.1.110", + "level": "warn", + "location": "vnudttwllunegitb", + "message": "Logging a request", + "os": "Windows", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 258, + "runtime": "nsz", + "session_id": "xyz", + "source_time": "2024-10-27T05:13:26.742", + "user_agent": "PearOS", + "user_id": 87865, + "uuid": "9f33b8e3-b875-4736-8c86-355a5dfc95a0", + "version": "1.1.0" + }, + { + "device_id": 780, + "host": "112.168.1.110", + "level": "info", + "message": "Application started", + "os": "Linux", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 365, + "response_time": 34, + "runtime": "qld", + "session_id": "xyz", + "source_time": "2024-10-27T05:13:26.742", + "status_code": 200, + "user_id": 71218, + "uuid": "54438529-5ad1-480f-b44b-f871e96afd39", + "version": "1.0.0" + }, + { + "device_id": 2772, + "host": "112.168.1.110", + "level": "warn", + "message": "Logging a request", + "os": "macOS", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 885, + "request_body": "wuoaqbuldmizuoxtjcohvytpjgcfkgrvqjmipzlhespctmmmmxszpjeadpurgeiifgcryqzqjxngohdyssxfkqzmyftqtlwikkrp", + "session_id": "pqr", + "source_time": "2024-10-27T05:13:26.742", + "timezone": "nhw", + "user_agent": "OrangeOS", + "user_id": 83057, + "uuid": "a93337df-0ca4-4d46-9b94-7a4ea3490a27", + "version": "1.0.0" + }, + { + "device_id": 2925, + "host": "112.168.1.110", + "level": "warn", + "message": "Logging a request", + "os": "Linux", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "process_id": 563, + "request_body": "dihfniynpucmgxhcknzxazlqbglzoygxmdpnnaabfgksxnirzwaojampaobiqxbcolgutjyvyuffkasvwqvtmazlzxdstjhszztf", + "response_time": 70, + "session_id": "abc", + "source_time": "2024-10-27T05:13:26.742", + "timezone": "ftj", + "user_id": 57348, + "uuid": "b816bf5d-3d26-4a9e-a7ae-51aac1662e06", + "version": "1.2.0" + }, + { + "app_meta": "qraspufqzwdvgafcmcmxkqmp", + "device_id": 1627, + "host": "172.162.1.120", + "level": "error", + "message": "Application started", + "os": "Windows", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "runtime": "ptp", + "session_id": "xyz", + "source_time": "2024-10-27T05:13:26.742", + "timezone": "tol", + "user_agent": "OrangeOS", + "user_id": 94706, + "uuid": "10048586-da52-41fb-889b-54a377ab6a9f", + "version": "1.2.0" + }, + { + "app_meta": "bkfmqbmmjzbhkxdjzzlaebqp", + "device_id": 42, + "host": "112.168.1.110", + "level": "warn", + "location": "ffxkmbwbtxplhgnz", + "message": "Logging a request", + "os": "Linux", + "p_metadata": "containername=log-generator^namespace=go-apasdp^host=10.116.0.3^source=quest-test^podlabels=app=go-app,pod-template-hash=6c87bc9cc9^containerimage=ghcr.io/parseablehq/quest", + "p_tags": "", + "p_timestamp": "2024-10-27T05:13:26.744", + "request_body": "ffywhsbtsgvraxjuixlsxtrgotcahkicyxnaermtqmfgzlwbqkxqmonrwojmawsyxsovcjlbkbvjsesfznpukicdtghnvvirtauo", + "session_id": "pqr", + "source_time": "2024-10-27T05:13:26.742", + "status_code": 300, + "user_id": 72278, + "uuid": "d679e104-778d-4bbe-b9b6-e6f2b48922ad", + "version": "1.1.0" + } +] +` + const FlogJsonSchema string = `{ "fields": [ { @@ -63,7 +339,12 @@ const FlogJsonSchema string = `{ }, { "name": "datetime", - "data_type": "Utf8", + "data_type": { + "Timestamp": [ + "Millisecond", + null + ] + }, "nullable": true, "dict_id": 0, "dict_is_ordered": false, @@ -159,190 +440,185 @@ const FlogJsonSchema string = `{ }` const SchemaBody string = `{ - "fields": [ - { - "name": "app_meta", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "device_id", - "data_type": "Int64", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "host", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "level", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "location", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "message", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "os", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "p_metadata", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "p_tags", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "p_timestamp", - "data_type": { - "Timestamp": [ - "Millisecond", - null - ] - }, - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "process_id", - "data_type": "Int64", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "request_body", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "response_time", - "data_type": "Int64", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "runtime", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "session_id", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "source_time", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "status_code", - "data_type": "Int64", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "timezone", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "user_agent", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "user_id", - "data_type": "Int64", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "uuid", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - }, - { - "name": "version", - "data_type": "Utf8", - "nullable": true, - "dict_id": 0, - "dict_is_ordered": false, - "metadata": {} - } - ], - "metadata": {} + "fields": [ + { + "name": "device_id", + "data_type": "Int64", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "host", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "level", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "message", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "os", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "p_metadata", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "p_tags", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "p_timestamp", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "process_id", + "data_type": "Int64", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "response_time", + "data_type": "Int64", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "session_id", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "source_time", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "status_code", + "data_type": "Int64", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "timezone", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "user_id", + "data_type": "Int64", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "uuid", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "version", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "request_body", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "runtime", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "user_agent", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "location", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + }, + { + "name": "app_meta", + "data_type": "Utf8", + "nullable": true, + "dict_id": 0, + "dict_is_ordered": false, + "metadata": {} + } + ], + "metadata": {} }` const RetentionBody string = `[ diff --git a/quest_test.go b/quest_test.go index a99dcb8..aac08f0 100644 --- a/quest_test.go +++ b/quest_test.go @@ -60,6 +60,10 @@ func TestSmokeCreateStream(t *testing.T) { DeleteStream(t, NewGlob.QueryClient, NewGlob.Stream) } +func TestSmokeDetectSchema(t *testing.T) { + DetectSchema(t, NewGlob.QueryClient, SampleJson, SchemaBody) +} + func TestSmokeIngestEventsToStream(t *testing.T) { CreateStream(t, NewGlob.QueryClient, NewGlob.Stream) if NewGlob.IngestorUrl.String() == "" { @@ -115,7 +119,7 @@ func TestTimePartition_IncorrectDateTimeFormatTimePartitionInLog(t *testing.T) { func TestLoadStream_StaticSchema_EventWithSameFields(t *testing.T) { staticSchemaStream := NewGlob.Stream + "staticschema" staticSchemaFlagHeader := map[string]string{"X-P-Static-Schema-Flag": "true"} - CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader) + CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader, SchemaPayload) if NewGlob.IngestorUrl.String() == "" { IngestOneEventForStaticSchemaStream_SameFieldsInLog(t, NewGlob.QueryClient, staticSchemaStream) } else { @@ -128,7 +132,7 @@ func TestLoadStreamBatchWithK6_StaticSchema(t *testing.T) { if NewGlob.Mode == "load" { staticSchemaStream := NewGlob.Stream + "staticschema" staticSchemaFlagHeader := map[string]string{"X-P-Static-Schema-Flag": "true"} - CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader) + CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader, SchemaPayload) if NewGlob.IngestorUrl.String() == "" { cmd := exec.Command("k6", "run", @@ -176,7 +180,7 @@ func TestLoadStreamBatchWithK6_StaticSchema(t *testing.T) { func TestLoadStream_StaticSchema_EventWithNewField(t *testing.T) { staticSchemaStream := NewGlob.Stream + "staticschema" staticSchemaFlagHeader := map[string]string{"X-P-Static-Schema-Flag": "true"} - CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader) + CreateStreamWithSchemaBody(t, NewGlob.QueryClient, staticSchemaStream, staticSchemaFlagHeader, SchemaPayload) if NewGlob.IngestorUrl.String() == "" { IngestOneEventForStaticSchemaStream_NewFieldInLog(t, NewGlob.QueryClient, staticSchemaStream) } else { diff --git a/test_utils.go b/test_utils.go index bd916c0..a5ed5ae 100644 --- a/test_utils.go +++ b/test_utils.go @@ -82,87 +82,8 @@ func CreateStreamWithHeader(t *testing.T, client HTTPClient, stream string, head require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s", response.Status) } -func CreateStreamWithSchemaBody(t *testing.T, client HTTPClient, stream string, header map[string]string) { - var schema_payload string = `{ - "fields":[ - { - "name": "source_time", - "data_type": "string" - }, - { - "name": "level", - "data_type": "string" - }, - { - "name": "message", - "data_type": "string" - }, - { - "name": "version", - "data_type": "string" - }, - { - "name": "user_id", - "data_type": "int" - }, - { - "name": "device_id", - "data_type": "int" - }, - { - "name": "session_id", - "data_type": "string" - }, - { - "name": "os", - "data_type": "string" - }, - { - "name": "host", - "data_type": "string" - }, - { - "name": "uuid", - "data_type": "string" - }, - { - "name": "location", - "data_type": "string" - }, - { - "name": "timezone", - "data_type": "string" - }, - { - "name": "user_agent", - "data_type": "string" - }, - { - "name": "runtime", - "data_type": "string" - }, - { - "name": "request_body", - "data_type": "string" - }, - { - "name": "status_code", - "data_type": "int" - }, - { - "name": "response_time", - "data_type": "int" - }, - { - "name": "process_id", - "data_type": "int" - }, - { - "name": "app_meta", - "data_type": "string" - } - ] - }` +func CreateStreamWithSchemaBody(t *testing.T, client HTTPClient, stream string, header map[string]string, schema_payload string) { + req, _ := client.NewRequest("PUT", "logstream/"+stream, bytes.NewBufferString(schema_payload)) for k, v := range header { req.Header.Add(k, v) @@ -172,6 +93,15 @@ func CreateStreamWithSchemaBody(t *testing.T, client HTTPClient, stream string, require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s", response.Status) } +func DetectSchema(t *testing.T, client HTTPClient, sampleJson string, schemaBody string) { + req, _ := client.NewRequest("POST", "logstream/schema/detect", bytes.NewBufferString(sampleJson)) + response, err := client.Do(req) + require.NoErrorf(t, err, "Request failed: %s", err) + body := readAsString(response.Body) + require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s", response.Status) + require.JSONEq(t, schemaBody, body, "Schema detection failed") +} + func DeleteStream(t *testing.T, client HTTPClient, stream string) { req, _ := client.NewRequest("DELETE", "logstream/"+stream, nil) response, err := client.Do(req)