From f0406667e57c76fce4b11ae09d8ca5cff8e148b4 Mon Sep 17 00:00:00 2001 From: kyouheicf <85217388+kyouheicf@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:31:26 +0900 Subject: [PATCH 1/4] Update schema-http.json Add the missing latest field and its data type --- logpush-to-bigquery/schema-http.json | 93 ++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/logpush-to-bigquery/schema-http.json b/logpush-to-bigquery/schema-http.json index 611896b..94f7c87 100644 --- a/logpush-to-bigquery/schema-http.json +++ b/logpush-to-bigquery/schema-http.json @@ -1,4 +1,9 @@ [ + { + "name": "BotDetectionIDs", + "type": "ARRAY", + "mode": "NULLABLE" + }, { "name": "BotScore", "type": "INTEGER", @@ -9,11 +14,21 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "BotTags", + "type": "ARRAY", + "mode": "NULLABLE" + }, { "name": "CacheCacheStatus", "type": "STRING", "mode": "NULLABLE" }, + { + "name": "CacheReserveUsed", + "type": "BOOLEAN", + "mode": "NULLABLE" + }, { "name": "CacheResponseBytes", "type": "INTEGER", @@ -64,6 +79,11 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "ClientRegionCode", + "type": "STRING", + "mode": "NULLABLE" + }, { "name": "ClientRequestBytes", "type": "INTEGER", @@ -139,6 +159,16 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "ContentScanObjResults", + "type": "ARRAY", + "mode": "NULLABLE" + }, + { + "name": "ContentScanObjTypes", + "type": "ARRAY", + "mode": "NULLABLE" + }, { "name": "EdgeCFConnectingO2O", "type": "BOOLEAN", @@ -183,7 +213,7 @@ "name": "EdgeRateLimitID", "type": "INTEGER", "mode": "NULLABLE" - }, + }, { "name": "EdgeRequestHost", "type": "STRING", @@ -229,6 +259,11 @@ "type": "INTEGER", "mode": "NULLABLE" }, + { + "name": "JA3Hash", + "type": "STRING", + "mode": "REPEATED" + }, { "name": "FirewallMatchesActions", "type": "STRING", @@ -263,7 +298,7 @@ "name": "OriginResponseBytes", "type": "INTEGER", "mode": "NULLABLE" - }, + }, { "name": "OriginResponseDurationMs", "type": "INTEGER", @@ -319,11 +354,41 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "SecurityAction", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "SecurityActions", + "type": "ARRAY", + "mode": "NULLABLE" + }, { "name": "SecurityLevel", "type": "STRING", "mode": "NULLABLE" }, + { + "name": "SecurityRuleDescription", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "SecurityRuleID", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "SecurityRuleIDs", + "type": "ARRAY", + "mode": "NULLABLE" + }, + { + "name": "SecuritySources", + "type": "ARRAY", + "mode": "NULLABLE" + }, { "name": "SmartRouteColoID", "type": "INTEGER", @@ -339,6 +404,11 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "WAFAttackScore", + "type": "INTEGER", + "mode": "NULLABLE" + }, { "name": "WAFFlags", "type": "INTEGER", @@ -354,6 +424,11 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "WAFRCEAttackScore", + "type": "INTEGER", + "mode": "NULLABLE" + }, { "name": "WAFRuleID", "type": "STRING", @@ -364,6 +439,16 @@ "type": "STRING", "mode": "NULLABLE" }, + { + "name": "WAFSQLiAttackScore", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "WAFXSSAttackScore", + "type": "INTEGER", + "mode": "NULLABLE" + }, { "name": "WorkerCPUTime", "type": "INTEGER", @@ -385,7 +470,7 @@ "mode": "NULLABLE" }, { - "name": "ZoneID", + "name": "WorkerWallTimeUs", "type": "INTEGER", "mode": "NULLABLE" }, @@ -394,4 +479,4 @@ "type": "STRING", "mode": "NULLABLE" } -] \ No newline at end of file +] From 91c03d4167ed87e727f63942026e543349e18faa Mon Sep 17 00:00:00 2001 From: kyouheicf <85217388+kyouheicf@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:39:25 +0900 Subject: [PATCH 2/4] Update schema-http.json Change from ARRAY to ARRAY --- logpush-to-bigquery/schema-http.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logpush-to-bigquery/schema-http.json b/logpush-to-bigquery/schema-http.json index 94f7c87..aa8fdf8 100644 --- a/logpush-to-bigquery/schema-http.json +++ b/logpush-to-bigquery/schema-http.json @@ -1,7 +1,7 @@ [ { "name": "BotDetectionIDs", - "type": "ARRAY", + "type": "ARRAY", "mode": "NULLABLE" }, { From 12bb5c1c6674245b1ea66bfeb0cf98e39b76e655 Mon Sep 17 00:00:00 2001 From: kyouheicf <85217388+kyouheicf@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:50:17 +0900 Subject: [PATCH 3/4] Update schema-http.json Use mode: "REPEATED" instead of ARRAY type --- logpush-to-bigquery/schema-http.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/logpush-to-bigquery/schema-http.json b/logpush-to-bigquery/schema-http.json index aa8fdf8..fa3492c 100644 --- a/logpush-to-bigquery/schema-http.json +++ b/logpush-to-bigquery/schema-http.json @@ -1,8 +1,8 @@ [ { "name": "BotDetectionIDs", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "INTEGER", + "mode": "REPEATED" }, { "name": "BotScore", @@ -16,8 +16,8 @@ }, { "name": "BotTags", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "CacheCacheStatus", @@ -161,13 +161,13 @@ }, { "name": "ContentScanObjResults", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "ContentScanObjTypes", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "EdgeCFConnectingO2O", @@ -361,8 +361,8 @@ }, { "name": "SecurityActions", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "SecurityLevel", @@ -381,13 +381,13 @@ }, { "name": "SecurityRuleIDs", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "SecuritySources", - "type": "ARRAY", - "mode": "NULLABLE" + "type": "STRING", + "mode": "REPEATED" }, { "name": "SmartRouteColoID", From d0ed37abf25588a7294f67087046606c4d7e3e1f Mon Sep 17 00:00:00 2001 From: kyouheicf <85217388+kyouheicf@users.noreply.github.com> Date: Sat, 26 Aug 2023 00:51:23 +0900 Subject: [PATCH 4/4] Update schema-http.json - INTEGER to INT64 - FLOAT to FLOAT64 --- logpush-to-bigquery/schema-http.json | 68 ++++++++++++++-------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/logpush-to-bigquery/schema-http.json b/logpush-to-bigquery/schema-http.json index fa3492c..e9a1001 100644 --- a/logpush-to-bigquery/schema-http.json +++ b/logpush-to-bigquery/schema-http.json @@ -1,12 +1,12 @@ [ { "name": "BotDetectionIDs", - "type": "INTEGER", + "type": "INT64", "mode": "REPEATED" }, { "name": "BotScore", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -31,12 +31,12 @@ }, { "name": "CacheResponseBytes", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "CacheResponseStatus", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -46,7 +46,7 @@ }, { "name": "ClientASN", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -86,7 +86,7 @@ }, { "name": "ClientRequestBytes", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -136,7 +136,7 @@ }, { "name": "ClientSrcPort", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -151,7 +151,7 @@ }, { "name": "ClientTCPRTTMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -181,7 +181,7 @@ }, { "name": "EdgeColoID", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -211,7 +211,7 @@ }, { "name": "EdgeRateLimitID", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -221,17 +221,17 @@ }, { "name": "EdgeResponseBodyBytes", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "EdgeResponseBytes", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "EdgeResponseCompressionRatio", - "type": "FLOAT", + "type": "FLOAT64", "mode": "NULLABLE" }, { @@ -241,7 +241,7 @@ }, { "name": "EdgeResponseStatus", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -256,7 +256,7 @@ }, { "name": "EdgeTimeToFirstByteMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -281,7 +281,7 @@ }, { "name": "OriginDNSResponseTimeMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -291,22 +291,22 @@ }, { "name": "OriginRequestHeaderSendDurationMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "OriginResponseBytes", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "OriginResponseDurationMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "OriginResponseHeaderReceiveDurationMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -321,12 +321,12 @@ }, { "name": "OriginResponseStatus", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "OriginResponseTime", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -336,12 +336,12 @@ }, { "name": "OriginTCPHandshakeDurationMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "OriginTLSHandshakeDurationMs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -391,12 +391,12 @@ }, { "name": "SmartRouteColoID", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "UpperTierColoID", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -406,12 +406,12 @@ }, { "name": "WAFAttackScore", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "WAFFlags", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -426,7 +426,7 @@ }, { "name": "WAFRCEAttackScore", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -441,17 +441,17 @@ }, { "name": "WAFSQLiAttackScore", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "WAFXSSAttackScore", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "WorkerCPUTime", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { @@ -466,12 +466,12 @@ }, { "name": "WorkerSubrequestCount", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, { "name": "WorkerWallTimeUs", - "type": "INTEGER", + "type": "INT64", "mode": "NULLABLE" }, {