Skip to content

Commit b8979dd

Browse files
efd6efd6
committed
o365: fix handling of file size values
O365 audit data may inclued decimal points and sizes may be represented with e-notation. These break the convert processor. To work around this, in the case of strings, non-digit characters were removed to allow parsing, but this was incorrect since it would add, e.g. significant zeros. Fix this all by using painless to conditionally convert/parse the values of the size field and render them in long.
1 parent 2be7f18 commit b8979dd

File tree

6 files changed

+732
-17
lines changed

6 files changed

+732
-17
lines changed

‎packages/o365/changelog.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "2.18.3"
3+
changes:
4+
- description: Fix handling of floating point encoded file sizes.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/14289
7+
- description: Fix handling of numeric IDs.
8+
type: bugfix
9+
link: https://github.com/elastic/integrations/pull/14289
210
- version: "2.18.2"
311
changes:
412
- description: Add temporary processor to remove the fields added by the Agentless policy.

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-common-config.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ numeric_keyword_fields:
1717
- "o365.audit.ExchangeMetaData.Sent"
1818
- "o365.audit.ExchangeMetaData.To"
1919
- "o365.audit.ExchangeMetaData.UniqueID"
20+
- "o365.audit.FileSize"
2021
- "o365.audit.Item.IsRecord"
2122
- "o365.audit.Item.SizeInBytes"
2223
- "o365.audit.SharePointMetaData.FileSize"
23-
- "o365.audit.SharePointMetaData.IsViewableByExternalUsers"
24+
- "o365.audit.SharePointMetaData.IsViewableByExternalUsers"

‎packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
{
2+
"events": [
3+
{
4+
"event": {
5+
"original": "{\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[{\\\"Name\\\":\\\"SystemArtifactType\\\",\\\"Value\\\":\\\"None\\\"}]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1e9}"
6+
},
7+
"o365audit": {
8+
"Activity": "CreateArtifact",
9+
"WorkspaceName": "obszar_robaczy",
10+
"OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
11+
"Operation": "CreateArtifact",
12+
"Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
13+
"CreationTime": "2024-01-30T14:23:40",
14+
"Timestamp": "2024-01-30T14:22:50",
15+
"UserId": "[email protected]",
16+
"ClientIP": "81.2.69.144",
17+
"RecordType": 20,
18+
"ResultStatus": "InProgress",
19+
"ObjectDisplayName": "test_lakehouse",
20+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
21+
"Experience": "Lakehouse",
22+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
23+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
24+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
25+
"Workload": "PowerBI",
26+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
27+
"OperationProperties": "[{\"Name\":\"SystemArtifactType\",\"Value\":\"None\"}]",
28+
"ObjectType": "Lakehouse",
29+
"UserType": 0,
30+
"UserKey": "xxxxxxxx",
31+
"FileSize": 1e9
32+
}
33+
},
34+
{
35+
"event": {
36+
"original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1000.0}"
37+
},
38+
"o365audit": {
39+
"AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}",
40+
"Activity": "CreateArtifact",
41+
"WorkspaceName": "obszar_robaczy",
42+
"OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
43+
"Operation": "CreateArtifact",
44+
"Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
45+
"CreationTime": "2024-01-30T14:23:40",
46+
"Timestamp": "2024-01-30T14:22:50",
47+
"UserId": "[email protected]",
48+
"ClientIP": "81.2.69.144",
49+
"RecordType": 20,
50+
"ResultStatus": "InProgress",
51+
"ObjectDisplayName": "test_lakehouse",
52+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
53+
"Experience": "Lakehouse",
54+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
55+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
56+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
57+
"Workload": "PowerBI",
58+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
59+
"OperationProperties": "[]",
60+
"ObjectType": "Lakehouse",
61+
"UserType": 0,
62+
"UserKey": "xxxxxxxx",
63+
"FileSize": 1000.0
64+
}
65+
},
66+
{
67+
"event": {
68+
"original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}"
69+
},
70+
"o365audit": {
71+
"AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}",
72+
"Activity": "CreateArtifact",
73+
"WorkspaceName": "obszar_robaczy",
74+
"OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
75+
"Operation": "CreateArtifact",
76+
"Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
77+
"CreationTime": "2024-01-30T14:23:40",
78+
"Timestamp": "2024-01-30T14:22:50",
79+
"UserId": "[email protected]",
80+
"ClientIP": "81.2.69.144",
81+
"RecordType": 20,
82+
"ResultStatus": "InProgress",
83+
"ObjectDisplayName": "test_lakehouse",
84+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
85+
"Experience": "Lakehouse",
86+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
87+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
88+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
89+
"Workload": "PowerBI",
90+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
91+
"OperationProperties": "[]",
92+
"ObjectType": "Lakehouse",
93+
"UserType": 0,
94+
"UserKey": "xxxxxxxx",
95+
"FileSize": "1000.0"
96+
}
97+
},
98+
{
99+
"event": {
100+
"original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}"
101+
},
102+
"o365audit": {
103+
"AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}",
104+
"Activity": "CreateArtifact",
105+
"WorkspaceName": "obszar_robaczy",
106+
"OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81",
107+
"Operation": "CreateArtifact",
108+
"Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22",
109+
"CreationTime": "2024-01-30T14:23:40",
110+
"Timestamp": "2024-01-30T14:22:50",
111+
"UserId": "[email protected]",
112+
"ClientIP": "81.2.69.144",
113+
"RecordType": 20,
114+
"ResultStatus": "InProgress",
115+
"ObjectDisplayName": "test_lakehouse",
116+
"OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756",
117+
"Experience": "Lakehouse",
118+
"WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c",
119+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
120+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
121+
"Workload": "PowerBI",
122+
"RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b",
123+
"OperationProperties": "[]",
124+
"ObjectType": "Lakehouse",
125+
"UserType": 0,
126+
"UserKey": "xxxxxxxx",
127+
"FileSize": 1000
128+
}
129+
},
130+
{
131+
"event": {
132+
"original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"[email protected]\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}"
133+
},
134+
"o365audit": {
135+
"AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}",
136+
"Activity": "CreateArtifact",
137+
"WorkspaceName": "obszar_robaczy",
138+
"OrganizationId": 2,
139+
"Operation": "CreateArtifact",
140+
"Id": 6,
141+
"CreationTime": "2024-01-30T14:23:40",
142+
"Timestamp": "2024-01-30T14:22:50",
143+
"UserId": 1,
144+
"ClientIP": "81.2.69.144",
145+
"RecordType": 20,
146+
"ResultStatus": "InProgress",
147+
"ObjectDisplayName": "test_lakehouse",
148+
"OperationId": 4,
149+
"Experience": "Lakehouse",
150+
"WorkspaceId": 3,
151+
"ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669",
152+
"UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
153+
"Workload": "PowerBI",
154+
"RequestId": 5,
155+
"OperationProperties": "[]",
156+
"ObjectType": "Lakehouse",
157+
"UserType": 0,
158+
"UserKey": "xxxxxxxx",
159+
"FileSize": 1000
160+
}
161+
}
162+
]
163+
}

0 commit comments

Comments
 (0)