diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index fd7a742e1bf..87299cefe21 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "2.18.3" + changes: + - description: Fix handling of floating point encoded file sizes. + type: bugfix + link: https://github.com/elastic/integrations/pull/14289 + - description: Fix handling of numeric IDs. + type: bugfix + link: https://github.com/elastic/integrations/pull/14289 - version: "2.18.2" changes: - description: Add temporary processor to remove the fields added by the Agentless policy. diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-common-config.yml b/packages/o365/data_stream/audit/_dev/test/pipeline/test-common-config.yml index db0258cd7ae..5902a7aacca 100644 --- a/packages/o365/data_stream/audit/_dev/test/pipeline/test-common-config.yml +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-common-config.yml @@ -11,13 +11,21 @@ fields: tags: - preserve_original_event numeric_keyword_fields: + - "event.id" + - "host.id" - "o365.audit.ExceptionInfo.FalsePositive" - "o365.audit.ExchangeMetaData.CC" - "o365.audit.ExchangeMetaData.MessageID" - "o365.audit.ExchangeMetaData.Sent" - "o365.audit.ExchangeMetaData.To" - "o365.audit.ExchangeMetaData.UniqueID" + - "o365.audit.FileSize" - "o365.audit.Item.IsRecord" - "o365.audit.Item.SizeInBytes" + - "o365.audit.OperationId" + - "o365.audit.RequestId" - "o365.audit.SharePointMetaData.FileSize" - "o365.audit.SharePointMetaData.IsViewableByExternalUsers" + - "o365.audit.UserId" + - "o365.audit.WorkspaceId" + - "organization.id" diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json new file mode 100644 index 00000000000..49953a2e9d5 --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json @@ -0,0 +1,163 @@ +{ + "events": [ + { + "event": { + "original": "{\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[{\\\"Name\\\":\\\"SystemArtifactType\\\",\\\"Value\\\":\\\"None\\\"}]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1e9}" + }, + "o365audit": { + "Activity": "CreateArtifact", + "WorkspaceName": "obszar_robaczy", + "OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "Operation": "CreateArtifact", + "Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "CreationTime": "2024-01-30T14:23:40", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "ClientIP": "81.2.69.144", + "RecordType": 20, + "ResultStatus": "InProgress", + "ObjectDisplayName": "test_lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "Experience": "Lakehouse", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Workload": "PowerBI", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "OperationProperties": "[{\"Name\":\"SystemArtifactType\",\"Value\":\"None\"}]", + "ObjectType": "Lakehouse", + "UserType": 0, + "UserKey": "xxxxxxxx", + "FileSize": 1e9 + } + }, + { + "event": { + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1000.0}" + }, + "o365audit": { + "AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}", + "Activity": "CreateArtifact", + "WorkspaceName": "obszar_robaczy", + "OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "Operation": "CreateArtifact", + "Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "CreationTime": "2024-01-30T14:23:40", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "ClientIP": "81.2.69.144", + "RecordType": 20, + "ResultStatus": "InProgress", + "ObjectDisplayName": "test_lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "Experience": "Lakehouse", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Workload": "PowerBI", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "OperationProperties": "[]", + "ObjectType": "Lakehouse", + "UserType": 0, + "UserKey": "xxxxxxxx", + "FileSize": 1000.0 + } + }, + { + "event": { + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}" + }, + "o365audit": { + "AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}", + "Activity": "CreateArtifact", + "WorkspaceName": "obszar_robaczy", + "OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "Operation": "CreateArtifact", + "Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "CreationTime": "2024-01-30T14:23:40", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "ClientIP": "81.2.69.144", + "RecordType": 20, + "ResultStatus": "InProgress", + "ObjectDisplayName": "test_lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "Experience": "Lakehouse", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Workload": "PowerBI", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "OperationProperties": "[]", + "ObjectType": "Lakehouse", + "UserType": 0, + "UserKey": "xxxxxxxx", + "FileSize": "1000.0" + } + }, + { + "event": { + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}" + }, + "o365audit": { + "AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}", + "Activity": "CreateArtifact", + "WorkspaceName": "obszar_robaczy", + "OrganizationId": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "Operation": "CreateArtifact", + "Id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "CreationTime": "2024-01-30T14:23:40", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "ClientIP": "81.2.69.144", + "RecordType": 20, + "ResultStatus": "InProgress", + "ObjectDisplayName": "test_lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "Experience": "Lakehouse", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Workload": "PowerBI", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "OperationProperties": "[]", + "ObjectType": "Lakehouse", + "UserType": 0, + "UserKey": "xxxxxxxx", + "FileSize": 1000 + } + }, + { + "event": { + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}" + }, + "o365audit": { + "AdditionalInfo": "{\"resourceDisplayName\":\"Track Pictures Viewer\"}", + "Activity": "CreateArtifact", + "WorkspaceName": "obszar_robaczy", + "OrganizationId": 2, + "Operation": "CreateArtifact", + "Id": 6, + "CreationTime": "2024-01-30T14:23:40", + "Timestamp": "2024-01-30T14:22:50", + "UserId": 1, + "ClientIP": "81.2.69.144", + "RecordType": 20, + "ResultStatus": "InProgress", + "ObjectDisplayName": "test_lakehouse", + "OperationId": 4, + "Experience": "Lakehouse", + "WorkspaceId": 3, + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "UserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Workload": "PowerBI", + "RequestId": 5, + "OperationProperties": "[]", + "ObjectType": "Lakehouse", + "UserType": 0, + "UserKey": "xxxxxxxx", + "FileSize": 1000 + } + } + ] +} \ No newline at end of file diff --git a/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json-expected.json b/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json-expected.json new file mode 100644 index 00000000000..c98aa13f0f0 --- /dev/null +++ b/packages/o365/data_stream/audit/_dev/test/pipeline/test-numbery-json-events.json-expected.json @@ -0,0 +1,535 @@ +{ + "expected": [ + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client": { + "address": "81.2.69.144", + "ip": "81.2.69.144" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "CreateArtifact", + "category": [ + "web" + ], + "code": "PowerBIAudit", + "id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "kind": "event", + "original": "{\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[{\\\"Name\\\":\\\"SystemArtifactType\\\",\\\"Value\\\":\\\"None\\\"}]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1e9}", + "outcome": "success", + "provider": "PowerBI", + "type": [ + "info" + ] + }, + "file": { + "size": 1000000000 + }, + "host": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "name": "domain.pl" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "Activity": "CreateArtifact", + "CreationTime": "2024-01-30T14:23:40", + "Experience": "Lakehouse", + "FileSize": 1.0E9, + "ObjectDisplayName": "test_lakehouse", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "ObjectType": "Lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "OperationProperties": [ + { + "Name": "SystemArtifactType", + "Value": "None" + } + ], + "RecordType": "20", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "ResultStatus": "InProgress", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "UserKey": "xxxxxxxx", + "UserType": "0", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "WorkspaceName": "obszar_robaczy" + } + }, + "organization": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "username" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "domain.pl", + "email": "username@domain.pl", + "id": "username@domain.pl", + "name": "username" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + }, + "version": "120.0.0.0" + } + }, + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client": { + "address": "81.2.69.144", + "ip": "81.2.69.144" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "CreateArtifact", + "category": [ + "web" + ], + "code": "PowerBIAudit", + "id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "kind": "event", + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":1000.0}", + "outcome": "success", + "provider": "PowerBI", + "type": [ + "info" + ] + }, + "file": { + "size": 1000 + }, + "host": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "name": "domain.pl" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "Activity": "CreateArtifact", + "AdditionalInfo": { + "resourceDisplayName": "Track Pictures Viewer" + }, + "CreationTime": "2024-01-30T14:23:40", + "Experience": "Lakehouse", + "FileSize": 1000.0, + "ObjectDisplayName": "test_lakehouse", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "ObjectType": "Lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "RecordType": "20", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "ResultStatus": "InProgress", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "UserKey": "xxxxxxxx", + "UserType": "0", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "WorkspaceName": "obszar_robaczy" + } + }, + "organization": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "username" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "domain.pl", + "email": "username@domain.pl", + "id": "username@domain.pl", + "name": "username" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + }, + "version": "120.0.0.0" + } + }, + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client": { + "address": "81.2.69.144", + "ip": "81.2.69.144" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "CreateArtifact", + "category": [ + "web" + ], + "code": "PowerBIAudit", + "id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "kind": "event", + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}", + "outcome": "success", + "provider": "PowerBI", + "type": [ + "info" + ] + }, + "file": { + "size": 1000 + }, + "host": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "name": "domain.pl" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "Activity": "CreateArtifact", + "AdditionalInfo": { + "resourceDisplayName": "Track Pictures Viewer" + }, + "CreationTime": "2024-01-30T14:23:40", + "Experience": "Lakehouse", + "FileSize": "1000.0", + "ObjectDisplayName": "test_lakehouse", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "ObjectType": "Lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "RecordType": "20", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "ResultStatus": "InProgress", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "UserKey": "xxxxxxxx", + "UserType": "0", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "WorkspaceName": "obszar_robaczy" + } + }, + "organization": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "username" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "domain.pl", + "email": "username@domain.pl", + "id": "username@domain.pl", + "name": "username" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + }, + "version": "120.0.0.0" + } + }, + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client": { + "address": "81.2.69.144", + "ip": "81.2.69.144" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "CreateArtifact", + "category": [ + "web" + ], + "code": "PowerBIAudit", + "id": "a4420e70-b7a1-xxx-xxx-11e3364acd22", + "kind": "event", + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}", + "outcome": "success", + "provider": "PowerBI", + "type": [ + "info" + ] + }, + "file": { + "size": 1000 + }, + "host": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81", + "name": "domain.pl" + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "Activity": "CreateArtifact", + "AdditionalInfo": { + "resourceDisplayName": "Track Pictures Viewer" + }, + "CreationTime": "2024-01-30T14:23:40", + "Experience": "Lakehouse", + "FileSize": 1000, + "ObjectDisplayName": "test_lakehouse", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "ObjectType": "Lakehouse", + "OperationId": "a84f7f73-xxxx-xxxx-8cf3-094f69c23756", + "RecordType": "20", + "RequestId": "fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b", + "ResultStatus": "InProgress", + "Timestamp": "2024-01-30T14:22:50", + "UserId": "username@domain.pl", + "UserKey": "xxxxxxxx", + "UserType": "0", + "WorkspaceId": "91dad513-xxxx-xxxx-94bb-f5cbf305691c", + "WorkspaceName": "obszar_robaczy" + } + }, + "organization": { + "id": "53d83e1d-xxx-xxx-84e9-01ec5045dd81" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "username" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "domain.pl", + "email": "username@domain.pl", + "id": "username@domain.pl", + "name": "username" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + }, + "version": "120.0.0.0" + } + }, + { + "@timestamp": "2024-01-30T14:23:40.000Z", + "client": { + "address": "81.2.69.144", + "ip": "81.2.69.144" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "CreateArtifact", + "category": [ + "web" + ], + "code": "PowerBIAudit", + "id": 6, + "kind": "event", + "original": "{\"AdditionalInfo\": \"{\\\"resourceDisplayName\\\":\\\"Track Pictures Viewer\\\"}\",\"Activity\":\"CreateArtifact\",\"WorkspaceName\":\"obszar_robaczy\",\"OrganizationId\":\"53d83e1d-xxx-xxx-84e9-01ec5045dd81\",\"Operation\":\"CreateArtifact\",\"Id\":\"a4420e70-b7a1-xxx-xxx-11e3364acd22\",\"CreationTime\":\"2024-01-30T14:23:40\",\"Timestamp\":\"2024-01-30T14:22:50\",\"UserId\":\"username@domain.pl\",\"ClientIP\":\"81.2.69.144\",\"RecordType\":20,\"ResultStatus\":\"InProgress\",\"ObjectDisplayName\":\"test_lakehouse\",\"OperationId\":\"a84f7f73-xxxx-xxxx-8cf3-094f69c23756\",\"Experience\":\"Lakehouse\",\"WorkspaceId\":\"91dad513-xxxx-xxxx-94bb-f5cbf305691c\",\"ObjectId\":\"0e00d1cf-825a-4d78-98ff-8a8199357669\",\"UserAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\"Workload\":\"PowerBI\",\"RequestId\":\"fcbbe282-xxx-xxxx-xxxx-dc1e6d9b090b\",\"OperationProperties\":\"[]\",\"ObjectType\":\"Lakehouse\",\"UserType\":0,\"UserKey\":\"xxxxxxxx\",\"FileSize\":\"1000.0\"}", + "outcome": "success", + "provider": "PowerBI", + "type": [ + "info" + ] + }, + "file": { + "size": 1000 + }, + "host": { + "id": 2 + }, + "network": { + "type": "ipv4" + }, + "o365": { + "audit": { + "Activity": "CreateArtifact", + "AdditionalInfo": { + "resourceDisplayName": "Track Pictures Viewer" + }, + "CreationTime": "2024-01-30T14:23:40", + "Experience": "Lakehouse", + "FileSize": 1000, + "ObjectDisplayName": "test_lakehouse", + "ObjectId": "0e00d1cf-825a-4d78-98ff-8a8199357669", + "ObjectType": "Lakehouse", + "OperationId": 4, + "RecordType": "20", + "RequestId": 5, + "ResultStatus": "InProgress", + "Timestamp": "2024-01-30T14:22:50", + "UserId": 1, + "UserKey": "xxxxxxxx", + "UserType": "0", + "WorkspaceId": 3, + "WorkspaceName": "obszar_robaczy" + } + }, + "organization": { + "id": 2 + }, + "related": { + "ip": [ + "81.2.69.144" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "1" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + }, + "version": "120.0.0.0" + } + } + ] +} diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index d6674755fa6..8619b91ba5b 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -81,17 +81,25 @@ processors: copy_from: o365audit.FileSize ignore_empty_value: true override: false - - gsub: - field: file.size - pattern: '\D+' - replacement: '' - if: ctx.file?.size instanceof String - ignore_missing: true - - convert: + - script: tag: convert_file_size_to_long - field: file.size - type: long - ignore_missing: true + if: ctx.file?.size != null + source: |- + if (ctx.file.size instanceof long || ctx.file.size instanceof int) { + return; + } + if (ctx.file.size instanceof double) { + ctx.file.size = (long) ctx.file.size; + return; + } + try { + // Attempt to parse as long to prevent inexact integer conversion. + ctx.file.size = Long.parseLong(ctx.file.size); + } + catch (NumberFormatException e) { + // But fall back to float parsing for cases where we failed. + ctx.file.size = (long) Double.parseDouble(ctx.file.size); + } on_failure: - remove: field: file.size @@ -1112,7 +1120,7 @@ processors: if: ctx.server?.ip != null - script: lang: painless - if: 'ctx.user?.id != null && ctx.user?.id.contains("@")' + if: 'ctx.user?.id instanceof String && ctx.user?.id.contains("@")' source: > String[] splitmail = ctx.user.id.splitOnToken("@"); if (splitmail.length != 2) { @@ -1123,7 +1131,7 @@ processors: ctx.user.name = splitmail[0]; - script: lang: painless - if: 'ctx.user?.target?.id != null && ctx.user?.target?.id.contains("@")' + if: 'ctx.user?.target?.id instanceof String && ctx.user?.target?.id.contains("@")' source: > String[] splitmail = ctx.user.target.id.splitOnToken("@"); if (splitmail.length != 2) { @@ -1134,7 +1142,7 @@ processors: ctx.user.target.name = splitmail[0]; - script: lang: painless - if: 'ctx.source?.user?.id != null && ctx.source?.user?.id.contains("@")' + if: 'ctx.source?.user?.id instanceof String && ctx.source?.user?.id.contains("@")' source: > String[] splitmail = ctx.source.user.id.splitOnToken("@"); if (splitmail.length != 2) { @@ -1145,7 +1153,7 @@ processors: ctx.source.user.name = splitmail[0]; - script: lang: painless - if: 'ctx.destination?.user?.id != null && ctx.destination?.user?.id.contains("@")' + if: 'ctx.destination?.user?.id instanceof String && ctx.destination?.user?.id.contains("@")' source: > String[] splitmail = ctx.destination.user.id.splitOnToken("@"); if (splitmail.length != 2) { @@ -1157,7 +1165,7 @@ processors: - set: field: network.type value: ipv6 - if: 'ctx.client?.ip != null && ctx.client?.ip.contains(":")' + if: 'ctx.client?.ip instanceof String && ctx.client?.ip.contains(":")' - set: field: network.type value: ipv4 diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index 247d4cb6c77..2cd725f853e 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,6 +1,6 @@ name: o365 title: Microsoft Office 365 -version: "2.18.2" +version: "2.18.3" description: Collect logs from Microsoft Office 365 with Elastic Agent. type: integration format_version: "3.2.3"