how to generate unique ID from ingest pipeline.

[TomonoriSoejima]

PUT _ingest/pipeline/generate_unique_id
{
  "description": "Generates a unique ID based on tunnelId, sessionStartTime, and sessionEndTime",
  "processors": [
    {
      "set": {
        "field": "unique_id_source",
        "value": "{{tunnelId}}-{{sessionStartTime}}-{{sessionEndTime}}"
      }
    },
    {
      "script": {
        "source": "ctx.unique_id = ctx.unique_id_source.hashCode().toString()"
      }
    },
    {
      "remove": {
        "field": "unique_id_source"
      }
    }
  ]
}


POST /your_index/_doc?pipeline=generate_unique_id
{
  "clientIpAddress": "x.x.x.x",
  "sessionEndTime": "2024-04-24T05:54:57.083Z",
  "sessionStartTime": "2024-04-24T05:54:23.020Z",
  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
  "message": "Successfully Disconnected",
  "userName": "test-mitch",
  "duration": 34063,
  "targetResourceId": "x",
  "protocol": "ssh",
  "tunnelId": "b09133e1-860d-479a-afcb-79c2d68bb84",
  "clientPort": 10034,
  "targetVMIPAddress": "x",
  "userEmail": "x",
  "subscriptionId": "c7aa95c2-c438-4b08-a6a7-c057e0d87a39",
  "resourceType": "VM"
}


GET your_index/_search

[TomonoriSoejima]

PUT _ingest/pipeline/logs-azure.activitylogs@custom
{
  "description": "Flatten nested properties to the document root with prefixed names.",
  "processors": [
    {
      "script": {
        "lang": "painless",
        "source": """
        // Access the nested 'azure.activitylogs.properties' map
        Map props = ctx['azure.activitylogs.properties'];
        
        // Move each field from the nested structure to the top level with a prefix
        for (String key : props.keySet()) {
          ctx['azure.activitylogs.properties.' + key] = props.get(key);
        }

        // Optionally, you can remove the original nested structure if no longer needed
        ctx.remove('azure.activitylogs.properties');

        // Construct the unique ID source string from relevant fields and add it at the top level
        String tunnelId = ctx.containsKey('azure.activitylogs.properties.tunnelId') ? ctx.get('azure.activitylogs.properties.tunnelId') : '';
        String sessionStart = ctx.containsKey('azure.activitylogs.properties.sessionStartTime') ? ctx.get('azure.activitylogs.properties.sessionStartTime') : '';
        String sessionEnd = ctx.containsKey('azure.activitylogs.properties.sessionEndTime') ? ctx.get('azure.activitylogs.properties.sessionEndTime') : '';
        String uniqueIdSource = tunnelId + '-' + sessionStart + '-' + sessionEnd;

        // Generate hash code and convert it to string for the unique ID
        if (!uniqueIdSource.isEmpty()) {
          ctx['azure.activitylogs.properties.unique_id'] = Integer.toString(uniqueIdSource.hashCode());
        }
        """
      }
    }
  ]
}


POST test/_doc/1?pipeline=logs-azure.activitylogs@custom
{
  "azure.activitylogs.properties": {
    "clientIpAddress": "x.x.x.x",
    "sessionEndTime": "2024-04-24T05:54:57.083Z",
    "sessionStartTime": "2024-04-24T05:54:23.020Z",
    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
    "message": "Successfully Disconnected",
    "userName": "test-mitch",
    "duration": 34063,
    "targetResourceId": "x",
    "protocol": "ssh",
    "tunnelId": "b09133e1-860d-479a-afcb-79c2d68bb84",
    "clientPort": 10034,
    "targetVMIPAddress": "x",
    "userEmail": "x",
    "subscriptionId": "c7aa95c2-c438-4b08-a6a7-c057e0d87a39",
    "resourceType": "VM"
  }
}

GET test/_search