meltano.yml

version: 1
default_environment: dev
environments:
- name: dev
send_anonymous_usage_stats: true
project_id: target-elasticsearch
plugins:
  extractors:
  - name: tap-smoke-test
    namespace: tap_smoke_test
    pip_url: git+https://github.com/meltano/tap-smoke-test.git
    executable: tap-smoke-test
    config:
      streams:
      - stream_name: animals
        input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl
      - stream_name: page_views
        input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/pageviews-data.jsonl
  loaders:
  - name: target-elasticsearch
    namespace: target_elasticsearch
    pip_url: -e .
    config:
      host: localhost
      index_format: "ecs-{{ stream_name }}-an-{{ foo }}-{{ current_timestamp_daily }}"
      stream_maps: # use stream maps to do more complex base record manipulation
        animals:
          "first_name": "description.split()[0]"
          'tz': "created_at" # this will copy created_at to tz INSIDE the record https://sdk.meltano.com/en/latest/stream_maps.html
          '@timestamp': "created_at"
      index_schema_fields: # This occurs after stream_maps, it should be used to pull out specific fields for index creation
        animals:
          'timestamp': 'tz' # Best not to name variables that will be used for index fmt that is not standard python
          "foo": 'animals' #Using keys that do not exist in the json object is fine, will be interp literally
        page_views:
          'timestamp': created_at
          "foo": "buzz"
      metadata_fields: # This occurs after stream_maps, it should be used to pull out specific fields to be used on for ecs index patters
        animals:
          '_id': "id" # this will add _id to a top level field to comply with ecs