diff --git a/docs/config.json b/docs/config.json index d93c873dd988a..ac304ad8e3aef 100644 --- a/docs/config.json +++ b/docs/config.json @@ -472,16 +472,6 @@ "title": "Docker", "slug": "/management/guides/docker/" }, - { - "title": "Fluentd", - "slug": "/management/guides/fluentd/", - "forScopes": ["enterprise", "cloud"] - }, - { - "title": "Monitor Audit Events with the Elastic Stack", - "slug": "/management/guides/elastic-stack/", - "forScopes": ["enterprise", "cloud"] - }, { "title": "EC2 Tags", "slug": "/management/guides/ec2-tags/" @@ -526,6 +516,27 @@ "forScopes": ["oss", "enterprise", "cloud"] } ] + }, + { + "title": "Exporting Audit Events", + "slug": "/management/export-audit-events/", + "entries": [ + { + "title": "Exporting Audit Events to Fluentd", + "slug": "/management/export-audit-events/fluentd/", + "forScopes": ["enterprise", "cloud"] + }, + { + "title": "Monitoring Audit Events with the Elastic Stack", + "slug": "/management/export-audit-events/elastic-stack/", + "forScopes": ["enterprise", "cloud"] + }, + { + "title": "Monitoring Audit Events with Splunk", + "slug": "/management/export-audit-events/splunk/", + "forScopes": ["enterprise", "cloud"] + } + ] } ] }, @@ -1875,7 +1886,7 @@ }, { "source": "/setup/guides/fluentd/", - "destination": "/management/guides/fluentd/", + "destination": "/management/export-audit-events/fluentd/", "permanent": true }, { diff --git a/docs/img/enterprise/plugins/splunk/new-index.png b/docs/img/enterprise/plugins/splunk/new-index.png new file mode 100644 index 0000000000000..5eeb93ad45338 Binary files /dev/null and b/docs/img/enterprise/plugins/splunk/new-index.png differ diff --git a/docs/img/enterprise/plugins/splunk/new-token.png b/docs/img/enterprise/plugins/splunk/new-token.png new file mode 100644 index 0000000000000..4a0d1f61215ab Binary files /dev/null and b/docs/img/enterprise/plugins/splunk/new-token.png differ diff --git a/docs/img/enterprise/plugins/splunk/splunk-dashboard.png b/docs/img/enterprise/plugins/splunk/splunk-dashboard.png new file mode 100644 index 0000000000000..b8f11e6253555 Binary files /dev/null and b/docs/img/enterprise/plugins/splunk/splunk-dashboard.png differ diff --git a/docs/pages/deploy-a-cluster/teleport-cloud/faq.mdx b/docs/pages/deploy-a-cluster/teleport-cloud/faq.mdx index 4c962a507ed35..a60688e8dbeca 100644 --- a/docs/pages/deploy-a-cluster/teleport-cloud/faq.mdx +++ b/docs/pages/deploy-a-cluster/teleport-cloud/faq.mdx @@ -209,7 +209,7 @@ The ability to download recordings for offline viewing will be available in a fu ## Is there a way to forward Teleport Cloud audit events to my company's internal Security Information and Event Management (SIEM)? -Yes. We recommend Teleport's [event handler plugin](../../management/guides/fluentd.mdx) to export Teleport Cloud audit events. +Yes. We recommend Teleport's [event handler plugin](../../management/export-audit-events/fluentd.mdx) to export Teleport Cloud audit events. ## I'm noticing high latency / terminal sessions are slow ? diff --git a/docs/pages/management/export-audit-events.mdx b/docs/pages/management/export-audit-events.mdx new file mode 100644 index 0000000000000..c287f63bb8c91 --- /dev/null +++ b/docs/pages/management/export-audit-events.mdx @@ -0,0 +1,27 @@ +--- +title: Exporting Teleport Audit Events +description: Learn how to export Teleport audit events to your log management solution. +--- + +The Teleport Auth Service emits audit logs when users and services interact with +your cluster. + +You can use Teleport's Event Handler plugin to export audit events from Teleport +so you can store them in a log management platform or custom backend. + +If you are new to exporting audit events with Teleport, read [Forwarding Events +with Fluentd](./export-audit-events/fluentd.mdx) to learn the basics of how our +Event Handler plugin works. While this guide focuses on Fluentd, the Event +Handler plugin can export audit events to any endpoint that ingests JSON +messages via HTTP. + +Next, read our guides to setting up the Event Handler plugin to export audit +events to your solution of choice: + +- [Monitor Teleport Audit Events with the Elastic + Stack](./export-audit-events/elastic-stack.mdx): How to configure the Event + Handler plugin to forward Teleport audit logs to Logstash for ingestion in + Elasticsearch so you can explore them in Kibana. +- [Monitor Teleport Audit Events with Splunk](./export-audit-events/splunk.mdx): + How to configure the Event Handler plugin to send logs to Splunk's Universal + Forwarder so you can explore your audit events in Splunk. diff --git a/docs/pages/management/guides/elastic-stack.mdx b/docs/pages/management/export-audit-events/elastic-stack.mdx similarity index 92% rename from docs/pages/management/guides/elastic-stack.mdx rename to docs/pages/management/export-audit-events/elastic-stack.mdx index 54bb59a06ff35..f4b1c410de163 100644 --- a/docs/pages/management/guides/elastic-stack.mdx +++ b/docs/pages/management/export-audit-events/elastic-stack.mdx @@ -233,8 +233,8 @@ clients and services connect to the Teleport Proxy Service instead. -The following `tctl auth sign` command impersonates the `access-plugin` user, -generates signed credentials, and writes an identity file to the local +The following `tctl auth sign` command impersonates the `teleport-event-handler` +user, generates signed credentials, and writes an identity file to the local directory: ```code @@ -651,39 +651,56 @@ authority, certificate, and private key. ### Start the Event Handler -To start the Event Handler, run the following command on your Event Handler -host: +Start the Teleport Teleport Event Handler as a daemon. To do so, create a +systemd service definition at the path +`/usr/lib/systemd/system/teleport-event-handler.service` with the following +content: + +```ini +[Unit] +Description=Teleport Event Handler +After=network.target + +[Service] +Type=simple +Restart=on-failure +ExecStart=/usr/local/bin/teleport-event-handler start --config=/etc/teleport-event-handler.toml +ExecReload=/bin/kill -HUP $MAINPID +PIDFile=/run/teleport-event-handler.pid + +[Install] +WantedBy=multi-user.target +``` + +Enable and start the plugin: ```code -$ teleport-event-handler start -d --config teleport-event-handler.toml +$ sudo systemctl enable teleport-event-handler +$ sudo systemctl start teleport-event-handler ``` -Use the `-d` flag to print debug logs and identify any issues with -starting the Event Handler. Once you have completed this guide and confirmed -that the Event Handler runs as expected, you can run the command without the -`-d` flag. -
-You can configure when you would like the Event Handler to begin exporting -events when you run the `start` command. This example will start exporting from -May 5th, 2021: +You can configure when you would like the Teleport Event Handler to begin +exporting events when you run the `start` command. This example will start +exporting from May 5th, 2021: ```code $ teleport-event-handler start --config teleport-event-handler.toml --start-time "2021-05-05T00:00:00Z" ``` -You can only determine the start time once, when first running the Event -Handler. If you want to change the time frame later, remove the plugin state -directory that you specified in the `storage` field of the handler's +You can only determine the start time once, when first running the Teleport +Event Handler. If you want to change the time frame later, remove the plugin +state directory that you specified in the `storage` field of the handler's configuration file.
-Once the Event Handler starts, you will see notifications about scanned and -forwarded events: +Once the Teleport Event Handler starts, you will see notifications about scanned +and forwarded events: -```txt +```code +$ sudo journalctl -u teleport-event-handler DEBU Event sent id:f19cf375-4da6-4338-bfdc-e38334c60fd1 index:0 ts:2022-09-21 18:51:04.849 +0000 UTC type:cert.create event-handler/app.go:140 ... @@ -715,8 +732,25 @@ the event types for your Teleport audit events over time: ![Creating a visualization](../../../img/enterprise/plugins/elasticsearch/lens.png) +## Troubleshooting connection issues + +If the Teleport Event Handler is displaying error logs while connecting to your +Teleport Cluster, ensure that: + +- The certificate the Teleport Event Handler is using to connect to your + Teleport cluster is not past its expiration date. This is the value of the + `--ttl` flag in the `tctl auth sign` command, which is 12 hours by default. +- Ensure that in your Teleport Evant Handler configuration file + (`teleport-event-handler.toml`), you have provided the correct host *and* port + for the Teleport Proxy Service or Auth Service. + ## Next steps Now that you are exporting your audit events to the Elastic Stack, consult our [audit event reference](../../reference/audit.mdx#event-types) so you can plan visualizations and alerts. + +While this guide uses the `tctl auth sign` command to issue credentials for the +Teleport Event Handler, production clusters should use Machine ID for safer, +more reliable renewals. Read [our guide](../../machine-id/getting-started.mdx) +to getting started with Machine ID. diff --git a/docs/pages/management/guides/fluentd.mdx b/docs/pages/management/export-audit-events/fluentd.mdx similarity index 91% rename from docs/pages/management/guides/fluentd.mdx rename to docs/pages/management/export-audit-events/fluentd.mdx index 6461b41823744..05aeb184c3348 100644 --- a/docs/pages/management/guides/fluentd.mdx +++ b/docs/pages/management/export-audit-events/fluentd.mdx @@ -1,5 +1,5 @@ --- -title: Forwarding events with Fluentd +title: Forwarding Events with Fluentd description: Forwarding events with Fluentd and Teleport event handler videoBanner: HAqxs4rBv2c --- @@ -446,11 +446,30 @@ INFO[0046] Event sent id=0b5f2a3e-faa5-4d77-ab6e-362bca0994fc ts="2021-06-08 11: ... ``` +## Troubleshooting connection issues + +If the Teleport Event Handler is displaying error logs while connecting to your +Teleport Cluster, ensure that: + +- The certificate the Teleport Event Handler is using to connect to your + Teleport cluster is not past its expiration date. This is the value of the + `--ttl` flag in the `tctl auth sign` command, which is 12 hours by default. +- Ensure that in your Teleport Evant Handler configuration file + (`teleport-event-handler.toml`), you have provided the correct host *and* port + for the Teleport Proxy Service or Auth Service. + ## Next Steps -- Read more about - [impersonation](../../access-controls/guides/impersonation.mdx) -- To see all of the options you can set in the values file for the - `teleport-plugin-event-handler` Helm chart, consult our [reference - guide](../../reference/helm-reference/teleport-plugin-event-handler.mdx). +Read more about +[impersonation](https://goteleport.com/docs/access-controls/guides/impersonation/) +here. + +While this guide uses the `tctl auth sign` command to issue credentials for the +Teleport Event Handler, production clusters should use Machine ID for safer, +more reliable renewals. Read [our guide](../../machine-id/getting-started.mdx) +to getting started with Machine ID. + +To see all of the options you can set in the values file for the +`teleport-plugin-event-handler` Helm chart, consult our [reference +guide](../../reference/helm-reference/teleport-plugin-event-handler.mdx). diff --git a/docs/pages/management/export-audit-events/splunk.mdx b/docs/pages/management/export-audit-events/splunk.mdx new file mode 100644 index 0000000000000..4a9f2f850047e --- /dev/null +++ b/docs/pages/management/export-audit-events/splunk.mdx @@ -0,0 +1,834 @@ +--- +title: Monitor Teleport Audit Events with Splunk +description: How to configure the Teleport Event Handler plugin to send audit logs to Splunk +--- + +Teleport's Event Handler plugin receives audit logs from the Teleport Auth +Service and forwards them to your log management solution, letting you perform +historical analysis, detect unusual behavior, and form a better understanding of +how users interact with your Teleport cluster. + +In this guide, we will show you how to configure the Teleport Event Handler +plugin to send your Teleport audit logs to Splunk. In this setup, the Teleport +Event Handler plugin forwards audit logs from Teleport to Splunk's Universal +Forwarder, which stores them in Splunk Cloud Platform or Splunk Enterprise for +visualization and alerting. + +## Prerequisites + +(!docs/pages/includes/commercial-prereqs-tabs.mdx!) + +- Splunk Cloud Platform or Splunk Enterprise v9.0.1 or above. + +- A Linux host where you will run the Teleport Event Handler plugin and Splunk + Universal Forwarder. The Universal Forwarder must be installed on the host. + +
+ + If you run the Teleport Event Handler and Universal Forwarder on the same + host, there is no need to open a port on the host for ingesting logs. However, + if you run the Universal Forwarder on a separate host from the Teleport Event + Handler, you will need to open a port on the Universal Forwarder host to + traffic from the Teleport Event Handler. This guide assumes that the Universal + Forwarder is listening on port `9061`. + +
+ +- On Splunk Enterprise, port `8088` should be open to traffic from the host + running the Teleport Event Handler and Universal Forwarder. + +(!docs/pages/includes/tctl.mdx!) + +## Step 1/4. Set up the Teleport Event Handler plugin + +The Event Handler plugin is a binary that runs independently of your Teleport +cluster. It authenticates to your Teleport cluster and your Splunk Universal +Forwarder using mutual TLS. In this section, you will install the Teleport Event +Handler plugin on the Linux host where you are running your Universal Forwarder +and generate credentials that the plugin will use for authentication. + +### Install the Teleport Event Handler plugin + +Follow the instructions for your environment to install the Teleport Event +Handler plugin on your Universal Forwarder host: + + + + + On the host where you are running the Universal Forwarder, execute the + following commands: + + ```code + $ curl -L -O https://get.gravitational.com/teleport-event-handler-v(=teleport.plugin.version=)-linux-amd64-bin.tar.gz + $ tar -zxvf teleport-event-handler-v(=teleport.plugin.version=)-linux-amd64-bin.tar.gz + ``` + + Move the `teleport-event-handler` binary to `/usr/local/bin`. + + + + + On the host where you are running the Universal Forwarder, execute the + following command: + + ```code + $ docker pull public.ecr.aws/gravitational/teleport-plugin-event-handler:(=teleport.plugin.version=) + ``` + + + + + You must have Docker installed on your Universal Forwarder host. + + Log in to the host and execute the following commands to build the plugin: + + ```code + $ git clone https://github.com/gravitational/teleport-plugins.git --depth 1 + $ cd teleport-plugins/event-handler/build.assets + $ make build + ``` + + You can find the compiled binary within your clone of the `teleport-plugins` + repo, with the file path, `event-handler/build/teleport-event-handler`. Move + this to `/usr/local/bin`. + + + + + You will need Go >= (=teleport.golang=) installed. + + Run the following commands on your Universal Forwarder host: + + ```code + $ git clone https://github.com/gravitational/teleport-plugins.git --depth 1 + $ cd teleport-plugins/event-handler + $ go build + ``` + + The resulting executable will have the name `event-handler`. To follow the + rest of this guide, rename this file to `teleport-event-handler` and move it + to `/usr/local/bin`. + + + + +### Generate a starter config file + +On the host where you are running the Universal Forwarder and the Teleport Event +Handler plugin, execute the following command to generate a sample configuration +that we will edit later: + +```code +$ teleport-event-handler configure . +``` + +The `teleport-event-handler configure` command generates several files that you +will refer to when generating credentials for the Teleport Event Handler plugin +and configuring the Universal Forwarder: + +|File(s)|Purpose| +|---|---| +| `ca.crt` | Self-signed CA certificate that the Teleport Event Handler plugin and the Universal Forwarder will use to validate one another's TLS certificates.| +| `server.crt` and `server.key`| TLS certificate and key for the Universal Forwarder.| +| `client.crt` and `client.key`| TLS certificate and key for the Teleport Event Handler plugin.| +| `teleport-event-handler-role.yaml`| `user` and `role` resource definitions for the Teleport Event Handler plugin.| +| `fluent.conf`| Fluentd configuration file. We will use parts of this to configure the Universal Forwarder.| +| `teleport-event-handler.toml` | Starter configuration file for the Teleport Event Handler plugin.| +| `ca.key` | Not used in this setup. | + +### Define RBAC resources + +The `teleport-event-handler configure` command generated a file called +`teleport-event-handler-role.yaml`. This file defines a `teleport-event-handler` +role and a user with read-only access to the `event` API: + +```yaml +kind: role +metadata: + name: teleport-event-handler +spec: + allow: + rules: + - resources: ['event', 'session'] + verbs: ['list','read'] +version: v5 +--- +kind: user +metadata: + name: teleport-event-handler +spec: + roles: ['teleport-event-handler'] +version: v2 +``` + +Move this file to your workstation (or recreate it by pasting the snippet above) +and use `tctl` on your workstation to create the role and the user: + +```code +$ tctl create -f teleport-event-handler-role.yaml +# user "teleport-event-handler" has been created +# role 'teleport-event-handler' has been created +``` + +### Enable impersonation of the Teleport Event Handler plugin user + +In order for the Teleport Event Handler plugin to forward events from your +Teleport cluster, it needs signed credentials from the cluster's certificate +authority. + +The `teleport-event-handler` user cannot request this itself, and requires +another user to **impersonate** this account in order to request credentials. We +will show you how to enable impersonation for your Teleport user so you can +retrieve credentials for the Teleport Event Handler. + +Create a role that enables your user to impersonate the `teleport-event-handler` +user. First, paste the following YAML document into a file called +`teleport-event-handler-impersonator.yaml`: + +```yaml +kind: role +version: v5 +metadata: + name: teleport-event-handler-impersonator +spec: + options: + # max_session_ttl defines the TTL (time to live) of SSH certificates + # issued to the users with this role. + max_session_ttl: 10h + + # This section declares a list of resource/verb combinations that are + # allowed for the users of this role. By default nothing is allowed. + allow: + impersonate: + users: ["teleport-event-handler"] + roles: ["teleport-event-handler"] +``` + +Next, create the role: + +```code +$ tctl create -f teleport-event-handler-impersonator.yaml +``` + +Assign this role to your Teleport user by running the following commands, +depending on whether you authenticate as a local Teleport user or via the +`github`, `saml`, or `oidc` authentication connectors: + + + + +Retrieve your local user's configuration resource: + +```code +$ tctl get users/$(tsh status -f json | jq -r '.active.username') > myuser.yaml +``` + +
+This command requires `jq` to be installed on your workstation, which you can do +via the [`jq` download page](https://stedolan.github.io/jq/download/). +
+ +Edit `myuser.yaml`, adding `teleport-event-handler-impersonator` to the list of +existing roles. + +Apply your changes: + +```code +$ tctl create -f myuser.yaml +``` + + + +Retrieve your `github` configuration resource: + +```code +$ tctl get github/github > github.yaml +``` + +Edit `github.yaml`, adding `teleport-event-handler-impersonator` to the +`teams_to_roles` section. The team you will map to this role will depend on how +you have designed your organization's RBAC, but it should be the smallest team +possible within your organization. This team must also include your user. + +Here is an example: + +```diff + teams_to_roles: + - organization: octocats + team: admins + roles: + - access ++ - teleport-event-handler-impersonator +``` + +Apply your changes: + +```code +$ tctl create -f github.yaml +``` + + + +Retrieve your `saml` configuration resource: + +```code +$ tctl get saml/mysaml > saml.yaml +``` + +Edit `saml.yaml`, adding `teleport-event-handler-impersonator` to the +`attributes_to_roles` section. The attribute you will map to this role will +depend on how you have designed your organization's RBAC, but it should be the +smallest group possible within your organization. This group must also include +your user. + +Here is an example: + +```diff + attributes_to_roles: + - name: "groups" + value: "my-group" + roles: + - access ++ - teleport-event-handler-impersonator +``` + +Apply your changes: + +```code +$ tctl create -f saml.yaml +``` + + + +Retrieve your `oidc` configuration resource: + +```code +$ tctl get oidc/myoidc > oidc.yaml +``` + +Edit `oidc.yaml`, adding `teleport-event-handler-impersonator` to the +`claims_to_roles` section. The claim you will map to this role will depend on +how you have designed your organization's RBAC, but it should be the smallest +group possible within your organization. This group must also include your +user. + +Here is an example: + +```diff + claims_to_roles: + - name: "groups" + value: "my-group" + roles: + - access ++ - teleport-event-handler-impersonator +``` + +Apply your changes: + +```code +$ tctl create -f saml.yaml +``` + + + +Log out of your Teleport cluster and log in again to assume the new role. + +### Export the access plugin identity + +Like all Teleport users, `teleport-event-handler` needs signed credentials in +order to connect to your Teleport cluster. You will use the `tctl auth sign` +command to request these credentials for the plugin. + + + +The format of the credentials depends on whether you have set up your Teleport +cluster so clients and services connect to the Teleport Proxy Service or to the +Teleport Auth Service instead. + + + + +The following `tctl auth sign` command impersonates the `teleport-event-handler` +user, generates signed credentials, and writes an identity file to the local +directory. It uses the `--ttl` flag to request a certificate with a lifetime of +10 days: + +```code +$ tctl auth sign --user=teleport-event-handler --out=auth.pem --ttl=240h +``` + +The Event Handler plugin listens for audit logs by connecting to the Teleport +Auth Service's gRPC endpoint over TLS. + +The identity file, `auth.pem`, includes both TLS and SSH credentials. Your +Event Handler plugin uses the SSH credentials to connect to the Proxy Service, +which establishes a reverse tunnel connection to the Auth Service. The plugin +uses this reverse tunnel, along with your TLS credentials, to connect to the +Auth Service's gRPC endpoint. + +You will refer to this file later when configuring the plugin. + + + + +If your network allows your plugin to access the Auth Service directly, e.g., +you are running the plugin on the Auth Service host, the plugin uses TLS +credentials to connect to the Auth Service's gRPC endpoint and listen for audit +events. + +You can generate TLS credentials with the following command, which uses the +`--ttl` flag to request a certificate with a lifetime of 10 days: + +```code +$ tctl auth sign --format=tls --user=teleport-event-handler --out=auth --ttl=240h +``` + +This command should result in three PEM-encoded files: `auth.crt`, +`auth.key`, and `auth.cas` (certificate, private key, and CA certs +respectively). Later, you will configure the plugin to use these credentials to +connect to the Auth Service. + + + + + + + + +The following `tctl auth sign` command impersonates the `teleport-event-handler` +user, generates signed credentials, and writes an identity file to the local +directory. It uses the `--ttl` flag to request a certificate with a lifetime of +10 days: + +```code +$ tctl auth sign --user=teleport-event-handler --out=auth.pem --ttl=240h +``` + +Teleport's Event Handler plugin listens for new and updated audit logs by +connecting to the Teleport Auth Service's gRPC endpoint over TLS. + +The identity file, `auth.pem`, includes both TLS and SSH credentials. The Event +Handler plugin uses the SSH credentials to connect to the Proxy Service, which +establishes a reverse tunnel connection to the Auth Service. The plugin uses +this reverse tunnel, along with your TLS credentials, to connect to the Auth +Service's gRPC endpoint. + +You will refer to this file later when configuring the plugin. + + + +Move the credentials you generated to the host where you are running the +Teleport Event Handler plugin. + + + +Once the Teleport Event Handler's certificate expires, you will need to renew it +by running the `tctl auth sign` command again. + + + +## Step 2/4. Configure the Universal Forwarder + +In this step, you will configure the Universal Forwarder to receive audit logs +from the Teleport Event Handler plugin and forward them to Splunk. The Event +Handler sends audit logs as HTTP POST requests with the content type +`application/json`. + +We will assume that you assigned `$SPLUNK_HOME` to `/opt/splunkforwarder` when +installing the Universal Forwarder. + +
+ +To find your `$SPLUNK_HOME`, run the following command to see the location of +your Universal Forwarder service definition, which the init system systemd uses +to run the Universal Forwarder: + +```code +$ sudo systemctl status SplunkForwarder.service +● SplunkForwarder.service - Systemd service file for Splunk, generated by 'splunk enable boot-start' + Loaded: loaded (/lib/systemd/system/SplunkForwarder.service; enabled; vendor preset: enabled) + Active: active (running) since Fri 2022-10-07 15:57:37 UTC; 2h 18min ago + Main PID: 1772 (splunkd) + Tasks: 53 (limit: 2309) + Memory: 70.8M (limit: 1.8G) + CGroup: /system.slice/SplunkForwarder.service + ├─1772 splunkd --under-systemd --systemd-delegate=yes -p 8089 _internal_launch_under_systemd + └─1810 [splunkd pid=1772] splunkd --under-systemd --systemd-delegate=yes -p 8089 _internal_launch_under_systemd [process-runner] +``` + +View the file at the path shown in the `Loaded:` field. Your `$SPLUNK_HOME` +will include the filepath segments in `ExecStart` before `/bin`. In this case, +`$SPLUNK_HOME` is `/opt/splunkforwarder/`: + +```text +ExecStart=/opt/splunkforwarder/bin/splunk _internal_launch_under_systemd +``` + +
+ +### Create an index for your audit logs + +Create an index for your Teleport audit logs by visiting the home page of the +Splunk UI and navigating to **Settings** > **Indexes**. Click **New Index**. +Name your index `teleport-audit-logs` and assign the **Index Data Type** field +to "Events". + +![Creating an Index](../../../../docs/img/enterprise/plugins/splunk/new-index.png) + +The values of the remaining fields, **Max raw data size** and **Searchable +retention (days)** depend on your organization's resources and practices for log +management. + +Click **Save** + +### Create a token for the Universal Forwarder + +The Universal Forwarder authenticates client traffic using a token. To generate +a token, visit the home page of the Splunk UI. Navigate to **Settings** > **Data +inputs** In the **Local inputs** table, find the **HTTP Event Collector** row and +click **Add new** + +Enter a name you can use to recognize the token later so you can +manage it, e.g., `Teleport Audit Events`. Click **Next**. + +![Create a Token](../../../../docs/img/enterprise/plugins/splunk/new-token.png) + +In the **Input Settings** view (above), next to the **Source type** field, click +**Select**. In the **Select Source Type** dropdown menu, click **Structured**, +then **_json**. Splunk will index incoming logs as JSON, which is the format the +Event Handler uses to send logs to the Universal Forwarder. + + +In the **Index** section, select the `teleport-audit-logs` index you created +earlier. Click **Review** then view the summary and click **Submit**. Copy the +**Token Value** field and keep it somewhere safe so you can use it later in this +guide. + +### Prepare a certificate file for the Universal Forwarder + +The Universal Forwarder signs TLS certificates using a file that contains both +an X.509-format certificate and an RSA private key. To prepare this, run the +following commands on the Universal Forwarder host, where `server.crt` and +`server.key` are two of the files you generated earlier with the +`teleport-event-handler configure` the command: + +```code +$ cp server.crt server.pem +$ cat server.key >> server.pem +``` + +Allow the Universal Forwarder to access the certificate file: + +```code +$ sudo chown splunk:splunk server.pem +``` + +### Configure the HTTP Event Collector + + +On your Universal Forwarder host, create a file at +`/opt/splunkforwarder/etc/system/local/inputs.conf` with the following content: + +```ini +[http] +port = 9061 +disabled = false +serverCert = server.pem +sslPassword = +requireClientCert = true + +[http://audit] +token = +index = teleport-audit-logs +allowQueryStringAuth = true +``` + +This configuration enables the HTTP input, which will listen on port `9061` and +receive logs from the Teleport Event Handler Plugin, assigning them to the +`teleport-audit-logs` index. + +Assign `serverCert` to the path to the `server.pem` file you generated earlier. + +To assign `sslPassword`, run the following command in the directory that +contains `fluent.conf`: + +```code +$ cat fluent.conf | grep passphrase +private_key_passphrase "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" +``` + +Copy the passphrase and paste it as the value of `sslPassword`. + +The `token` field in the `[http://audit]` section enables the Universal +Forwarder to collect logs from HTTP clients that present a token. Assign `token` +to the token you generated earlier. + +`allowQueryStringAuth` enables the Teleport Event Handler to include the token +in a query string, rather than the `Authorization` HTTP header (the default). +This is necessary because the Teleport Event Handler does not currently support +custom HTTP headers. + +### Configure TLS + +To configure secure communications between the Universal Forwarder and the +Teleport Event Handler, create a file called +`/opt/splunkforwarder/etc/system/local/server.conf` with the following content +(if this file already exists, add the following field in the `[sslConfig]` +section): + +```ini +[sslConfig] +sslRootCAPath = +``` + +Assign `sslRootCAPath` to the path of the `ca.crt` file you generated earlier. + +Ensure that the Universal Forwarder can read the CA certificate: + +```code +$ sudo chmod +r ca.crt +``` + +### Configure an output + +Instruct the Universal Forwarder to send the logs it collects to Splunk. + +Create a file at the path `/opt/splunkforwarder/etc/system/local/outputs.conf` +with the following content: + +```ini +[tcpout] +sslVerifyServerCert = true + +[httpout] +httpEventCollectorToken = +uri = +``` + +Fill in `httpEventCollectorToken` with the token you generated earlier. + +Assign `uri` to the following, replacing `MYHOST` with the hostname of your +Splunk instance and `8088` with the port you are using for your Splunk HTTP +Event Collector. + +```text +https://MYHOST:8088 +``` + +The format of the URL to use will depend on your Splunk deployment. See the +[list of acceptable URL +formats](https://docs.splunk.com/Documentation/Splunk/9.0.1/Data/UsetheHTTPEventCollector#Send_data_to_HTTP_Event_Collector) +in the Splunk documentation. + +Note that you must only include the scheme, host, and port of the URL. The +Universal Forwarder will append the correct URL path of the Splunk ingestion +API when forwarding logs. + +Finally, restart the Universal Forwarder: + +```code +$ sudo systemctl restart SplunkForwarder +``` + +## Step 3/4. Run the Teleport Event Handler plugin + +Now that you have configured your Universal Forwarder to receive logs via HTTP +and forward them to Splunk, you will ensure that the Teleport Event Handler +plugin is configured to authenticate to the Universal Forwarder and your +Teleport cluster, then run the Teleport Event Handler. + +### Complete the Teleport Event Handler configuration + +Earlier, we generated a file called `teleport-event-handler.toml` to configure +the Teleport Event Handler plugin. This file includes settings similar to the +following: + +```toml +storage = "./storage" +timeout = "10s" +batch = 20 +namespace = "default" + +[forward.fluentd] +ca = "/home/ca.crt" +cert = "/home/client.crt" +key = "/home/client.key" +url = "https://localhost:9061/test.log" + +[teleport] +addr = "example.teleport.com:443" +identity = "identity" +``` + +Update the configuration file as follows. + +Change `forward.fluentd.url` to the following: + +```text +url = "https://localhost:9061/services/collector/raw?token=MYTOKEN" +``` + +Ensure the URL includes the scheme, host and port of your Universal Forwarder's +HTTP intput, plus the URL path that the Universal Forwarder uses for raw data +(`/services/collector/raw`). + +Replace `MYTOKEN` with the token you generated earlier for the Splunk Universal +Forwarder. If you are running the Universal Forwarder and Event Handler on +separate hosts, replace `localhost` with your Universal Forwarder's IP address +or domain name. + +Change `forward.fluentd.session-url` to the same value as `forward.fluentd.url`, +but with the query parameter key `&noop=` appended to the end: + +```text +session-url = "https://localhost:9061/services/collector/raw?token=MYTOKEN&noop=" +``` + +For audit logs related to Teleport sessions, the Teleport Event Handler appends +routing information to the URL that our HTTP input configuration does not use. +Adding the `noop` query parameter causes the Teleport Event Handler to append +the routing information as the parameter's value so the Universal Forwarder can +discard it. + +Change `teleport.addr` to the host and port of your Teleport Proxy Service, or +the Auth Service if you have configured the Teleport Event Handler to connect to +it directly, e.g., `mytenant.teleport.sh:443`. + + + + +Assign `teleport.identity` to a path to the identity file you exported earlier, +e.g., `/home/auth.pem`. + +Ensure that the Teleport Event Handler can read these credentials: + +```code +$ chmod +r auth.pem +``` + + + + +Assign `teleport.ca`, `teleport.cert`, and `teleport.key` to the paths of the +TLS credentials you generated earlier. Respectively, these are the certificate +authority, certificate, and private key. + +Ensure that the Teleport Event Handler can read these credentials: + +```code +$ chmod +r auth.cas auth.crt auth.key +``` + + + + +### Start the Teleport Event Handler + +Start the Teleport Teleport Event Handler as a daemon. To do so, create a +systemd service definition at the path +`/usr/lib/systemd/system/teleport-event-handler.service` with the following +content: + +```ini +[Unit] +Description=Teleport Event Handler +After=network.target + +[Service] +Type=simple +Restart=on-failure +ExecStart=/usr/local/bin/teleport-event-handler start --config=/etc/teleport-event-handler.toml +ExecReload=/bin/kill -HUP $MAINPID +PIDFile=/run/teleport-event-handler.pid + +[Install] +WantedBy=multi-user.target +``` + +Enable and start the plugin: + +```code +$ sudo systemctl enable teleport-event-handler +$ sudo systemctl start teleport-event-handler +``` + +
+ +You can configure when you would like the Teleport Event Handler to begin +exporting events when you run the `start` command. This example will start +exporting from May 5th, 2021: + +```code +$ teleport-event-handler start --config teleport-event-handler.toml --start-time "2021-05-05T00:00:00Z" +``` + +You can only determine the start time once, when first running the Teleport +Event Handler. If you want to change the time frame later, remove the plugin +state directory that you specified in the `storage` field of the handler's +configuration file. + +
+ +Once the Teleport Event Handler starts, you will see notifications about scanned +and forwarded events: + +```code +$ sudo journalctl -u teleport-event-handler +DEBU Event sent id:f19cf375-4da6-4338-bfdc-e38334c60fd1 index:0 ts:2022-09-21 +18:51:04.849 +0000 UTC type:cert.create event-handler/app.go:140 +... +``` + +## Step 4/4. Visualize your audit logs in Splunk + +Since our setup forwards audit logs to Splunk in the structured JSON format, +Splunk automatically indexes them, so fields will be available immediately for +use in visualizations. You can use these fields to create dashboards that track +the way users are interacting with your Teleport cluster. + +For example, from the Splunk UI home page, navigate to **Search & Reporting** > +**Dashboards** > **Create New Dashboard**. Enter "Teleport Audit Log Types" for +the title of your dashboard and click **Classic Dashboards**. Click **Create** +then, in the **Edit Dashboard** view, click **Add Panel**. + +In the **Add Panel** sidebar, click **New** > **Column Chart**. For the **Search +String** field, enter the following: + +```text +index="teleport-audit-logs" | timechart count by event +``` + +Once you click **Add to Dashboard** you will see a count of Teleport event types +over time, which gives you a general sense of how users are interacting with +Teleport: + +![Event Types over Time](../../../../docs/img/enterprise/plugins/splunk/splunk-dashboard.png) + +## Troubleshooting connection issues + +If the Teleport Event Handler is displaying error logs while connecting to your +Teleport Cluster, ensure that: + +- The certificate the Teleport Event Handler is using to connect to your + Teleport cluster is not past its expiration date. This is the value of the + `--ttl` flag in the `tctl auth sign` command, which is 12 hours by default. +- Ensure that in your Teleport Evant Handler configuration file + (`teleport-event-handler.toml`), you have provided the correct host *and* port + for the Teleport Proxy Service or Auth Service. + +## Next steps + +Now that you are exporting your audit logs to Splunk, consult our [audit log +reference](../../reference/audit.mdx#event-types) so you can plan visualizations +and alerts. + +In this guide, we made use of impersonation to supply credentials to the +Teleport Event Handler to communicat with your Teleport cluster. To learn more +about impersonation, read [our +guide](../../access-controls/guides/impersonation.mdx). + +While this guide uses the `tctl auth sign` command to issue credentials for the +Teleport Event Handler, production clusters should use Machine ID for safer, +more reliable renewals. Read [our guide](../../machine-id/getting-started.mdx) +to getting started with Machine ID. diff --git a/docs/pages/management/guides.mdx b/docs/pages/management/guides.mdx index 520fc39d1fd7f..b069d2a283d4d 100644 --- a/docs/pages/management/guides.mdx +++ b/docs/pages/management/guides.mdx @@ -7,10 +7,6 @@ layout: tocless-doc - [Kubernetes Operator (Preview)](./guides/teleport-operator.mdx). How to add the Teleport Kubernetes Operator to your Kubernetes cluster. - [Terraform Provider](./guides/terraform-provider.mdx). How to configure Teleport Cloud, Open Source, and Enterprise with the Terraform Provider for Teleport. - [Docker](./guides/docker.mdx). Getting started with Teleport Open Source using Docker. - - [Fluentd Event Forwarder](./guides/fluentd.mdx). Forwarding events with Fluentd and the Teleport Events Handler. - - [Monitor Audit Events with the Elastic Stack](./guides/elastic-stack.mdx). - How to send audit events to Elastcsearch for visualization and alerting in - Kibana. - [EC2 tags as Teleport Nodes](./guides/ec2-tags.mdx). How to set up Teleport Node labels based on EC2 tags. - [Joining Nodes via AWS IAM Role](./guides/joining-nodes-aws-iam.mdx). Use the IAM join method to add Nodes to your Teleport cluster on AWS. diff --git a/docs/pages/reference/metrics.mdx b/docs/pages/reference/metrics.mdx index 7eb4f269ede04..c18b0a9d4c880 100644 --- a/docs/pages/reference/metrics.mdx +++ b/docs/pages/reference/metrics.mdx @@ -14,9 +14,9 @@ monitor Teleport usage, consider utilizing our Event Handler plugin to push Audi preferred logging aggregation system (Elastic, Splunk, Sumo Logic, etc). - [Audit Events and Records](../reference/audit.mdx) -- [Forwarding events with Fluentd](../management/guides/fluentd.mdx) -- [Monitor Teleport Audit Events with the Elastic Stack](../management/guides/elastic-stack.mdx) +- [Forwarding events with Fluentd](../management/export-audit-events/fluentd.mdx) +- [Monitor Teleport Audit Events with the Elastic Stack](../management/export-audit-events/elastic-stack.mdx) The following metrics are available: -(!docs/pages/includes/metrics.mdx!) \ No newline at end of file +(!docs/pages/includes/metrics.mdx!)