-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathlogging-lambda-firehose-metric-stream.yaml
251 lines (245 loc) · 10.7 KB
/
logging-lambda-firehose-metric-stream.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: New Relic Template to forward logs using firehose/S3 bucket and metrics using streaming from AWS to New Relic
Metadata:
AWS::ServerlessRepo::Application:
Name: NewRelic-log-forwarding
Description: Send log data from AWS to New Relic.
Author: New Relic
SpdxLicenseId: Apache-2.0
LicenseUrl: LICENSE.txt
ReadmeUrl: README.md
HomePageUrl: https://github.com/newrelic/aws-unified-logging #TODO change this url once the repo is public
SemanticVersion: 0.0.17
SourceCodeUrl: https://github.com/newrelic/aws-unified-logging #TODO change this url once the repo is public
AWS::CloudFormation::Interface:
ParameterGroups:
-
Label:
default: 'AWS Integration configuration'
Parameters:
- NewRelicAccountId
- NewRelicRegion
- NewRelicLicenseKey
-
Label:
default: 'AWS Metric Stream Configuration'
Parameters:
- IAMRoleName
- NewRelicAPIKey
- IntegrationName
- MetricCollectionMode
- PollingIntegrationSlugs
- CloudWatchMetricStreamName
- FirehoseStreamName
- S3BackupBucketName
-
Label:
default: 'AWS Logging Configuration'
Parameters:
- LogGroupConfig
- S3BucketNames
- CommonAttributes
- LoggingFirehoseStreamName
- LoggingS3BackupBucketName
- EnableCloudWatchLoggingForFirehose
- StoreNRLicenseKeyInSecretManager
-
Label:
default: 'AWS Config Service Configuration (Optional)'
Parameters:
- CreateConfigService
- S3ConfigBucketName
ParameterLabels:
IntegrationName:
default: 'Name of integration in NewRelic'
NewRelicAccountId:
default: 'NewRelic account ID'
IAMRoleName:
default: 'Role name for NewRelic'
NewRelicAPIKey:
default: 'NewRelic API Key'
MetricCollectionMode:
default: 'Metric collection mode'
CloudWatchMetricStreamName:
default: 'CloudWatch Metric Stream name'
FirehoseStreamName:
default: 'Kinesis Data Firehose name'
NewRelicRegion:
default: 'New Relic Datacenter Region'
NewRelicLicenseKey:
default: 'New Relic Ingest License Key'
S3BackupBucketName:
default: 'Firehose S3 backup bucket name'
CreateConfigService:
default: 'Enrich metrics with resource metadata from AWS Config?'
S3ConfigBucketName:
default: 'Config S3 backup bucket name'
LogGroupConfig:
default: 'Log Group name'
S3BucketNames:
default: 'S3 bucket name'
CommonAttributes:
default: 'Common Attributes'
LoggingFirehoseStreamName:
default: 'Name of Logging Firehose Delivery Stream'
LoggingS3BackupBucketName:
default: 'S3 Bucket Destination for failed Logging events'
EnableCloudWatchLoggingForFirehose:
default: 'Enables CloudWatch logging for Logging Firehose stream'
StoreNRLicenseKeyInSecretManager:
default: 'Store New Relic License Key in AWS Secrets Manager'
Parameters:
IAMRoleName:
Description: Customize the base name of IAM role for New relic AWS integration (the final role name will contain some id at the end)
Type: String
Default: NewRelicInfrastructure-Integrations
NewRelicAccountId:
Type: String
Description: Id of the account in New relic
AllowedPattern: "[0-9]+"
ConstraintDescription: must only contain numbers
NewRelicRegion:
Type: String
Description: Datacenter where the data will be sent (US/EU), DO NOT TOUCH
AllowedValues: [US, EU]
Default: US
IntegrationName:
Type: String
Description: Name of the AWS integration that will be created in New relic
MinLength: 1
MaxLength: 255
AllowedPattern: "[a-zA-Z0-9_-]+"
ConstraintDescription: must only container letters (uppercase and lowercase), numbers, and characters '_', and '-' with max length of 255 total characters
NewRelicAPIKey:
Type: String
Description: New relic user key to use NerdGraph, our GraphQL API, create or get an existing key in https://one.newrelic.com/launcher/api-keys-ui.api-keys-launcher
NoEcho: true
MaxLength: 40
AllowedPattern: "^[^\\s]+$"
ConstraintDescription: "The parameter value cannot be empty, contain spaces, and must be alphanumeric and can contain symbols."
PollingIntegrationSlugs:
Type: CommaDelimitedList
Description: "Comma-delimited list of the polling integrations to configure"
MetricCollectionMode:
Type: String
Description: "Metrics Collection Mode to be chosen by customer between PUSH or PULL"
AllowedValues: [ PUSH, PULL ]
Default: PUSH
NewRelicLicenseKey:
Type: String
Description: New relic license key, create or get and existing key in https://one.newrelic.com/launcher/api-keys-ui.api-keys-launcher
NoEcho: true
MaxLength: 40
AllowedPattern: "^[^\\s]+$"
ConstraintDescription: "The parameter value cannot be empty, contain spaces, and must be alphanumeric and can contain symbols."
FirehoseStreamName:
Type: String
Description: Name of new Kinesis Firehose Delivery Stream (must be unique per AWS account in the same AWS Region)
Default: NewRelic-Delivery-Stream
MaxLength: 64
AllowedPattern: "[a-zA-Z0-9_.-]+"
ConstraintDescription: must only container letters (uppercase and lowercase), numbers, and characters '.', '_', and '-' with max length of 64 total characters
CloudWatchMetricStreamName:
Type: String
Description: Name of new CloudWatch Metric Stream (must be unique per AWS account in the same AWS Region)
Default: NewRelic-Metric-Stream
MaxLength: 255
AllowedPattern: "[a-zA-Z0-9_-]+"
ConstraintDescription: must only container letters (uppercase and lowercase), numbers, and characters '_', and '-' with max length of 255 total characters
S3BackupBucketName:
Type: String
Description: S3 Bucket Destination for failed events (must be globally unique across all AWS accounts in all AWS Regions within a partition)
Default: firehose-backup
MinLength: 3
MaxLength: 63
AllowedPattern: (?!(^xn--|-s3alias$))^[a-z0-9.][a-z0-9-.]{1,61}[a-z0-9.]$
ConstraintDescription: must adhere to the S3 bucket naming rules - https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
CreateConfigService:
Type: String
Description: Enable and configure AWS Config to track resource changes. This enables additional discovery and monitoring capabilities. (Optional)
Default: "false"
AllowedValues: [ "true", "false" ]
S3ConfigBucketName:
Type: String
Description: S3 Bucket Destination for delivery channel configuration (must be globally unique across all AWS accounts in all AWS Regions within a partition)
Default: config-backup
MinLength: 3
MaxLength: 63
AllowedPattern: (?!(^xn--|-s3alias$))^[a-z0-9.][a-z0-9-.]{1,61}[a-z0-9.]$
ConstraintDescription: must adhere to the S3 bucket naming rules - https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
S3BucketNames:
Type: String
Description: "JSON array of objects representing your S3Bucketname and prefixes (if applicable) For example: [{\"bucket\":\"bucket1\",\"prefix\":\"prefix/\"}]"
Default: ""
LogGroupConfig:
Description: "JSON array of objects representing your LogGroup and Filters (if applicable). For example: [{\"LogGroupName\":\"logGroup1\",\"FilterPattern\":\"filter1\"}]"
Type: String
CommonAttributes:
Type: String
Description: "JSON array of objects representing your custom attributes. For example: [{\"AttributeName\":\"key1\",\"AttributeValue\":\"value1\"}]"
Default: ""
LoggingFirehoseStreamName:
Type: String
Description: Name of new Data Firehose Delivery Stream (must be unique per AWS account in the same AWS Region)
Default: NewRelic-Logging-Delivery-Stream
MaxLength: 64
AllowedPattern: "[a-zA-Z0-9_.-]+"
ConstraintDescription: must only contain letters (uppercase and lowercase), numbers, and characters '.', '_', and '-' with max length of 64 total characters
LoggingS3BackupBucketName:
Type: String
Description: S3 Bucket Destination for failed events (must be globally unique across all AWS accounts in all AWS Regions within a partition)
Default: firehose-logging-backup
MinLength: 3
MaxLength: 63
AllowedPattern: (?!(^xn--|-s3alias$))^[a-z0-9.][a-z0-9-.]{1,61}[a-z0-9.]$
ConstraintDescription: must adhere to the S3 bucket naming rules - https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
EnableCloudWatchLoggingForFirehose:
Type: String
Description: Enables CloudWatch logging for the Firehose stream.
Default: "false"
AllowedValues:
- "true"
- "false"
StoreNRLicenseKeyInSecretManager:
Type: String
Description: Should we store the New Relic license key in AWS Secrets Manager. Defaults to true.
Default: "true"
AllowedValues:
- "true"
- "false"
Resources:
NewRelicLogsLoggingLambdaFirehose:
Type: 'AWS::CloudFormation::Stack'
Properties:
TemplateURL: logging-lambda-firehose-template.yaml
Parameters:
LicenseKey: !Ref NewRelicLicenseKey
NewRelicRegion: !Ref NewRelicRegion
NewRelicAccountId: !Ref NewRelicAccountId
S3BucketNames: !Ref S3BucketNames
LogGroupConfig: !Ref LogGroupConfig
CommonAttributes: !Ref CommonAttributes
LoggingFirehoseStreamName: !Ref LoggingFirehoseStreamName
LoggingS3BackupBucketName: !Ref LoggingS3BackupBucketName
EnableCloudWatchLoggingForFirehose: !Ref EnableCloudWatchLoggingForFirehose
StoreNRLicenseKeyInSecretManager: !Ref StoreNRLicenseKeyInSecretManager
# New Relic Metrics Stack
NewrelicMetricsStack:
Type: AWS::CloudFormation::Stack
Properties:
TemplateURL: https://nr-downloads-main.s3.amazonaws.com/cloud_integrations/aws/cloudformation/newrelic-cloudformation-mstreams.yml
Parameters:
IAMRoleName: !Ref IAMRoleName
NewRelicAccountId: !Ref NewRelicAccountId
NewRelicRegion: !Ref NewRelicRegion
IntegrationName: !Ref IntegrationName
NewRelicAPIKey: !Ref NewRelicAPIKey
PollingIntegrationSlugs: !Join [ "," ,!Ref PollingIntegrationSlugs ]
MetricCollectionMode: !Ref MetricCollectionMode
NewRelicLicenseKey: !Ref NewRelicLicenseKey
FirehoseStreamName: !Ref FirehoseStreamName
CloudWatchMetricStreamName: !Ref CloudWatchMetricStreamName
S3BackupBucketName: !Ref S3BackupBucketName
CreateConfigService: !Ref CreateConfigService
S3ConfigBucketName: !Ref S3ConfigBucketName