diff --git a/cloud/aws/apigateway/README.md b/cloud/aws/apigateway/README.md index 439aae4..d20226b 100644 --- a/cloud/aws/apigateway/README.md +++ b/cloud/aws/apigateway/README.md @@ -27,17 +27,23 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `5` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Environment | string | - | yes | -| filter_tags | Tags used for custom filtering | string | `*` | no | +| filter_tags | Tags used for filtering | string | `*` | no | +| http_4xx_requests_message | Custom message for API Gateway HTTP 4xx requests monitor | string | `` | no | +| http_4xx_requests_silenced | Groups to mute for API Gateway HTTP 4xx requests monitor | map | `` | no | | http_4xx_requests_threshold_critical | Maximum critical acceptable percent of 4xx errors | string | `30` | no | | http_4xx_requests_threshold_warning | Maximum warning acceptable percent of 4xx errors | string | `15` | no | +| http_5xx_requests_message | Custom message for API Gateway HTTP 5xx requests monitor | string | `` | no | +| http_5xx_requests_silenced | Groups to mute for API Gateway HTTP 5xx requests monitor | map | `` | no | | http_5xx_requests_threshold_critical | Maximum critical acceptable percent of 5xx errors | string | `20` | no | | http_5xx_requests_threshold_warning | Maximum warning acceptable percent of 5xx errors | string | `10` | no | +| latency_message | Custom message for API Gateway latency monitor | string | `` | no | +| latency_silenced | Groups to mute for API Gateway latency monitor | map | `` | no | +| latency_threshold_critical | Alerting threshold in milliseconds | string | `800` | no | +| latency_threshold_warning | Warning threshold in milliseconds | string | `400` | no | | message | Message sent when a monitor is triggered | string | - | yes | -| latency_threshold_critical | Alerting threshold in miliseconds | string | `800` | no | -| latency_threshold_warning | Warning threshold in miliseconds | string | `400` | no | -| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `0` | no | Related documentation --------------------- diff --git a/cloud/aws/apigateway/inputs.tf b/cloud/aws/apigateway/inputs.tf index 780f754..917c7ed 100644 --- a/cloud/aws/apigateway/inputs.tf +++ b/cloud/aws/apigateway/inputs.tf @@ -21,6 +21,18 @@ variable "delay" { ### LATENCY VARIABLES ### ################################### +variable "latency_silenced" { + description = "Groups to mute for API Gateway latency monitor" + type = "map" + default = {} +} + +variable "latency_message" { + description = "Custom message for API Gateway latency monitor" + type = "string" + default = "" +} + variable "latency_threshold_critical" { default = 800 description = "Alerting threshold in milliseconds" @@ -35,6 +47,18 @@ variable "latency_threshold_warning" { ### HTTP 5xx status pages ### ################################# +variable "http_5xx_requests_silenced" { + description = "Groups to mute for API Gateway HTTP 5xx requests monitor" + type = "map" + default = {} +} + +variable "http_5xx_requests_message" { + description = "Custom message for API Gateway HTTP 5xx requests monitor" + type = "string" + default = "" +} + variable "http_5xx_requests_threshold_critical" { default = 20 description = "Maximum critical acceptable percent of 5xx errors" @@ -49,6 +73,18 @@ variable "http_5xx_requests_threshold_warning" { ### HTTP 4xx status pages ### ################################# +variable "http_4xx_requests_silenced" { + description = "Groups to mute for API Gateway HTTP 4xx requests monitor" + type = "map" + default = {} +} + +variable "http_4xx_requests_message" { + description = "Custom message for API Gateway HTTP 4xx requests monitor" + type = "string" + default = "" +} + variable "http_4xx_requests_threshold_critical" { default = 30 description = "Maximum critical acceptable percent of 4xx errors" diff --git a/cloud/aws/apigateway/monitors-api.tf b/cloud/aws/apigateway/monitors-api.tf index 54e6bee..b25c3d2 100644 --- a/cloud/aws/apigateway/monitors-api.tf +++ b/cloud/aws/apigateway/monitors-api.tf @@ -2,7 +2,7 @@ resource "datadog_monitor" "API_Gateway_latency" { name = "[${var.environment}] API Gateway latency {{#is_alert}}{{comparator}} {{threshold}}ms ({{value}}ms){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}ms ({{value}}ms){{/is_warning}}" type = "metric alert" - message = "${var.message}" + message = "${coalesce(var.latency_message, var.message)}" query = < ${var.http_5xx_requests_threshold_critical} + default( + avg:aws.apigateway.5xxerror{${var.filter_tags}} by {region,apiname}.as_count() / + (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}), + 0) * 100 + ) > ${var.http_5xx_requests_threshold_critical} EOF evaluation_delay = "${var.delay}" @@ -54,6 +58,8 @@ resource "datadog_monitor" "API_http_5xx_errors_count" { timeout_h = 1 include_tags = true + silenced = "${var.http_5xx_requests_silenced}" + tags = ["env:${var.environment}", "resource:apigateway", "team:aws", "provider:aws"] } @@ -61,13 +67,15 @@ resource "datadog_monitor" "API_http_5xx_errors_count" { resource "datadog_monitor" "API_http_4xx_errors_count" { name = "[${var.environment}] API Gateway HTTP 4xx errors {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" type = "metric alert" - message = "${var.message}" + message = "${coalesce(var.http_4xx_requests_message, var.message)}" query = < ${var.http_4xx_requests_threshold_critical} + default( + avg:aws.apigateway.4xxerror{${var.filter_tags}} by {region,apiname}.as_count() / + (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}), + 0) * 100 + ) > ${var.http_4xx_requests_threshold_critical} EOF evaluation_delay = "${var.delay}" @@ -84,5 +92,7 @@ resource "datadog_monitor" "API_http_4xx_errors_count" { timeout_h = 1 include_tags = true + silenced = "${var.http_4xx_requests_silenced}" + tags = ["env:${var.environment}", "resource:apigateway", "team:aws", "provider:aws"] } diff --git a/cloud/aws/elasticsearch/README.md b/cloud/aws/elasticsearch/README.md index 770e237..8e7e45a 100644 --- a/cloud/aws/elasticsearch/README.md +++ b/cloud/aws/elasticsearch/README.md @@ -29,11 +29,17 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cpu_message | Custom message for ES cluster cpu monitor | string | `` | no | +| cpu_silenced | Groups to mute for ES cluster cpu monitor | map | `` | no | | cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no | | cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no | +| diskspace_message | Custom message for ES cluster diskspace monitor | string | `` | no | +| diskspace_silenced | Groups to mute for ES cluster diskspace monitor | map | `` | no | | diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no | | diskspace_threshold_warning | Disk free space in percent (warning threshold) | string | `20` | no | | environment | Architecture Environment | string | - | yes | +| es_cluster_status_message | Custom message for ES cluster status monitor | string | `` | no | +| es_cluster_status_silenced | Groups to mute for ES cluster status monitor | map | `` | no | | es_cluster_volume_size | ElasticSearch Domain volume size (in GB) | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | diff --git a/cloud/aws/elasticsearch/inputs.tf b/cloud/aws/elasticsearch/inputs.tf index a676154..a23da20 100644 --- a/cloud/aws/elasticsearch/inputs.tf +++ b/cloud/aws/elasticsearch/inputs.tf @@ -25,10 +25,35 @@ variable "filter_tags_custom" { } # AWS ElasticSearch Service specific + +variable "es_cluster_status_silenced" { + description = "Groups to mute for ES cluster status monitor" + type = "map" + default = {} +} + +variable "es_cluster_status_message" { + description = "Custom message for ES cluster status monitor" + type = "string" + default = "" +} + variable "es_cluster_volume_size" { description = "ElasticSearch Domain volume size (in GB)" } +variable "diskspace_silenced" { + description = "Groups to mute for ES cluster diskspace monitor" + type = "map" + default = {} +} + +variable "diskspace_message" { + description = "Custom message for ES cluster diskspace monitor" + type = "string" + default = "" +} + variable "diskspace_threshold_warning" { description = "Disk free space in percent (warning threshold)" default = "20" @@ -39,6 +64,18 @@ variable "diskspace_threshold_critical" { default = "10" } +variable "cpu_silenced" { + description = "Groups to mute for ES cluster cpu monitor" + type = "map" + default = {} +} + +variable "cpu_message" { + description = "Custom message for ES cluster cpu monitor" + type = "string" + default = "" +} + variable "cpu_threshold_warning" { description = "CPU usage in percent (warning threshold)" default = "80" diff --git a/cloud/aws/elasticsearch/monitors-elasticsearch.tf b/cloud/aws/elasticsearch/monitors-elasticsearch.tf index a9f2173..d51f50b 100644 --- a/cloud/aws/elasticsearch/monitors-elasticsearch.tf +++ b/cloud/aws/elasticsearch/monitors-elasticsearch.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### Elasticsearch cluster status monitor ### resource "datadog_monitor" "es_cluster_status" { name = "[${var.environment}] ElasticSearch cluster status is not green" - message = "${var.message}" + message = "${coalesce(var.es_cluster_status_message, var.message)}" type = "query alert" @@ -37,13 +37,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.es_cluster_status_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } ### Elasticsearch cluster free storage space monitor ### resource "datadog_monitor" "es_free_space_low" { name = "[${var.environment}] ElasticSearch cluster free storage space {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.diskspace_message, var.message)}" type = "query alert" @@ -70,13 +72,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.diskspace_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } ### Elasticsearch cluster CPU monitor ### resource "datadog_monitor" "es_cpu_90_15min" { name = "[${var.environment}] ElasticSearch cluster CPU high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_message, var.message)}" type = "query alert" @@ -102,5 +106,7 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.cpu_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } diff --git a/cloud/aws/elb/README.md b/cloud/aws/elb/README.md index 2943b8a..f3e5da5 100644 --- a/cloud/aws/elb/README.md +++ b/cloud/aws/elb/README.md @@ -31,16 +31,28 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | dd_aws_elb | # ELB | string | `disable` | no | +| elb_4xx_message | Custom message for ELB 4xx errors monitor | string | `` | no | +| elb_4xx_silenced | Groups to mute for ELB 4xx errors monitor | map | `` | no | | elb_4xx_threshold_critical | loadbalancer 4xx critical threshold in percentage | string | `10` | no | | elb_4xx_threshold_warning | loadbalancer 4xx warning threshold in percentage | string | `5` | no | +| elb_5xx_message | Custom message for ELB 5xx errors monitor | string | `` | no | +| elb_5xx_silenced | Groups to mute for ELB 5xx errors monitor | map | `` | no | | elb_5xx_threshold_critical | loadbalancer 5xx critical threshold in percentage | string | `10` | no | | elb_5xx_threshold_warning | loadbalancer 5xx warning threshold in percentage | string | `5` | no | +| elb_backend_4xx_message | Custom message for ELB backend 4xx errors monitor | string | `` | no | +| elb_backend_4xx_silenced | Groups to mute for ELB backend 4xx errors monitor | map | `` | no | | elb_backend_4xx_threshold_critical | loadbalancer backend 4xx critical threshold in percentage | string | `10` | no | | elb_backend_4xx_threshold_warning | loadbalancer backend 4xx warning threshold in percentage | string | `5` | no | +| elb_backend_5xx_message | Custom message for ELB backend 5xx errors monitor | string | `` | no | +| elb_backend_5xx_silenced | Groups to mute for ELB backend 5xx errors monitor | map | `` | no | | elb_backend_5xx_threshold_critical | loadbalancer backend 5xx critical threshold in percentage | string | `10` | no | | elb_backend_5xx_threshold_warning | loadbalancer backend 5xx warning threshold in percentage | string | `5` | no | | elb_backend_latency_critical | latency critical threshold in seconds | string | `5` | no | +| elb_backend_latency_message | Custom message for ELB backend latency monitor | string | `` | no | +| elb_backend_latency_silenced | Groups to mute for ELB backend latency monitor | map | `` | no | | elb_backend_latency_warning | latency warning threshold in seconds | string | `1` | no | +| elb_no_healthy_instance_message | Custom message for ELB no healty instance monitor | string | `` | no | +| elb_no_healthy_instance_silenced | Groups to mute for ELB no healty instance monitor | map | `` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf index f2056c8..53f122b 100644 --- a/cloud/aws/elb/inputs.tf +++ b/cloud/aws/elb/inputs.tf @@ -29,6 +29,30 @@ variable "dd_aws_elb" { default = "disable" } +variable "elb_no_healthy_instance_silenced" { + description = "Groups to mute for ELB no healty instance monitor" + type = "map" + default = {} +} + +variable "elb_no_healthy_instance_message" { + description = "Custom message for ELB no healty instance monitor" + type = "string" + default = "" +} + +variable "elb_4xx_silenced" { + description = "Groups to mute for ELB 4xx errors monitor" + type = "map" + default = {} +} + +variable "elb_4xx_message" { + description = "Custom message for ELB 4xx errors monitor" + type = "string" + default = "" +} + variable "elb_4xx_threshold_warning" { description = "loadbalancer 4xx warning threshold in percentage" default = 5 @@ -39,6 +63,18 @@ variable "elb_4xx_threshold_critical" { default = 10 } +variable "elb_5xx_silenced" { + description = "Groups to mute for ELB 5xx errors monitor" + type = "map" + default = {} +} + +variable "elb_5xx_message" { + description = "Custom message for ELB 5xx errors monitor" + type = "string" + default = "" +} + variable "elb_5xx_threshold_warning" { description = "loadbalancer 5xx warning threshold in percentage" default = 5 @@ -49,6 +85,18 @@ variable "elb_5xx_threshold_critical" { default = 10 } +variable "elb_backend_4xx_silenced" { + description = "Groups to mute for ELB backend 4xx errors monitor" + type = "map" + default = {} +} + +variable "elb_backend_4xx_message" { + description = "Custom message for ELB backend 4xx errors monitor" + type = "string" + default = "" +} + variable "elb_backend_4xx_threshold_warning" { description = "loadbalancer backend 4xx warning threshold in percentage" default = 5 @@ -59,6 +107,18 @@ variable "elb_backend_4xx_threshold_critical" { default = 10 } +variable "elb_backend_5xx_silenced" { + description = "Groups to mute for ELB backend 5xx errors monitor" + type = "map" + default = {} +} + +variable "elb_backend_5xx_message" { + description = "Custom message for ELB backend 5xx errors monitor" + type = "string" + default = "" +} + variable "elb_backend_5xx_threshold_warning" { description = "loadbalancer backend 5xx warning threshold in percentage" default = 5 @@ -69,6 +129,18 @@ variable "elb_backend_5xx_threshold_critical" { default = 10 } +variable "elb_backend_latency_silenced" { + description = "Groups to mute for ELB backend latency monitor" + type = "map" + default = {} +} + +variable "elb_backend_latency_message" { + description = "Custom message for ELB backend latency monitor" + type = "string" + default = "" +} + variable "elb_backend_latency_warning" { description = "latency warning threshold in seconds" default = 1 diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index 441d207..fc12fa5 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "ELB_no_healthy_instances" { name = "[${var.environment}] ELB no healthy instances" - message = "${var.message}" + message = "${coalesce(var.elb_no_healthy_instance_message, var.message)}" query = < ${var.elb_4xx_threshold_critical} + default( + avg:aws.elb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_4xx_threshold_critical} EOF type = "metric alert" @@ -61,18 +65,22 @@ resource "datadog_monitor" "ELB_too_much_4xx" { new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.elb_4xx_silenced}" + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_5xx" { name = "[${var.environment}] ELB 5xx errors too high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.elb_5xx_message, var.message)}" query = < ${var.elb_5xx_threshold_critical} + default( + avg:aws.elb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_5xx_threshold_critical} EOF type = "metric alert" @@ -93,18 +101,22 @@ resource "datadog_monitor" "ELB_too_much_5xx" { new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.elb_5xx_silenced}" + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_4xx_backend" { name = "[${var.environment}] ELB backend 4xx errors too high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.elb_backend_4xx_message, var.message)}" query = < ${var.elb_backend_4xx_threshold_critical} + default( + avg:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_backend_4xx_threshold_critical} EOF type = "metric alert" @@ -125,18 +137,22 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.elb_backend_4xx_silenced}" + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_5xx_backend" { name = "[${var.environment}] ELB backend 5xx errors too high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.elb_backend_5xx_message, var.message)}" query = < ${var.elb_backend_5xx_threshold_critical} + default( + avg:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_backend_5xx_threshold_critical} EOF type = "metric alert" @@ -157,12 +173,14 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.elb_backend_5xx_silenced}" + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_backend_latency" { name = "[${var.environment}] ELB latency too high {{#is_alert}}{{comparator}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.elb_backend_latency_message, var.message)}" query = <` | no | | incoming_records_timeframe | Monitor timeframe for incoming records metrics evaluation [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_15m` | no | | message | Message sent when an alert is triggered | string | - | yes | diff --git a/cloud/aws/kinesis-firehose/inputs.tf b/cloud/aws/kinesis-firehose/inputs.tf index eeef880..831045e 100644 --- a/cloud/aws/kinesis-firehose/inputs.tf +++ b/cloud/aws/kinesis-firehose/inputs.tf @@ -25,6 +25,19 @@ variable "filter_tags_custom" { } # Kinesis-Firehose + +variable "incoming_records_silenced" { + description = "Groups to mute for Kinesis Firehorse incoming records monitor" + type = "map" + default = {} +} + +variable "incoming_records_message" { + description = "Custom message for Kinesis Firehorse incoming records monitor" + type = "string" + default = "" +} + variable "incoming_records_timeframe" { description = "Monitor timeframe for incoming records metrics evaluation [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" default = "last_15m" diff --git a/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf b/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf index 0035472..5163d82 100644 --- a/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf +++ b/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### Kinesis Firehose Incoming records ### resource "datadog_monitor" "firehose_incoming_records" { name = "[${var.environment}] Kinesis Firehose No incoming records" - message = "${var.message}" + message = "${coalesce(var.incoming_records_message, var.message)}" type = "metric alert" @@ -34,5 +34,7 @@ EOF new_host_delay = "${var.delay}" no_data_timeframe = 20 + silenced = "${var.incoming_records_silenced}" + tags = ["env:${var.environment}", "resource:kinesis-firehose", "team:aws", "provider:aws"] } diff --git a/cloud/aws/rds/README.md b/cloud/aws/rds/README.md index b6337a1..737f08e 100644 --- a/cloud/aws/rds/README.md +++ b/cloud/aws/rds/README.md @@ -25,8 +25,12 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cpu_message | Custom message for RDS CPU usage monitor | string | `` | no | +| cpu_silenced | Groups to mute for RDS CPU usage monitor | map | `` | no | | cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no | | cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no | +| diskspace_message | Custom message for RDS free diskspace monitor | string | `` | no | +| diskspace_silenced | Groups to mute for RDS free diskspace monitor | map | `` | no | | diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no | | diskspace_threshold_warning | Disk free space in percent (warning threshold) | string | `20` | no | | environment | Architecture Environment | string | - | yes | diff --git a/cloud/aws/rds/inputs.tf b/cloud/aws/rds/inputs.tf index cac076f..7795cad 100644 --- a/cloud/aws/rds/inputs.tf +++ b/cloud/aws/rds/inputs.tf @@ -26,6 +26,18 @@ variable "filter_tags_custom" { # AWS RDS instance specific +variable "cpu_silenced" { + description = "Groups to mute for RDS CPU usage monitor" + type = "map" + default = {} +} + +variable "cpu_message" { + description = "Custom message for RDS CPU usage monitor" + type = "string" + default = "" +} + variable "cpu_threshold_warning" { description = "CPU usage in percent (warning threshold)" default = "80" @@ -36,6 +48,18 @@ variable "cpu_threshold_critical" { default = "90" } +variable "diskspace_silenced" { + description = "Groups to mute for RDS free diskspace monitor" + type = "map" + default = {} +} + +variable "diskspace_message" { + description = "Custom message for RDS free diskspace monitor" + type = "string" + default = "" +} + variable "diskspace_threshold_warning" { description = "Disk free space in percent (warning threshold)" default = "20" diff --git a/cloud/aws/rds/monitors-rds.tf b/cloud/aws/rds/monitors-rds.tf index 5965c52..deaf9aa 100644 --- a/cloud/aws/rds/monitors-rds.tf +++ b/cloud/aws/rds/monitors-rds.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### RDS instance CPU monitor ### resource "datadog_monitor" "rds_cpu_90_15min" { name = "[${var.environment}] RDS instance CPU high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_message, var.message)}" type = "metric alert" @@ -34,13 +34,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.cpu_silenced}" + tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"] } ### RDS instance free space monitor ### resource "datadog_monitor" "rds_free_space_low" { name = "[${var.environment}] RDS instance free space {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.diskspace_message, var.message)}" type = "metric alert" @@ -66,5 +68,7 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.diskspace_silenced}" + tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"] } diff --git a/cloud/aws/vpn/README.md b/cloud/aws/vpn/README.md index 4876c79..0bae76f 100644 --- a/cloud/aws/vpn/README.md +++ b/cloud/aws/vpn/README.md @@ -29,3 +29,5 @@ Inputs | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when an alert is triggered | string | - | yes | +| vpn_status_message | Custom message for VPN status monitor | string | `` | no | +| vpn_status_silenced | Groups to mute for VPN status monitor | map | `` | no | \ No newline at end of file diff --git a/cloud/aws/vpn/inputs.tf b/cloud/aws/vpn/inputs.tf index ff336dc..ba86c7a 100644 --- a/cloud/aws/vpn/inputs.tf +++ b/cloud/aws/vpn/inputs.tf @@ -23,3 +23,15 @@ variable "filter_tags_custom" { description = "Tags used for custom filtering when filter_tags_use_defaults is false" default = "*" } + +variable "vpn_status_silenced" { + description = "Groups to mute for VPN status monitor" + type = "map" + default = {} +} + +variable "vpn_status_message" { + description = "Custom message for VPN status monitor" + type = "string" + default = "" +} diff --git a/cloud/aws/vpn/monitors-vpn.tf b/cloud/aws/vpn/monitors-vpn.tf index c86e01c..4c36b9c 100644 --- a/cloud/aws/vpn/monitors-vpn.tf +++ b/cloud/aws/vpn/monitors-vpn.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "VPN_status" { name = "[${var.environment}] VPN Down" - message = "${var.message}" + message = "${coalesce(var.vpn_status_message, var.message)}" query = <` | no | diff --git a/databases/mongodb/inputs.tf b/databases/mongodb/inputs.tf index b698157..e526871 100644 --- a/databases/mongodb/inputs.tf +++ b/databases/mongodb/inputs.tf @@ -23,3 +23,15 @@ variable "filter_tags_custom" { description = "Tags used for custom filtering when filter_tags_use_defaults is false" default = "*" } + +variable "mongodb_replicaset_silenced" { + description = "Groups to mute for Mongodb replicaset monitor" + type = "map" + default = {} +} + +variable "mongodb_replicaset_message" { + description = "Custom message for Mongodb replicaset monitor" + type = "string" + default = "" +} diff --git a/databases/mongodb/monitors-mongo.tf b/databases/mongodb/monitors-mongo.tf index 03b55b8..1c06c51 100644 --- a/databases/mongodb/monitors-mongo.tf +++ b/databases/mongodb/monitors-mongo.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "mongodb_replicaset_state" { name = "[${var.environment}] Member down in the replica set" - message = "${var.message}" + message = "${coalesce(var.mongodb_replicaset_message, var.message)}" query = <` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | -| message | Message sent when an alert is triggered | string | - | yes | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when an alert is triggered | string | - | yes | \ No newline at end of file diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf index 1812e3d..d300534 100644 --- a/middleware/apache/inputs.tf +++ b/middleware/apache/inputs.tf @@ -14,5 +14,26 @@ variable "message" { description = "Message sent when an alert is triggered" } +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + # Apache Middleware specific +variable "apache_connect_silenced" { + description = "Groups to mute for Apache process monitor" + type = "map" + default = {} +} + +variable "apache_connect_message" { + description = "Custom message for Apache process monitor" + type = "string" + default = "" +} diff --git a/middleware/apache/monitors-apache.tf b/middleware/apache/monitors-apache.tf index d31cfd6..3406d3b 100644 --- a/middleware/apache/monitors-apache.tf +++ b/middleware/apache/monitors-apache.tf @@ -1,9 +1,20 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_apache:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}" + } +} + resource "datadog_monitor" "datadog_apache_process" { name = "[${var.environment}] Can't connect to apache vhost status" - message = "${var.message}" + message = "${coalesce(var.apache_connect_message, var.message)}" - type = "service check" - query = "\"apache.can_connect\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <` | no | \ No newline at end of file diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf index a82f6c2..b4ca2f7 100644 --- a/middleware/nginx/inputs.tf +++ b/middleware/nginx/inputs.tf @@ -14,5 +14,26 @@ variable "message" { description = "Message sent when an alert is triggered" } +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + # Nginx Middleware specific +variable "nginx_connect_silenced" { + description = "Groups to mute for Nginx process monitor" + type = "map" + default = {} +} + +variable "nginx_connect_message" { + description = "Custom message for Nginx process monitor" + type = "string" + default = "" +} diff --git a/middleware/nginx/monitors-nginx.tf b/middleware/nginx/monitors-nginx.tf index 3db1c0f..b578fca 100644 --- a/middleware/nginx/monitors-nginx.tf +++ b/middleware/nginx/monitors-nginx.tf @@ -1,9 +1,20 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_nginx:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}" + } +} + resource "datadog_monitor" "datadog_nginx_process" { name = "[${var.environment}] Can't connect to nginx vhost status" - message = "${var.message}" + message = "${coalesce(var.nginx_connect_message, var.message)}" - type = "service check" - query = "\"nginx.can_connect\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <` | no | | php_fpm_busy_threshold_critical | php fpm busy critical threshold | string | `0.9` | no | | php_fpm_busy_threshold_warning | php fpm busy warning threshold | string | `0.8` | no | +| php_fpm_connect_message | Custom message for PHP FPM process monitor | string | `` | no | +| php_fpm_connect_silenced | Groups to mute for PHP FPM process monitor | map | `` | no | \ No newline at end of file diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf index 726bd04..49ac556 100644 --- a/middleware/php-fpm/inputs.tf +++ b/middleware/php-fpm/inputs.tf @@ -31,6 +31,18 @@ variable "filter_tags_custom" { # PHP FPM Middleware specific +variable "php_fpm_busy_silenced" { + description = "Groups to mute for PHP FPM busy worker monitor" + type = "map" + default = {} +} + +variable "php_fpm_busy_message" { + description = "Custom message for PHP FPM busy worker monitor" + type = "string" + default = "" +} + variable "php_fpm_busy_threshold_warning" { description = "php fpm busy warning threshold" default = 0.8 @@ -40,3 +52,15 @@ variable "php_fpm_busy_threshold_critical" { description = "php fpm busy critical threshold" default = 0.9 } + +variable "php_fpm_connect_silenced" { + description = "Groups to mute for PHP FPM process monitor" + type = "map" + default = {} +} + +variable "php_fpm_connect_message" { + description = "Custom message for PHP FPM process monitor" + type = "string" + default = "" +} diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf index 51f582d..16a944d 100644 --- a/middleware/php-fpm/monitors-fpm.tf +++ b/middleware/php-fpm/monitors-fpm.tf @@ -6,9 +6,9 @@ data "template_file" "filter" { } } -resource "datadog_monitor" "datadog_php_fpm_process_idle" { +resource "datadog_monitor" "datadog_php_fpm_connect_idle" { name = "[${var.environment}] php_fpm busy worker {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.php_fpm_busy_message, var.message)}" type = "metric alert" @@ -35,15 +35,20 @@ resource "datadog_monitor" "datadog_php_fpm_process_idle" { require_full_window = true no_data_timeframe = 20 + silenced = "${var.php_fpm_busy_silenced}" + tags = ["env:${var.environment}", "resource:php-fpm"] } resource "datadog_monitor" "datadog_fpm_process" { name = "[${var.environment}] Can't connect to php-fpm" - message = "${var.message}" + message = "${coalesce(var.php_fpm_connect_message, var.message)}" - type = "service check" - query = "\"php_fpm.can_ping\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <` | no | | cpu_high_threshold_critical | CPU high critical threshold | string | `95` | no | | cpu_high_threshold_warning | CPU high warning threshold | string | `80` | no | | cpu_high_timeframe | CPU high timeframe | string | `last_5m` | no | @@ -31,10 +33,16 @@ Inputs | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| free_disk_inodes_message | Custom message for Free disk inodes monitor | string | `` | no | +| free_disk_inodes_silenced | Groups to mute for Free disk inodes monitor | map | `` | no | | free_disk_inodes_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_disk_inodes_threshold_warning | Free disk space warning threshold | string | `10` | no | +| free_disk_space_message | Custom message for Free diskspace monitor | string | `` | no | +| free_disk_space_silenced | Groups to mute for Free diskspace monitor | map | `` | no | | free_disk_space_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_disk_space_threshold_warning | Free disk space warning threshold | string | `10` | no | +| free_memory_message | Custom message for Free memory monitor | string | `` | no | +| free_memory_silenced | Groups to mute for Free memory monitor | map | `` | no | | free_memory_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_memory_threshold_warning | Free disk space warning threshold | string | `10` | no | -| message | Message sent when an alert is triggered | string | - | yes | +| message | Message sent when an alert is triggered | string | - | yes | \ No newline at end of file diff --git a/system/generic/inputs.tf b/system/generic/inputs.tf index 5b34296..a9a7a98 100644 --- a/system/generic/inputs.tf +++ b/system/generic/inputs.tf @@ -26,6 +26,18 @@ variable "filter_tags_custom" { # Custom CPU instance specific +variable "cpu_high_silenced" { + description = "Groups to mute for CPU high monitor" + type = "map" + default = {} +} + +variable "cpu_high_message" { + description = "Custom message for CPU high monitor" + type = "string" + default = "" +} + variable "cpu_high_timeframe" { description = "CPU high timeframe" default = "last_5m" @@ -41,6 +53,18 @@ variable "cpu_high_threshold_critical" { default = 95 } +variable "free_disk_space_silenced" { + description = "Groups to mute for Free diskspace monitor" + type = "map" + default = {} +} + +variable "free_disk_space_message" { + description = "Custom message for Free diskspace monitor" + type = "string" + default = "" +} + variable "free_disk_space_threshold_warning" { description = "Free disk space warning threshold" default = 10 @@ -51,6 +75,18 @@ variable "free_disk_space_threshold_critical" { default = 5 } +variable "free_disk_inodes_silenced" { + description = "Groups to mute for Free disk inodes monitor" + type = "map" + default = {} +} + +variable "free_disk_inodes_message" { + description = "Custom message for Free disk inodes monitor" + type = "string" + default = "" +} + variable "free_disk_inodes_threshold_warning" { description = "Free disk space warning threshold" default = 10 @@ -61,6 +97,18 @@ variable "free_disk_inodes_threshold_critical" { default = 5 } +variable "free_memory_silenced" { + description = "Groups to mute for Free memory monitor" + type = "map" + default = {} +} + +variable "free_memory_message" { + description = "Custom message for Free memory monitor" + type = "string" + default = "" +} + variable "free_memory_threshold_warning" { description = "Free disk space warning threshold" default = 10 diff --git a/system/generic/monitors-system.tf b/system/generic/monitors-system.tf index ce70cd2..9908b0e 100644 --- a/system/generic/monitors-system.tf +++ b/system/generic/monitors-system.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "datadog_cpu_too_high" { name = "[${var.environment}] CPU usage {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_high_message, var.message)}" query = <