From bb8824805353be8e59103c67524e5149561dd42d Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Tue, 20 Mar 2018 12:20:31 +0100 Subject: [PATCH 1/5] MON-96 - Updated aws monitors with inputs best practice --- cloud/aws/apigateway/README.md | 14 ++-- cloud/aws/apigateway/inputs.tf | 36 ++++++++++ cloud/aws/apigateway/monitors-api.tf | 12 +++- cloud/aws/elasticsearch/README.md | 6 ++ cloud/aws/elasticsearch/inputs.tf | 37 ++++++++++ .../elasticsearch/monitors-elasticsearch.tf | 12 +++- cloud/aws/elb/README.md | 12 ++++ cloud/aws/elb/inputs.tf | 72 +++++++++++++++++++ cloud/aws/elb/monitors-elb.tf | 24 +++++-- cloud/aws/kinesis-firehose/README.md | 2 + cloud/aws/kinesis-firehose/inputs.tf | 13 ++++ .../monitors-kinesis-firehose.tf | 4 +- cloud/aws/rds/README.md | 4 ++ cloud/aws/rds/inputs.tf | 24 +++++++ cloud/aws/rds/monitors-rds.tf | 8 ++- cloud/aws/vpn/README.md | 2 + cloud/aws/vpn/inputs.tf | 12 ++++ cloud/aws/vpn/monitors-vpn.tf | 4 +- 18 files changed, 278 insertions(+), 20 deletions(-) diff --git a/cloud/aws/apigateway/README.md b/cloud/aws/apigateway/README.md index 439aae4..d20226b 100644 --- a/cloud/aws/apigateway/README.md +++ b/cloud/aws/apigateway/README.md @@ -27,17 +27,23 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `5` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Environment | string | - | yes | -| filter_tags | Tags used for custom filtering | string | `*` | no | +| filter_tags | Tags used for filtering | string | `*` | no | +| http_4xx_requests_message | Custom message for API Gateway HTTP 4xx requests monitor | string | `` | no | +| http_4xx_requests_silenced | Groups to mute for API Gateway HTTP 4xx requests monitor | map | `` | no | | http_4xx_requests_threshold_critical | Maximum critical acceptable percent of 4xx errors | string | `30` | no | | http_4xx_requests_threshold_warning | Maximum warning acceptable percent of 4xx errors | string | `15` | no | +| http_5xx_requests_message | Custom message for API Gateway HTTP 5xx requests monitor | string | `` | no | +| http_5xx_requests_silenced | Groups to mute for API Gateway HTTP 5xx requests monitor | map | `` | no | | http_5xx_requests_threshold_critical | Maximum critical acceptable percent of 5xx errors | string | `20` | no | | http_5xx_requests_threshold_warning | Maximum warning acceptable percent of 5xx errors | string | `10` | no | +| latency_message | Custom message for API Gateway latency monitor | string | `` | no | +| latency_silenced | Groups to mute for API Gateway latency monitor | map | `` | no | +| latency_threshold_critical | Alerting threshold in milliseconds | string | `800` | no | +| latency_threshold_warning | Warning threshold in milliseconds | string | `400` | no | | message | Message sent when a monitor is triggered | string | - | yes | -| latency_threshold_critical | Alerting threshold in miliseconds | string | `800` | no | -| latency_threshold_warning | Warning threshold in miliseconds | string | `400` | no | -| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `0` | no | Related documentation --------------------- diff --git a/cloud/aws/apigateway/inputs.tf b/cloud/aws/apigateway/inputs.tf index 780f754..917c7ed 100644 --- a/cloud/aws/apigateway/inputs.tf +++ b/cloud/aws/apigateway/inputs.tf @@ -21,6 +21,18 @@ variable "delay" { ### LATENCY VARIABLES ### ################################### +variable "latency_silenced" { + description = "Groups to mute for API Gateway latency monitor" + type = "map" + default = {} +} + +variable "latency_message" { + description = "Custom message for API Gateway latency monitor" + type = "string" + default = "" +} + variable "latency_threshold_critical" { default = 800 description = "Alerting threshold in milliseconds" @@ -35,6 +47,18 @@ variable "latency_threshold_warning" { ### HTTP 5xx status pages ### ################################# +variable "http_5xx_requests_silenced" { + description = "Groups to mute for API Gateway HTTP 5xx requests monitor" + type = "map" + default = {} +} + +variable "http_5xx_requests_message" { + description = "Custom message for API Gateway HTTP 5xx requests monitor" + type = "string" + default = "" +} + variable "http_5xx_requests_threshold_critical" { default = 20 description = "Maximum critical acceptable percent of 5xx errors" @@ -49,6 +73,18 @@ variable "http_5xx_requests_threshold_warning" { ### HTTP 4xx status pages ### ################################# +variable "http_4xx_requests_silenced" { + description = "Groups to mute for API Gateway HTTP 4xx requests monitor" + type = "map" + default = {} +} + +variable "http_4xx_requests_message" { + description = "Custom message for API Gateway HTTP 4xx requests monitor" + type = "string" + default = "" +} + variable "http_4xx_requests_threshold_critical" { default = 30 description = "Maximum critical acceptable percent of 4xx errors" diff --git a/cloud/aws/apigateway/monitors-api.tf b/cloud/aws/apigateway/monitors-api.tf index 54e6bee..f6d2cd9 100644 --- a/cloud/aws/apigateway/monitors-api.tf +++ b/cloud/aws/apigateway/monitors-api.tf @@ -2,7 +2,7 @@ resource "datadog_monitor" "API_Gateway_latency" { name = "[${var.environment}] API Gateway latency {{#is_alert}}{{comparator}} {{threshold}}ms ({{value}}ms){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}ms ({{value}}ms){{/is_warning}}" type = "metric alert" - message = "${var.message}" + message = "${coalesce(var.latency_message, var.message)}" query = <` | no | | cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no | | cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no | +| diskspace_message | Custom message for ES cluster diskspace monitor | string | `` | no | +| diskspace_silenced | Groups to mute for ES cluster diskspace monitor | map | `` | no | | diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no | | diskspace_threshold_warning | Disk free space in percent (warning threshold) | string | `20` | no | | environment | Architecture Environment | string | - | yes | +| es_cluster_status_message | Custom message for ES cluster status monitor | string | `` | no | +| es_cluster_status_silenced | Groups to mute for ES cluster status monitor | map | `` | no | | es_cluster_volume_size | ElasticSearch Domain volume size (in GB) | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | diff --git a/cloud/aws/elasticsearch/inputs.tf b/cloud/aws/elasticsearch/inputs.tf index a676154..a23da20 100644 --- a/cloud/aws/elasticsearch/inputs.tf +++ b/cloud/aws/elasticsearch/inputs.tf @@ -25,10 +25,35 @@ variable "filter_tags_custom" { } # AWS ElasticSearch Service specific + +variable "es_cluster_status_silenced" { + description = "Groups to mute for ES cluster status monitor" + type = "map" + default = {} +} + +variable "es_cluster_status_message" { + description = "Custom message for ES cluster status monitor" + type = "string" + default = "" +} + variable "es_cluster_volume_size" { description = "ElasticSearch Domain volume size (in GB)" } +variable "diskspace_silenced" { + description = "Groups to mute for ES cluster diskspace monitor" + type = "map" + default = {} +} + +variable "diskspace_message" { + description = "Custom message for ES cluster diskspace monitor" + type = "string" + default = "" +} + variable "diskspace_threshold_warning" { description = "Disk free space in percent (warning threshold)" default = "20" @@ -39,6 +64,18 @@ variable "diskspace_threshold_critical" { default = "10" } +variable "cpu_silenced" { + description = "Groups to mute for ES cluster cpu monitor" + type = "map" + default = {} +} + +variable "cpu_message" { + description = "Custom message for ES cluster cpu monitor" + type = "string" + default = "" +} + variable "cpu_threshold_warning" { description = "CPU usage in percent (warning threshold)" default = "80" diff --git a/cloud/aws/elasticsearch/monitors-elasticsearch.tf b/cloud/aws/elasticsearch/monitors-elasticsearch.tf index a9f2173..d51f50b 100644 --- a/cloud/aws/elasticsearch/monitors-elasticsearch.tf +++ b/cloud/aws/elasticsearch/monitors-elasticsearch.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### Elasticsearch cluster status monitor ### resource "datadog_monitor" "es_cluster_status" { name = "[${var.environment}] ElasticSearch cluster status is not green" - message = "${var.message}" + message = "${coalesce(var.es_cluster_status_message, var.message)}" type = "query alert" @@ -37,13 +37,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.es_cluster_status_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } ### Elasticsearch cluster free storage space monitor ### resource "datadog_monitor" "es_free_space_low" { name = "[${var.environment}] ElasticSearch cluster free storage space {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.diskspace_message, var.message)}" type = "query alert" @@ -70,13 +72,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.diskspace_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } ### Elasticsearch cluster CPU monitor ### resource "datadog_monitor" "es_cpu_90_15min" { name = "[${var.environment}] ElasticSearch cluster CPU high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_message, var.message)}" type = "query alert" @@ -102,5 +106,7 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.cpu_silenced}" + tags = ["env:${var.environment}", "resource:elasticsearch", "team:aws", "provider:aws"] } diff --git a/cloud/aws/elb/README.md b/cloud/aws/elb/README.md index 2943b8a..f3e5da5 100644 --- a/cloud/aws/elb/README.md +++ b/cloud/aws/elb/README.md @@ -31,16 +31,28 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | dd_aws_elb | # ELB | string | `disable` | no | +| elb_4xx_message | Custom message for ELB 4xx errors monitor | string | `` | no | +| elb_4xx_silenced | Groups to mute for ELB 4xx errors monitor | map | `` | no | | elb_4xx_threshold_critical | loadbalancer 4xx critical threshold in percentage | string | `10` | no | | elb_4xx_threshold_warning | loadbalancer 4xx warning threshold in percentage | string | `5` | no | +| elb_5xx_message | Custom message for ELB 5xx errors monitor | string | `` | no | +| elb_5xx_silenced | Groups to mute for ELB 5xx errors monitor | map | `` | no | | elb_5xx_threshold_critical | loadbalancer 5xx critical threshold in percentage | string | `10` | no | | elb_5xx_threshold_warning | loadbalancer 5xx warning threshold in percentage | string | `5` | no | +| elb_backend_4xx_message | Custom message for ELB backend 4xx errors monitor | string | `` | no | +| elb_backend_4xx_silenced | Groups to mute for ELB backend 4xx errors monitor | map | `` | no | | elb_backend_4xx_threshold_critical | loadbalancer backend 4xx critical threshold in percentage | string | `10` | no | | elb_backend_4xx_threshold_warning | loadbalancer backend 4xx warning threshold in percentage | string | `5` | no | +| elb_backend_5xx_message | Custom message for ELB backend 5xx errors monitor | string | `` | no | +| elb_backend_5xx_silenced | Groups to mute for ELB backend 5xx errors monitor | map | `` | no | | elb_backend_5xx_threshold_critical | loadbalancer backend 5xx critical threshold in percentage | string | `10` | no | | elb_backend_5xx_threshold_warning | loadbalancer backend 5xx warning threshold in percentage | string | `5` | no | | elb_backend_latency_critical | latency critical threshold in seconds | string | `5` | no | +| elb_backend_latency_message | Custom message for ELB backend latency monitor | string | `` | no | +| elb_backend_latency_silenced | Groups to mute for ELB backend latency monitor | map | `` | no | | elb_backend_latency_warning | latency warning threshold in seconds | string | `1` | no | +| elb_no_healthy_instance_message | Custom message for ELB no healty instance monitor | string | `` | no | +| elb_no_healthy_instance_silenced | Groups to mute for ELB no healty instance monitor | map | `` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf index f2056c8..53f122b 100644 --- a/cloud/aws/elb/inputs.tf +++ b/cloud/aws/elb/inputs.tf @@ -29,6 +29,30 @@ variable "dd_aws_elb" { default = "disable" } +variable "elb_no_healthy_instance_silenced" { + description = "Groups to mute for ELB no healty instance monitor" + type = "map" + default = {} +} + +variable "elb_no_healthy_instance_message" { + description = "Custom message for ELB no healty instance monitor" + type = "string" + default = "" +} + +variable "elb_4xx_silenced" { + description = "Groups to mute for ELB 4xx errors monitor" + type = "map" + default = {} +} + +variable "elb_4xx_message" { + description = "Custom message for ELB 4xx errors monitor" + type = "string" + default = "" +} + variable "elb_4xx_threshold_warning" { description = "loadbalancer 4xx warning threshold in percentage" default = 5 @@ -39,6 +63,18 @@ variable "elb_4xx_threshold_critical" { default = 10 } +variable "elb_5xx_silenced" { + description = "Groups to mute for ELB 5xx errors monitor" + type = "map" + default = {} +} + +variable "elb_5xx_message" { + description = "Custom message for ELB 5xx errors monitor" + type = "string" + default = "" +} + variable "elb_5xx_threshold_warning" { description = "loadbalancer 5xx warning threshold in percentage" default = 5 @@ -49,6 +85,18 @@ variable "elb_5xx_threshold_critical" { default = 10 } +variable "elb_backend_4xx_silenced" { + description = "Groups to mute for ELB backend 4xx errors monitor" + type = "map" + default = {} +} + +variable "elb_backend_4xx_message" { + description = "Custom message for ELB backend 4xx errors monitor" + type = "string" + default = "" +} + variable "elb_backend_4xx_threshold_warning" { description = "loadbalancer backend 4xx warning threshold in percentage" default = 5 @@ -59,6 +107,18 @@ variable "elb_backend_4xx_threshold_critical" { default = 10 } +variable "elb_backend_5xx_silenced" { + description = "Groups to mute for ELB backend 5xx errors monitor" + type = "map" + default = {} +} + +variable "elb_backend_5xx_message" { + description = "Custom message for ELB backend 5xx errors monitor" + type = "string" + default = "" +} + variable "elb_backend_5xx_threshold_warning" { description = "loadbalancer backend 5xx warning threshold in percentage" default = 5 @@ -69,6 +129,18 @@ variable "elb_backend_5xx_threshold_critical" { default = 10 } +variable "elb_backend_latency_silenced" { + description = "Groups to mute for ELB backend latency monitor" + type = "map" + default = {} +} + +variable "elb_backend_latency_message" { + description = "Custom message for ELB backend latency monitor" + type = "string" + default = "" +} + variable "elb_backend_latency_warning" { description = "latency warning threshold in seconds" default = 1 diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index 441d207..df8cf26 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "ELB_no_healthy_instances" { name = "[${var.environment}] ELB no healthy instances" - message = "${var.message}" + message = "${coalesce(var.elb_no_healthy_instance_message, var.message)}" query = <` | no | | incoming_records_timeframe | Monitor timeframe for incoming records metrics evaluation [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_15m` | no | | message | Message sent when an alert is triggered | string | - | yes | diff --git a/cloud/aws/kinesis-firehose/inputs.tf b/cloud/aws/kinesis-firehose/inputs.tf index eeef880..831045e 100644 --- a/cloud/aws/kinesis-firehose/inputs.tf +++ b/cloud/aws/kinesis-firehose/inputs.tf @@ -25,6 +25,19 @@ variable "filter_tags_custom" { } # Kinesis-Firehose + +variable "incoming_records_silenced" { + description = "Groups to mute for Kinesis Firehorse incoming records monitor" + type = "map" + default = {} +} + +variable "incoming_records_message" { + description = "Custom message for Kinesis Firehorse incoming records monitor" + type = "string" + default = "" +} + variable "incoming_records_timeframe" { description = "Monitor timeframe for incoming records metrics evaluation [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" default = "last_15m" diff --git a/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf b/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf index 0035472..5163d82 100644 --- a/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf +++ b/cloud/aws/kinesis-firehose/monitors-kinesis-firehose.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### Kinesis Firehose Incoming records ### resource "datadog_monitor" "firehose_incoming_records" { name = "[${var.environment}] Kinesis Firehose No incoming records" - message = "${var.message}" + message = "${coalesce(var.incoming_records_message, var.message)}" type = "metric alert" @@ -34,5 +34,7 @@ EOF new_host_delay = "${var.delay}" no_data_timeframe = 20 + silenced = "${var.incoming_records_silenced}" + tags = ["env:${var.environment}", "resource:kinesis-firehose", "team:aws", "provider:aws"] } diff --git a/cloud/aws/rds/README.md b/cloud/aws/rds/README.md index b6337a1..737f08e 100644 --- a/cloud/aws/rds/README.md +++ b/cloud/aws/rds/README.md @@ -25,8 +25,12 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cpu_message | Custom message for RDS CPU usage monitor | string | `` | no | +| cpu_silenced | Groups to mute for RDS CPU usage monitor | map | `` | no | | cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no | | cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no | +| diskspace_message | Custom message for RDS free diskspace monitor | string | `` | no | +| diskspace_silenced | Groups to mute for RDS free diskspace monitor | map | `` | no | | diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no | | diskspace_threshold_warning | Disk free space in percent (warning threshold) | string | `20` | no | | environment | Architecture Environment | string | - | yes | diff --git a/cloud/aws/rds/inputs.tf b/cloud/aws/rds/inputs.tf index cac076f..7795cad 100644 --- a/cloud/aws/rds/inputs.tf +++ b/cloud/aws/rds/inputs.tf @@ -26,6 +26,18 @@ variable "filter_tags_custom" { # AWS RDS instance specific +variable "cpu_silenced" { + description = "Groups to mute for RDS CPU usage monitor" + type = "map" + default = {} +} + +variable "cpu_message" { + description = "Custom message for RDS CPU usage monitor" + type = "string" + default = "" +} + variable "cpu_threshold_warning" { description = "CPU usage in percent (warning threshold)" default = "80" @@ -36,6 +48,18 @@ variable "cpu_threshold_critical" { default = "90" } +variable "diskspace_silenced" { + description = "Groups to mute for RDS free diskspace monitor" + type = "map" + default = {} +} + +variable "diskspace_message" { + description = "Custom message for RDS free diskspace monitor" + type = "string" + default = "" +} + variable "diskspace_threshold_warning" { description = "Disk free space in percent (warning threshold)" default = "20" diff --git a/cloud/aws/rds/monitors-rds.tf b/cloud/aws/rds/monitors-rds.tf index 5965c52..deaf9aa 100644 --- a/cloud/aws/rds/monitors-rds.tf +++ b/cloud/aws/rds/monitors-rds.tf @@ -9,7 +9,7 @@ data "template_file" "filter" { ### RDS instance CPU monitor ### resource "datadog_monitor" "rds_cpu_90_15min" { name = "[${var.environment}] RDS instance CPU high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_message, var.message)}" type = "metric alert" @@ -34,13 +34,15 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.cpu_silenced}" + tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"] } ### RDS instance free space monitor ### resource "datadog_monitor" "rds_free_space_low" { name = "[${var.environment}] RDS instance free space {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.diskspace_message, var.message)}" type = "metric alert" @@ -66,5 +68,7 @@ EOF new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 + silenced = "${var.diskspace_silenced}" + tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"] } diff --git a/cloud/aws/vpn/README.md b/cloud/aws/vpn/README.md index 4876c79..0bae76f 100644 --- a/cloud/aws/vpn/README.md +++ b/cloud/aws/vpn/README.md @@ -29,3 +29,5 @@ Inputs | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when an alert is triggered | string | - | yes | +| vpn_status_message | Custom message for VPN status monitor | string | `` | no | +| vpn_status_silenced | Groups to mute for VPN status monitor | map | `` | no | \ No newline at end of file diff --git a/cloud/aws/vpn/inputs.tf b/cloud/aws/vpn/inputs.tf index ff336dc..ba86c7a 100644 --- a/cloud/aws/vpn/inputs.tf +++ b/cloud/aws/vpn/inputs.tf @@ -23,3 +23,15 @@ variable "filter_tags_custom" { description = "Tags used for custom filtering when filter_tags_use_defaults is false" default = "*" } + +variable "vpn_status_silenced" { + description = "Groups to mute for VPN status monitor" + type = "map" + default = {} +} + +variable "vpn_status_message" { + description = "Custom message for VPN status monitor" + type = "string" + default = "" +} diff --git a/cloud/aws/vpn/monitors-vpn.tf b/cloud/aws/vpn/monitors-vpn.tf index c86e01c..4c36b9c 100644 --- a/cloud/aws/vpn/monitors-vpn.tf +++ b/cloud/aws/vpn/monitors-vpn.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "VPN_status" { name = "[${var.environment}] VPN Down" - message = "${var.message}" + message = "${coalesce(var.vpn_status_message, var.message)}" query = < Date: Tue, 20 Mar 2018 14:16:23 +0100 Subject: [PATCH 2/5] MON-96 - Updated databases, middleware and system monitors with new best practice --- databases/mongodb/README.md | 2 ++ databases/mongodb/inputs.tf | 12 +++++++ databases/mongodb/monitors-mongo.tf | 4 ++- middleware/apache/README.md | 2 ++ middleware/apache/inputs.tf | 11 +++++++ middleware/apache/monitors-apache.tf | 4 ++- middleware/nginx/README.md | 2 ++ middleware/nginx/inputs.tf | 11 +++++++ middleware/nginx/monitors-nginx.tf | 4 ++- middleware/php-fpm/README.md | 4 +++ middleware/php-fpm/inputs.tf | 24 ++++++++++++++ middleware/php-fpm/monitors-fpm.tf | 8 +++-- system/generic/README.md | 10 +++++- system/generic/inputs.tf | 48 ++++++++++++++++++++++++++++ system/generic/monitors-system.tf | 16 +++++++--- 15 files changed, 152 insertions(+), 10 deletions(-) diff --git a/databases/mongodb/README.md b/databases/mongodb/README.md index 029ce14..1828671 100644 --- a/databases/mongodb/README.md +++ b/databases/mongodb/README.md @@ -79,3 +79,5 @@ Inputs | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when an alert is triggered | string | - | yes | +| mongodb_replicaset_message | Custom message for Mongodb replicaset monitor | string | `` | no | +| mongodb_replicaset_silenced | Groups to mute for Mongodb replicaset monitor | map | `` | no | diff --git a/databases/mongodb/inputs.tf b/databases/mongodb/inputs.tf index b698157..e526871 100644 --- a/databases/mongodb/inputs.tf +++ b/databases/mongodb/inputs.tf @@ -23,3 +23,15 @@ variable "filter_tags_custom" { description = "Tags used for custom filtering when filter_tags_use_defaults is false" default = "*" } + +variable "mongodb_replicaset_silenced" { + description = "Groups to mute for Mongodb replicaset monitor" + type = "map" + default = {} +} + +variable "mongodb_replicaset_message" { + description = "Custom message for Mongodb replicaset monitor" + type = "string" + default = "" +} diff --git a/databases/mongodb/monitors-mongo.tf b/databases/mongodb/monitors-mongo.tf index 03b55b8..1c06c51 100644 --- a/databases/mongodb/monitors-mongo.tf +++ b/databases/mongodb/monitors-mongo.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "mongodb_replicaset_state" { name = "[${var.environment}] Member down in the replica set" - message = "${var.message}" + message = "${coalesce(var.mongodb_replicaset_message, var.message)}" query = <` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | | message | Message sent when an alert is triggered | string | - | yes | diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf index 1812e3d..7d14445 100644 --- a/middleware/apache/inputs.tf +++ b/middleware/apache/inputs.tf @@ -16,3 +16,14 @@ variable "message" { # Apache Middleware specific +variable "apache_process_silenced" { + description = "Groups to mute for Apache process monitor" + type = "map" + default = {} +} + +variable "apache_process_message" { + description = "Custom message for Apache process monitor" + type = "string" + default = "" +} diff --git a/middleware/apache/monitors-apache.tf b/middleware/apache/monitors-apache.tf index d31cfd6..4a9d79f 100644 --- a/middleware/apache/monitors-apache.tf +++ b/middleware/apache/monitors-apache.tf @@ -1,6 +1,6 @@ resource "datadog_monitor" "datadog_apache_process" { name = "[${var.environment}] Can't connect to apache vhost status" - message = "${var.message}" + message = "${coalesce(var.apache_process_message, var.message)}" type = "service check" query = "\"apache.can_connect\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -22,5 +22,7 @@ resource "datadog_monitor" "datadog_apache_process" { require_full_window = true no_data_timeframe = 20 + silenced = "${var.apache_process_silenced}" + tags = ["env:${var.environment}", "resource:apache"] } diff --git a/middleware/nginx/README.md b/middleware/nginx/README.md index ccf7772..1753648 100644 --- a/middleware/nginx/README.md +++ b/middleware/nginx/README.md @@ -27,3 +27,5 @@ Inputs | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | | message | Message sent when an alert is triggered | string | - | yes | +| nginx_process_message | Custom message for Nginx process monitor | string | `` | no | +| nginx_process_silenced | Groups to mute for Nginx process monitor | map | `` | no | diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf index a82f6c2..e8fea80 100644 --- a/middleware/nginx/inputs.tf +++ b/middleware/nginx/inputs.tf @@ -16,3 +16,14 @@ variable "message" { # Nginx Middleware specific +variable "nginx_process_silenced" { + description = "Groups to mute for Nginx process monitor" + type = "map" + default = {} +} + +variable "nginx_process_message" { + description = "Custom message for Nginx process monitor" + type = "string" + default = "" +} diff --git a/middleware/nginx/monitors-nginx.tf b/middleware/nginx/monitors-nginx.tf index 3db1c0f..3b03808 100644 --- a/middleware/nginx/monitors-nginx.tf +++ b/middleware/nginx/monitors-nginx.tf @@ -1,6 +1,6 @@ resource "datadog_monitor" "datadog_nginx_process" { name = "[${var.environment}] Can't connect to nginx vhost status" - message = "${var.message}" + message = "${coalesce(var.nginx_process_message, var.message)}" type = "service check" query = "\"nginx.can_connect\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -22,5 +22,7 @@ resource "datadog_monitor" "datadog_nginx_process" { require_full_window = true no_data_timeframe = 20 + silenced = "${var.nginx_process_silenced}" + tags = ["env:${var.environment}", "resource:nginx"] } diff --git a/middleware/php-fpm/README.md b/middleware/php-fpm/README.md index f15a56e..9d22794 100644 --- a/middleware/php-fpm/README.md +++ b/middleware/php-fpm/README.md @@ -31,5 +31,9 @@ Inputs | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when an alert is triggered | string | - | yes | +| php_fpm_busy_message | Custom message for PHP FPM busy worker monitor | string | `` | no | +| php_fpm_busy_silenced | Groups to mute for PHP FPM busy worker monitor | map | `` | no | | php_fpm_busy_threshold_critical | php fpm busy critical threshold | string | `0.9` | no | | php_fpm_busy_threshold_warning | php fpm busy warning threshold | string | `0.8` | no | +| php_fpm_process_message | Custom message for PHP FPM process monitor | string | `` | no | +| php_fpm_process_silenced | Groups to mute for PHP FPM process monitor | map | `` | no | \ No newline at end of file diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf index 726bd04..83b1efa 100644 --- a/middleware/php-fpm/inputs.tf +++ b/middleware/php-fpm/inputs.tf @@ -31,6 +31,18 @@ variable "filter_tags_custom" { # PHP FPM Middleware specific +variable "php_fpm_busy_silenced" { + description = "Groups to mute for PHP FPM busy worker monitor" + type = "map" + default = {} +} + +variable "php_fpm_busy_message" { + description = "Custom message for PHP FPM busy worker monitor" + type = "string" + default = "" +} + variable "php_fpm_busy_threshold_warning" { description = "php fpm busy warning threshold" default = 0.8 @@ -40,3 +52,15 @@ variable "php_fpm_busy_threshold_critical" { description = "php fpm busy critical threshold" default = 0.9 } + +variable "php_fpm_process_silenced" { + description = "Groups to mute for PHP FPM process monitor" + type = "map" + default = {} +} + +variable "php_fpm_process_message" { + description = "Custom message for PHP FPM process monitor" + type = "string" + default = "" +} diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf index 51f582d..aec13b0 100644 --- a/middleware/php-fpm/monitors-fpm.tf +++ b/middleware/php-fpm/monitors-fpm.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "datadog_php_fpm_process_idle" { name = "[${var.environment}] php_fpm busy worker {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.php_fpm_busy_message, var.message)}" type = "metric alert" @@ -35,12 +35,14 @@ resource "datadog_monitor" "datadog_php_fpm_process_idle" { require_full_window = true no_data_timeframe = 20 + silenced = "${var.php_fpm_busy_silenced}" + tags = ["env:${var.environment}", "resource:php-fpm"] } resource "datadog_monitor" "datadog_fpm_process" { name = "[${var.environment}] Can't connect to php-fpm" - message = "${var.message}" + message = "${coalesce(var.php_fpm_process_message, var.message)}" type = "service check" query = "\"php_fpm.can_ping\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -62,5 +64,7 @@ resource "datadog_monitor" "datadog_fpm_process" { require_full_window = true no_data_timeframe = 20 + silenced = "${var.php_fpm_process_silenced}" + tags = ["env:${var.environment}", "resource:php-fpm"] } diff --git a/system/generic/README.md b/system/generic/README.md index abf80e0..cfcdda0 100644 --- a/system/generic/README.md +++ b/system/generic/README.md @@ -24,6 +24,8 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cpu_high_message | Custom message for CPU high monitor | string | `` | no | +| cpu_high_silenced | Groups to mute for CPU high monitor | map | `` | no | | cpu_high_threshold_critical | CPU high critical threshold | string | `95` | no | | cpu_high_threshold_warning | CPU high warning threshold | string | `80` | no | | cpu_high_timeframe | CPU high timeframe | string | `last_5m` | no | @@ -31,10 +33,16 @@ Inputs | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| free_disk_inodes_message | Custom message for Free disk inodes monitor | string | `` | no | +| free_disk_inodes_silenced | Groups to mute for Free disk inodes monitor | map | `` | no | | free_disk_inodes_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_disk_inodes_threshold_warning | Free disk space warning threshold | string | `10` | no | +| free_disk_space_message | Custom message for Free diskspace monitor | string | `` | no | +| free_disk_space_silenced | Groups to mute for Free diskspace monitor | map | `` | no | | free_disk_space_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_disk_space_threshold_warning | Free disk space warning threshold | string | `10` | no | +| free_memory_message | Custom message for Free memory monitor | string | `` | no | +| free_memory_silenced | Groups to mute for Free memory monitor | map | `` | no | | free_memory_threshold_critical | Free disk space critical threshold | string | `5` | no | | free_memory_threshold_warning | Free disk space warning threshold | string | `10` | no | -| message | Message sent when an alert is triggered | string | - | yes | +| message | Message sent when an alert is triggered | string | - | yes | \ No newline at end of file diff --git a/system/generic/inputs.tf b/system/generic/inputs.tf index 5b34296..a9a7a98 100644 --- a/system/generic/inputs.tf +++ b/system/generic/inputs.tf @@ -26,6 +26,18 @@ variable "filter_tags_custom" { # Custom CPU instance specific +variable "cpu_high_silenced" { + description = "Groups to mute for CPU high monitor" + type = "map" + default = {} +} + +variable "cpu_high_message" { + description = "Custom message for CPU high monitor" + type = "string" + default = "" +} + variable "cpu_high_timeframe" { description = "CPU high timeframe" default = "last_5m" @@ -41,6 +53,18 @@ variable "cpu_high_threshold_critical" { default = 95 } +variable "free_disk_space_silenced" { + description = "Groups to mute for Free diskspace monitor" + type = "map" + default = {} +} + +variable "free_disk_space_message" { + description = "Custom message for Free diskspace monitor" + type = "string" + default = "" +} + variable "free_disk_space_threshold_warning" { description = "Free disk space warning threshold" default = 10 @@ -51,6 +75,18 @@ variable "free_disk_space_threshold_critical" { default = 5 } +variable "free_disk_inodes_silenced" { + description = "Groups to mute for Free disk inodes monitor" + type = "map" + default = {} +} + +variable "free_disk_inodes_message" { + description = "Custom message for Free disk inodes monitor" + type = "string" + default = "" +} + variable "free_disk_inodes_threshold_warning" { description = "Free disk space warning threshold" default = 10 @@ -61,6 +97,18 @@ variable "free_disk_inodes_threshold_critical" { default = 5 } +variable "free_memory_silenced" { + description = "Groups to mute for Free memory monitor" + type = "map" + default = {} +} + +variable "free_memory_message" { + description = "Custom message for Free memory monitor" + type = "string" + default = "" +} + variable "free_memory_threshold_warning" { description = "Free disk space warning threshold" default = 10 diff --git a/system/generic/monitors-system.tf b/system/generic/monitors-system.tf index ce70cd2..9908b0e 100644 --- a/system/generic/monitors-system.tf +++ b/system/generic/monitors-system.tf @@ -8,7 +8,7 @@ data "template_file" "filter" { resource "datadog_monitor" "datadog_cpu_too_high" { name = "[${var.environment}] CPU usage {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${var.message}" + message = "${coalesce(var.cpu_high_message, var.message)}" query = < Date: Wed, 21 Mar 2018 12:28:49 +0100 Subject: [PATCH 3/5] MON-140 - Added default function to ELB and Api Gateway monitors --- cloud/aws/apigateway/monitors-api.tf | 16 ++++++++------ cloud/aws/elb/monitors-elb.tf | 32 +++++++++++++++++----------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/cloud/aws/apigateway/monitors-api.tf b/cloud/aws/apigateway/monitors-api.tf index f6d2cd9..b25c3d2 100644 --- a/cloud/aws/apigateway/monitors-api.tf +++ b/cloud/aws/apigateway/monitors-api.tf @@ -37,9 +37,11 @@ resource "datadog_monitor" "API_http_5xx_errors_count" { query = < ${var.http_5xx_requests_threshold_critical} + default( + avg:aws.apigateway.5xxerror{${var.filter_tags}} by {region,apiname}.as_count() / + (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}), + 0) * 100 + ) > ${var.http_5xx_requests_threshold_critical} EOF evaluation_delay = "${var.delay}" @@ -69,9 +71,11 @@ resource "datadog_monitor" "API_http_4xx_errors_count" { query = < ${var.http_4xx_requests_threshold_critical} + default( + avg:aws.apigateway.4xxerror{${var.filter_tags}} by {region,apiname}.as_count() / + (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}), + 0) * 100 + ) > ${var.http_4xx_requests_threshold_critical} EOF evaluation_delay = "${var.delay}" diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index df8cf26..fc12fa5 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -40,9 +40,11 @@ resource "datadog_monitor" "ELB_too_much_4xx" { query = < ${var.elb_4xx_threshold_critical} + default( + avg:aws.elb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_4xx_threshold_critical} EOF type = "metric alert" @@ -74,9 +76,11 @@ resource "datadog_monitor" "ELB_too_much_5xx" { query = < ${var.elb_5xx_threshold_critical} + default( + avg:aws.elb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_5xx_threshold_critical} EOF type = "metric alert" @@ -108,9 +112,11 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { query = < ${var.elb_backend_4xx_threshold_critical} + default( + avg:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_backend_4xx_threshold_critical} EOF type = "metric alert" @@ -142,9 +148,11 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { query = < ${var.elb_backend_5xx_threshold_critical} + default( + avg:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} / + avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}, + 0) * 100 + ) > ${var.elb_backend_5xx_threshold_critical} EOF type = "metric alert" From 999019339c895cc73cf55aadb0c2b1f25298f90a Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Thu, 22 Mar 2018 20:36:55 +0100 Subject: [PATCH 4/5] MON-96 rename middleware status variables --- middleware/apache/inputs.tf | 4 ++-- middleware/apache/monitors-apache.tf | 4 ++-- middleware/nginx/inputs.tf | 4 ++-- middleware/nginx/monitors-nginx.tf | 4 ++-- middleware/php-fpm/inputs.tf | 4 ++-- middleware/php-fpm/monitors-fpm.tf | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf index 7d14445..36beec8 100644 --- a/middleware/apache/inputs.tf +++ b/middleware/apache/inputs.tf @@ -16,13 +16,13 @@ variable "message" { # Apache Middleware specific -variable "apache_process_silenced" { +variable "apache_connect_silenced" { description = "Groups to mute for Apache process monitor" type = "map" default = {} } -variable "apache_process_message" { +variable "apache_connect_message" { description = "Custom message for Apache process monitor" type = "string" default = "" diff --git a/middleware/apache/monitors-apache.tf b/middleware/apache/monitors-apache.tf index 4a9d79f..139a052 100644 --- a/middleware/apache/monitors-apache.tf +++ b/middleware/apache/monitors-apache.tf @@ -1,6 +1,6 @@ resource "datadog_monitor" "datadog_apache_process" { name = "[${var.environment}] Can't connect to apache vhost status" - message = "${coalesce(var.apache_process_message, var.message)}" + message = "${coalesce(var.apache_connect_message, var.message)}" type = "service check" query = "\"apache.can_connect\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -22,7 +22,7 @@ resource "datadog_monitor" "datadog_apache_process" { require_full_window = true no_data_timeframe = 20 - silenced = "${var.apache_process_silenced}" + silenced = "${var.apache_connect_silenced}" tags = ["env:${var.environment}", "resource:apache"] } diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf index e8fea80..8ab7f01 100644 --- a/middleware/nginx/inputs.tf +++ b/middleware/nginx/inputs.tf @@ -16,13 +16,13 @@ variable "message" { # Nginx Middleware specific -variable "nginx_process_silenced" { +variable "nginx_connect_silenced" { description = "Groups to mute for Nginx process monitor" type = "map" default = {} } -variable "nginx_process_message" { +variable "nginx_connect_message" { description = "Custom message for Nginx process monitor" type = "string" default = "" diff --git a/middleware/nginx/monitors-nginx.tf b/middleware/nginx/monitors-nginx.tf index 3b03808..9ab5daf 100644 --- a/middleware/nginx/monitors-nginx.tf +++ b/middleware/nginx/monitors-nginx.tf @@ -1,6 +1,6 @@ resource "datadog_monitor" "datadog_nginx_process" { name = "[${var.environment}] Can't connect to nginx vhost status" - message = "${coalesce(var.nginx_process_message, var.message)}" + message = "${coalesce(var.nginx_connect_message, var.message)}" type = "service check" query = "\"nginx.can_connect\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -22,7 +22,7 @@ resource "datadog_monitor" "datadog_nginx_process" { require_full_window = true no_data_timeframe = 20 - silenced = "${var.nginx_process_silenced}" + silenced = "${var.nginx_connect_silenced}" tags = ["env:${var.environment}", "resource:nginx"] } diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf index 83b1efa..49ac556 100644 --- a/middleware/php-fpm/inputs.tf +++ b/middleware/php-fpm/inputs.tf @@ -53,13 +53,13 @@ variable "php_fpm_busy_threshold_critical" { default = 0.9 } -variable "php_fpm_process_silenced" { +variable "php_fpm_connect_silenced" { description = "Groups to mute for PHP FPM process monitor" type = "map" default = {} } -variable "php_fpm_process_message" { +variable "php_fpm_connect_message" { description = "Custom message for PHP FPM process monitor" type = "string" default = "" diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf index aec13b0..52b4a92 100644 --- a/middleware/php-fpm/monitors-fpm.tf +++ b/middleware/php-fpm/monitors-fpm.tf @@ -6,7 +6,7 @@ data "template_file" "filter" { } } -resource "datadog_monitor" "datadog_php_fpm_process_idle" { +resource "datadog_monitor" "datadog_php_fpm_connect_idle" { name = "[${var.environment}] php_fpm busy worker {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.php_fpm_busy_message, var.message)}" @@ -42,7 +42,7 @@ resource "datadog_monitor" "datadog_php_fpm_process_idle" { resource "datadog_monitor" "datadog_fpm_process" { name = "[${var.environment}] Can't connect to php-fpm" - message = "${coalesce(var.php_fpm_process_message, var.message)}" + message = "${coalesce(var.php_fpm_connect_message, var.message)}" type = "service check" query = "\"php_fpm.can_ping\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" @@ -64,7 +64,7 @@ resource "datadog_monitor" "datadog_fpm_process" { require_full_window = true no_data_timeframe = 20 - silenced = "${var.php_fpm_process_silenced}" + silenced = "${var.php_fpm_connect_silenced}" tags = ["env:${var.environment}", "resource:php-fpm"] } From 84f4e8d5ba3496a0de6e7fcaf756b445b3ee7779 Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Fri, 23 Mar 2018 14:53:39 +0100 Subject: [PATCH 5/5] MON-96 - Update middleware monitors with EOF best practice --- middleware/apache/README.md | 8 +++++--- middleware/apache/inputs.tf | 10 ++++++++++ middleware/apache/monitors-apache.tf | 15 +++++++++++++-- middleware/nginx/README.md | 6 ++++-- middleware/nginx/inputs.tf | 10 ++++++++++ middleware/nginx/monitors-nginx.tf | 15 +++++++++++++-- middleware/php-fpm/README.md | 4 ++-- middleware/php-fpm/monitors-fpm.tf | 7 +++++-- 8 files changed, 62 insertions(+), 13 deletions(-) diff --git a/middleware/apache/README.md b/middleware/apache/README.md index e4da4f0..d3f5010 100644 --- a/middleware/apache/README.md +++ b/middleware/apache/README.md @@ -24,8 +24,10 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| apache_process_message | Custom message for Apache process monitor | string | `` | no | -| apache_process_silenced | Groups to mute for Apache process monitor | map | `` | no | +| apache_connect_message | Custom message for Apache process monitor | string | `` | no | +| apache_connect_silenced | Groups to mute for Apache process monitor | map | `` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | -| message | Message sent when an alert is triggered | string | - | yes | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when an alert is triggered | string | - | yes | \ No newline at end of file diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf index 36beec8..d300534 100644 --- a/middleware/apache/inputs.tf +++ b/middleware/apache/inputs.tf @@ -14,6 +14,16 @@ variable "message" { description = "Message sent when an alert is triggered" } +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + # Apache Middleware specific variable "apache_connect_silenced" { diff --git a/middleware/apache/monitors-apache.tf b/middleware/apache/monitors-apache.tf index 139a052..3406d3b 100644 --- a/middleware/apache/monitors-apache.tf +++ b/middleware/apache/monitors-apache.tf @@ -1,9 +1,20 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_apache:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}" + } +} + resource "datadog_monitor" "datadog_apache_process" { name = "[${var.environment}] Can't connect to apache vhost status" message = "${coalesce(var.apache_connect_message, var.message)}" - type = "service check" - query = "\"apache.can_connect\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <` | no | +| nginx_connect_message | Custom message for Nginx process monitor | string | `` | no | +| nginx_connect_silenced | Groups to mute for Nginx process monitor | map | `` | no | \ No newline at end of file diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf index 8ab7f01..b4ca2f7 100644 --- a/middleware/nginx/inputs.tf +++ b/middleware/nginx/inputs.tf @@ -14,6 +14,16 @@ variable "message" { description = "Message sent when an alert is triggered" } +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + # Nginx Middleware specific variable "nginx_connect_silenced" { diff --git a/middleware/nginx/monitors-nginx.tf b/middleware/nginx/monitors-nginx.tf index 9ab5daf..b578fca 100644 --- a/middleware/nginx/monitors-nginx.tf +++ b/middleware/nginx/monitors-nginx.tf @@ -1,9 +1,20 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_nginx:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}" + } +} + resource "datadog_monitor" "datadog_nginx_process" { name = "[${var.environment}] Can't connect to nginx vhost status" message = "${coalesce(var.nginx_connect_message, var.message)}" - type = "service check" - query = "\"nginx.can_connect\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <` | no | | php_fpm_busy_threshold_critical | php fpm busy critical threshold | string | `0.9` | no | | php_fpm_busy_threshold_warning | php fpm busy warning threshold | string | `0.8` | no | -| php_fpm_process_message | Custom message for PHP FPM process monitor | string | `` | no | -| php_fpm_process_silenced | Groups to mute for PHP FPM process monitor | map | `` | no | \ No newline at end of file +| php_fpm_connect_message | Custom message for PHP FPM process monitor | string | `` | no | +| php_fpm_connect_silenced | Groups to mute for PHP FPM process monitor | map | `` | no | \ No newline at end of file diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf index 52b4a92..16a944d 100644 --- a/middleware/php-fpm/monitors-fpm.tf +++ b/middleware/php-fpm/monitors-fpm.tf @@ -44,8 +44,11 @@ resource "datadog_monitor" "datadog_fpm_process" { name = "[${var.environment}] Can't connect to php-fpm" message = "${coalesce(var.php_fpm_connect_message, var.message)}" - type = "service check" - query = "\"php_fpm.can_ping\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"env:${var.environment}\").by(\"host\",\"port\").last(6).count_by_status()" + type = "service check" + + query = <