From 69792127452169c0f8e7cee7771fd1f65e49021d Mon Sep 17 00:00:00 2001 From: Ahmed Fourti Date: Mon, 4 Dec 2017 11:34:12 +0100 Subject: [PATCH 1/5] MON-93 Make elb_tags customizable --- README.md | 13 +++++++++++++ cloud/aws/elb/monitors-elb.tf | 28 ++++++++++++++++++---------- inputs.tf | 11 +++++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 88e7dda..4423d4b 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,19 @@ variable "warning_escalation_group" { variable "datadog_app_key" {} variable "datadog_api_key" {} +##Set this variable to true if you want to keep the default tags (tags in query directive) +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +##If you have to set custom tags, you have to set the previous input to false and add your tags in this variable +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + + variable "dd_linux_basics" { default = "enabled" } diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index 1f4d548..6b2935f 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -1,9 +1,17 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_elb:enabled,env:%s", var.env) : "${var.filter_tags_custom}"}" + } +} + resource "datadog_monitor" "ELB_no_healthy_instances" { name = "[${var.env}] ELB no healthy instances on {{host.identifier}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} == 0" + query = "avg(last_5m):avg:aws.elb.healthy_host_count{${data.template_file.filter.rendered}} by {loadbalancername,region} == 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -17,7 +25,7 @@ resource "datadog_monitor" "ELB_no_healthy_instances" { new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 - tags = ["*"] + tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_unhealthy_instances" { @@ -25,7 +33,7 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > 0" + query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{${data.template_file.filter.rendered}} by {loadbalancername,region} > 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -39,7 +47,7 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 - tags = ["*"] + tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_5xx_backend" { @@ -47,7 +55,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_5xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {loadbalancername,region} / avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {loadbalancername,region} * 100 > ${var.elb_5xx_threshold["critical"]}" type = "query alert" thresholds { @@ -66,7 +74,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 - tags = ["*"] + tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_4xx_backend" { @@ -74,7 +82,7 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_4xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {loadbalancername,region} / avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {loadbalancername,region} * 100 > ${var.elb_4xx_threshold["critical"]}" type = "query alert" thresholds { @@ -93,7 +101,7 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 - tags = ["*"] + tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_backend_latency" { @@ -101,7 +109,7 @@ resource "datadog_monitor" "ELB_backend_latency" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.latency{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > ${var.elb_backend_latency["critical"]}" + query = "avg(last_5m):avg:aws.elb.latency{${data.template_file.filter.rendered}} by {loadbalancername,region} > ${var.elb_backend_latency["critical"]}" type = "query alert" thresholds { @@ -120,5 +128,5 @@ resource "datadog_monitor" "ELB_backend_latency" { new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 - tags = ["*"] + tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] } diff --git a/inputs.tf b/inputs.tf index 7fd4b4b..6266b41 100644 --- a/inputs.tf +++ b/inputs.tf @@ -4,6 +4,17 @@ variable "ho_escalation_group" {} variable env {} variable region {} +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + + ##linux variable "dd_linux_basics" { default = "disabled" From ce55b16b86996dc033ae9f78128366049d00315c Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Mon, 19 Feb 2018 19:25:07 +0100 Subject: [PATCH 2/5] MON-96 - Update ELB with new best practices and follow recommendations --- cloud/aws/elb/README.md | 43 +++++++++++ cloud/aws/elb/inputs.tf | 66 ++++++++++++++++- cloud/aws/elb/monitors-elb.tf | 136 ++++++++++++++++++++-------------- inputs.tf | 37 --------- 4 files changed, 187 insertions(+), 95 deletions(-) create mode 100644 cloud/aws/elb/README.md mode change 120000 => 100644 cloud/aws/elb/inputs.tf diff --git a/cloud/aws/elb/README.md b/cloud/aws/elb/README.md new file mode 100644 index 0000000..645453d --- /dev/null +++ b/cloud/aws/elb/README.md @@ -0,0 +1,43 @@ +AWS ELB DataDog monitors +======================== + +How to use this module +---------------------- + +``` +module "datadog-monitors-aws-elb" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/aws/elb?ref={revision}" + + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` + +Purpose +------- +Creates DataDog monitors with the following checks : + +* ELB no healthy hosts +* ELB latency too high +* ELB http code 5xx percent to high +* ELB http code 4xx percent to high + +Inputs +------ + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| dd_aws_elb | # ELB | string | `disable` | no | +| elb_4xx_threshold_critical | loadbalancer 4xx critical threshold in percentage | string | `10` | no | +| elb_4xx_threshold_warning | loadbalancer 4xx warning threshold in percentage | string | `5` | no | +| elb_5xx_threshold_critical | loadbalancer 5xx critical threshold in percentage | string | `10` | no | +| elb_5xx_threshold_warning | loadbalancer 5xx warning threshold in percentage | string | `5` | no | +| elb_backend_latency_critical | latency critical threshold in seconds | string | `5` | no | +| elb_backend_latency_warning | latency warning threshold in seconds | string | `1` | no | +| elb_notify_no_data | Use this variable to disable notify no data | string | `true` | no | +| environment | Architecture Environment | string | - | yes | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when an alert is triggered | string | - | yes | diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf deleted file mode 120000 index a68ace3..0000000 --- a/cloud/aws/elb/inputs.tf +++ /dev/null @@ -1 +0,0 @@ -../../../inputs.tf \ No newline at end of file diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf new file mode 100644 index 0000000..3b9ab9e --- /dev/null +++ b/cloud/aws/elb/inputs.tf @@ -0,0 +1,65 @@ +# Global Terraform +variable "environment" { + description = "Architecture Environment" + type = "string" +} + +# Global DataDog +variable "evaluation_delay" { + description = "Delay in seconds for the metric evaluation" + default = 600 +} + +variable "message" { + description = "Message sent when an alert is triggered" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +## ELB +variable "dd_aws_elb" { + default = "disable" +} + +variable "elb_notify_no_data" { + description = "Use this variable to disable notify no data" + default = true +} + +variable "elb_5xx_threshold_warning" { + description = "loadbalancer 5xx warning threshold in percentage" + default = 5 +} + +variable "elb_5xx_threshold_critical" { + description = "loadbalancer 5xx critical threshold in percentage" + default = 10 +} + +variable "elb_4xx_threshold_warning" { + description = "loadbalancer 4xx warning threshold in percentage" + default = 5 +} + +variable "elb_4xx_threshold_critical" { + description = "loadbalancer 4xx critical threshold in percentage" + default = 10 +} + +variable "elb_backend_latency_warning" { + description = "latency warning threshold in seconds" + default = 1 +} + +variable "elb_backend_latency_critical" { + description = "latency critical threshold in seconds" + default = 5 +} diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index 6b2935f..a8f04e8 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -2,131 +2,153 @@ data "template_file" "filter" { template = "$${filter}" vars { - filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_elb:enabled,env:%s", var.env) : "${var.filter_tags_custom}"}" + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_elb:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}" } } resource "datadog_monitor" "ELB_no_healthy_instances" { - name = "[${var.env}] ELB no healthy instances on {{host.identifier}}" - message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" + name = "[${var.environment}] ELB no healthy instances on {{host.identifier}}" + message = "${var.message}" - count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.healthy_host_count{${data.template_file.filter.rendered}} by {loadbalancername,region} == 0" - type = "query alert" + query = < 0" + EOF - notify_no_data = "${var.elb_config["notify_no_data"]}" - evaluation_delay = "${var.elb_config["delay"]}" - renotify_interval = 60 + type = "metric alert" + + notify_no_data = "${var.elb_notify_no_data}" + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false require_full_window = true - new_host_delay = "${var.elb_config["delay"]}" + new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 - tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_5xx_backend" { - name = "[${var.env}] ELB too much 5xx backend err on {{host.identifier}}" - message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" + name = "[${var.environment}] ELB too much 5xx backend err on {{host.identifier}}" + message = "${var.message}" - count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {loadbalancername,region} / avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {loadbalancername,region} * 100 > ${var.elb_5xx_threshold["critical"]}" - type = "query alert" + query = < ${var.elb_5xx_threshold_critical}" + EOF + + type = "metric alert" thresholds { - warning = "${var.elb_5xx_threshold["warning"]}" - critical = "${var.elb_5xx_threshold["critical"]}" + warning = "${var.elb_5xx_threshold_warning}" + critical = "${var.elb_5xx_threshold_critical}" } - notify_no_data = "${var.elb_config["notify_no_data"]}" - evaluation_delay = "${var.elb_config["delay"]}" - renotify_interval = 60 + notify_no_data = "${var.elb_notify_no_data}" + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false require_full_window = true - new_host_delay = "${var.elb_config["delay"]}" + new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 - tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_too_much_4xx_backend" { - name = "[${var.env}] ELB too much 4xx backend err on {{host.identifier}}" - message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" + name = "[${var.environment}] ELB too much 4xx backend err on {{host.identifier}}" + message = "${var.message}" - count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {loadbalancername,region} / avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {loadbalancername,region} * 100 > ${var.elb_4xx_threshold["critical"]}" - type = "query alert" + query = < ${var.elb_4xx_threshold_critical}" + EOF + + type = "metric alert" thresholds { - warning = "${var.elb_4xx_threshold["warning"]}" - critical = "${var.elb_4xx_threshold["critical"]}" + warning = "${var.elb_4xx_threshold_warning}" + critical = "${var.elb_4xx_threshold_critical}" } - notify_no_data = "${var.elb_config["notify_no_data"]}" - evaluation_delay = "${var.elb_config["delay"]}" - renotify_interval = 60 + notify_no_data = "${var.elb_notify_no_data}" + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false require_full_window = true - new_host_delay = "${var.elb_config["delay"]}" + new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 - tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } resource "datadog_monitor" "ELB_backend_latency" { - name = "[${var.env}] ELB latency to high on {{host.identifier}}" - message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" + name = "[${var.environment}] ELB latency to high on {{host.identifier}}" + message = "${var.message}" - count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.latency{${data.template_file.filter.rendered}} by {loadbalancername,region} > ${var.elb_backend_latency["critical"]}" - type = "query alert" + query = < ${var.elb_backend_latency_critical}}" + EOF + + type = "metric alert" thresholds { - warning = "${var.elb_backend_latency["warning"]}" - critical = "${var.elb_backend_latency["critical"]}" + warning = "${var.elb_backend_latency_warning}" + critical = "${var.elb_backend_latency_critical}" } - notify_no_data = "${var.elb_config["notify_no_data"]}" - evaluation_delay = "${var.elb_config["delay"]}" - renotify_interval = 60 + notify_no_data = "${var.elb_notify_no_data}" + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false require_full_window = true - new_host_delay = "${var.elb_config["delay"]}" + new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 - tags = ["env:${var.env}", "resource:elb", "team:aws", "provider:aws"] + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] } diff --git a/inputs.tf b/inputs.tf index 6266b41..e4ace8c 100644 --- a/inputs.tf +++ b/inputs.tf @@ -77,43 +77,6 @@ variable "rds_mem_threshold" { } } -## ELB -variable "dd_aws_elb" { - default = "disable" -} - -variable "elb_config" { - type = "map" - - default = { - notify_no_data = false - delay = 900 - } -} - -variable "elb_5xx_threshold" { - default = { - warning = 5 - critical = 10 - } -} - -variable "elb_4xx_threshold" { - default = { - warning = 5 - critical = 10 - } -} - -variable "elb_backend_latency" { - description = "Average time elapsed after the request leaves the load balancer until a response is received. In seconds" - - default = { - warning = 1 - critical = 5 - } -} - ##apache nginx php variable "dd_nginx" { default = "disabled" From 5b68882cfe451909c5a1cb547f95de6c595c29de Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Mon, 19 Feb 2018 19:28:35 +0100 Subject: [PATCH 3/5] MON-96 - Clean general inputs and README --- README.md | 13 ------------- inputs.tf | 11 ----------- 2 files changed, 24 deletions(-) diff --git a/README.md b/README.md index 4423d4b..88e7dda 100644 --- a/README.md +++ b/README.md @@ -54,19 +54,6 @@ variable "warning_escalation_group" { variable "datadog_app_key" {} variable "datadog_api_key" {} -##Set this variable to true if you want to keep the default tags (tags in query directive) -variable "filter_tags_use_defaults" { - description = "Use default filter tags convention" - default = "true" -} - -##If you have to set custom tags, you have to set the previous input to false and add your tags in this variable -variable "filter_tags_custom" { - description = "Tags used for custom filtering when filter_tags_use_defaults is false" - default = "*" -} - - variable "dd_linux_basics" { default = "enabled" } diff --git a/inputs.tf b/inputs.tf index e4ace8c..9b30566 100644 --- a/inputs.tf +++ b/inputs.tf @@ -4,17 +4,6 @@ variable "ho_escalation_group" {} variable env {} variable region {} -variable "filter_tags_use_defaults" { - description = "Use default filter tags convention" - default = "true" -} - -variable "filter_tags_custom" { - description = "Tags used for custom filtering when filter_tags_use_defaults is false" - default = "*" -} - - ##linux variable "dd_linux_basics" { default = "disabled" From a4b225f798cb1a0f6956664a662c978fea105280 Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Wed, 21 Feb 2018 15:24:25 +0100 Subject: [PATCH 4/5] MON-93 - Add some monitors, and update names to respect new best practice --- cloud/aws/elb/README.md | 11 +- cloud/aws/elb/inputs.tf | 29 ++++-- cloud/aws/elb/monitors-elb.tf | 184 ++++++++++++++++++++-------------- 3 files changed, 141 insertions(+), 83 deletions(-) diff --git a/cloud/aws/elb/README.md b/cloud/aws/elb/README.md index 645453d..2943b8a 100644 --- a/cloud/aws/elb/README.md +++ b/cloud/aws/elb/README.md @@ -20,8 +20,10 @@ Creates DataDog monitors with the following checks : * ELB no healthy hosts * ELB latency too high -* ELB http code 5xx percent to high * ELB http code 4xx percent to high +* ELB http code 5xx percent to high +* ELB backend http code 4xx percent to high +* ELB backend http code 5xx percent to high Inputs ------ @@ -33,11 +35,14 @@ Inputs | elb_4xx_threshold_warning | loadbalancer 4xx warning threshold in percentage | string | `5` | no | | elb_5xx_threshold_critical | loadbalancer 5xx critical threshold in percentage | string | `10` | no | | elb_5xx_threshold_warning | loadbalancer 5xx warning threshold in percentage | string | `5` | no | +| elb_backend_4xx_threshold_critical | loadbalancer backend 4xx critical threshold in percentage | string | `10` | no | +| elb_backend_4xx_threshold_warning | loadbalancer backend 4xx warning threshold in percentage | string | `5` | no | +| elb_backend_5xx_threshold_critical | loadbalancer backend 5xx critical threshold in percentage | string | `10` | no | +| elb_backend_5xx_threshold_warning | loadbalancer backend 5xx warning threshold in percentage | string | `5` | no | | elb_backend_latency_critical | latency critical threshold in seconds | string | `5` | no | | elb_backend_latency_warning | latency warning threshold in seconds | string | `1` | no | -| elb_notify_no_data | Use this variable to disable notify no data | string | `true` | no | | environment | Architecture Environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | -| message | Message sent when an alert is triggered | string | - | yes | +| message | Message sent when an alert is triggered | string | - | yes | \ No newline at end of file diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf index 3b9ab9e..f2056c8 100644 --- a/cloud/aws/elb/inputs.tf +++ b/cloud/aws/elb/inputs.tf @@ -29,9 +29,14 @@ variable "dd_aws_elb" { default = "disable" } -variable "elb_notify_no_data" { - description = "Use this variable to disable notify no data" - default = true +variable "elb_4xx_threshold_warning" { + description = "loadbalancer 4xx warning threshold in percentage" + default = 5 +} + +variable "elb_4xx_threshold_critical" { + description = "loadbalancer 4xx critical threshold in percentage" + default = 10 } variable "elb_5xx_threshold_warning" { @@ -44,13 +49,23 @@ variable "elb_5xx_threshold_critical" { default = 10 } -variable "elb_4xx_threshold_warning" { - description = "loadbalancer 4xx warning threshold in percentage" +variable "elb_backend_4xx_threshold_warning" { + description = "loadbalancer backend 4xx warning threshold in percentage" default = 5 } -variable "elb_4xx_threshold_critical" { - description = "loadbalancer 4xx critical threshold in percentage" +variable "elb_backend_4xx_threshold_critical" { + description = "loadbalancer backend 4xx critical threshold in percentage" + default = 10 +} + +variable "elb_backend_5xx_threshold_warning" { + description = "loadbalancer backend 5xx warning threshold in percentage" + default = 5 +} + +variable "elb_backend_5xx_threshold_critical" { + description = "loadbalancer backend 5xx critical threshold in percentage" default = 10 } diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index a8f04e8..835dc3e 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -7,97 +7,39 @@ data "template_file" "filter" { } resource "datadog_monitor" "ELB_no_healthy_instances" { - name = "[${var.environment}] ELB no healthy instances on {{host.identifier}}" + name = "[${var.environment}] ELB no healthy instances" message = "${var.message}" query = < 0" - EOF - - type = "metric alert" - - notify_no_data = "${var.elb_notify_no_data}" - evaluation_delay = "${var.evaluation_delay}" - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - include_tags = true - locked = false - require_full_window = true - new_host_delay = "${var.evaluation_delay}" - no_data_timeframe = 20 - - tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] -} - -resource "datadog_monitor" "ELB_too_much_5xx_backend" { - name = "[${var.environment}] ELB too much 5xx backend err on {{host.identifier}}" - message = "${var.message}" - - query = < ${var.elb_5xx_threshold_critical}" - EOF - - type = "metric alert" - - thresholds { - warning = "${var.elb_5xx_threshold_warning}" - critical = "${var.elb_5xx_threshold_critical}" - } - - notify_no_data = "${var.elb_notify_no_data}" - evaluation_delay = "${var.evaluation_delay}" - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - include_tags = true - locked = false - require_full_window = true - new_host_delay = "${var.evaluation_delay}" - no_data_timeframe = 20 - - tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] -} - -resource "datadog_monitor" "ELB_too_much_4xx_backend" { - name = "[${var.environment}] ELB too much 4xx backend err on {{host.identifier}}" - message = "${var.message}" - - query = < ${var.elb_4xx_threshold_critical}" EOF @@ -108,14 +50,110 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { critical = "${var.elb_4xx_threshold_critical}" } - notify_no_data = "${var.elb_notify_no_data}" + notify_no_data = true evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false - require_full_window = true + require_full_window = false + new_host_delay = "${var.evaluation_delay}" + no_data_timeframe = 20 + + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] +} + +resource "datadog_monitor" "ELB_too_much_5xx" { + name = "[${var.environment}] ELB 5xx errors too high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)" + message = "${var.message}" + + query = < ${var.elb_5xx_threshold_critical}" + EOF + + type = "metric alert" + + thresholds { + warning = "${var.elb_5xx_threshold_warning}" + critical = "${var.elb_5xx_threshold_critical}" + } + + notify_no_data = true + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = false + new_host_delay = "${var.evaluation_delay}" + no_data_timeframe = 20 + + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] +} + +resource "datadog_monitor" "ELB_too_much_4xx_backend" { + name = "[${var.environment}] ELB backend 4xx errors too high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)" + message = "${var.message}" + + query = < ${var.elb_backend_4xx_threshold_critical}" + EOF + + type = "metric alert" + + thresholds { + warning = "${var.elb_backend_4xx_threshold_warning}" + critical = "${var.elb_backend_4xx_threshold_critical}" + } + + notify_no_data = true + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = false + new_host_delay = "${var.evaluation_delay}" + no_data_timeframe = 20 + + tags = ["env:${var.environment}", "resource:elb", "team:aws", "provider:aws"] +} + +resource "datadog_monitor" "ELB_too_much_5xx_backend" { + name = "[${var.environment}] ELB backend 5xx errors too high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)" + message = "${var.message}" + + query = < ${var.elb_backend_5xx_threshold_critical}" + EOF + + type = "metric alert" + + thresholds { + warning = "${var.elb_backend_5xx_threshold_warning}" + critical = "${var.elb_backend_5xx_threshold_critical}" + } + + notify_no_data = true + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = false new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 @@ -123,12 +161,12 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { } resource "datadog_monitor" "ELB_backend_latency" { - name = "[${var.environment}] ELB latency to high on {{host.identifier}}" + name = "[${var.environment}] ELB latency too high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)" message = "${var.message}" query = < ${var.elb_backend_latency_critical}}" EOF @@ -139,14 +177,14 @@ resource "datadog_monitor" "ELB_backend_latency" { critical = "${var.elb_backend_latency_critical}" } - notify_no_data = "${var.elb_notify_no_data}" + notify_no_data = true evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false timeout_h = 0 include_tags = true locked = false - require_full_window = true + require_full_window = false new_host_delay = "${var.evaluation_delay}" no_data_timeframe = 20 From 681a0418c3bee06b1bb8d7a5d8be51146c6ff37c Mon Sep 17 00:00:00 2001 From: Alexandre Gaillet Date: Thu, 22 Feb 2018 11:30:10 +0100 Subject: [PATCH 5/5] MON-93 - Update monitors --- cloud/aws/elb/monitors-elb.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf index 835dc3e..29c7736 100644 --- a/cloud/aws/elb/monitors-elb.tf +++ b/cloud/aws/elb/monitors-elb.tf @@ -50,7 +50,7 @@ resource "datadog_monitor" "ELB_too_much_4xx" { critical = "${var.elb_4xx_threshold_critical}" } - notify_no_data = true + notify_no_data = false evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false @@ -82,7 +82,7 @@ resource "datadog_monitor" "ELB_too_much_5xx" { critical = "${var.elb_5xx_threshold_critical}" } - notify_no_data = true + notify_no_data = false evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false @@ -114,7 +114,7 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { critical = "${var.elb_backend_4xx_threshold_critical}" } - notify_no_data = true + notify_no_data = false evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false @@ -146,7 +146,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { critical = "${var.elb_backend_5xx_threshold_critical}" } - notify_no_data = true + notify_no_data = false evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false @@ -165,7 +165,7 @@ resource "datadog_monitor" "ELB_backend_latency" { message = "${var.message}" query = < ${var.elb_backend_latency_critical}}" EOF @@ -177,7 +177,7 @@ resource "datadog_monitor" "ELB_backend_latency" { critical = "${var.elb_backend_latency_critical}" } - notify_no_data = true + notify_no_data = false evaluation_delay = "${var.evaluation_delay}" renotify_interval = 0 notify_audit = false