diff --git a/inputs.tf b/inputs.tf index 03b1829..5e6720b 100644 --- a/inputs.tf +++ b/inputs.tf @@ -23,6 +23,15 @@ variable "dd_custom_cpu" { } } +#cpu threshold + +variable "cpu_5_critical" { + default = 95 +} + +variable "cpu_15_critical" { + default = 80 +} ## RDS variable "dd_aws_rds" { diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index f854caf..ee92cd5 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -1,11 +1,17 @@ resource "datadog_monitor" "cpu_80_15min" { - name = "[${var.env}] CPU High > 80% for 15 min on {{host.name}}" + name = "[${var.env}] CPU High > ${var.cpu_15_critical} for 15 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" - query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > 80" + query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}" type = "query alert" + thresholds { + critical = "${var.cpu_15_critical}" + } + + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -19,13 +25,19 @@ resource "datadog_monitor" "cpu_80_15min" { } resource "datadog_monitor" "cpu_95_5min" { - name = "[${var.env}] CPU High > 95% for 5 min on {{host.name}}" + name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,region,stack} > 95" + query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = "${var.cpu_5_critical}" + } + + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -42,10 +54,16 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { name = "[${var.env}] Free disk space < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -62,7 +80,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { name = "[${var.env}] Free disk space < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -71,6 +89,8 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { critical = 10 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -87,10 +107,16 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { name = "[${var.env}] Free disk inodes < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}" - query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5" + query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -107,7 +133,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { name = "[${var.env}] Free disk inodes < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" + query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -116,6 +142,8 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { critical = 10 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -154,10 +182,16 @@ resource "datadog_monitor" "datadog_free_memory" { name = "[${var.env}] Free memory < 5% on {{host.name}}" message = "Debugging alert - no escalation" - query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5" + query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" diff --git a/monitors-rds_mysql-basics.tf b/monitors-rds_mysql-basics.tf index 0d1bbf3..95557e0 100644 --- a/monitors-rds_mysql-basics.tf +++ b/monitors-rds_mysql-basics.tf @@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" - query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} > 90" + query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90" type = "query alert" thresholds { @@ -16,6 +16,8 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { critical = "${var.rds_cpu_threshold["critical"]}" } + tags = ["*"] + notify_no_data = "${var.rds_config["notify_no_data"]}" evaluation_delay = "${var.rds_config["delay"]}" renotify_interval = 60 @@ -33,7 +35,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" type = "query alert" - query = "avg(last_15m): avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_rds-mysql_basics:enabled,env:${var.env},!dd_custom_rds-mysql:enabled} by {identifier,region} * 100 < 10" + query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10" count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" @@ -42,6 +44,8 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { critical = "${var.rds_mem_threshold["critical"]}" } + tags = ["*"] + notify_no_data = "${var.rds_config["notify_no_data"]}" evaluation_delay = "${var.rds_config["delay"]}" renotify_interval = 60 diff --git a/monitors_apache.tf b/monitors_apache.tf index b318de2..d807312 100644 --- a/monitors_apache.tf +++ b/monitors_apache.tf @@ -23,4 +23,6 @@ resource "datadog_monitor" "Apache_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_elb.tf b/monitors_elb.tf index a66fb84..acabd1a 100644 --- a/monitors_elb.tf +++ b/monitors_elb.tf @@ -16,6 +16,8 @@ resource "datadog_monitor" "ELB_no_healthy_instances" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_unhealthy_instances" { @@ -36,6 +38,8 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_too_much_5xx_backend" { @@ -61,6 +65,8 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_too_much_4xx_backend" { @@ -86,6 +92,8 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_backend_latency" { @@ -111,6 +119,8 @@ resource "datadog_monitor" "ELB_backend_latency" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_fpm.tf b/monitors_fpm.tf index cff10a0..844d6ac 100644 --- a/monitors_fpm.tf +++ b/monitors_fpm.tf @@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" { type = "query alert" - query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.90" + query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9" count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }" thresholds { @@ -23,6 +23,8 @@ resource "datadog_monitor" "php-fpm_process_idle" { require_full_window = true renotify_interval = 0 no_data_timeframe = 20 + + tags = ["*"] } @@ -51,4 +53,6 @@ resource "datadog_monitor" "FPM_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_nginx.tf b/monitors_nginx.tf index 01a95fe..8baf1cc 100644 --- a/monitors_nginx.tf +++ b/monitors_nginx.tf @@ -23,4 +23,6 @@ resource "datadog_monitor" "Nginx_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] }