Merged in MON-81 (pull request #9)

MON-81 fix monitors delta from nexperience

Approved-by: Kevin Pecquet <kevin.pecquet@fr.clara.net>
Approved-by: Quentin Manfroi <quentin.manfroi@yahoo.fr>
Approved-by: vincent.elkhatib@morea.fr <vincent.elkhatib@morea.fr>

inputs.tf

@@ -23,6 +23,15 @@ variable "dd_custom_cpu" {
   }
 }
 
+#cpu threshold
+
+variable "cpu_5_critical" {
+  default = 95
+}
+
+variable "cpu_15_critical" {
+  default = 80
+}
 
 ## RDS
 variable "dd_aws_rds" {

monitors-linux-basics.tf

@@ -1,11 +1,17 @@
 resource "datadog_monitor" "cpu_80_15min" {
-  name    = "[${var.env}] CPU High > 80% for 15 min on {{host.name}}"
+  name    = "[${var.env}] CPU High > ${var.cpu_15_critical} for 15 min on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
   count   = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
-  query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > 80"
+  query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}"
   type  = "query alert"
 
+  thresholds {
+    critical = "${var.cpu_15_critical}"
+  }
+
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -19,13 +25,19 @@ resource "datadog_monitor" "cpu_80_15min" {
 }
 
 resource "datadog_monitor" "cpu_95_5min" {
-  name    = "[${var.env}] CPU High > 95% for 5 min on {{host.name}}"
+  name    = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
 
-  query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,region,stack} > 95"
+  query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}"
   type  = "query alert"
   count   = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
+  thresholds {
+    critical = "${var.cpu_5_critical}"
+  }
+
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -42,10 +54,16 @@ resource "datadog_monitor" "datadog_free_disk_space_5" {
   name   = "[${var.env}] Free disk space < 5%  on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
 
-  query   = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5"
+  query   = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5"
   type  = "query alert"
   count   = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
+  thresholds {
+    critical = 5
+  }
+
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -62,7 +80,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" {
   name    = "[${var.env}] Free disk space < 10% on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
 
-  query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10"
+  query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 10"
   type  = "query alert"
   count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
@@ -71,6 +89,8 @@ resource "datadog_monitor" "datadog_free_disk_space_10" {
     critical = 10
   }
 
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -87,10 +107,16 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
   name    = "[${var.env}] Free disk inodes < 5% on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
 
-  query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5"
+  query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5"
   type  = "query alert"
   count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
+  thresholds {
+    critical = 5
+  }
+
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -107,7 +133,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
   name    = "[${var.env}] Free disk inodes < 10% on {{host.name}}"
   message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
 
-  query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10"
+  query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10"
   type  = "query alert"
   count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
@@ -116,6 +142,8 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
     critical = 10
   }
 
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"
@@ -154,10 +182,16 @@ resource "datadog_monitor" "datadog_free_memory" {
   name   = "[${var.env}] Free memory < 5% on {{host.name}}"
   message = "Debugging alert - no escalation"
 
-  query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5"
+  query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5"
   type  = "query alert"
   count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
 
+  thresholds {
+    critical = 5
+  }
+
+  tags = ["*"]
+
   notify_no_data      = "${var.linux_basics_config["notify_no_data"]}"
   evaluation_delay    = "${var.linux_basics_config["delay"]}"
   new_host_delay      = "${var.linux_basics_config["delay"]}"

monitors-rds_mysql-basics.tf

@@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" {
   count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }"
 
 
-  query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} > 90"
+  query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90"
   type  = "query alert"
 
   thresholds {
@@ -16,6 +16,8 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" {
     critical = "${var.rds_cpu_threshold["critical"]}"
   }
 
+  tags = ["*"]
+
   notify_no_data      = "${var.rds_config["notify_no_data"]}"
   evaluation_delay    = "${var.rds_config["delay"]}"
   renotify_interval   = 60
@@ -33,7 +35,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" {
   message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
   
   type  = "query alert"
-  query = "avg(last_15m): avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} /  avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_rds-mysql_basics:enabled,env:${var.env},!dd_custom_rds-mysql:enabled} by {identifier,region}  * 100 < 10"
+  query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10"
   count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }"
 
 
@@ -42,6 +44,8 @@ resource "datadog_monitor" "mysql_rds_free_space_low" {
     critical = "${var.rds_mem_threshold["critical"]}"
   }
 
+  tags = ["*"]
+
   notify_no_data      = "${var.rds_config["notify_no_data"]}"
   evaluation_delay    = "${var.rds_config["delay"]}"
   renotify_interval   = 60

monitors_apache.tf

@@ -23,4 +23,6 @@ resource "datadog_monitor" "Apache_process" {
   locked              = false
   require_full_window = true
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }

monitors_elb.tf

@@ -16,6 +16,8 @@ resource "datadog_monitor" "ELB_no_healthy_instances" {
   require_full_window = true
   new_host_delay      = "${var.elb_config["delay"]}"
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 resource "datadog_monitor" "ELB_unhealthy_instances" {
@@ -36,6 +38,8 @@ resource "datadog_monitor" "ELB_unhealthy_instances" {
   require_full_window = true
   new_host_delay      = "${var.elb_config["delay"]}"
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 resource "datadog_monitor" "ELB_too_much_5xx_backend" {
@@ -61,6 +65,8 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" {
   require_full_window = true
   new_host_delay      = "${var.elb_config["delay"]}"
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 resource "datadog_monitor" "ELB_too_much_4xx_backend" {
@@ -86,6 +92,8 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" {
   require_full_window = true
   new_host_delay      = "${var.elb_config["delay"]}"
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 resource "datadog_monitor" "ELB_backend_latency" {
@@ -111,6 +119,8 @@ resource "datadog_monitor" "ELB_backend_latency" {
   require_full_window = true
   new_host_delay      = "${var.elb_config["delay"]}"
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 

monitors_fpm.tf

@@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" {
 
 
   type  = "query alert"
-  query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.90"
+  query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9"
   count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }"
 
   thresholds {
@@ -23,6 +23,8 @@ resource "datadog_monitor" "php-fpm_process_idle" {
   require_full_window = true
   renotify_interval   = 0
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }
 
 
@@ -51,4 +53,6 @@ resource "datadog_monitor" "FPM_process" {
   locked              = false
   require_full_window = true
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }

monitors_nginx.tf

@@ -23,4 +23,6 @@ resource "datadog_monitor" "Nginx_process" {
   locked              = false
   require_full_window = true
   no_data_timeframe   = 20
+
+  tags = ["*"]
 }