From 7d2ad5114edbc7f75c0aa957e5712b560041658f Mon Sep 17 00:00:00 2001 From: vincent EL KHATIB Date: Mon, 30 Oct 2017 15:04:49 +0100 Subject: [PATCH 1/9] MON-81 fix delta from nexperience MON-81 fix strconv.ParseInt MON-81 fix hardcoded env MON-81 added custom cpu tag & variabilize cpu threshold MON-81 fixed dd_custom missing MON-81 added region --- inputs.tf | 9 +++++++++ monitors-linux-basics.tf | 38 +++++++++++++++++++++++++++--------- monitors-rds_mysql-basics.tf | 4 ++-- monitors_fpm.tf | 2 +- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/inputs.tf b/inputs.tf index 03b1829..5e6720b 100644 --- a/inputs.tf +++ b/inputs.tf @@ -23,6 +23,15 @@ variable "dd_custom_cpu" { } } +#cpu threshold + +variable "cpu_5_critical" { + default = 95 +} + +variable "cpu_15_critical" { + default = 80 +} ## RDS variable "dd_aws_rds" { diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index f854caf..961a070 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -1,11 +1,15 @@ resource "datadog_monitor" "cpu_80_15min" { - name = "[${var.env}] CPU High > 80% for 15 min on {{host.name}}" + name = "[${var.env}] CPU High > ${var.cpu_15_critical} for 15 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" - query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > 80" + query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}" type = "query alert" + thresholds { + critical = "${var.cpu_15_critical}" + } + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -19,13 +23,17 @@ resource "datadog_monitor" "cpu_80_15min" { } resource "datadog_monitor" "cpu_95_5min" { - name = "[${var.env}] CPU High > 95% for 5 min on {{host.name}}" + name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,region,stack} > 95" + query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = "${var.cpu_5_critical}" + } + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -42,10 +50,14 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { name = "[${var.env}] Free disk space < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -62,7 +74,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { name = "[${var.env}] Free disk space < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -87,10 +99,14 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { name = "[${var.env}] Free disk inodes < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}" - query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5" + query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -107,7 +123,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { name = "[${var.env}] Free disk inodes < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" + query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -154,10 +170,14 @@ resource "datadog_monitor" "datadog_free_memory" { name = "[${var.env}] Free memory < 5% on {{host.name}}" message = "Debugging alert - no escalation" - query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5" + query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" + thresholds { + critical = 5 + } + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" diff --git a/monitors-rds_mysql-basics.tf b/monitors-rds_mysql-basics.tf index 0d1bbf3..9d81b7c 100644 --- a/monitors-rds_mysql-basics.tf +++ b/monitors-rds_mysql-basics.tf @@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" - query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} > 90" + query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90" type = "query alert" thresholds { @@ -33,7 +33,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" type = "query alert" - query = "avg(last_15m): avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_rds-mysql_basics:enabled,env:${var.env},!dd_custom_rds-mysql:enabled} by {identifier,region} * 100 < 10" + query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10" count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" diff --git a/monitors_fpm.tf b/monitors_fpm.tf index cff10a0..0218212 100644 --- a/monitors_fpm.tf +++ b/monitors_fpm.tf @@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" { type = "query alert" - query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.90" + query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9" count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }" thresholds { From 272ab84244d8057fbaeb344bacdf8c6d1e38d265 Mon Sep 17 00:00:00 2001 From: vincent EL KHATIB Date: Tue, 31 Oct 2017 16:00:47 +0100 Subject: [PATCH 2/9] MON-81 added stack --- monitors-linux-basics.tf | 14 +++++++------- monitors-rds_mysql-basics.tf | 4 ++-- monitors_elb.tf | 10 +++++----- monitors_fpm.tf | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index 961a070..223c6ef 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -3,7 +3,7 @@ resource "datadog_monitor" "cpu_80_15min" { message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" - query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}" + query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} > ${var.cpu_15_critical}" type = "query alert" thresholds { @@ -26,7 +26,7 @@ resource "datadog_monitor" "cpu_95_5min" { name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}" + query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} > ${var.cpu_5_critical}" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -50,7 +50,7 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { name = "[${var.env}] Free disk space < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -74,7 +74,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { name = "[${var.env}] Free disk space < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 10" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -99,7 +99,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { name = "[${var.env}] Free disk inodes < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}" - query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5" + query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -123,7 +123,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { name = "[${var.env}] Free disk inodes < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10" + query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -170,7 +170,7 @@ resource "datadog_monitor" "datadog_free_memory" { name = "[${var.env}] Free memory < 5% on {{host.name}}" message = "Debugging alert - no escalation" - query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5" + query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" diff --git a/monitors-rds_mysql-basics.tf b/monitors-rds_mysql-basics.tf index 9d81b7c..e45cc1b 100644 --- a/monitors-rds_mysql-basics.tf +++ b/monitors-rds_mysql-basics.tf @@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" - query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90" + query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} > 90" type = "query alert" thresholds { @@ -33,7 +33,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" type = "query alert" - query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10" + query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} * 100 < 10" count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" diff --git a/monitors_elb.tf b/monitors_elb.tf index a66fb84..ccb72fb 100644 --- a/monitors_elb.tf +++ b/monitors_elb.tf @@ -3,7 +3,7 @@ resource "datadog_monitor" "ELB_no_healthy_instances" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} == 0" + query = "avg(last_5m):avg:aws.elb.healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} == 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -23,7 +23,7 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > 0" + query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} > 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -43,7 +43,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_5xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} * 100 > ${var.elb_5xx_threshold["critical"]}" type = "query alert" thresholds { @@ -68,7 +68,7 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_4xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} * 100 > ${var.elb_4xx_threshold["critical"]}" type = "query alert" thresholds { @@ -93,7 +93,7 @@ resource "datadog_monitor" "ELB_backend_latency" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.latency{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > ${var.elb_backend_latency["critical"]}" + query = "avg(last_5m):avg:aws.elb.latency{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} > ${var.elb_backend_latency["critical"]}" type = "query alert" thresholds { diff --git a/monitors_fpm.tf b/monitors_fpm.tf index 0218212..9929229 100644 --- a/monitors_fpm.tf +++ b/monitors_fpm.tf @@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" { type = "query alert" - query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9" + query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.9" count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }" thresholds { From 7481485da1d6d91b8396a8ea506367d4abb4b6e4 Mon Sep 17 00:00:00 2001 From: vincent EL KHATIB Date: Tue, 31 Oct 2017 16:27:25 +0100 Subject: [PATCH 3/9] MON-81 remove space --- monitors-linux-basics.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index 223c6ef..1ed76a8 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -3,7 +3,7 @@ resource "datadog_monitor" "cpu_80_15min" { message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" - query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} > ${var.cpu_15_critical}" + query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > ${var.cpu_15_critical}" type = "query alert" thresholds { @@ -26,7 +26,7 @@ resource "datadog_monitor" "cpu_95_5min" { name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, stack} > ${var.cpu_5_critical}" + query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > ${var.cpu_5_critical}" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -50,7 +50,7 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { name = "[${var.env}] Free disk space < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 5" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -74,7 +74,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { name = "[${var.env}] Free disk space < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 10" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -99,7 +99,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { name = "[${var.env}] Free disk inodes < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}" - query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device,stack} * 100 < 5" + query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" From 64be248b9eff70649c9cb2d6b1d0f5381adc5110 Mon Sep 17 00:00:00 2001 From: vincent EL KHATIB Date: Thu, 2 Nov 2017 15:47:47 +0100 Subject: [PATCH 4/9] MON-81 added tag * --- monitors-linux-basics.tf | 14 ++++++++++++++ monitors-rds_mysql-basics.tf | 4 ++++ monitors_apache.tf | 2 ++ monitors_elb.tf | 10 ++++++++++ monitors_fpm.tf | 4 ++++ monitors_nginx.tf | 2 ++ 6 files changed, 36 insertions(+) diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index 1ed76a8..d23a02e 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -10,6 +10,8 @@ resource "datadog_monitor" "cpu_80_15min" { critical = "${var.cpu_15_critical}" } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -34,6 +36,8 @@ resource "datadog_monitor" "cpu_95_5min" { critical = "${var.cpu_5_critical}" } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -58,6 +62,8 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { critical = 5 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -83,6 +89,8 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { critical = 10 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -107,6 +115,8 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { critical = 5 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -132,6 +142,8 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { critical = 10 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" @@ -178,6 +190,8 @@ resource "datadog_monitor" "datadog_free_memory" { critical = 5 } + tags = ["*"] + notify_no_data = "${var.linux_basics_config["notify_no_data"]}" evaluation_delay = "${var.linux_basics_config["delay"]}" new_host_delay = "${var.linux_basics_config["delay"]}" diff --git a/monitors-rds_mysql-basics.tf b/monitors-rds_mysql-basics.tf index e45cc1b..c0c44d1 100644 --- a/monitors-rds_mysql-basics.tf +++ b/monitors-rds_mysql-basics.tf @@ -16,6 +16,8 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { critical = "${var.rds_cpu_threshold["critical"]}" } + tags = ["*"] + notify_no_data = "${var.rds_config["notify_no_data"]}" evaluation_delay = "${var.rds_config["delay"]}" renotify_interval = 60 @@ -42,6 +44,8 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { critical = "${var.rds_mem_threshold["critical"]}" } + tags = ["*"] + notify_no_data = "${var.rds_config["notify_no_data"]}" evaluation_delay = "${var.rds_config["delay"]}" renotify_interval = 60 diff --git a/monitors_apache.tf b/monitors_apache.tf index b318de2..d807312 100644 --- a/monitors_apache.tf +++ b/monitors_apache.tf @@ -23,4 +23,6 @@ resource "datadog_monitor" "Apache_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_elb.tf b/monitors_elb.tf index ccb72fb..788037a 100644 --- a/monitors_elb.tf +++ b/monitors_elb.tf @@ -16,6 +16,8 @@ resource "datadog_monitor" "ELB_no_healthy_instances" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_unhealthy_instances" { @@ -36,6 +38,8 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_too_much_5xx_backend" { @@ -61,6 +65,8 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_too_much_4xx_backend" { @@ -86,6 +92,8 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } resource "datadog_monitor" "ELB_backend_latency" { @@ -111,6 +119,8 @@ resource "datadog_monitor" "ELB_backend_latency" { require_full_window = true new_host_delay = "${var.elb_config["delay"]}" no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_fpm.tf b/monitors_fpm.tf index 9929229..957e1b9 100644 --- a/monitors_fpm.tf +++ b/monitors_fpm.tf @@ -23,6 +23,8 @@ resource "datadog_monitor" "php-fpm_process_idle" { require_full_window = true renotify_interval = 0 no_data_timeframe = 20 + + tags = ["*"] } @@ -51,4 +53,6 @@ resource "datadog_monitor" "FPM_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] } diff --git a/monitors_nginx.tf b/monitors_nginx.tf index 01a95fe..8baf1cc 100644 --- a/monitors_nginx.tf +++ b/monitors_nginx.tf @@ -23,4 +23,6 @@ resource "datadog_monitor" "Nginx_process" { locked = false require_full_window = true no_data_timeframe = 20 + + tags = ["*"] } From 63ba5e54d1cc439ca15bf5c34ee3767b3c9b0931 Mon Sep 17 00:00:00 2001 From: vincent EL KHATIB Date: Thu, 2 Nov 2017 15:54:51 +0100 Subject: [PATCH 5/9] MON-81 remove stack --- monitors-linux-basics.tf | 14 +++++++------- monitors-rds_mysql-basics.tf | 4 ++-- monitors_elb.tf | 10 +++++----- monitors_fpm.tf | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/monitors-linux-basics.tf b/monitors-linux-basics.tf index d23a02e..ee92cd5 100644 --- a/monitors-linux-basics.tf +++ b/monitors-linux-basics.tf @@ -3,7 +3,7 @@ resource "datadog_monitor" "cpu_80_15min" { message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" - query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > ${var.cpu_15_critical}" + query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}" type = "query alert" thresholds { @@ -28,7 +28,7 @@ resource "datadog_monitor" "cpu_95_5min" { name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > ${var.cpu_5_critical}" + query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -54,7 +54,7 @@ resource "datadog_monitor" "datadog_free_disk_space_5" { name = "[${var.env}] Free disk space < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 5" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -80,7 +80,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" { name = "[${var.env}] Free disk space < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 10" + query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -107,7 +107,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" { name = "[${var.env}] Free disk inodes < 5% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}" - query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device,stack} * 100 < 5" + query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -133,7 +133,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" { name = "[${var.env}] Free disk inodes < 10% on {{host.name}}" message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" - query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10" + query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" @@ -182,7 +182,7 @@ resource "datadog_monitor" "datadog_free_memory" { name = "[${var.env}] Free memory < 5% on {{host.name}}" message = "Debugging alert - no escalation" - query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5" + query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5" type = "query alert" count = "${var.dd_linux_basics == "enabled" ? 1 : 0}" diff --git a/monitors-rds_mysql-basics.tf b/monitors-rds_mysql-basics.tf index c0c44d1..95557e0 100644 --- a/monitors-rds_mysql-basics.tf +++ b/monitors-rds_mysql-basics.tf @@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" { count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" - query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} > 90" + query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90" type = "query alert" thresholds { @@ -35,7 +35,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}" type = "query alert" - query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name,stack} * 100 < 10" + query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10" count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }" diff --git a/monitors_elb.tf b/monitors_elb.tf index 788037a..acabd1a 100644 --- a/monitors_elb.tf +++ b/monitors_elb.tf @@ -3,7 +3,7 @@ resource "datadog_monitor" "ELB_no_healthy_instances" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} == 0" + query = "avg(last_5m):avg:aws.elb.healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} == 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -25,7 +25,7 @@ resource "datadog_monitor" "ELB_unhealthy_instances" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} > 0" + query = "avg(last_5m):avg:aws.elb.un_healthy_host_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > 0" type = "query alert" notify_no_data = "${var.elb_config["notify_no_data"]}" @@ -47,7 +47,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} * 100 > ${var.elb_5xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_5xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_5xx_threshold["critical"]}" type = "query alert" thresholds { @@ -74,7 +74,7 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" { message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} * 100 > ${var.elb_4xx_threshold["critical"]}" + query = "avg(last_5m): avg:aws.elb.httpcode_backend_4xx{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} / avg:aws.elb.request_count{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} * 100 > ${var.elb_4xx_threshold["critical"]}" type = "query alert" thresholds { @@ -101,7 +101,7 @@ resource "datadog_monitor" "ELB_backend_latency" { message = "{{#is_alert}}\n${var.ho_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.ho_escalation_group}\n{{/is_recovery}}" count = "${var.dd_aws_elb == "enabled" ? 1 : 0 }" - query = "avg(last_5m):avg:aws.elb.latency{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region,stack} > ${var.elb_backend_latency["critical"]}" + query = "avg(last_5m):avg:aws.elb.latency{dd_monitoring:enabled,dd_aws_elb:enabled,env:${var.env}} by {loadbalancername,region} > ${var.elb_backend_latency["critical"]}" type = "query alert" thresholds { diff --git a/monitors_fpm.tf b/monitors_fpm.tf index 957e1b9..844d6ac 100644 --- a/monitors_fpm.tf +++ b/monitors_fpm.tf @@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" { type = "query alert" - query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.9" + query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9" count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }" thresholds { From 27455866a758929aa51ffc7cf99f01dd127b1641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Respaut?= Date: Fri, 27 Oct 2017 15:11:53 +0200 Subject: [PATCH 6/9] MON-66: Update testing module url --- testing/modules.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/modules.tf b/testing/modules.tf index 5574a18..258d85b 100644 --- a/testing/modules.tf +++ b/testing/modules.tf @@ -1,5 +1,5 @@ module "datadog-monitors" { - source = "git::ssh://git@bitbucket.org/morea/terraform.datadog.monitors.git" + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git" env = "${var.environment}" region = "${var.region}" @@ -12,4 +12,3 @@ module "datadog-monitors" { dd_linux_basics = "${var.dd_linux_basics}" } - From 66577a13e2b737a1ee9a2632756ce24179a5961c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Respaut?= Date: Fri, 27 Oct 2017 15:18:43 +0200 Subject: [PATCH 7/9] MON-66: Dummy tesing subdir --- testing/dummy/dummy.tf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 testing/dummy/dummy.tf diff --git a/testing/dummy/dummy.tf b/testing/dummy/dummy.tf new file mode 100644 index 0000000..f5cc69c --- /dev/null +++ b/testing/dummy/dummy.tf @@ -0,0 +1,3 @@ + + +resource "null_resource" "test-migration" {} \ No newline at end of file From a86de69c5e47ae6999093085100a19641fb6b770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Respaut?= Date: Fri, 27 Oct 2017 16:08:06 +0200 Subject: [PATCH 8/9] MON-66: New tree directory and README updates --- README.md | 16 +++++++--------- cloud/aws/elb/inputs.tf | 1 + monitors_elb.tf => cloud/aws/elb/monitors-elb.tf | 0 cloud/aws/rds-mysql/inputs.tf | 1 + .../aws/rds-mysql/monitors-rds_mysql-basics.tf | 0 middleware/apache/inputs.tf | 1 + .../apache/monitors-apache.tf | 0 middleware/nginx/inputs.tf | 1 + .../nginx/monitors-nginx.tf | 0 middleware/php-fpm/inputs.tf | 1 + .../php-fpm/monitors-fpm.tf | 0 os/linux/inputs.tf | 1 + .../linux/monitors-linux-basics.tf | 0 system/cpu/inputs.tf | 1 + .../cpu/monitors-custom-cpu.tf | 0 15 files changed, 14 insertions(+), 9 deletions(-) create mode 120000 cloud/aws/elb/inputs.tf rename monitors_elb.tf => cloud/aws/elb/monitors-elb.tf (100%) create mode 120000 cloud/aws/rds-mysql/inputs.tf rename monitors-rds_mysql-basics.tf => cloud/aws/rds-mysql/monitors-rds_mysql-basics.tf (100%) create mode 120000 middleware/apache/inputs.tf rename monitors_apache.tf => middleware/apache/monitors-apache.tf (100%) create mode 120000 middleware/nginx/inputs.tf rename monitors_nginx.tf => middleware/nginx/monitors-nginx.tf (100%) create mode 120000 middleware/php-fpm/inputs.tf rename monitors_fpm.tf => middleware/php-fpm/monitors-fpm.tf (100%) create mode 120000 os/linux/inputs.tf rename monitors-linux-basics.tf => os/linux/monitors-linux-basics.tf (100%) create mode 120000 system/cpu/inputs.tf rename monitors-custom-cpu.tf => system/cpu/monitors-custom-cpu.tf (100%) diff --git a/README.md b/README.md index 78e83cf..88e7dda 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# README # +# README - DataDog Monitors # This repository is used to store all our monitors templates. -These templates have to be used with dedicated agent configurations and ressource tagging to works as expected. +These templates have to be used with dedicated agent configurations and resource tagging to works as expected. ### How to contribute ? ### @@ -10,13 +10,11 @@ The easiest way to contribute on this repository is to add monitors file inside ### Important notes ### -*This repository will be included as a Terraform module source. - -*Each monitors have to be disabled by default, so you have to manage this condition when you create your monitors. +* This repository will be included as a Terraform module source. +* Each monitors have to be disabled by default, so you have to manage this condition when you create your monitors. +* If you override a basic monitor with a custom one, you have to use tag filters on you query. (Example dd_cpu_high, if you want to override default cpu monitors) -*If you override a basic monitor with a custom one, you have to use tag filters on you query. (Example dd_cpu_high, if you want to override default cpu monitors) - -### Main.tf : add the datadog provider ### +### Main.tf : add the DataDog provider ### ``` provider "datadog" { @@ -29,7 +27,7 @@ provider "datadog" { ``` module "datadog-monitors" { - source = "git::ssh://git@bitbucket.org/morea/terraform.datadog.monitors.git" + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//{module-path}?ref={version}" critical_escalation_group = "${var.critical_escalation_group}" warning_escalation_group = "${var.warning_escalation_group}" diff --git a/cloud/aws/elb/inputs.tf b/cloud/aws/elb/inputs.tf new file mode 120000 index 0000000..a68ace3 --- /dev/null +++ b/cloud/aws/elb/inputs.tf @@ -0,0 +1 @@ +../../../inputs.tf \ No newline at end of file diff --git a/monitors_elb.tf b/cloud/aws/elb/monitors-elb.tf similarity index 100% rename from monitors_elb.tf rename to cloud/aws/elb/monitors-elb.tf diff --git a/cloud/aws/rds-mysql/inputs.tf b/cloud/aws/rds-mysql/inputs.tf new file mode 120000 index 0000000..a68ace3 --- /dev/null +++ b/cloud/aws/rds-mysql/inputs.tf @@ -0,0 +1 @@ +../../../inputs.tf \ No newline at end of file diff --git a/monitors-rds_mysql-basics.tf b/cloud/aws/rds-mysql/monitors-rds_mysql-basics.tf similarity index 100% rename from monitors-rds_mysql-basics.tf rename to cloud/aws/rds-mysql/monitors-rds_mysql-basics.tf diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf new file mode 120000 index 0000000..cdfc6c6 --- /dev/null +++ b/middleware/apache/inputs.tf @@ -0,0 +1 @@ +../../inputs.tf \ No newline at end of file diff --git a/monitors_apache.tf b/middleware/apache/monitors-apache.tf similarity index 100% rename from monitors_apache.tf rename to middleware/apache/monitors-apache.tf diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf new file mode 120000 index 0000000..cdfc6c6 --- /dev/null +++ b/middleware/nginx/inputs.tf @@ -0,0 +1 @@ +../../inputs.tf \ No newline at end of file diff --git a/monitors_nginx.tf b/middleware/nginx/monitors-nginx.tf similarity index 100% rename from monitors_nginx.tf rename to middleware/nginx/monitors-nginx.tf diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf new file mode 120000 index 0000000..cdfc6c6 --- /dev/null +++ b/middleware/php-fpm/inputs.tf @@ -0,0 +1 @@ +../../inputs.tf \ No newline at end of file diff --git a/monitors_fpm.tf b/middleware/php-fpm/monitors-fpm.tf similarity index 100% rename from monitors_fpm.tf rename to middleware/php-fpm/monitors-fpm.tf diff --git a/os/linux/inputs.tf b/os/linux/inputs.tf new file mode 120000 index 0000000..cdfc6c6 --- /dev/null +++ b/os/linux/inputs.tf @@ -0,0 +1 @@ +../../inputs.tf \ No newline at end of file diff --git a/monitors-linux-basics.tf b/os/linux/monitors-linux-basics.tf similarity index 100% rename from monitors-linux-basics.tf rename to os/linux/monitors-linux-basics.tf diff --git a/system/cpu/inputs.tf b/system/cpu/inputs.tf new file mode 120000 index 0000000..cdfc6c6 --- /dev/null +++ b/system/cpu/inputs.tf @@ -0,0 +1 @@ +../../inputs.tf \ No newline at end of file diff --git a/monitors-custom-cpu.tf b/system/cpu/monitors-custom-cpu.tf similarity index 100% rename from monitors-custom-cpu.tf rename to system/cpu/monitors-custom-cpu.tf From 5758f0f03f7b0201cd5f72d92ab438e556908899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Respaut?= Date: Fri, 27 Oct 2017 17:08:32 +0200 Subject: [PATCH 9/9] MON-66: Some more rename --- {monitors-unready => incubator}/monitors-apache-nat.tf | 0 {monitors-unready => incubator}/monitors-cassandra.tf | 0 {monitors-unready => incubator}/monitors-gcp-cloud-sql.tf | 0 {monitors-unready => incubator}/monitors-gcp-lb.tf | 0 {monitors-unready => incubator}/monitors-gcp-vpn.tf | 0 {monitors-unready => incubator}/monitors-kubernetes.tf | 0 {monitors-unready => incubator}/monitors-linux-basics.log | 0 {monitors-unready => incubator}/monitors-linux-basics.tf | 0 {monitors-unready => incubator}/monitors-redis-containers.tf | 0 {monitors-unready => incubator}/monitors-redis.tf | 0 {os/linux => system/generic}/inputs.tf | 0 system/{cpu => generic}/monitors-custom-cpu.tf | 0 system/{cpu => linux}/inputs.tf | 0 {os => system}/linux/monitors-linux-basics.tf | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename {monitors-unready => incubator}/monitors-apache-nat.tf (100%) rename {monitors-unready => incubator}/monitors-cassandra.tf (100%) rename {monitors-unready => incubator}/monitors-gcp-cloud-sql.tf (100%) rename {monitors-unready => incubator}/monitors-gcp-lb.tf (100%) rename {monitors-unready => incubator}/monitors-gcp-vpn.tf (100%) rename {monitors-unready => incubator}/monitors-kubernetes.tf (100%) rename {monitors-unready => incubator}/monitors-linux-basics.log (100%) rename {monitors-unready => incubator}/monitors-linux-basics.tf (100%) rename {monitors-unready => incubator}/monitors-redis-containers.tf (100%) rename {monitors-unready => incubator}/monitors-redis.tf (100%) rename {os/linux => system/generic}/inputs.tf (100%) rename system/{cpu => generic}/monitors-custom-cpu.tf (100%) rename system/{cpu => linux}/inputs.tf (100%) rename {os => system}/linux/monitors-linux-basics.tf (100%) diff --git a/monitors-unready/monitors-apache-nat.tf b/incubator/monitors-apache-nat.tf similarity index 100% rename from monitors-unready/monitors-apache-nat.tf rename to incubator/monitors-apache-nat.tf diff --git a/monitors-unready/monitors-cassandra.tf b/incubator/monitors-cassandra.tf similarity index 100% rename from monitors-unready/monitors-cassandra.tf rename to incubator/monitors-cassandra.tf diff --git a/monitors-unready/monitors-gcp-cloud-sql.tf b/incubator/monitors-gcp-cloud-sql.tf similarity index 100% rename from monitors-unready/monitors-gcp-cloud-sql.tf rename to incubator/monitors-gcp-cloud-sql.tf diff --git a/monitors-unready/monitors-gcp-lb.tf b/incubator/monitors-gcp-lb.tf similarity index 100% rename from monitors-unready/monitors-gcp-lb.tf rename to incubator/monitors-gcp-lb.tf diff --git a/monitors-unready/monitors-gcp-vpn.tf b/incubator/monitors-gcp-vpn.tf similarity index 100% rename from monitors-unready/monitors-gcp-vpn.tf rename to incubator/monitors-gcp-vpn.tf diff --git a/monitors-unready/monitors-kubernetes.tf b/incubator/monitors-kubernetes.tf similarity index 100% rename from monitors-unready/monitors-kubernetes.tf rename to incubator/monitors-kubernetes.tf diff --git a/monitors-unready/monitors-linux-basics.log b/incubator/monitors-linux-basics.log similarity index 100% rename from monitors-unready/monitors-linux-basics.log rename to incubator/monitors-linux-basics.log diff --git a/monitors-unready/monitors-linux-basics.tf b/incubator/monitors-linux-basics.tf similarity index 100% rename from monitors-unready/monitors-linux-basics.tf rename to incubator/monitors-linux-basics.tf diff --git a/monitors-unready/monitors-redis-containers.tf b/incubator/monitors-redis-containers.tf similarity index 100% rename from monitors-unready/monitors-redis-containers.tf rename to incubator/monitors-redis-containers.tf diff --git a/monitors-unready/monitors-redis.tf b/incubator/monitors-redis.tf similarity index 100% rename from monitors-unready/monitors-redis.tf rename to incubator/monitors-redis.tf diff --git a/os/linux/inputs.tf b/system/generic/inputs.tf similarity index 100% rename from os/linux/inputs.tf rename to system/generic/inputs.tf diff --git a/system/cpu/monitors-custom-cpu.tf b/system/generic/monitors-custom-cpu.tf similarity index 100% rename from system/cpu/monitors-custom-cpu.tf rename to system/generic/monitors-custom-cpu.tf diff --git a/system/cpu/inputs.tf b/system/linux/inputs.tf similarity index 100% rename from system/cpu/inputs.tf rename to system/linux/inputs.tf diff --git a/os/linux/monitors-linux-basics.tf b/system/linux/monitors-linux-basics.tf similarity index 100% rename from os/linux/monitors-linux-basics.tf rename to system/linux/monitors-linux-basics.tf