wbyc/terraform-datadog-old-monitors
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MON-81 fix delta from nexperience

Browse Source 
MON-81 fix strconv.ParseInt

MON-81 fix hardcoded env

MON-81 added custom cpu tag & variabilize cpu threshold

MON-81 fixed dd_custom missing

MON-81 added region
This commit is contained in:
vincent EL KHATIB 2017-10-30 15:04:49 +01:00
parent 21fa0cd3c9
commit 7d2ad5114e
4 changed files with 41 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
inputs.tf
View File

@ -23,6 +23,15 @@ variable "dd_custom_cpu" {
}
}
#cpu threshold
variable "cpu_5_critical" {
default = 95
}
variable "cpu_15_critical" {
default = 80
}
## RDS
variable "dd_aws_rds" {

38
monitors-linux-basics.tf
View File

@ -1,11 +1,15 @@
resource "datadog_monitor" "cpu_80_15min" {
name = "[${var.env}] CPU High > 80% for 15 min on {{host.name}}"
name = "[${var.env}] CPU High > ${var.cpu_15_critical} for 15 min on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} > 80"
query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}"
type = "query alert"
thresholds {
critical = "${var.cpu_15_critical}"
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
@ -19,13 +23,17 @@ resource "datadog_monitor" "cpu_80_15min" {
}
resource "datadog_monitor" "cpu_95_5min" {
name = "[${var.env}] CPU High > 95% for 5 min on {{host.name}}"
name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,stack} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,region,stack} > 95"
query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = "${var.cpu_5_critical}"
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
@ -42,10 +50,14 @@ resource "datadog_monitor" "datadog_free_disk_space_5" {
name = "[${var.env}] Free disk space < 5% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = 5
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
@ -62,7 +74,7 @@ resource "datadog_monitor" "datadog_free_disk_space_10" {
name = "[${var.env}] Free disk space < 10% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 10"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
@ -87,10 +99,14 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
name = "[${var.env}] Free disk inodes < 5% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 5"
query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region, device} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = 5
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
@ -107,7 +123,7 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
name = "[${var.env}] Free disk inodes < 10% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region,stack} * 100 < 10"
query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
@ -154,10 +170,14 @@ resource "datadog_monitor" "datadog_free_memory" {
name = "[${var.env}] Free memory < 5% on {{host.name}}"
message = "Debugging alert - no escalation"
query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region,stack} * 100 < 5"
query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = 5
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"

4
monitors-rds_mysql-basics.tf
View File

@ -8,7 +8,7 @@ resource "datadog_monitor" "rds-mysql_cpu_80_15min" {
count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }"
query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} > 90"
query = "avg(last_15m):avg:aws.rds.cpuutilization{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} > 90"
type = "query alert"
thresholds {
@ -33,7 +33,7 @@ resource "datadog_monitor" "mysql_rds_free_space_low" {
message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
type = "query alert"
query = "avg(last_15m): avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {name,region} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_rds-mysql_basics:enabled,env:${var.env},!dd_custom_rds-mysql:enabled} by {identifier,region} * 100 < 10"
query = "avg(last_15m):avg:aws.rds.free_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} / avg:aws.rds.total_storage_space{dd_monitoring:enabled,dd_aws_rds:enabled,env:${var.env}} by {region,name} * 100 < 10"
count = "${var.dd_aws_rds == "enabled" ? 1 : 0 }"

2
monitors_fpm.tf
View File

@ -4,7 +4,7 @@ resource "datadog_monitor" "php-fpm_process_idle" {
type = "query alert"
query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,app} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region,stack} ) > 0.90"
query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9"
count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }"
thresholds {