wbyc/terraform-datadog-old-monitors
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MON-96 - Update system directory and update inputs-declaration sample

Browse Source
This commit is contained in:
Alexandre Gaillet 2018-02-21 17:34:17 +01:00
parent febc38eec7
commit dd729b90e5
7 changed files with 333 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
datadog-samples/inputs-declaration.sample
View File

@ -1,35 +1,47 @@
variable environment {}
variable region {}
variable "critical_escalation_group" {
default = "@pagerduty_HODummy"
}
variable "warning_escalation_group" {
default = "@pagerduty_HNODummy"
# Global Terraform
variable "environment" {
description = "Architecture Environment"
type = "string"
}
variable "datadog_api_key" {}
variable "datadog_app_key" {}
variable "dd_linux_basics" {
default = "enabled"
# Global DataDog
variable "evaluation_delay" {
description = "Delay in seconds for the metric evaluation"
default = 600
}
variable "dd_aws_rds" {
default = "enabled"
variable "message" {
description = "Message sent when an alert is triggered"
}
variable "dd_custom_cpu" {
type = "map"
default = {
status = "enabled"
name = "CPU High > 95% during 1 hour"
period = "last_1h"
critical_threshold = 95
warning_threshold = 90
}
variable "filter_tags_use_defaults" {
description = "Use default filter tags convention"
default = "true"
}
variable "filter_tags_custom" {
description = "Tags used for custom filtering when filter_tags_use_defaults is false"
default = "*"
}
# instance specific
variable "cpu_threshold_warning" {
description = "CPU usage in percent (warning threshold)"
default = "80"
}
variable "cpu_threshold_critical" {
description = "CPU usage in percent (critical threshold)"
default = "90"
}
variable "diskspace_threshold_warning" {
description = "Disk free space in percent (warning threshold)"
default = "20"
}
variable "diskspace_threshold_critical" {
description = "Disk free space in percent (critical threshold)"
default = "10"
}

34
system/generic/README.md Normal file
View File

@ -0,0 +1,34 @@
System Generic DataDog monitors
===============================
How to use this module
----------------------
```
module "datadog-monitors-system-generic" {
source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//system/generic?ref={revision}"
message = "${module.datadog-message-alerting.alerting-message}"
environment = "${var.environment}"
}
```
Purpose
-------
Creates a DataDog monitors with the following checks :
* System CPU High
Inputs
------
| Name | Description | Type | Default | Required |
|------|-------------|:----:|:-----:|:-----:|
| custom_cpu_period | Set up period for the query | string | `last_5m` | no |
| custom_cpu_threshold_critical | Custom CPU critical threshold | string | `95` | no |
| custom_cpu_threshold_warning | Custom CPU warning threshold | string | `80` | no |
| environment | Architecture Environment | string | - | yes |
| evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no |
| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no |
| message | Message sent when an alert is triggered | string | - | yes |

1
system/generic/inputs.tf
View File

@ -1 +0,0 @@
../../inputs.tf

42
system/generic/inputs.tf Normal file
View File

@ -0,0 +1,42 @@
# Global Terraform
variable "environment" {
description = "Architecture Environment"
type = "string"
}
# Global DataDog
variable "evaluation_delay" {
description = "Delay in seconds for the metric evaluation"
default = 600
}
variable "message" {
description = "Message sent when an alert is triggered"
}
variable "filter_tags_use_defaults" {
description = "Use default filter tags convention"
default = "true"
}
variable "filter_tags_custom" {
description = "Tags used for custom filtering when filter_tags_use_defaults is false"
default = "*"
}
# Custom CPU instance specific
variable "custom_cpu_period" {
description = "Set up period for the query"
default = "last_5m"
}
variable "custom_cpu_threshold_warning" {
description = "Custom CPU warning threshold"
default = 80
}
variable "custom_cpu_threshold_critical" {
description = "Custom CPU critical threshold"
default = 95
}

35
system/generic/monitors-custom-cpu.tf
View File

@ -1,19 +1,32 @@
resource "datadog_monitor" "cpu_custom" {
name = "${var.dd_custom_cpu["name"]}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
count = "${var.dd_custom_cpu["status"] == "enabled" ? 1 : 0}"
data "template_file" "filter" {
template = "$${filter}"
query = "min(${var.dd_custom_cpu["period"]}):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu.monitoring:enabled} by {host} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host} > ${var.dd_custom_cpu["critical_threshold"]}"
type = "query alert"
vars {
filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_rds:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}"
}
}
resource "datadog_monitor" "cpu_custom" {
name = "[${var.environment}] CPU too High {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
message = "${var.message}"
query = <<EOF
min(${var.custom_cpu_period}): (
avg:system.cpu.system{${data.template_file.filter.rendered}} by {region,host} +
avg:system.cpu.user{${data.template_file.filter.rendered}} by {region,host}
) > ${var.custom_cpu_threshold_critical}"
EOF
type = "metric alert"
thresholds = {
warning = "${var.dd_custom_cpu["warning_threshold"]}"
critical = "${var.dd_custom_cpu["critical_threshold"]}"
warning = "${var.custom_cpu_threshold_warning}"
critical = "${var.custom_cpu_threshold_critical}"
}
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.evaluation_delay}"
renotify_interval = 60
notify_audit = false
timeout_h = 0

43
system/linux/README.md Normal file
View File

@ -0,0 +1,43 @@
System Linux DataDog monitors
=============================
How to use this module
----------------------
```
module "datadog-monitors-system-generic" {
source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//system/linux?ref={revision}"
message = "${module.datadog-message-alerting.alerting-message}"
environment = "${var.environment}"
}
```
Purpose
-------
Creates a DataDog monitors with the following checks :
* System CPU High
* System Free disk space
* System Free disk inodes
* System Free memory
Inputs
------
| Name | Description | Type | Default | Required |
|------|-------------|:----:|:-----:|:-----:|
| cpu_high_threshold_critical | CPU high critical threshold | string | `95` | no |
| cpu_high_threshold_warning | CPU high warning threshold | string | `80` | no |
| custom_cpu_period | Set up period for the query | string | `last_5m` | no |
| environment | Architecture Environment | string | - | yes |
| evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no |
| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no |
| free_disk_inodes_threshold_critical | Free disk space critical threshold | string | `5` | no |
| free_disk_inodes_threshold_warning | Free disk space warning threshold | string | `10` | no |
| free_disk_space_threshold_critical | Free disk space critical threshold | string | `5` | no |
| free_disk_space_threshold_warning | Free disk space warning threshold | string | `10` | no |
| free_memory_threshold_critical | Free disk space critical threshold | string | `5` | no |
| free_memory_threshold_warning | Free disk space warning threshold | string | `10` | no |
| message | Message sent when an alert is triggered | string | - | yes |

1
system/linux/inputs.tf
View File

@ -1 +0,0 @@
../../inputs.tf

72
system/linux/inputs.tf Normal file
View File

@ -0,0 +1,72 @@
# Global Terraform
variable "environment" {
description = "Architecture Environment"
type = "string"
}
# Global DataDog
variable "evaluation_delay" {
description = "Delay in seconds for the metric evaluation"
default = 600
}
variable "message" {
description = "Message sent when an alert is triggered"
}
variable "filter_tags_use_defaults" {
description = "Use default filter tags convention"
default = "true"
}
variable "filter_tags_custom" {
description = "Tags used for custom filtering when filter_tags_use_defaults is false"
default = "*"
}
# Custom CPU instance specific
variable "custom_cpu_period" {
description = "Set up period for the query"
default = "last_5m"
}
variable "cpu_high_threshold_warning" {
description = "CPU high warning threshold"
default = 80
}
variable "cpu_high_threshold_critical" {
description = "CPU high critical threshold"
default = 95
}
variable "free_disk_space_threshold_warning" {
description = "Free disk space warning threshold"
default = 10
}
variable "free_disk_space_threshold_critical" {
description = "Free disk space critical threshold"
default = 5
}
variable "free_disk_inodes_threshold_warning" {
description = "Free disk space warning threshold"
default = 10
}
variable "free_disk_inodes_threshold_critical" {
description = "Free disk space critical threshold"
default = 5
}
variable "free_memory_threshold_warning" {
description = "Free disk space warning threshold"
default = 10
}
variable "free_memory_threshold_critical" {
description = "Free disk space critical threshold"
default = 5
}

204
system/linux/monitors-linux-basics.tf
View File

@ -1,21 +1,34 @@
resource "datadog_monitor" "cpu_80_15min" {
name = "[${var.env}] CPU High > ${var.cpu_15_critical} for 15 min on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
data "template_file" "filter" {
template = "$${filter}"
query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_15_critical}"
type = "query alert"
vars {
filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_rds:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}"
}
}
resource "datadog_monitor" "datadog_cpu_too_high" {
name = "[${var.environment}] CPU High {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
message = "${var.message}"
query = <<EOF
min(last_15m): (
avg:system.cpu.system{${data.template_file.filter.rendered}} by {region,host} +
avg:system.cpu.user{${data.template_file.filter.rendered}} by {region,host}
) > ${var.cpu_high_threshold_critical}
EOF
type = "metric alert"
thresholds {
critical = "${var.cpu_15_critical}"
warning = "${var.cpu_high_threshold_warning}"
critical = "${var.cpu_high_threshold_critical}"
}
tags = ["env:${var.env}", "type:system"]
tags = ["env:${var.environment}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.evaluation_delay}"
notify_audit = false
timeout_h = 0
include_tags = true
@ -24,24 +37,29 @@ resource "datadog_monitor" "cpu_80_15min" {
no_data_timeframe = 20
}
resource "datadog_monitor" "cpu_95_5min" {
name = "[${var.env}] CPU High > ${var.cpu_5_critical} for 5 min on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
resource "datadog_monitor" "datadog_free_disk_space_too_low" {
name = "[${var.environment}] Free disk space {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
message = "${var.message}"
query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region} > ${var.cpu_5_critical}"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
query = <<EOF
sum(last_5m): (
avg:system.disk.free{${data.template_file.filter.rendered}} by {region,host,device} /
avg:system.disk.total{${data.template_file.filter.rendered}} by {region,host,device} * 100
) < ${var.free_disk_space_threshold_critical}
EOF
type = "metric alert"
thresholds {
critical = "${var.cpu_5_critical}"
warning = "${var.free_disk_space_threshold_warning}"
critical = "${var.free_disk_space_threshold_critical}"
}
tags = ["env:${var.env}", "type:system"]
tags = ["env:${var.environment}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.evaluation_delay}"
notify_audit = false
timeout_h = 0
include_tags = true
@ -50,24 +68,29 @@ resource "datadog_monitor" "cpu_95_5min" {
no_data_timeframe = 20
}
resource "datadog_monitor" "datadog_free_disk_space_5" {
name = "[${var.env}] Free disk space < 5% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
resource "datadog_monitor" "datadog_free_disk_space_inodes_too_low" {
name = "[${var.environment}] Free disk inodes {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
message = "${var.message}"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
query = <<EOF
sum(last_5m): (
avg:system.fs.inodes.free{${data.template_file.filter.rendered}} by {region,host,device} /
avg:system.fs.inodes.total{${data.template_file.filter.rendered}} by {region,host,device} * 100
) < ${var.free_disk_inodes_threshold_critical}
EOF
type = "metric alert"
thresholds {
critical = 5
warning = "${var.free_disk_inodes_threshold_warning}"
critical = "${var.free_disk_inodes_threshold_critical}"
}
tags = ["env:${var.env}", "type:system"]
tags = ["env:${var.environment}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.evaluation_delay}"
notify_audit = false
timeout_h = 0
include_tags = true
@ -76,77 +99,29 @@ resource "datadog_monitor" "datadog_free_disk_space_5" {
no_data_timeframe = 20
}
resource "datadog_monitor" "datadog_free_disk_space_10" {
name = "[${var.env}] Free disk space < 10% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
resource "datadog_monitor" "datadog_free_memory" {
name = "[${var.environment}] Free memory {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
message = "Debugging alert - no escalation"
query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 10"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
query = <<EOF
sum(last_1m): (
avg:system.mem.free{${data.template_file.filter.rendered}} by {region,host} /
avg:system.mem.total{${data.template_file.filter.rendered}} by {region,host} * 100
) < ${var.free_memory_threshold_critical}
EOF
type = "metric alert"
thresholds {
warning = 20
critical = 10
warning = "${var.free_memory_threshold_warning}"
critical = "${var.free_memory_threshold_critical}"
}
tags = ["env:${var.env}", "type:system"]
tags = ["env:${var.environment}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = true
no_data_timeframe = 20
}
resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
name = "[${var.env}] Free disk inodes < 5% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,region,device} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = 5
}
tags = ["env:${var.env}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = true
no_data_timeframe = 20
}
resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
name = "[${var.env}] Free disk inodes < 10% on {{host.name}}"
message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_cpu:enabled} by {host,device,region} * 100 < 10"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
warning = 20
critical = 10
}
tags = ["env:${var.env}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.evaluation_delay}"
renotify_interval = 60
notify_audit = false
timeout_h = 0
@ -160,10 +135,12 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
# name = "CPU Load > 2"
# message = "Debugging alert - no escalation"
# query = "min(last_5m):avg:system.load.5{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {instance-id} / avg:gcp.gce.instance.cpu.reserved_cores{*} by {instance-id} > 2"
# type = "query alert"
# count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
# notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
# evaluation_delay = "${var.linux_basics_config["delay"]}"
# new_host_delay = "${var.linux_basics_config["delay"]}"
@ -176,31 +153,6 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
# no_data_timeframe = 20
# }
resource "datadog_monitor" "datadog_free_memory" {
name = "[${var.env}] Free memory < 5% on {{host.name}}"
message = "Debugging alert - no escalation"
query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} / avg:system.mem.total{dd_monitoring:enabled,dd_linux_basics:enabled,env:${var.env},!dd_custom_memory:enabled} by {host,region} * 100 < 5"
type = "query alert"
count = "${var.dd_linux_basics == "enabled" ? 1 : 0}"
thresholds {
critical = 5
}
tags = ["env:${var.env}", "type:system"]
notify_no_data = "${var.linux_basics_config["notify_no_data"]}"
evaluation_delay = "${var.linux_basics_config["delay"]}"
new_host_delay = "${var.linux_basics_config["delay"]}"
renotify_interval = 60
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = true
no_data_timeframe = 20
}
# resource "datadog_monitor" "datadog_host_unreachable" {
# name = "Host unreachable"