reworking datadog directory

.gitignore

@@ -0,0 +1 @@
+testing/*

datadog-samples/inputs-declaration.sample

@@ -0,0 +1,19 @@
+variable "ho_escalation_group" {
+  default = "@pagerduty_HODummy"
+}
+
+variable "hno_escalation_group" {
+  default = "@pagerduty_HNODummy"
+}
+
+variable "linux_basics" {
+  default = "enabled"
+}
+
+variable "datadog_api_key" {
+  default = "4878503d31a00ad7af4486581d0f98e8"
+}
+
+variable "datadog_app_key" {
+  default = "e1a10feedbb095e7d28ef966c4a2c00d50f53272"
+}

datadog-samples/modules-declaration.sample

@@ -0,0 +1,19 @@
+module "datadog-monitors" {
+  source = "../monitors"
+
+  ho_escalation_group  = "${var.ho_escalation_group}"
+  hno_escalation_group = "${var.hno_escalation_group}"
+
+  #default monitors templates integrations examples
+  linux_basics  = "${var.linux_basics}"
+  #nginx         = "false"
+  #aws_rds_mysql = "false"
+
+  #custom monitors template integrations examples
+  #cpu_custom                        = "true"  #Will match the tag dd_cpu_high (NB: it will replace the basic cpu alert)
+  #cpu_custom_period                 = "60"    
+  #cpu_custom_warning_threshold      = "90"
+  #cpu_custom_warning_escalation_msg = ${var.ho_escalation_group}
+  #cpu_custom_alert_threshold        = "95"
+  #cpu_custom_alert_escalation_msg   = ${var.hno_escalation_group}
+}

inputs.tf

@@ -1,9 +1,11 @@
-variable "warning_HO" {
+variable "dd_custom_ignores" {
-  type = "string"
+  default = ""
 }
 
-variable "alert_HNO" {
+variable "ho_escalation_group" {}
-  type = "string"
-}
 
-variable "project_id" {}
+variable "hno_escalation_group" {}
+
+variable "linux_basics" {
+  default = "disabled"
+}

module-datadog.sample

@@ -1,19 +0,0 @@
-module "datadog-monitors" {
-  source = "git::ssh://git@bitbucket.org/morea/terraform.datadog.monitors.git"
-
-  ho_escalation_group  = "@pagerduty-Public_Cloud_FR_-CustomerName_HO"
-  hno_escalation_group = "@pagerduty-Public_Cloud_FR_-CustomerName_HNO"
-
-  #default monitors templates integrations examples
-  linux_basics  = "true"
-  nginx         = "true"
-  aws_rds_mysql = "true"
-
-  #custom monitors template integrations examples
-  cpu_custom                        = "true"  #Will match the tag dd_cpu_high (NB: it will replace the basic cpu alert)
-  cpu_custom_period                 = "60"    
-  cpu_custom_warning_threshold      = "90"
-  cpu_custom_warning_escalation_msg = ${var.ho_escalation_group}
-  cpu_custom_alert_threshold        = "95"
-  cpu_custom_alert_escalation_msg   = ${var.hno_escalation_group}
-}

modules.tf

@@ -1 +0,0 @@
-#TODO

monitors-linux-basics.tf

@@ -0,0 +1,198 @@
+resource "datadog_monitor" "cpu_80_15min" {
+  name    = "CPU High > 80% for 15 min"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
+  count   = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host} > 80"
+  type  = "query alert"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "cpu_95_5min" {
+  name    = "CPU High > 95% for 5 min"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
+
+  query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host} + avg:system.cpu.user{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host} > 95"
+  type  = "query alert"
+  count   = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_free_disk_space_5" {
+  name   = "Free disk space < 5%"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
+
+  query   = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} * 100 < 5"
+  type  = "query alert"
+  count   = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_free_disk_space_10" {
+  name    = "Free disk space < 10%"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
+
+  query = "sum(last_5m):avg:system.disk.free{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} / avg:system.disk.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} * 100 < 10"
+  type  = "query alert"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  thresholds {
+    warning  = 20
+    critical = 10
+  }
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
+  name    = "Free disk inodes < 5%"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
+
+  query = "sum(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} * 100 < 5"
+  type  = "query alert"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
+  name    = "Free disk inodes < 10%"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
+
+  query = "max(last_5m):avg:system.fs.inodes.free{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} / avg:system.fs.inodes.total{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {host,device} * 100 < 10"
+  type  = "query alert"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  thresholds {
+    warning  = 20
+    critical = 10
+  }
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_cpu_load" {
+  name    = "CPU Load > 2"
+  message = "Debugging alert - no escalation"
+
+  query = "min(last_5m):avg:system.load.5{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled} by {instance-id} / avg:gcp.gce.instance.cpu.reserved_cores{*} by {instance-id} > 2"
+  type  = "query alert"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_free_memory" {
+  name   = "Free memory < 5%"
+  message = "Debugging alert - no escalation"
+
+  query = "sum(last_1m):avg:system.mem.free{dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_memory:enabled} by {host} / avg:system.mem.total{*} by {host} * 100 < 5"
+  type  = "query alert"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = false
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}
+
+resource "datadog_monitor" "datadog_host_unreachable" {
+  name    = "Host unreachable"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
+
+  query = "\"datadog.agent.up\".over(\"dd_monitoring:enabled,dd_linux_basics:enabled,!dd_custom_cpu:enabled\").last(1).count_by_status()"
+  type  = "service check"
+  count = "${var.linux_basics == "enabled" ? 1 : 0}"
+
+  notify_no_data      = false
+  renotify_interval   = 60
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+  new_host_delay      = 300
+  notify_no_data      = true
+  renotify_interval   = 0
+  no_data_timeframe   = 20
+}

monitors-unready/monitors-linux-basics.tf

@@ -1,8 +1,9 @@
 resource "datadog_monitor" "cpu_80_15min" {
   name    = "CPU High > 80% for 15 min"
-  message = "{{#is_alert}}\n${var.alert_HNO}\n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO}\n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
+  count   = "${var.linux-basics == "enabled" ? 1 : 0}"
 
-  query = "min(last_15m):avg:system.cpu.system{!goog-gke-node} by {host} + avg:system.cpu.user{!goog-gke-node} by {host} > 80"
+  query = "min(last_15m):avg:system.cpu.system{dd_monitoring:enabled,!dd_custom_cpu:enabled} by {host} + avg:system.cpu.user{dd_monitoring:enabled,!dd_custom_cpu:enabled} by {host} > 80"
   type  = "query alert"
 
   notify_no_data      = false
@@ -20,10 +21,11 @@ resource "datadog_monitor" "cpu_80_15min" {
 
 resource "datadog_monitor" "cpu_95_5min" {
   name    = "CPU High > 95% for 5 min"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO}\n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
 
-  query = "min(last_5m):avg:system.cpu.system{!goog-gke-node} by {host} + avg:system.cpu.user{!goog-gke-node} by {host} > 95"
+  query = "min(last_5m):avg:system.cpu.system{dd_monitoring:enabled,!dd_custom_cpu:enabled} by {host} + avg:system.cpu.user{dd_monitoring:enabled,!dd_custom_cpu:enabled} by {host} > 95"
   type  = "query alert"
+  count   = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -40,9 +42,11 @@ resource "datadog_monitor" "cpu_95_5min" {
 
 resource "datadog_monitor" "datadog_free_disk_space_5" {
   name   = "Free disk space < 5%"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO}\n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}"
+
   query   = "sum(last_5m):avg:system.disk.free{*} by {host,device} / avg:system.disk.total{*} by {host,device} * 100 < 5"
   type  = "query alert"
+  count   = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -58,29 +62,12 @@ resource "datadog_monitor" "datadog_free_disk_space_5" {
 }
 
 resource "datadog_monitor" "datadog_free_disk_space_10" {
-  name   = "Free disk space < 10%"
+  name    = "Free disk space < 10%"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO}\n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group}\n{{/is_alert}}\n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group}\n{{/is_warning}}\n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
-  query   = "sum(last_5m):avg:system.disk.free{*} by {host,device} / avg:system.disk.total{*} by {host,device} * 100 < 10"
-  type  = "query alert"
 
-  notify_no_data      = false
+  query = "sum(last_5m):avg:system.disk.free{*} by {host,device} / avg:system.disk.total{*} by {host,device} * 100 < 10"
-  renotify_interval   = 60
-  notify_audit        = false
-  timeout_h           = 0
-  include_tags        = true
-  locked              = false
-  require_full_window = true
-  new_host_delay      = 300
-  notify_no_data      = false
-  renotify_interval   = 0
-  no_data_timeframe   = 20
-}
-
-resource "datadog_monitor" "datadog_free_disk_space_20" {
-  name   = "Free disk space < 20%"
-  message = "${var.warning_HO}"
-  query   = "sum(last_5m):avg:system.disk.free{*} by {host,device} / avg:system.disk.total{*} by {host,device} * 100 < 20"
   type  = "query alert"
+  count = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -96,10 +83,12 @@ resource "datadog_monitor" "datadog_free_disk_space_20" {
 }
 
 resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
-  name   = "Free disk inodes < 5%"
+  name    = "Free disk inodes < 5%"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO} \n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
-  query   = "sum(last_5m):avg:system.fs.inodes.free{*} by {host,device} / avg:system.fs.inodes.total{*} by {host,device} * 100 < 5"
+
+  query = "sum(last_5m):avg:system.fs.inodes.free{*} by {host,device} / avg:system.fs.inodes.total{*} by {host,device} * 100 < 5"
   type  = "query alert"
+  count = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -115,29 +104,12 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_5" {
 }
 
 resource "datadog_monitor" "datadog_free_disk_space_inodes_10" {
-  name   = "Free disk inodes < 10%"
+  name    = "Free disk inodes < 10%"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO} \n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
-  query   = "max(last_5m):avg:system.fs.inodes.free{*} by {host,device} / avg:system.fs.inodes.total{*} by {host,device} * 100 < 10"
-  type  = "query alert"
 
-  notify_no_data      = false
+  query = "max(last_5m):avg:system.fs.inodes.free{*} by {host,device} / avg:system.fs.inodes.total{*} by {host,device} * 100 < 10"
-  renotify_interval   = 60
-  notify_audit        = false
-  timeout_h           = 0
-  include_tags        = true
-  locked              = false
-  require_full_window = true
-  new_host_delay      = 300
-  notify_no_data      = false
-  renotify_interval   = 0
-  no_data_timeframe   = 20
-}
-
-resource "datadog_monitor" "datadog_free_disk_space_inodes_20" {
-  name   = "Free disk inodes < 20%"
-  message = "{{#is_alert}}\n${var.warning_HO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.warning_HO} \n{{/is_recovery}}"
-  query   = "max(last_5m):avg:system.fs.inodes.free{*} by {host,device} / avg:system.fs.inodes.total{*} by {host,device} * 100 < 20"
   type  = "query alert"
+  count = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -153,10 +125,12 @@ resource "datadog_monitor" "datadog_free_disk_space_inodes_20" {
 }
 
 resource "datadog_monitor" "datadog_cpu_load" {
-  name   = "CPU Load > 2"
+  name    = "CPU Load > 2"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO} \n{{/is_recovery}}"
+  message = "Debugging alert - no escalation"
-  query   = "min(last_5m):avg:system.load.5{*} by {instance-id} / avg:gcp.gce.instance.cpu.reserved_cores{*} by {instance-id} > 2"
+
+  query = "min(last_5m):avg:system.load.5{*} by {instance-id} / avg:gcp.gce.instance.cpu.reserved_cores{*} by {instance-id} > 2"
   type  = "query alert"
+  count = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60
@@ -173,7 +147,7 @@ resource "datadog_monitor" "datadog_cpu_load" {
 
 resource "datadog_monitor" "datadog_free_memory" {
   name   = "Free memory < 5%"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO} \n{{/is_recovery}}"
+  message = "Debugging alert - no escalation"
   query   = "sum(last_1m):avg:system.mem.free{*} by {host} / avg:system.mem.total{*} by {host} * 100 < 5"
   type  = "query alert"
 
@@ -191,10 +165,12 @@ resource "datadog_monitor" "datadog_free_memory" {
 }
 
 resource "datadog_monitor" "datadog_host_unreachable" {
-  name   = "Host unreachable"
+  name    = "Host unreachable"
-  message = "{{#is_alert}}\n${var.alert_HNO} \n{{/is_alert}} \n{{#is_recovery}}\n${var.alert_HNO} \n{{/is_recovery}}"
+  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group} \n{{/is_recovery}}"
-  query   = "\"datadog.agent.up\".over(\"*\").last(1).count_by_status()"
+
+  query = "\"datadog.agent.up\".over(\"*\").last(1).count_by_status()"
   type  = "service check"
+  count = "${var.linux-basics == "enabled" ? 1 : 0}"
 
   notify_no_data      = false
   renotify_interval   = 60

outputs.tf

@@ -1,5 +0,0 @@
-output service_accounts_emails {
-  value = {
-    monitoring_datadog = "${google_service_account.monitoring_datadog.email}"
-  }
-}

resources.sample

@@ -1,4 +0,0 @@
-resource "google_service_account" "monitoring_datadog" {
-  account_id   = "monitoring-datadog"
-  display_name = "monitoring datadog"
-}