From febc38eec7e284dedc92f6629612b5e0da210a49 Mon Sep 17 00:00:00 2001
From: Alexandre Gaillet <alexandre.gaillet@fr.clara.net>
Date: Wed, 21 Feb 2018 16:55:06 +0100
Subject: [PATCH] MON-96 - Apply conventions and update middleware directory

---
 .../elasticsearch/monitors-elasticsearch.tf   |  7 ++-
 middleware/apache/README.md                   | 30 ++++++++++
 middleware/apache/inputs.tf                   | 22 +++++++-
 middleware/apache/monitors-apache.tf          | 15 +++--
 middleware/nginx/README.md                    | 30 ++++++++++
 middleware/nginx/inputs.tf                    | 22 +++++++-
 middleware/nginx/monitors-nginx.tf            | 15 +++--
 middleware/php-fpm/README.md                  | 35 ++++++++++++
 middleware/php-fpm/inputs.tf                  | 43 ++++++++++++++-
 middleware/php-fpm/monitors-fpm.tf            | 55 +++++++++++--------
 10 files changed, 230 insertions(+), 44 deletions(-)
 create mode 100644 middleware/apache/README.md
 mode change 120000 => 100644 middleware/apache/inputs.tf
 create mode 100644 middleware/nginx/README.md
 mode change 120000 => 100644 middleware/nginx/inputs.tf
 create mode 100644 middleware/php-fpm/README.md
 mode change 120000 => 100644 middleware/php-fpm/inputs.tf

diff --git a/cloud/aws/elasticsearch/monitors-elasticsearch.tf b/cloud/aws/elasticsearch/monitors-elasticsearch.tf
index 7ba3aa2..8880a02 100644
--- a/cloud/aws/elasticsearch/monitors-elasticsearch.tf
+++ b/cloud/aws/elasticsearch/monitors-elasticsearch.tf
@@ -42,14 +42,15 @@ EOF
 
 ### Elasticsearch cluster free storage space monitor ###
 resource "datadog_monitor" "es_free_space_low" {
-  name    = "[${var.environment}] ElasticSearch cluster free storage space < ${var.diskspace_threshold_critical}%"
+  name    = "[${var.environment}] ElasticSearch cluster free storage space {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
   message = "${var.message}"
 
   type = "query alert"
 
   query = <<EOF
   avg(last_15m): (
-    avg:aws.es.free_storage_space{${data.template_file.filter.rendered}} by {region,name} / (${var.es_cluster_volume_size}*1000) * 100
+    avg:aws.es.free_storage_space{${data.template_file.filter.rendered}} by {region,name} /
+    (${var.es_cluster_volume_size}*1000) * 100
   ) < ${var.diskspace_threshold_critical}
 EOF
 
@@ -74,7 +75,7 @@ EOF
 
 ### Elasticsearch cluster CPU monitor ###
 resource "datadog_monitor" "es_cpu_90_15min" {
-  name    = "[${var.environment}] ElasticSearch cluster CPU high > ${var.cpu_threshold_critical}%"
+  name    = "[${var.environment}] ElasticSearch cluster CPU high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
   message = "${var.message}"
 
   type = "query alert"
diff --git a/middleware/apache/README.md b/middleware/apache/README.md
new file mode 100644
index 0000000..4d9c00f
--- /dev/null
+++ b/middleware/apache/README.md
@@ -0,0 +1,30 @@
+Apache Middleware DataDog monitors
+==================================
+
+How to use this module
+----------------------
+
+```
+module "datadog-monitors-apache-middleware" {
+  source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//middleware/apache?ref={revision}"
+
+  message     = "${module.datadog-message-alerting.alerting-message}"
+  environment = "${var.environment}"
+}
+```
+
+Purpose
+-------
+Creates a DataDog monitors with the following checks :
+
+* Apache process
+
+Inputs
+------
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| dd_apache |  | string | `disabled` | no |
+| environment | Architecture Environment | string | - | yes |
+| evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no |
+| message | Message sent when an alert is triggered | string | - | yes |
diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf
deleted file mode 120000
index cdfc6c6..0000000
--- a/middleware/apache/inputs.tf
+++ /dev/null
@@ -1 +0,0 @@
-../../inputs.tf
\ No newline at end of file
diff --git a/middleware/apache/inputs.tf b/middleware/apache/inputs.tf
new file mode 100644
index 0000000..0c3b10e
--- /dev/null
+++ b/middleware/apache/inputs.tf
@@ -0,0 +1,21 @@
+# Global Terraform
+variable "environment" {
+  description = "Architecture Environment"
+  type        = "string"
+}
+
+# Global DataDog
+variable "evaluation_delay" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 15
+}
+
+variable "message" {
+  description = "Message sent when an alert is triggered"
+}
+
+# Apache Middleware specific
+
+variable "dd_apache" {
+  default = "disabled"
+}
diff --git a/middleware/apache/monitors-apache.tf b/middleware/apache/monitors-apache.tf
index b542aa4..11f4624 100644
--- a/middleware/apache/monitors-apache.tf
+++ b/middleware/apache/monitors-apache.tf
@@ -1,10 +1,9 @@
 resource "datadog_monitor" "Apache_process" {
-  name    = "[${var.env}] Apache process is down on {{host.name}}"
-  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
+  name    = "[${var.environment}] Apache process is down on {{host.name}}"
+  message = "${var.message}"
 
   type  = "service check"
-  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"process:apache\",\"env:${var.env}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
-  count = "${var.dd_apache == "enabled" ? 1 : 0 }"
+  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_apache:enabled\",\"process:apache\",\"env:${var.environment}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
 
   thresholds = {
     ok       = 1
@@ -12,9 +11,9 @@ resource "datadog_monitor" "Apache_process" {
     critical = 4
   }
 
-  notify_no_data      = "${var.apache_nginx_fpm_config["notify_no_data"]}"
-  evaluation_delay    = "${var.apache_nginx_fpm_config["delay"]}"
-  new_host_delay      = "${var.apache_nginx_fpm_config["delay"]}"
+  notify_no_data      = false
+  evaluation_delay    = "${var.evaluation_delay}"
+  new_host_delay      = "${var.evaluation_delay}"
   renotify_interval   = 60
   notify_audit        = false
   timeout_h           = 0
@@ -23,5 +22,5 @@ resource "datadog_monitor" "Apache_process" {
   require_full_window = true
   no_data_timeframe   = 20
 
-  tags = ["env:${var.env}", "type:apache"]
+  tags = ["env:${var.environment}", "type:apache"]
 }
diff --git a/middleware/nginx/README.md b/middleware/nginx/README.md
new file mode 100644
index 0000000..435dde6
--- /dev/null
+++ b/middleware/nginx/README.md
@@ -0,0 +1,30 @@
+Nginx Middleware DataDog monitors
+=================================
+
+How to use this module
+----------------------
+
+```
+module "datadog-monitors-nginx-middleware" {
+  source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//middleware/nginx?ref={revision}"
+
+  message     = "${module.datadog-message-alerting.alerting-message}"
+  environment = "${var.environment}"
+}
+```
+
+Purpose
+-------
+Creates a DataDog monitors with the following checks :
+
+* Nginx process
+
+Inputs
+------
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| dd_nginx |  | string | `disabled` | no |
+| environment | Architecture Environment | string | - | yes |
+| evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no |
+| message | Message sent when an alert is triggered | string | - | yes |
diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf
deleted file mode 120000
index cdfc6c6..0000000
--- a/middleware/nginx/inputs.tf
+++ /dev/null
@@ -1 +0,0 @@
-../../inputs.tf
\ No newline at end of file
diff --git a/middleware/nginx/inputs.tf b/middleware/nginx/inputs.tf
new file mode 100644
index 0000000..f823913
--- /dev/null
+++ b/middleware/nginx/inputs.tf
@@ -0,0 +1,21 @@
+# Global Terraform
+variable "environment" {
+  description = "Architecture Environment"
+  type        = "string"
+}
+
+# Global DataDog
+variable "evaluation_delay" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 15
+}
+
+variable "message" {
+  description = "Message sent when an alert is triggered"
+}
+
+# Nginx Middleware specific
+
+variable "dd_nginx" {
+  default = "disabled"
+}
diff --git a/middleware/nginx/monitors-nginx.tf b/middleware/nginx/monitors-nginx.tf
index 03595bf..b018ec3 100644
--- a/middleware/nginx/monitors-nginx.tf
+++ b/middleware/nginx/monitors-nginx.tf
@@ -1,10 +1,9 @@
 resource "datadog_monitor" "Nginx_process" {
-  name    = "[${var.env}] Nginx process is down on {{host.name}}"
-  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
+  name    = "[${var.environment}] Nginx process is down on {{host.name}}"
+  message = "${var.message}"
 
   type  = "service check"
-  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"process:nginx\",\"env:${var.env}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
-  count = "${var.dd_nginx == "enabled" ? 1 : 0 }"
+  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_nginx:enabled\",\"process:nginx\",\"env:${var.environment}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
 
   thresholds = {
     ok       = 1
@@ -12,9 +11,9 @@ resource "datadog_monitor" "Nginx_process" {
     critical = 4
   }
 
-  notify_no_data      = "${var.apache_nginx_fpm_config["notify_no_data"]}"
-  evaluation_delay    = "${var.apache_nginx_fpm_config["delay"]}"
-  new_host_delay      = "${var.apache_nginx_fpm_config["delay"]}"
+  notify_no_data      = false
+  evaluation_delay    = "${var.evaluation_delay}"
+  new_host_delay      = "${var.evaluation_delay}"
   renotify_interval   = 60
   notify_audit        = false
   timeout_h           = 0
@@ -23,5 +22,5 @@ resource "datadog_monitor" "Nginx_process" {
   require_full_window = true
   no_data_timeframe   = 20
 
-  tags = ["env:${var.env}", "type:nginx"]
+  tags = ["env:${var.environment}", "type:nginx"]
 }
diff --git a/middleware/php-fpm/README.md b/middleware/php-fpm/README.md
new file mode 100644
index 0000000..f15a56e
--- /dev/null
+++ b/middleware/php-fpm/README.md
@@ -0,0 +1,35 @@
+PHP FPM Middleware DataDog monitors
+===================================
+
+How to use this module
+----------------------
+
+```
+module "datadog-monitors-php-fpm-middleware" {
+  source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//middleware/php-fpm?ref={revision}"
+
+  message     = "${module.datadog-message-alerting.alerting-message}"
+  environment = "${var.environment}"
+}
+```
+
+Purpose
+-------
+Creates a DataDog monitors with the following checks :
+
+* PHP FPM process
+* PHP FPM load
+
+Inputs
+------
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| environment | Architecture Environment | string | - | yes |
+| evaluation_delay_metric | Delay in seconds for the metric evaluation | string | `600` | no |
+| evaluation_delay_service | Delay in seconds for the metric evaluation | string | `15` | no |
+| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
+| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no |
+| message | Message sent when an alert is triggered | string | - | yes |
+| php_fpm_busy_threshold_critical | php fpm busy critical threshold | string | `0.9` | no |
+| php_fpm_busy_threshold_warning | php fpm busy warning threshold | string | `0.8` | no |
diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf
deleted file mode 120000
index cdfc6c6..0000000
--- a/middleware/php-fpm/inputs.tf
+++ /dev/null
@@ -1 +0,0 @@
-../../inputs.tf
\ No newline at end of file
diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf
new file mode 100644
index 0000000..726bd04
--- /dev/null
+++ b/middleware/php-fpm/inputs.tf
@@ -0,0 +1,42 @@
+# Global Terraform
+variable "environment" {
+  description = "Architecture Environment"
+  type        = "string"
+}
+
+# Global DataDog
+variable "evaluation_delay_service" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 15
+}
+
+variable "evaluation_delay_metric" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 600
+}
+
+variable "message" {
+  description = "Message sent when an alert is triggered"
+}
+
+variable "filter_tags_use_defaults" {
+  description = "Use default filter tags convention"
+  default     = "true"
+}
+
+variable "filter_tags_custom" {
+  description = "Tags used for custom filtering when filter_tags_use_defaults is false"
+  default     = "*"
+}
+
+# PHP FPM Middleware specific
+
+variable "php_fpm_busy_threshold_warning" {
+  description = "php fpm busy warning threshold"
+  default     = 0.8
+}
+
+variable "php_fpm_busy_threshold_critical" {
+  description = "php fpm busy critical threshold"
+  default     = 0.9
+}
diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf
index 8bf654e..95289e8 100644
--- a/middleware/php-fpm/monitors-fpm.tf
+++ b/middleware/php-fpm/monitors-fpm.tf
@@ -1,38 +1,49 @@
-resource "datadog_monitor" "php-fpm_process_idle" {
-  name    = "[${var.env}] php_fpm busy worker > 90% on {{host.name}}"
-  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
+data "template_file" "filter" {
+  template = "$${filter}"
 
-  type  = "query alert"
-  query = "avg(last_10m):avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} / ( avg:php_fpm.processes.idle{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} + avg:php_fpm.processes.active{dd_monitoring:enabled,dd_php_fpm:enabled,env:${var.env}} by {host,region} ) > 0.9"
-  count = "${var.dd_php_fpm == "enabled" ? 1 : 0 }"
+  vars {
+    filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_rds:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}"
+  }
+}
+
+resource "datadog_monitor" "php-fpm_process_idle" {
+  name    = "[${var.environment}] php_fpm busy worker {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
+  message = "${var.message}"
+
+  type = "metric alert"
+
+  query = <<EOF
+    avg(last_10m): (
+      avg:php_fpm.processes.active{${data.template_file.filter.rendered}} by {region, host} /
+      ( avg:php_fpm.processes.idle{${data.template_file.filter.rendered}} by {region, host} +
+       avg:php_fpm.processes.active{${data.template_file.filter.rendered}} by {region, host} )
+    ) > ${var.php_fpm_busy_threshold_critical}
+  EOF
 
   thresholds {
-    warning  = "${var.php_fpm_busy_threshold["warning"]}"
-    critical = "${var.php_fpm_busy_threshold["critical"]}"
+    warning  = "${var.php_fpm_busy_threshold_warning}"
+    critical = "${var.php_fpm_busy_threshold_critical}"
   }
 
-  notify_no_data      = "${var.apache_nginx_fpm_config["notify_no_data"]}"
-  evaluation_delay    = "${var.apache_nginx_fpm_config["delay"]}"
-  new_host_delay      = "${var.apache_nginx_fpm_config["delay"]}"
-  renotify_interval   = 60
+  notify_no_data      = false
+  evaluation_delay    = "${var.evaluation_delay_metric}"
+  new_host_delay      = "${var.evaluation_delay_metric}"
   notify_audit        = false
   timeout_h           = 0
   include_tags        = true
   locked              = false
   require_full_window = true
-  renotify_interval   = 0
   no_data_timeframe   = 20
 
-  tags = ["env:${var.env}", "type:php-fpm"]
+  tags = ["env:${var.environment}", "type:php-fpm"]
 }
 
 resource "datadog_monitor" "FPM_process" {
-  name    = "[${var.env}] FPM process is down on {{host.name}}"
-  message = "{{#is_alert}}\n${var.hno_escalation_group} \n{{/is_alert}} \n{{#is_recovery}}\n${var.hno_escalation_group}\n{{/is_recovery}}\n{{#is_warning}}\n${var.ho_escalation_group} \n{{/is_warning}} \n{{#is_warning_recovery}}\n${var.ho_escalation_group}\n{{/is_warning_recovery}}"
+  name    = "[${var.environment}] FPM process is down on {{host.name}}"
+  message = "${var.message}"
 
   type  = "service check"
-  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"process:php_fpm\",\"env:${var.env}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
-  count = "${var.dd_nginx == "enabled" ? 1 : 0 }"
+  query = "\"process.up\".over(\"dd_monitoring:enabled\",\"dd_php_fpm:enabled\",\"process:php_fpm\",\"env:${var.environment}\").by(\"host\",\"process\", \"app\").last(4).count_by_status()"
 
   thresholds = {
     ok       = 1
@@ -40,9 +51,9 @@ resource "datadog_monitor" "FPM_process" {
     critical = 4
   }
 
-  notify_no_data      = "${var.apache_nginx_fpm_config["notify_no_data"]}"
-  evaluation_delay    = "${var.apache_nginx_fpm_config["delay"]}"
-  new_host_delay      = "${var.apache_nginx_fpm_config["delay"]}"
+  notify_no_data      = true
+  evaluation_delay    = "${var.evaluation_delay_service}"
+  new_host_delay      = "${var.evaluation_delay_service}"
   renotify_interval   = 60
   notify_audit        = false
   timeout_h           = 0
@@ -51,5 +62,5 @@ resource "datadog_monitor" "FPM_process" {
   require_full_window = true
   no_data_timeframe   = 20
 
-  tags = ["env:${var.env}", "type:php-fpm"]
+  tags = ["env:${var.environment}", "type:php-fpm"]
 }