From c3ef50e1b900acde209559be2fafe8acca3d51e7 Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Tue, 11 Sep 2018 13:25:28 +0200
Subject: [PATCH] MON-290 make the status monitor more tolerant

---
 middleware/php-fpm/README.md       | 10 +++--
 middleware/php-fpm/inputs.tf       | 20 ++++++++--
 middleware/php-fpm/monitors-fpm.tf | 62 +++++++++++++++---------------
 middleware/php-fpm/outputs.tf      | 10 ++---
 4 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/middleware/php-fpm/README.md b/middleware/php-fpm/README.md
index fbf269b..4b4ab24 100644
--- a/middleware/php-fpm/README.md
+++ b/middleware/php-fpm/README.md
@@ -37,10 +37,12 @@ Creates DataDog monitors with the following checks:
 | php_fpm_busy_threshold_warning | php fpm busy warning threshold | string | `80` | no |
 | php_fpm_busy_time_aggregator | Monitor aggregator for PHP FPM busy worker [available values: min, max or avg] | string | `avg` | no |
 | php_fpm_busy_timeframe | Monitor timeframe for PHP FPM busy worker [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_10m` | no |
-| php_fpm_connect_enabled | Flag to enable PHP FPM process monitor | string | `true` | no |
-| php_fpm_connect_extra_tags | Extra tags for PHP FPM process monitor | list | `<list>` | no |
-| php_fpm_connect_message | Custom message for PHP FPM process monitor | string | `` | no |
-| php_fpm_connect_silenced | Groups to mute for PHP FPM process monitor | map | `<map>` | no |
+| php_fpm_connect_enabled | Flag to enable PHP FPM status monitor | string | `true` | no |
+| php_fpm_connect_extra_tags | Extra tags for PHP FPM status monitor | list | `<list>` | no |
+| php_fpm_connect_message | Custom message for PHP FPM status monitor | string | `` | no |
+| php_fpm_connect_no_data_timeframe | PHP FPM status monitor no data timeframe | string | `10` | no |
+| php_fpm_connect_silenced | Groups to mute for PHP FPM status monitor | map | `<map>` | no |
+| php_fpm_connect_threshold_warning | PHP FPM status monitor (warning threshold) | string | `3` | no |
 
 ## Outputs
 
diff --git a/middleware/php-fpm/inputs.tf b/middleware/php-fpm/inputs.tf
index 9fbeb84..96c6e5e 100644
--- a/middleware/php-fpm/inputs.tf
+++ b/middleware/php-fpm/inputs.tf
@@ -78,25 +78,37 @@ variable "php_fpm_busy_threshold_critical" {
 }
 
 variable "php_fpm_connect_silenced" {
-  description = "Groups to mute for PHP FPM process monitor"
+  description = "Groups to mute for PHP FPM status monitor"
   type        = "map"
   default     = {}
 }
 
 variable "php_fpm_connect_enabled" {
-  description = "Flag to enable PHP FPM process monitor"
+  description = "Flag to enable PHP FPM status monitor"
   type        = "string"
   default     = "true"
 }
 
 variable "php_fpm_connect_extra_tags" {
-  description = "Extra tags for PHP FPM process monitor"
+  description = "Extra tags for PHP FPM status monitor"
   type        = "list"
   default     = []
 }
 
 variable "php_fpm_connect_message" {
-  description = "Custom message for PHP FPM process monitor"
+  description = "Custom message for PHP FPM status monitor"
   type        = "string"
   default     = ""
 }
+
+variable "php_fpm_connect_threshold_warning" {
+  description = "PHP FPM status monitor (warning threshold)"
+  type        = "string"
+  default     = 3
+}
+
+variable "php_fpm_connect_no_data_timeframe" {
+  description = "PHP FPM status monitor no data timeframe"
+  type        = "string"
+  default     = 10
+}
diff --git a/middleware/php-fpm/monitors-fpm.tf b/middleware/php-fpm/monitors-fpm.tf
index cb74a08..15ce0d1 100644
--- a/middleware/php-fpm/monitors-fpm.tf
+++ b/middleware/php-fpm/monitors-fpm.tf
@@ -1,3 +1,34 @@
+resource "datadog_monitor" "datadog_fpm_process" {
+  count   = "${var.php_fpm_connect_enabled ? 1 : 0}"
+  name    = "[${var.environment}] Php-fpm ping url does not respond"
+  message = "${coalesce(var.php_fpm_connect_message, var.message)}"
+
+  type = "service check"
+
+  query = <<EOF
+    "php_fpm.can_ping".over${module.filter-tags.service_check}.by("ping_url").last(6).count_by_status()
+  EOF
+
+  thresholds = {
+    warning  = "${var.php_fpm_connect_threshold_warning}"
+    critical = 5
+  }
+
+  notify_no_data      = true
+  no_data_timeframe   = "${var.php_fpm_connect_no_data_timeframe}"
+  new_host_delay      = "${var.new_host_delay}"
+  renotify_interval   = 0
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+
+  silenced = "${var.php_fpm_connect_silenced}"
+
+  tags = ["env:${var.environment}", "type:middleware", "provider:php-fpm", "resource:php-fpm", "team:claranet", "created-by:terraform", "${var.php_fpm_connect_extra_tags}"]
+}
+
 resource "datadog_monitor" "datadog_php_fpm_connect_idle" {
   count   = "${var.php_fpm_busy_enabled ? 1 : 0}"
   name    = "[${var.environment}] Php-fpm busy worker {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
@@ -31,34 +62,3 @@ resource "datadog_monitor" "datadog_php_fpm_connect_idle" {
 
   tags = ["env:${var.environment}", "type:middleware", "provider:php-fpm", "resource:php-fpm", "team:claranet", "created-by:terraform", "${var.php_fpm_busy_extra_tags}"]
 }
-
-resource "datadog_monitor" "datadog_fpm_process" {
-  count   = "${var.php_fpm_connect_enabled ? 1 : 0}"
-  name    = "[${var.environment}] Php-fpm ping url does not respond"
-  message = "${coalesce(var.php_fpm_connect_message, var.message)}"
-
-  type = "service check"
-
-  query = <<EOF
-    "php_fpm.can_ping".over${module.filter-tags.service_check}.by("host","ping_url").last(6).count_by_status()
-  EOF
-
-  thresholds = {
-    ok       = 1
-    warning  = 2
-    critical = 5
-  }
-
-  notify_no_data      = true
-  new_host_delay      = "${var.new_host_delay}"
-  renotify_interval   = 0
-  notify_audit        = false
-  timeout_h           = 0
-  include_tags        = true
-  locked              = false
-  require_full_window = true
-
-  silenced = "${var.php_fpm_connect_silenced}"
-
-  tags = ["env:${var.environment}", "type:middleware", "provider:php-fpm", "resource:php-fpm", "team:claranet", "created-by:terraform", "${var.php_fpm_connect_extra_tags}"]
-}
diff --git a/middleware/php-fpm/outputs.tf b/middleware/php-fpm/outputs.tf
index 53cdd1a..f70420c 100644
--- a/middleware/php-fpm/outputs.tf
+++ b/middleware/php-fpm/outputs.tf
@@ -1,9 +1,9 @@
-output "datadog_php_fpm_connect_idle_id" {
-  description = "id for monitor datadog_php_fpm_connect_idle"
-  value       = "${datadog_monitor.datadog_php_fpm_connect_idle.*.id}"
-}
-
 output "datadog_fpm_process_id" {
   description = "id for monitor datadog_fpm_process"
   value       = "${datadog_monitor.datadog_fpm_process.*.id}"
 }
+
+output "datadog_php_fpm_connect_idle_id" {
+  description = "id for monitor datadog_php_fpm_connect_idle"
+  value       = "${datadog_monitor.datadog_php_fpm_connect_idle.*.id}"
+}