From f0e252df33b469b1a600e01e27d3237083095b5b Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Tue, 11 Sep 2018 15:56:40 +0200
Subject: [PATCH 1/3] MON-228 improve service check

---
 database/elasticsearch/README.md              |  6 +-
 database/elasticsearch/inputs.tf              | 27 ++----
 .../elasticsearch/monitors-elasticsearch.tf   | 83 ++++++++++---------
 database/elasticsearch/outputs.tf             | 10 +--
 4 files changed, 57 insertions(+), 69 deletions(-)

diff --git a/database/elasticsearch/README.md b/database/elasticsearch/README.md
index 00a66eb..b4c8ad7 100644
--- a/database/elasticsearch/README.md
+++ b/database/elasticsearch/README.md
@@ -170,13 +170,11 @@ Creates DataDog monitors with the following checks:
 | node_free_space_threshold_warning | Cluster Status warning threshold | string | `20` | no |
 | node_free_space_time_aggregator | Time aggregator for the Cluster Status monitor | string | `sum` | no |
 | node_free_space_timeframe | Timeframe for the Cluster Status monitor | string | `last_5m` | no |
-| not_responding_by | Group by for the service check | string | `"port","server"` | no |
 | not_responding_extra_tags | Extra tags for Elasticsearch does not respond monitor | list | `<list>` | no |
-| not_responding_last | Parameter 'last' for the service check | string | `1` | no |
 | not_responding_message | Custom message for Elasticsearch does not respond monitor | string | `` | no |
+| not_responding_no_data_timeframe | Elasticsearch not responding monitor no data timeframe | string | `10` | no |
 | not_responding_silenced | Groups to mute for Elasticsearch does not respond monitor | map | `<map>` | no |
-| not_responding_threshold_critical | Not responding limit (critical threshold) | string | `5` | no |
-| not_responding_threshold_warning | Not responding limit (warning threshold) | string | `0` | no |
+| not_responding_threshold_warning | Elasticsearch not responding limit (warning threshold) | string | `3` | no |
 | query_cache_evictions_change_extra_tags | Extra tags for Cluster Status monitor | list | `<list>` | no |
 | query_cache_evictions_change_message | Custom message for the Cluster Status monitor | string | `` | no |
 | query_cache_evictions_change_silenced | Groups to mute for Cluster Status monitor | map | `<map>` | no |
diff --git a/database/elasticsearch/inputs.tf b/database/elasticsearch/inputs.tf
index 706fff5..f2739e0 100644
--- a/database/elasticsearch/inputs.tf
+++ b/database/elasticsearch/inputs.tf
@@ -1068,26 +1068,15 @@ variable "not_responding_message" {
   default     = ""
 }
 
-variable "not_responding_by" {
-  description = "Group by for the service check"
-  type        = "string"
-  default     = "\"port\",\"server\""
-}
-
-variable "not_responding_last" {
-  description = "Parameter 'last' for the service check"
-  type        = "string"
-  default     = 1
-}
-
-variable "not_responding_threshold_critical" {
-  description = "Not responding limit (critical threshold)"
-  default     = 5
-}
-
 variable "not_responding_threshold_warning" {
-  description = "Not responding limit (warning threshold)"
-  default     = 0
+  description = "Elasticsearch not responding limit (warning threshold)"
+  default     = 3
+}
+
+variable "not_responding_no_data_timeframe" {
+  description = "Elasticsearch not responding monitor no data timeframe"
+  type        = "string"
+  default     = 10
 }
 
 variable "not_responding_extra_tags" {
diff --git a/database/elasticsearch/monitors-elasticsearch.tf b/database/elasticsearch/monitors-elasticsearch.tf
index 76f3ea8..60eebbf 100644
--- a/database/elasticsearch/monitors-elasticsearch.tf
+++ b/database/elasticsearch/monitors-elasticsearch.tf
@@ -1,3 +1,45 @@
+#
+# Service Check
+#
+resource "datadog_monitor" "not_responding" {
+  name    = "[${var.environment}] ElasticSearch does not respond"
+  message = "${coalesce(var.not_responding_message, var.message)}"
+
+  query = <<EOL
+    "elasticsearch.can_connect".over${module.filter-tags.service_check}.by("server","port").last(6).count_by_status()
+EOL
+
+  type = "service check"
+
+  thresholds {
+    warning  = "${var.not_responding_threshold_warning}"
+    critical = 5
+  }
+
+  silenced = "${var.not_responding_silenced}"
+
+  no_data_timeframe   = "${var.not_responding_no_data_timeframe}"
+  notify_no_data      = true
+  notify_audit        = false
+  locked              = false
+  timeout_h           = 0
+  include_tags        = true
+  require_full_window = true
+  renotify_interval   = 0
+
+  new_host_delay = "${var.new_host_delay}"
+
+  tags = [
+    "created-by:terraform",
+    "team:claranet",
+    "type:databases",
+    "provider:elasticsearch",
+    "env:${var.environment}",
+    "resource:elasticsearch",
+    "${var.not_responding_extra_tags}",
+  ]
+}
+
 #
 # Cluster Status Not Green
 #
@@ -877,44 +919,3 @@ EOF
     "${var.task_time_in_queue_change_extra_tags}",
   ]
 }
-
-#
-# Service Check
-#
-resource "datadog_monitor" "not_responding" {
-  name    = "[${var.environment}] ElasticSearch does not respond"
-  message = "${coalesce(var.not_responding_message, var.message)}"
-
-  query = <<EOL
-    "elasticsearch.can_connect".over${module.filter-tags.service_check}.by(${var.not_responding_by}).last(${var.not_responding_last}).pct_by_status()
-EOL
-
-  type = "service check"
-
-  thresholds {
-    warning  = "${var.not_responding_threshold_warning}"
-    critical = "${var.not_responding_threshold_critical}"
-  }
-
-  silenced = "${var.not_responding_silenced}"
-
-  notify_audit        = false
-  locked              = false
-  timeout_h           = 0
-  include_tags        = true
-  require_full_window = true
-  notify_no_data      = true
-  renotify_interval   = 0
-
-  new_host_delay = "${var.new_host_delay}"
-
-  tags = [
-    "created-by:terraform",
-    "team:claranet",
-    "type:databases",
-    "provider:elasticsearch",
-    "env:${var.environment}",
-    "resource:elasticsearch",
-    "${var.not_responding_extra_tags}",
-  ]
-}
diff --git a/database/elasticsearch/outputs.tf b/database/elasticsearch/outputs.tf
index 6a0fd13..9c90913 100644
--- a/database/elasticsearch/outputs.tf
+++ b/database/elasticsearch/outputs.tf
@@ -1,3 +1,8 @@
+output "not_responding_id" {
+  description = "id for monitor not_responding"
+  value       = "${datadog_monitor.not_responding.*.id}"
+}
+
 output "cluster_status_not_green_id" {
   description = "id for monitor cluster_status_not_green"
   value       = "${datadog_monitor.cluster_status_not_green.*.id}"
@@ -102,8 +107,3 @@ output "task_time_in_queue_change_id" {
   description = "id for monitor task_time_in_queue_change"
   value       = "${datadog_monitor.task_time_in_queue_change.*.id}"
 }
-
-output "not_responding_id" {
-  description = "id for monitor not_responding"
-  value       = "${datadog_monitor.not_responding.*.id}"
-}

From 4b7c6b0be6e8cec474bea91b23acf9468712ee9a Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Tue, 11 Sep 2018 16:13:18 +0200
Subject: [PATCH 2/3] MON-228 remove notify no data because there is a service
 check for that

---
 .../elasticsearch/monitors-elasticsearch.tf   | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/database/elasticsearch/monitors-elasticsearch.tf b/database/elasticsearch/monitors-elasticsearch.tf
index 60eebbf..25efd6c 100644
--- a/database/elasticsearch/monitors-elasticsearch.tf
+++ b/database/elasticsearch/monitors-elasticsearch.tf
@@ -65,7 +65,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -106,7 +106,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -147,7 +147,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -188,7 +188,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -232,7 +232,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -273,7 +273,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -314,7 +314,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -355,7 +355,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -396,7 +396,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -437,7 +437,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -479,7 +479,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -521,7 +521,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -570,7 +570,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -612,7 +612,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -654,7 +654,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -695,7 +695,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -736,7 +736,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -778,7 +778,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -820,7 +820,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -862,7 +862,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 
@@ -903,7 +903,7 @@ EOF
   locked              = false
   include_tags        = true
   require_full_window = true
-  notify_no_data      = true
+  notify_no_data      = false
 
   evaluation_delay = "${var.evaluation_delay}"
 

From 52fa68b4575708163bc81f7bb877e95673cc11cc Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Fri, 14 Sep 2018 12:33:33 +0200
Subject: [PATCH 3/3] MON-228 fix tag type value

---
 .../elasticsearch/monitors-elasticsearch.tf   | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/database/elasticsearch/monitors-elasticsearch.tf b/database/elasticsearch/monitors-elasticsearch.tf
index 25efd6c..52bccfb 100644
--- a/database/elasticsearch/monitors-elasticsearch.tf
+++ b/database/elasticsearch/monitors-elasticsearch.tf
@@ -32,7 +32,7 @@ EOL
   tags = [
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "env:${var.environment}",
     "resource:elasticsearch",
@@ -76,7 +76,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.cluster_status_not_green_extra_tags}",
   ]
@@ -117,7 +117,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.cluster_initializing_shards_extra_tags}",
   ]
@@ -158,7 +158,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.cluster_relocating_shards_extra_tags}",
   ]
@@ -199,7 +199,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.cluster_unassigned_shards_extra_tags}",
   ]
@@ -243,7 +243,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.node_free_space_extra_tags}",
   ]
@@ -284,7 +284,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.jvm_heap_memory_usage_extra_tags}",
   ]
@@ -325,7 +325,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.jvm_memory_young_usage_extra_tags}",
   ]
@@ -366,7 +366,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.jvm_memory_old_usage_extra_tags}",
   ]
@@ -407,7 +407,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.jvm_gc_old_collection_latency_extra_tags}",
   ]
@@ -448,7 +448,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.jvm_gc_young_collection_latency_extra_tags}",
   ]
@@ -490,7 +490,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.indexing_latency_extra_tags}",
   ]
@@ -532,7 +532,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.flush_latency_extra_tags}",
   ]
@@ -581,7 +581,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.http_connections_anomaly_extra_tags}",
   ]
@@ -623,7 +623,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.search_query_latency_extra_tags}",
   ]
@@ -665,7 +665,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.fetch_latency_extra_tags}",
   ]
@@ -706,7 +706,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.search_query_change_extra_tags}",
   ]
@@ -747,7 +747,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.fetch_change_extra_tags}",
   ]
@@ -789,7 +789,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.field_data_evictions_change_extra_tags}",
   ]
@@ -831,7 +831,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.query_cache_evictions_change_extra_tags}",
   ]
@@ -873,7 +873,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.request_cache_evictions_change_extra_tags}",
   ]
@@ -914,7 +914,7 @@ EOF
     "env:${var.environment}",
     "created-by:terraform",
     "team:claranet",
-    "type:databases",
+    "type:database",
     "provider:elasticsearch",
     "${var.task_time_in_queue_change_extra_tags}",
   ]