From dd09e1aedc53c1c51c2e23ac05eef8906c90b43a Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Fri, 13 Sep 2019 09:55:40 +0200
Subject: [PATCH] MON-512 fix queries for elasticsearch latencies monitors

---
 .../elasticsearch/monitors-elasticsearch.tf   | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/database/elasticsearch/monitors-elasticsearch.tf b/database/elasticsearch/monitors-elasticsearch.tf
index 740f20d..6f512ed 100644
--- a/database/elasticsearch/monitors-elasticsearch.tf
+++ b/database/elasticsearch/monitors-elasticsearch.tf
@@ -329,7 +329,10 @@ resource "datadog_monitor" "jvm_gc_old_collection_latency" {
 
   query = <<EOQ
   ${var.jvm_gc_old_collection_latency_time_aggregator}(${var.jvm_gc_old_collection_latency_timeframe}):
-    avg:jvm.gc.collectors.old.collection_time${module.filter-tags.query_alert} by {node_name} / avg:jvm.gc.collectors.old.count${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:jvm.gc.collectors.old.collection_time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:jvm.gc.collectors.old.count${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
   > ${var.jvm_gc_old_collection_latency_threshold_critical}
 EOQ
 
@@ -364,7 +367,10 @@ resource "datadog_monitor" "jvm_gc_young_collection_latency" {
 
   query = <<EOQ
   ${var.jvm_gc_young_collection_latency_time_aggregator}(${var.jvm_gc_young_collection_latency_timeframe}):
-    avg:jvm.gc.collectors.young.collection_time${module.filter-tags.query_alert} by {node_name} / avg:jvm.gc.collectors.young.count${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:jvm.gc.collectors.young.collection_time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:jvm.gc.collectors.young.count${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
   > ${var.jvm_gc_young_collection_latency_threshold_critical}
 EOQ
 
@@ -400,7 +406,10 @@ resource "datadog_monitor" "indexing_latency" {
   // TODO add tags to filter by node type and do not apply this monitor on non-data nodes
   query = <<EOQ
   ${var.indexing_latency_time_aggregator}(${var.indexing_latency_timeframe}):
-    avg:elasticsearch.indexing.index.time${module.filter-tags.query_alert} by {node_name}/ avg:elasticsearch.indexing.index.total${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:elasticsearch.indexing.index.time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:elasticsearch.indexing.index.total${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
    > ${var.indexing_latency_threshold_critical}
 EOQ
 
@@ -436,7 +445,10 @@ resource "datadog_monitor" "flush_latency" {
   // TODO add tags to filter by node type and do not apply this monitor on non-data nodes
   query = <<EOQ
   ${var.flush_latency_time_aggregator}(${var.flush_latency_timeframe}):
-    avg:elasticsearch.flush.total.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.flush.total${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:elasticsearch.flush.total.time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:elasticsearch.flush.total${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
   > ${var.flush_latency_threshold_critical}
 EOQ
 
@@ -520,7 +532,10 @@ resource "datadog_monitor" "search_query_latency" {
   // TODO add tags to filter by node type and do not apply this monitor on non-data nodes
   query = <<EOQ
   ${var.search_query_latency_time_aggregator}(${var.search_query_latency_timeframe}):
-    avg:elasticsearch.search.query.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.search.query.total${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:elasticsearch.search.query.time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:elasticsearch.search.query.total${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
   > ${var.search_query_latency_threshold_critical}
 EOQ
 
@@ -556,7 +571,10 @@ resource "datadog_monitor" "fetch_latency" {
   // TODO add tags to filter by node type and do not apply this monitor on non-data nodes
   query = <<EOQ
   ${var.fetch_latency_time_aggregator}(${var.fetch_latency_timeframe}):
-    avg:elasticsearch.search.fetch.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.search.fetch.total${module.filter-tags.query_alert} by {node_name} * 1000
+    default(
+      diff(avg:elasticsearch.search.fetch.time${module.filter-tags.query_alert} by {node_name}) /
+      diff(avg:elasticsearch.search.fetch.total${module.filter-tags.query_alert} by {node_name})
+    * 1000, 0)
   > ${var.fetch_latency_threshold_critical}
 EOQ