From 57913a7385324dcf1c70ef5b7dd15c165266cb89 Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Wed, 26 Sep 2018 19:04:53 +0200
Subject: [PATCH] MON-308 add warning for part of unhealthy instances

---
 cloud/aws/alb/monitors-alb.tf |  9 ++++++---
 cloud/aws/elb/monitors-elb.tf | 13 ++++++++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/cloud/aws/alb/monitors-alb.tf b/cloud/aws/alb/monitors-alb.tf
index 4b6fcd3..70631f7 100644
--- a/cloud/aws/alb/monitors-alb.tf
+++ b/cloud/aws/alb/monitors-alb.tf
@@ -1,13 +1,15 @@
 resource "datadog_monitor" "ALB_no_healthy_instances" {
   count   = "${var.alb_no_healthy_instances_enabled ? 1 : 0}"
-  name    = "[${var.environment}] ALB no healthy instances"
+  name    = "[${var.environment}] ALB {{#is_alert}}no healthy instances{{/is_alert}}{{#is_warning}}{{value}}% of healthy instances{{/is_warning}}"
   type    = "metric alert"
   message = "${coalesce(var.alb_no_healthy_instances_message, var.message)}"
 
   query = <<EOF
     ${var.alb_no_healthy_instances_time_aggregator}(${var.alb_no_healthy_instances_timeframe}): (
-      sum:aws.applicationelb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancer}
-    ) < 1
+      sum:aws.applicationelb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancer} / (
+      sum:aws.applicationelb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancer} +
+      sum:aws.applicationelb.un_healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancer} )
+    ) * 100 < 1
   EOF
 
   evaluation_delay = "${var.evaluation_delay}"
@@ -15,6 +17,7 @@ resource "datadog_monitor" "ALB_no_healthy_instances" {
 
   thresholds {
     critical = 1
+    warning = 100
   }
 
   notify_no_data      = true
diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf
index 6c6955f..7f6dccd 100644
--- a/cloud/aws/elb/monitors-elb.tf
+++ b/cloud/aws/elb/monitors-elb.tf
@@ -1,16 +1,23 @@
 resource "datadog_monitor" "ELB_no_healthy_instances" {
   count   = "${var.elb_no_healthy_instance_enabled ? 1 : 0}"
-  name    = "[${var.environment}] ELB no healthy instances"
+  name    = "[${var.environment}] ELB {{#is_alert}}no healthy instances{{/is_alert}}{{#is_warning}}{{value}}% of healthy instances{{/is_warning}}"
   message = "${coalesce(var.elb_no_healthy_instance_message, var.message)}"
 
   query = <<EOF
     ${var.elb_no_healthy_instance_time_aggregator}(${var.elb_no_healthy_instance_timeframe}): (
-      sum:aws.elb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancername}
-    ) < 1
+      sum:aws.elb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancername} / (
+      sum:aws.elb.healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancername} +
+      sum:aws.elb.un_healthy_host_count.maximum${module.filter-tags.query_alert} by {region,loadbalancername} )
+    ) * 100 < 1
   EOF
 
   type = "metric alert"
 
+  thresholds {
+    critical = 1
+    warning = 100
+  }
+
   notify_no_data      = true
   evaluation_delay    = "${var.evaluation_delay}"
   renotify_interval   = 0