From a2bc4d40f023e9c8d8cece9d5e0bc07000ddcd94 Mon Sep 17 00:00:00 2001
From: Quentin Manfroi <quentin.manfroi@fr.clara.net>
Date: Thu, 29 Mar 2018 16:34:04 +0200
Subject: [PATCH] MON-96 replace avg by min on elb, alb and api gateway to
 avoid false positive

---
 cloud/aws/alb/monitors-alb.tf        | 20 ++++++++++----------
 cloud/aws/apigateway/monitors-api.tf | 12 ++++++------
 cloud/aws/elb/monitors-elb.tf        | 22 +++++++++++-----------
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/cloud/aws/alb/monitors-alb.tf b/cloud/aws/alb/monitors-alb.tf
index 6b77e39..9b39352 100644
--- a/cloud/aws/alb/monitors-alb.tf
+++ b/cloud/aws/alb/monitors-alb.tf
@@ -15,7 +15,7 @@ resource "datadog_monitor" "ALB_no_healthy_instances" {
 
   query = <<EOF
     min(last_1m): (
-      sum:aws.applicationelb.healthy_host_count{${data.template_file.filter.rendered}} by {region,loadbalancer}
+      min:aws.applicationelb.healthy_host_count{${data.template_file.filter.rendered}} by {region,loadbalancer}
     ) <= 0
   EOF
 
@@ -44,7 +44,7 @@ resource "datadog_monitor" "ALB_latency" {
 
   query = <<EOF
     min(last_5m): (
-      avg:aws.applicationelb.target_response_time.average{${data.template_file.filter.rendered}} by {region,loadbalancer}
+      min:aws.applicationelb.target_response_time.average{${data.template_file.filter.rendered}} by {region,loadbalancer}
     ) > ${var.latency_threshold_critical}
   EOF
 
@@ -75,8 +75,8 @@ resource "datadog_monitor" "ALB_httpcode_elb_5xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.applicationelb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
-        (avg:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
+        min:aws.applicationelb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
+        (min:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.httpcode_elb_5xx_threshold_critical}
   EOF
@@ -108,8 +108,8 @@ resource "datadog_monitor" "ALB_httpcode_elb_4xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.applicationelb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
-        (avg:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
+        min:aws.applicationelb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
+        (min:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.httpcode_elb_4xx_threshold_critical}
   EOF
@@ -141,8 +141,8 @@ resource "datadog_monitor" "ALB_httpcode_target_5xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.applicationelb.httpcode_target_5xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
-        (avg:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
+        min:aws.applicationelb.httpcode_target_5xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
+        (min:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.httpcode_target_5xx_threshold_critical}
   EOF
@@ -174,8 +174,8 @@ resource "datadog_monitor" "ALB_httpcode_target_4xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.applicationelb.httpcode_target_4xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
-        (avg:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
+        min:aws.applicationelb.httpcode_target_4xx{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() /
+        (min:aws.applicationelb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancer}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.httpcode_target_4xx_threshold_critical}
   EOF
diff --git a/cloud/aws/apigateway/monitors-api.tf b/cloud/aws/apigateway/monitors-api.tf
index 17a515f..2300294 100644
--- a/cloud/aws/apigateway/monitors-api.tf
+++ b/cloud/aws/apigateway/monitors-api.tf
@@ -5,8 +5,8 @@ resource "datadog_monitor" "API_Gateway_latency" {
   message = "${coalesce(var.latency_message, var.message)}"
 
   query = <<EOF
-    avg(last_5m): (
-      avg:aws.apigateway.latency{${var.filter_tags}} by {region,apiname}
+    min(last_5m): (
+      min:aws.apigateway.latency{${var.filter_tags}} by {region,apiname}
     ) > ${var.latency_threshold_critical}
   EOF
 
@@ -38,8 +38,8 @@ resource "datadog_monitor" "API_http_5xx_errors_count" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.apigateway.5xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
-        (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}),
+        min:aws.apigateway.5xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
+        (min:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.http_5xx_requests_threshold_critical}
   EOF
@@ -72,8 +72,8 @@ resource "datadog_monitor" "API_http_4xx_errors_count" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.apigateway.4xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
-        (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}),
+        min:aws.apigateway.4xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
+        (min:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.http_4xx_requests_threshold_critical}
   EOF
diff --git a/cloud/aws/elb/monitors-elb.tf b/cloud/aws/elb/monitors-elb.tf
index 90d4750..f4824fa 100644
--- a/cloud/aws/elb/monitors-elb.tf
+++ b/cloud/aws/elb/monitors-elb.tf
@@ -11,8 +11,8 @@ resource "datadog_monitor" "ELB_no_healthy_instances" {
   message = "${coalesce(var.elb_no_healthy_instance_message, var.message)}"
 
   query = <<EOF
-    avg(last_5m): (
-      avg:aws.elb.healthy_host_count{${data.template_file.filter.rendered}} by {region,loadbalancername}
+    min(last_5m): (
+      min:aws.elb.healthy_host_count{${data.template_file.filter.rendered}} by {region,loadbalancername}
     ) < 1
   EOF
 
@@ -41,8 +41,8 @@ resource "datadog_monitor" "ELB_too_much_4xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.elb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername}.as_count() /
-        (avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}.as_count() + ${var.artificial_requests_count}),
+        min:aws.elb.httpcode_elb_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername}.as_count() /
+        (min:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername}.as_count() + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.elb_4xx_threshold_critical}
   EOF
@@ -77,8 +77,8 @@ resource "datadog_monitor" "ELB_too_much_5xx" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.elb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
-        (avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
+        min:aws.elb.httpcode_elb_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
+        (min:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.elb_5xx_threshold_critical}
   EOF
@@ -113,8 +113,8 @@ resource "datadog_monitor" "ELB_too_much_4xx_backend" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
-        (avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
+        min:aws.elb.httpcode_backend_4xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
+        (min:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.elb_backend_4xx_threshold_critical}
   EOF
@@ -149,8 +149,8 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" {
   query = <<EOF
     min(last_5m): (
       default(
-        avg:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
-        (avg:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
+        min:aws.elb.httpcode_backend_5xx{${data.template_file.filter.rendered}} by {region,loadbalancername} /
+        (min:aws.elb.request_count{${data.template_file.filter.rendered}} by {region,loadbalancername} + ${var.artificial_requests_count}),
       0) * 100
     ) > ${var.elb_backend_5xx_threshold_critical}
   EOF
@@ -184,7 +184,7 @@ resource "datadog_monitor" "ELB_backend_latency" {
 
   query = <<EOF
     min(last_5m): (
-      avg:aws.elb.latency{${data.template_file.filter.rendered}} by {region,loadbalancername}
+      min:aws.elb.latency{${data.template_file.filter.rendered}} by {region,loadbalancername}
     ) > ${var.elb_backend_latency_critical}
   EOF