Merge branch 'MON-386_caas_nging_ingress_fix_query' into 'master'

MON-386 fix ingress monitors and update for VTS

See merge request claranet/cloudnative/projects/datadog/terraform/monitors!12

README.md

@@ -77,6 +77,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi
 	- [kubernetes](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/)
 		- [ark](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ark/)
 		- [ingress](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ingress/)
+			- [vts](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ingress/vts/)
 - [cloud](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/)
 	- [aws](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/aws/)
 		- [alb](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/aws/alb/)

caas/kubernetes/ingress/vts/README.md

@@ -1,10 +1,10 @@
-# CAAS KUBERNETES INGRESS DataDog monitors
+# CAAS KUBERNETES INGRESS VTS DataDog monitors
 
 ## How to use this module
 
 ```
-module "datadog-monitors-caas-kubernetes-ingress" {
-  source = "git::ssh://git@git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors.git//caas/kubernetes/ingress?ref={revision}"
+module "datadog-monitors-caas-kubernetes-ingress-vts" {
+  source = "git::ssh://git@git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors.git//caas/kubernetes/ingress/vts?ref={revision}"
 
   environment = "${var.environment}"
   message     = "${module.datadog-message-alerting.alerting-message}"
@@ -63,6 +63,9 @@ https://github.com/kubernetes/ingress-nginx/pull/423/commits/1d38e3a38425f08de2f
 
 Nginx Ingress Controller setup
 ------------------------------
+This configuration and monitors only work for ingress controller version :
+- >= 0.10 because ingress is beta before that and metrics naming convention not stable
+- <= 0.15 because ingress does not use VTS metrics since 0.16
 Enable the following flags in the Nginx Ingress Controller chart
 controller.stats.enabled=true,controller.metrics.enabled=true
 and the following Datadog agent configuration for each ingress controller:
@@ -87,5 +90,5 @@ datadog:
               - dd_monitoring:enabled
               - dd_ingress:enabled
               - dd_ingress_class:nginx
-              - env:prod
+              - env:ENV
 ```

caas/kubernetes/ingress/vts/modules.tf

@@ -1,5 +1,5 @@
 module "filter-tags" {
-  source = "../../../common/filter-tags"
+  source = "../../../../common/filter-tags"
 
   environment                 = "${var.environment}"
   resource                    = "ingress"
@@ -10,7 +10,7 @@ module "filter-tags" {
 }
 
 module "filter-tags-5xx" {
-  source = "../../../common/filter-tags"
+  source = "../../../../common/filter-tags"
 
   environment                 = "${var.environment}"
   resource                    = "ingress"
@@ -21,7 +21,7 @@ module "filter-tags-5xx" {
 }
 
 module "filter-tags-4xx" {
-  source = "../../../common/filter-tags"
+  source = "../../../../common/filter-tags"
 
   environment                 = "${var.environment}"
   resource                    = "ingress"

caas/kubernetes/ingress/vts/monitors-ingress.tf

@@ -4,10 +4,10 @@ resource "datadog_monitor" "nginx_ingress_too_many_5xx" {
   message = "${coalesce(var.ingress_5xx_message, var.message)}"
 
   query = <<EOF
-    ${var.ingress_5xx_time_aggregator}(${var.ingress_5xx_timeframe}): (
-      default(avg:nginx_ingress.nginx_upstream_responses_total{module.filter-tags-5xx.query_alert} by {upstream,ingress_class}.as_rate(), 0) / (
-      default(avg:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}, 1))
-      * 100 > ${var.ingress_5xx_threshold_critical}
+    ${var.ingress_5xx_time_aggregator}(${var.ingress_5xx_timeframe}): default(
+      sum:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-5xx.query_alert} by {upstream,ingress_class}.as_rate() /
+      (sum:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count})
+      * 100, 0) > ${var.ingress_5xx_threshold_critical}
   EOF
 
   type = "metric alert"
@@ -38,10 +38,10 @@ resource "datadog_monitor" "nginx_ingress_too_many_4xx" {
   message = "${coalesce(var.ingress_4xx_message, var.message)}"
 
   query = <<EOF
-    ${var.ingress_4xx_time_aggregator}(${var.ingress_4xx_timeframe}): (
-      default(avg:nginx_ingress.nginx_upstream_responses_total{module.filter-tags-4xx.query_alert} by {upstream,ingress_class}.as_rate(), 0) / (
-      default(avg:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}, 1))
-      * 100 > ${var.ingress_4xx_threshold_critical}
+    ${var.ingress_4xx_time_aggregator}(${var.ingress_4xx_timeframe}): default(
+    sum:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-4xx.query_alert} by {upstream,ingress_class}.as_rate() /
+    (sum:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count})
+    * 100, 0) > ${var.ingress_4xx_threshold_critical}
   EOF
 
   type = "metric alert"