From 0593f75d84aa0075d20be296cc7a825476ec449a Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 22 Jan 2019 11:00:38 +0100 Subject: [PATCH 1/5] MON-386 Query fixed to avoid error when deploying. --- caas/kubernetes/ingress/monitors-ingress.tf | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/caas/kubernetes/ingress/monitors-ingress.tf b/caas/kubernetes/ingress/monitors-ingress.tf index 5be3c67..cff209d 100644 --- a/caas/kubernetes/ingress/monitors-ingress.tf +++ b/caas/kubernetes/ingress/monitors-ingress.tf @@ -4,10 +4,12 @@ resource "datadog_monitor" "nginx_ingress_too_many_5xx" { message = "${coalesce(var.ingress_5xx_message, var.message)}" query = < ${var.ingress_5xx_threshold_critical} + ${var.ingress_5xx_time_aggregator}(${var.ingress_5xx_timeframe}): + (default(avg:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-5xx.query_alert} by {upstream,ingress_class}.as_rate(), 0)) + / + (default(avg:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}, 1)) + * 100 + > ${var.ingress_5xx_threshold_critical} EOF type = "metric alert" @@ -38,10 +40,12 @@ resource "datadog_monitor" "nginx_ingress_too_many_4xx" { message = "${coalesce(var.ingress_4xx_message, var.message)}" query = < ${var.ingress_4xx_threshold_critical} + ${var.ingress_4xx_time_aggregator}(${var.ingress_4xx_timeframe}): + (default(avg:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-4xx.query_alert} by {upstream,ingress_class}.as_rate(), 0)) + / + (default(avg:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}, 1)) + * 100 + > ${var.ingress_4xx_threshold_critical} EOF type = "metric alert" From 74e0f804b736939915c908009876956f24708825 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 22 Jan 2019 13:19:05 +0100 Subject: [PATCH 2/5] MON-386 update readme --- caas/kubernetes/ingress/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/caas/kubernetes/ingress/README.md b/caas/kubernetes/ingress/README.md index 3e61685..78165d0 100644 --- a/caas/kubernetes/ingress/README.md +++ b/caas/kubernetes/ingress/README.md @@ -63,6 +63,7 @@ https://github.com/kubernetes/ingress-nginx/pull/423/commits/1d38e3a38425f08de2f Nginx Ingress Controller setup ------------------------------ +This configuration and monitors only works for ingress controller >= 0.10 and <= 0.15. Enable the following flags in the Nginx Ingress Controller chart controller.stats.enabled=true,controller.metrics.enabled=true and the following Datadog agent configuration for each ingress controller: @@ -87,5 +88,5 @@ datadog: - dd_monitoring:enabled - dd_ingress:enabled - dd_ingress_class:nginx - - env:prod + - env:ENV ``` From a46abb4a68862b1efd39b03cfc62d2829ad01b11 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 22 Jan 2019 13:22:45 +0100 Subject: [PATCH 3/5] MON-386 change space aggregator for sum and fix default function --- caas/kubernetes/ingress/monitors-ingress.tf | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/caas/kubernetes/ingress/monitors-ingress.tf b/caas/kubernetes/ingress/monitors-ingress.tf index cff209d..812e3a8 100644 --- a/caas/kubernetes/ingress/monitors-ingress.tf +++ b/caas/kubernetes/ingress/monitors-ingress.tf @@ -4,12 +4,10 @@ resource "datadog_monitor" "nginx_ingress_too_many_5xx" { message = "${coalesce(var.ingress_5xx_message, var.message)}" query = < ${var.ingress_5xx_threshold_critical} + ${var.ingress_5xx_time_aggregator}(${var.ingress_5xx_timeframe}): default( + sum:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-5xx.query_alert} by {upstream,ingress_class}.as_rate() / + (sum:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}) + * 100, 0) > ${var.ingress_5xx_threshold_critical} EOF type = "metric alert" @@ -40,12 +38,10 @@ resource "datadog_monitor" "nginx_ingress_too_many_4xx" { message = "${coalesce(var.ingress_4xx_message, var.message)}" query = < ${var.ingress_4xx_threshold_critical} + ${var.ingress_4xx_time_aggregator}(${var.ingress_4xx_timeframe}): default( + sum:nginx_ingress.nginx_upstream_responses_total${module.filter-tags-4xx.query_alert} by {upstream,ingress_class}.as_rate() / + (sum:nginx_ingress.nginx_upstream_requests_total${module.filter-tags.query_alert} by {upstream,ingress_class}.as_rate() + ${var.artificial_requests_count}) + * 100, 0) > ${var.ingress_4xx_threshold_critical} EOF type = "metric alert" From c72a4fe1bd99951744ac0b3c0391a337a30515f4 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 22 Jan 2019 13:43:55 +0100 Subject: [PATCH 4/5] MON-386 move ingress to specific vts sub directory --- caas/kubernetes/ingress/{ => vts}/README.md | 4 +++- caas/kubernetes/ingress/{ => vts}/inputs.tf | 0 caas/kubernetes/ingress/{ => vts}/modules.tf | 0 caas/kubernetes/ingress/{ => vts}/monitors-ingress.tf | 0 caas/kubernetes/ingress/{ => vts}/outputs.tf | 0 5 files changed, 3 insertions(+), 1 deletion(-) rename caas/kubernetes/ingress/{ => vts}/README.md (95%) rename caas/kubernetes/ingress/{ => vts}/inputs.tf (100%) rename caas/kubernetes/ingress/{ => vts}/modules.tf (100%) rename caas/kubernetes/ingress/{ => vts}/monitors-ingress.tf (100%) rename caas/kubernetes/ingress/{ => vts}/outputs.tf (100%) diff --git a/caas/kubernetes/ingress/README.md b/caas/kubernetes/ingress/vts/README.md similarity index 95% rename from caas/kubernetes/ingress/README.md rename to caas/kubernetes/ingress/vts/README.md index 78165d0..fe900d4 100644 --- a/caas/kubernetes/ingress/README.md +++ b/caas/kubernetes/ingress/vts/README.md @@ -63,7 +63,9 @@ https://github.com/kubernetes/ingress-nginx/pull/423/commits/1d38e3a38425f08de2f Nginx Ingress Controller setup ------------------------------ -This configuration and monitors only works for ingress controller >= 0.10 and <= 0.15. +This configuration and monitors only work for ingress controller version : +- >= 0.10 because ingress is beta before that and metrics naming convention not stable +- <= 0.15 because ingress does not use VTS metrics since 0.16 Enable the following flags in the Nginx Ingress Controller chart controller.stats.enabled=true,controller.metrics.enabled=true and the following Datadog agent configuration for each ingress controller: diff --git a/caas/kubernetes/ingress/inputs.tf b/caas/kubernetes/ingress/vts/inputs.tf similarity index 100% rename from caas/kubernetes/ingress/inputs.tf rename to caas/kubernetes/ingress/vts/inputs.tf diff --git a/caas/kubernetes/ingress/modules.tf b/caas/kubernetes/ingress/vts/modules.tf similarity index 100% rename from caas/kubernetes/ingress/modules.tf rename to caas/kubernetes/ingress/vts/modules.tf diff --git a/caas/kubernetes/ingress/monitors-ingress.tf b/caas/kubernetes/ingress/vts/monitors-ingress.tf similarity index 100% rename from caas/kubernetes/ingress/monitors-ingress.tf rename to caas/kubernetes/ingress/vts/monitors-ingress.tf diff --git a/caas/kubernetes/ingress/outputs.tf b/caas/kubernetes/ingress/vts/outputs.tf similarity index 100% rename from caas/kubernetes/ingress/outputs.tf rename to caas/kubernetes/ingress/vts/outputs.tf From f75d230044d7876df55e8e22d4d7e6c614aca578 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 22 Jan 2019 13:48:43 +0100 Subject: [PATCH 5/5] MON-386 auto update --- README.md | 1 + caas/kubernetes/ingress/vts/README.md | 6 +++--- caas/kubernetes/ingress/vts/modules.tf | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e61f84..7d5d678 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi - [kubernetes](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/) - [ark](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ark/) - [ingress](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ingress/) + - [vts](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ingress/vts/) - [cloud](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/) - [aws](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/aws/) - [alb](https://git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors/tree/master/cloud/aws/alb/) diff --git a/caas/kubernetes/ingress/vts/README.md b/caas/kubernetes/ingress/vts/README.md index fe900d4..9a620f6 100644 --- a/caas/kubernetes/ingress/vts/README.md +++ b/caas/kubernetes/ingress/vts/README.md @@ -1,10 +1,10 @@ -# CAAS KUBERNETES INGRESS DataDog monitors +# CAAS KUBERNETES INGRESS VTS DataDog monitors ## How to use this module ``` -module "datadog-monitors-caas-kubernetes-ingress" { - source = "git::ssh://git@git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors.git//caas/kubernetes/ingress?ref={revision}" +module "datadog-monitors-caas-kubernetes-ingress-vts" { + source = "git::ssh://git@git.fr.clara.net/claranet/cloudnative/projects/datadog/terraform/monitors.git//caas/kubernetes/ingress/vts?ref={revision}" environment = "${var.environment}" message = "${module.datadog-message-alerting.alerting-message}" diff --git a/caas/kubernetes/ingress/vts/modules.tf b/caas/kubernetes/ingress/vts/modules.tf index fcdceb0..288bd39 100644 --- a/caas/kubernetes/ingress/vts/modules.tf +++ b/caas/kubernetes/ingress/vts/modules.tf @@ -1,5 +1,5 @@ module "filter-tags" { - source = "../../../common/filter-tags" + source = "../../../../common/filter-tags" environment = "${var.environment}" resource = "ingress" @@ -10,7 +10,7 @@ module "filter-tags" { } module "filter-tags-5xx" { - source = "../../../common/filter-tags" + source = "../../../../common/filter-tags" environment = "${var.environment}" resource = "ingress" @@ -21,7 +21,7 @@ module "filter-tags-5xx" { } module "filter-tags-4xx" { - source = "../../../common/filter-tags" + source = "../../../../common/filter-tags" environment = "${var.environment}" resource = "ingress"