diff --git a/README.md b/README.md index a8706f7..e8767bc 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,9 @@ The `//` is very important, it's a terraform specific syntax used to separate gi ### Monitors summary ### +- [caas](https://bitbucket.org/morea/terraform.feature.datadog/src/master/caas/) + - [kubernetes](https://bitbucket.org/morea/terraform.feature.datadog/src/master/caas/kubernetes/) + - [ingress](https://bitbucket.org/morea/terraform.feature.datadog/src/master/caas/kubernetes/ingress/) - [cloud](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/) - [aws](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/aws/) - [alb](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/aws/alb/) diff --git a/caas/kubernetes/ingress/README.md b/caas/kubernetes/ingress/README.md new file mode 100644 index 0000000..0a7ba2a --- /dev/null +++ b/caas/kubernetes/ingress/README.md @@ -0,0 +1,88 @@ +# CAAS KUBERNETES INGRESS DataDog monitors + +## How to use this module + +``` +module "datadog-monitors-caas-kubernetes-ingress" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//caas/kubernetes/ingress?ref={revision}" + + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` + +## Purpose + +Creates DataDog monitors with the following checks: + +- Nginx Ingress 4xx errors +- Nginx Ingress 5xx errors + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `5` | no | +| environment | Architecture Environment | string | - | yes | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| ingress_4xx_enabled | Flag to enable Ingress 4xx errors monitor | string | `true` | no | +| ingress_4xx_extra_tags | Extra tags for Ingress 4xx errors monitor | list | `` | no | +| ingress_4xx_message | Message sent when an alert is triggered | string | `` | no | +| ingress_4xx_silenced | Groups to mute for Ingress 4xx errors monitor | map | `` | no | +| ingress_4xx_threshold_critical | | string | `40` | no | +| ingress_4xx_threshold_warning | | string | `20` | no | +| ingress_4xx_timeframe | Monitor timeframe for Ingress 4xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | +| ingress_5xx_enabled | Flag to enable Ingress 5xx errors monitor | string | `true` | no | +| ingress_5xx_extra_tags | Extra tags for Ingress 5xx errors monitor | list | `` | no | +| ingress_5xx_message | Message sent when an alert is triggered | string | `` | no | +| ingress_5xx_silenced | Groups to mute for Ingress 5xx errors monitor | map | `` | no | +| ingress_5xx_threshold_critical | | string | `20` | no | +| ingress_5xx_threshold_warning | | string | `10` | no | +| ingress_5xx_timeframe | Monitor timeframe for Ingress 5xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | +| message | Message sent when an alert is triggered | string | - | yes | +| new_host_delay | Delay in seconds before monitor new resource | string | `300` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| nginx_ingress_too_many_4xx_id | id for monitor nginx_ingress_too_many_4xx | +| nginx_ingress_too_many_5xx_id | id for monitor nginx_ingress_too_many_5xx | + +Related documentation +--------------------- + +DataDog blog: https://www.datadoghq.com/blog/monitor-prometheus-metrics +https://github.com/kubernetes/ingress-nginx/pull/423/commits/1d38e3a38425f08de2f75fcae13896a3fec4d144 + +Nginx Ingress Controller setup +------------------------------ +Enable the following flags in the Nginx Ingress Controller chart +controller.stats.enabled=true,controller.metrics.enabled=true +and the following Datadog agent configuration for each ingress controller: +``` +datadog: + confd: + prometheus.yaml: |- + #nginx_upstream_responses_total{ingress_class,namespace,server,status_code:{1xx,2xx,3xx,4xx,5xx},upstream} + #nginx_upstream_requests_total{ingress_class,namespace,server,upstream} + init_config: + instances: + # The prometheus endpoint to query from + - prometheus_url: http://nginx-ingress-controller-metrics:9913/metrics + # This is NOT the ingress namespace, it is the prefix that will be used for the custom metrics + namespace: nginx-ingress + # Filter on the following metrics only + metrics: + - "nginx_upstream_requests_total" + - "nginx_upstream_responses_total" + # Adapt the tags to the current convention and verify that the monitor will match + tags: + - dd_monitoring:enabled + - dd_ingress:enabled + - dd_ingress_class:nginx + - env:prod +``` diff --git a/caas/kubernetes/ingress/inputs.tf b/caas/kubernetes/ingress/inputs.tf new file mode 100644 index 0000000..7be1590 --- /dev/null +++ b/caas/kubernetes/ingress/inputs.tf @@ -0,0 +1,114 @@ +# Global Terraform +variable "environment" { + description = "Architecture Environment" + type = "string" +} + +# Global DataDog +variable "evaluation_delay" { + description = "Delay in seconds for the metric evaluation" + default = 15 +} + +variable "new_host_delay" { + description = "Delay in seconds before monitor new resource" + default = 300 +} + +variable "message" { + description = "Message sent when an alert is triggered" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +#Ingress +variable "ingress_5xx_silenced" { + description = "Groups to mute for Ingress 5xx errors monitor" + type = "map" + default = {} +} + +variable "ingress_5xx_enabled" { + description = "Flag to enable Ingress 5xx errors monitor" + type = "string" + default = "true" +} + +variable "ingress_5xx_extra_tags" { + description = "Extra tags for Ingress 5xx errors monitor" + type = "list" + default = [] +} + +variable "ingress_5xx_message" { + description = "Message sent when an alert is triggered" + default = "" +} + +variable "ingress_5xx_timeframe" { + description = "Monitor timeframe for Ingress 5xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} + +variable "ingress_5xx_threshold_critical" { + type = "string" + default = "20" +} + +variable "ingress_5xx_threshold_warning" { + type = "string" + default = "10" +} + +variable "ingress_4xx_silenced" { + description = "Groups to mute for Ingress 4xx errors monitor" + type = "map" + default = {} +} + +variable "ingress_4xx_enabled" { + description = "Flag to enable Ingress 4xx errors monitor" + type = "string" + default = "true" +} + +variable "ingress_4xx_extra_tags" { + description = "Extra tags for Ingress 4xx errors monitor" + type = "list" + default = [] +} + +variable "ingress_4xx_message" { + description = "Message sent when an alert is triggered" + default = "" +} + +variable "ingress_4xx_timeframe" { + description = "Monitor timeframe for Ingress 4xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} + +variable "ingress_4xx_threshold_critical" { + type = "string" + default = "40" +} + +variable "ingress_4xx_threshold_warning" { + type = "string" + default = "20" +} + +variable "artificial_requests_count" { + default = 5 + description = "Number of false requests used to mitigate false positive in case of low trafic" +} diff --git a/caas/kubernetes/ingress/modules.tf b/caas/kubernetes/ingress/modules.tf new file mode 100644 index 0000000..ee1e148 --- /dev/null +++ b/caas/kubernetes/ingress/modules.tf @@ -0,0 +1,29 @@ +module "filter-tags" { + source = "../../../common/filter-tags" + + environment = "${var.environment}" + resource = "ingress" + filter_tags_use_defaults = "${var.filter_tags_use_defaults}" + filter_tags_custom = "${var.filter_tags_custom}" + extra_tags = ["!upstream:upstream-default-backend"] +} + +module "filter-tags-5xx" { + source = "../../../common/filter-tags" + + environment = "${var.environment}" + resource = "ingress" + filter_tags_use_defaults = "${var.filter_tags_use_defaults}" + filter_tags_custom = "${var.filter_tags_custom}" + extra_tags = ["!upstream:upstream-default-backend,status_code:5xx"] +} + +module "filter-tags-4xx" { + source = "../../../common/filter-tags" + + environment = "${var.environment}" + resource = "ingress" + filter_tags_use_defaults = "${var.filter_tags_use_defaults}" + filter_tags_custom = "${var.filter_tags_custom}" + extra_tags = ["!upstream:upstream-default-backend,status_code:4xx"] +} diff --git a/caas/kubernetes/ingress/monitors-ingress.tf b/caas/kubernetes/ingress/monitors-ingress.tf new file mode 100644 index 0000000..ab44196 --- /dev/null +++ b/caas/kubernetes/ingress/monitors-ingress.tf @@ -0,0 +1,71 @@ +resource "datadog_monitor" "nginx_ingress_too_many_5xx" { + count = "${var.ingress_5xx_enabled ? 1 : 0}" + name = "[${var.environment}] Nginx Ingress 5xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.ingress_5xx_message, var.message)}" + + query = < ${var.ingress_5xx_threshold_critical} + EOF + + type = "metric alert" + + thresholds { + warning = "${var.ingress_5xx_threshold_warning}" + critical = "${var.ingress_5xx_threshold_critical}" + } + + notify_no_data = false + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + + silenced = "${var.ingress_5xx_silenced}" + + tags = ["env:${var.environment}", "type:caas", "provider:prometheus", "resource:nginx-ingress-controller", "team:claranet", "created-by:terraform", "${var.ingress_5xx_extra_tags}"] +} + +resource "datadog_monitor" "nginx_ingress_too_many_4xx" { + count = "${var.ingress_4xx_enabled ? 1 : 0}" + name = "[${var.environment}] Nginx Ingress 4xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.ingress_4xx_message, var.message)}" + + query = < ${var.ingress_4xx_threshold_critical} + EOF + + type = "metric alert" + + thresholds { + warning = "${var.ingress_4xx_threshold_warning}" + critical = "${var.ingress_4xx_threshold_critical}" + } + + notify_no_data = false + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + + silenced = "${var.ingress_4xx_silenced}" + + tags = ["env:${var.environment}", "type:caas", "provider:prometheus", "resource:nginx-ingress-controller", "team:claranet", "created-by:terraform", "${var.ingress_4xx_extra_tags}"] +} diff --git a/caas/kubernetes/ingress/outputs.tf b/caas/kubernetes/ingress/outputs.tf new file mode 100644 index 0000000..28582f8 --- /dev/null +++ b/caas/kubernetes/ingress/outputs.tf @@ -0,0 +1,9 @@ +output "nginx_ingress_too_many_5xx_id" { + description = "id for monitor nginx_ingress_too_many_5xx" + value = "${datadog_monitor.nginx_ingress_too_many_5xx.*.id}" +} + +output "nginx_ingress_too_many_4xx_id" { + description = "id for monitor nginx_ingress_too_many_4xx" + value = "${datadog_monitor.nginx_ingress_too_many_4xx.*.id}" +}