From e019c9056370b3d9e0b2a50bf7908cfcba25c6f4 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 20 Aug 2019 09:16:04 +0100 Subject: [PATCH 1/4] MON-497 Docker monitors: service check and memory used --- README.md | 1 + caas/docker/README.md | 22 ++++++ caas/docker/inputs.tf | 130 +++++++++++++++++++++++++++++++++ caas/docker/modules.tf | 9 +++ caas/docker/monitors-docker.tf | 69 +++++++++++++++++ caas/docker/outputs.tf | 10 +++ 6 files changed, 241 insertions(+) create mode 100644 caas/docker/README.md create mode 100644 caas/docker/inputs.tf create mode 100644 caas/docker/modules.tf create mode 100644 caas/docker/monitors-docker.tf create mode 100644 caas/docker/outputs.tf diff --git a/README.md b/README.md index d30b9fe..e749418 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi ### Monitors summary ### - [caas](https://git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors/tree/master/caas/) + - [docker](https://git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors/tree/master/caas/docker/) - [kubernetes](https://git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/) - [ark](https://git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/ark/) - [cluster](https://git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors/tree/master/caas/kubernetes/cluster/) diff --git a/caas/docker/README.md b/caas/docker/README.md new file mode 100644 index 0000000..b20dd74 --- /dev/null +++ b/caas/docker/README.md @@ -0,0 +1,22 @@ +# CAAS DOCKER DataDog monitors + +## How to use this module + +``` +module "datadog-monitors-caas-docker" { + source = "git::ssh://git@git.fr.clara.net/claranet/pt-monitoring/projects/datadog/terraform/monitors.git//caas/docker?ref={revision}" + + environment = var.environment + message = module.datadog-message-alerting.alerting-message +} + +``` + +## Purpose + +Creates DataDog monitors with the following checks: + +- Docker Container Memory Used (disabled by default) +- Service Docker does not respond + + diff --git a/caas/docker/inputs.tf b/caas/docker/inputs.tf new file mode 100644 index 0000000..8f06b2a --- /dev/null +++ b/caas/docker/inputs.tf @@ -0,0 +1,130 @@ +# Global Terraform +variable "environment" { + description = "Architecture Environment" + type = string +} + +# Global DataDog +variable "evaluation_delay" { + description = "Delay in seconds for the metric evaluation" + default = 15 +} + +variable "new_host_delay" { + description = "Delay in seconds before monitor new resource" + default = 300 +} + +variable "prefix_slug" { + description = "Prefix string to prepend between brackets on every monitors names" + default = "" +} + +variable "message" { + description = "Message sent when an alert is triggered" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +variable "filter_tags_custom_excluded" { + description = "Tags excluded for custom filtering when filter_tags_use_defaults is false" + default = "" +} + +# +# Not Responding +# +variable "not_responding_enabled" { + description = "Flag to enable Docker does not respond monitor" + type = string + default = "true" +} + +variable "not_responding_message" { + description = "Custom message for Docker does not respond monitor" + type = string + default = "" +} + +variable "not_responding_last" { + description = "Parameter 'last' for the service check" + type = string + default = 6 +} + +variable "not_responding_threshold_warning" { + description = "Docker does not respond monitor (warning threshold)" + type = string + default = 3 +} + +variable "not_responding_threshold_critical" { + description = "Docker does not respond monitor (warning threshold)" + type = string + default = 5 +} + +variable "not_responding_no_data_timeframe" { + description = "Docker does not respond monitor no data timeframe" + type = string + default = 10 +} + +variable "not_responding_extra_tags" { + description = "Extra tags for Docker does not respond monitor" + type = list(string) + default = [] +} + +# +# Container Memory Usage +# +variable "memory_used_enabled" { + description = "Flag to enable Container Memory Usage monitor" + type = string + default = "false" +} + +variable "memory_used_message" { + description = "Custom message for the Container Memory Usage monitor" + type = string + default = "" +} + +variable "memory_used_time_aggregator" { + description = "Time aggregator for the Container Memory Usage monitor" + type = string + default = "min" +} + +variable "memory_used_timeframe" { + description = "Timeframe for the Container Memory Usage monitor" + type = string + default = "last_5m" +} + +variable "memory_used_threshold_warning" { + description = "Container Memory Usage warning threshold" + type = string + default = 80 +} + +variable "memory_used_threshold_critical" { + description = "Container Memory Usage critical threshold" + type = string + default = 90 +} + +variable "memory_used_extra_tags" { + description = "Extra tags for Container Memory Usage monitor" + type = list(string) + default = [] +} diff --git a/caas/docker/modules.tf b/caas/docker/modules.tf new file mode 100644 index 0000000..4038380 --- /dev/null +++ b/caas/docker/modules.tf @@ -0,0 +1,9 @@ +module "filter-tags" { + source = "../../common/filter-tags" + + environment = var.environment + resource = "docker" + filter_tags_use_defaults = var.filter_tags_use_defaults + filter_tags_custom = var.filter_tags_custom + filter_tags_custom_excluded = var.filter_tags_custom_excluded +} diff --git a/caas/docker/monitors-docker.tf b/caas/docker/monitors-docker.tf new file mode 100644 index 0000000..39329fb --- /dev/null +++ b/caas/docker/monitors-docker.tf @@ -0,0 +1,69 @@ +# +# Service Check +# +resource "datadog_monitor" "not_responding" { + count = var.not_responding_enabled == "true" ? 1 : 0 + name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Service Docker does not respond" + message = coalesce(var.not_responding_message, var.message) + type = "service check" + + query = < ${var.memory_used_threshold_critical} +EOQ + + thresholds = { + warning = var.memory_used_threshold_warning + critical = var.memory_used_threshold_critical + } + + evaluation_delay = var.evaluation_delay + new_host_delay = var.new_host_delay + notify_no_data = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + + tags = concat(["env:${var.environment}", "type:docker", "provider:docker", "resource:docker", "team:claranet", "created-by:terraform"], var.memory_used_extra_tags) + + lifecycle { + ignore_changes = ["silenced"] + } +} + diff --git a/caas/docker/outputs.tf b/caas/docker/outputs.tf new file mode 100644 index 0000000..6da69d5 --- /dev/null +++ b/caas/docker/outputs.tf @@ -0,0 +1,10 @@ +output "not_responding_id" { + description = "id for monitor not_responding" + value = datadog_monitor.not_responding.*.id +} + +output "memory_used_id" { + description = "id for monitor memory_used" + value = datadog_monitor.memory_used.*.id +} + From bded526f07690e74e802f7d57ebfb5e78cb867e4 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 20 Aug 2019 12:15:56 +0100 Subject: [PATCH 2/4] MON-497 Readme ready --- caas/docker/README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/caas/docker/README.md b/caas/docker/README.md index b20dd74..c1af2f2 100644 --- a/caas/docker/README.md +++ b/caas/docker/README.md @@ -19,4 +19,40 @@ Creates DataDog monitors with the following checks: - Docker Container Memory Used (disabled by default) - Service Docker does not respond +## Inputs +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| environment | Architecture Environment | string | n/a | yes | +| evaluation\_delay | Delay in seconds for the metric evaluation | string | `"15"` | no | +| filter\_tags\_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `"*"` | no | +| filter\_tags\_custom\_excluded | Tags excluded for custom filtering when filter_tags_use_defaults is false | string | `""` | no | +| filter\_tags\_use\_defaults | Use default filter tags convention | string | `"true"` | no | +| memory\_used\_enabled | Flag to enable Container Memory Usage monitor | string | `"false"` | no | +| memory\_used\_extra\_tags | Extra tags for Container Memory Usage monitor | list(string) | `[]` | no | +| memory\_used\_message | Custom message for the Container Memory Usage monitor | string | `""` | no | +| memory\_used\_threshold\_critical | Container Memory Usage critical threshold | string | `"90"` | no | +| memory\_used\_threshold\_warning | Container Memory Usage warning threshold | string | `"80"` | no | +| memory\_used\_time\_aggregator | Time aggregator for the Container Memory Usage monitor | string | `"min"` | no | +| memory\_used\_timeframe | Timeframe for the Container Memory Usage monitor | string | `"last_5m"` | no | +| message | Message sent when an alert is triggered | string | n/a | yes | +| new\_host\_delay | Delay in seconds before monitor new resource | string | `"300"` | no | +| not\_responding\_enabled | Flag to enable Docker does not respond monitor | string | `"true"` | no | +| not\_responding\_extra\_tags | Extra tags for Docker does not respond monitor | list(string) | `[]` | no | +| not\_responding\_last | Parameter 'last' for the service check | string | `"6"` | no | +| not\_responding\_message | Custom message for Docker does not respond monitor | string | `""` | no | +| not\_responding\_no\_data\_timeframe | Docker does not respond monitor no data timeframe | string | `"10"` | no | +| not\_responding\_threshold\_critical | Docker does not respond monitor (warning threshold) | string | `"5"` | no | +| not\_responding\_threshold\_warning | Docker does not respond monitor (warning threshold) | string | `"3"` | no | +| prefix\_slug | Prefix string to prepend between brackets on every monitors names | string | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| memory\_used\_id | id for monitor memory_used | +| not\_responding\_id | id for monitor not_responding | + +## Related documentation + +* [Datadog Docker integration](https://docs.datadoghq.com/integrations/docker_daemon/) From 5f2a430d450de6a107c31369075a51fc726e4c3d Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Thu, 29 Aug 2019 13:08:34 +0100 Subject: [PATCH 3/4] MON-497 Warning threshold for memory limit increased from 80 to 85%. Removed the inputs for the critical threshold and the last of the service check. --- caas/docker/README.md | 4 +--- caas/docker/inputs.tf | 14 +------------- caas/docker/monitors-docker.tf | 4 ++-- caas/docker/outputs.tf | 10 +++++----- 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/caas/docker/README.md b/caas/docker/README.md index c1af2f2..7188671 100644 --- a/caas/docker/README.md +++ b/caas/docker/README.md @@ -32,17 +32,15 @@ Creates DataDog monitors with the following checks: | memory\_used\_extra\_tags | Extra tags for Container Memory Usage monitor | list(string) | `[]` | no | | memory\_used\_message | Custom message for the Container Memory Usage monitor | string | `""` | no | | memory\_used\_threshold\_critical | Container Memory Usage critical threshold | string | `"90"` | no | -| memory\_used\_threshold\_warning | Container Memory Usage warning threshold | string | `"80"` | no | +| memory\_used\_threshold\_warning | Container Memory Usage warning threshold | string | `"85"` | no | | memory\_used\_time\_aggregator | Time aggregator for the Container Memory Usage monitor | string | `"min"` | no | | memory\_used\_timeframe | Timeframe for the Container Memory Usage monitor | string | `"last_5m"` | no | | message | Message sent when an alert is triggered | string | n/a | yes | | new\_host\_delay | Delay in seconds before monitor new resource | string | `"300"` | no | | not\_responding\_enabled | Flag to enable Docker does not respond monitor | string | `"true"` | no | | not\_responding\_extra\_tags | Extra tags for Docker does not respond monitor | list(string) | `[]` | no | -| not\_responding\_last | Parameter 'last' for the service check | string | `"6"` | no | | not\_responding\_message | Custom message for Docker does not respond monitor | string | `""` | no | | not\_responding\_no\_data\_timeframe | Docker does not respond monitor no data timeframe | string | `"10"` | no | -| not\_responding\_threshold\_critical | Docker does not respond monitor (warning threshold) | string | `"5"` | no | | not\_responding\_threshold\_warning | Docker does not respond monitor (warning threshold) | string | `"3"` | no | | prefix\_slug | Prefix string to prepend between brackets on every monitors names | string | `""` | no | diff --git a/caas/docker/inputs.tf b/caas/docker/inputs.tf index 8f06b2a..b78c3df 100644 --- a/caas/docker/inputs.tf +++ b/caas/docker/inputs.tf @@ -54,24 +54,12 @@ variable "not_responding_message" { default = "" } -variable "not_responding_last" { - description = "Parameter 'last' for the service check" - type = string - default = 6 -} - variable "not_responding_threshold_warning" { description = "Docker does not respond monitor (warning threshold)" type = string default = 3 } -variable "not_responding_threshold_critical" { - description = "Docker does not respond monitor (warning threshold)" - type = string - default = 5 -} - variable "not_responding_no_data_timeframe" { description = "Docker does not respond monitor no data timeframe" type = string @@ -114,7 +102,7 @@ variable "memory_used_timeframe" { variable "memory_used_threshold_warning" { description = "Container Memory Usage warning threshold" type = string - default = 80 + default = 85 } variable "memory_used_threshold_critical" { diff --git a/caas/docker/monitors-docker.tf b/caas/docker/monitors-docker.tf index 39329fb..84678b7 100644 --- a/caas/docker/monitors-docker.tf +++ b/caas/docker/monitors-docker.tf @@ -8,12 +8,12 @@ resource "datadog_monitor" "not_responding" { type = "service check" query = < Date: Thu, 29 Aug 2019 13:39:50 +0100 Subject: [PATCH 4/4] MON-497 The name of the service check is now Docker does not respond --- caas/docker/README.md | 2 +- caas/docker/monitors-docker.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/caas/docker/README.md b/caas/docker/README.md index 7188671..08d7a19 100644 --- a/caas/docker/README.md +++ b/caas/docker/README.md @@ -17,7 +17,7 @@ module "datadog-monitors-caas-docker" { Creates DataDog monitors with the following checks: - Docker Container Memory Used (disabled by default) -- Service Docker does not respond +- Docker does not respond ## Inputs diff --git a/caas/docker/monitors-docker.tf b/caas/docker/monitors-docker.tf index 84678b7..c68cb5d 100644 --- a/caas/docker/monitors-docker.tf +++ b/caas/docker/monitors-docker.tf @@ -3,7 +3,7 @@ # resource "datadog_monitor" "not_responding" { count = var.not_responding_enabled == "true" ? 1 : 0 - name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Service Docker does not respond" + name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Docker does not respond" message = coalesce(var.not_responding_message, var.message) type = "service check"