From 39ec9b169434a8a05727e2a9650bb3587bc4735d Mon Sep 17 00:00:00 2001 From: Boris Rousseau Date: Fri, 20 Apr 2018 16:50:00 +0200 Subject: [PATCH 01/14] MON-105: Add postgresql monitors --- databases/postgresql/README.md | 45 +++++++++++++++ databases/postgresql/inputs.tf | 77 ++++++++++++++++++++++++++ databases/postgresql/mon-postgresql.tf | 67 ++++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 databases/postgresql/README.md create mode 100644 databases/postgresql/inputs.tf create mode 100644 databases/postgresql/mon-postgresql.tf diff --git a/databases/postgresql/README.md b/databases/postgresql/README.md new file mode 100644 index 0000000..e96d886 --- /dev/null +++ b/databases/postgresql/README.md @@ -0,0 +1,45 @@ +PostgreSQL DataDog monitors +====================== + +How to use this module +---------------------- + +``` +module "datadog-monitors-postgresql" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//databases/postgresql?ref={revision}" + + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` +Purpose +------- +Creates DataDog monitors with the following checks : + +* PostgreSQL not enough available connections +* PostgreSQL too many locks + +Inputs +------ + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| delay | Delay in seconds for the metric evaluation | string | `15` | no | +| environment | Environment | string | - | yes | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| postgresql_connection_threshold_critical | Maximum critical acceptable percent of connections | string | `80` | no | +| postgresql_connection_threshold_warning | Maximum warning acceptable percent of connections | string | `70` | no | +| postgresql_connection_silenced | Groups to mute PostgreSQL connection monitor | map | `` | no | +| postgresql_connection_message | Custom message for PostgreSQL connection monitor | string | `` | no | +| postgresql_thread_threshold_critical | Maximum critical acceptable number of threads | string | `500` | no | +| postgresql_thread_threshold_warning | Maximum warning acceptable number of threads | string | `400` | no | +| postgresql_thread_silenced | Groups to mute PostgreSQL threads monitor | map | `` | no | +| postgresql_thread_message | Custom message for PostgreSQL thread monitor | string | `` | no | +| message | Message sent when a monitor is triggered | string | - | yes | + +Related documentation +--------------------- + +DataDog documentation: [https://docs.datadoghq.com/integrations/postgres/](https://docs.datadoghq.com/integrations/postgres/) diff --git a/databases/postgresql/inputs.tf b/databases/postgresql/inputs.tf new file mode 100644 index 0000000..9c884f5 --- /dev/null +++ b/databases/postgresql/inputs.tf @@ -0,0 +1,77 @@ +variable "environment" { + description = "Environment" + type = "string" +} + +# Global DataDog +variable "delay" { + description = "Delay in seconds for the metric evaluation" + default = 15 +} + +variable "message" { + description = "Message sent when an alert is triggered" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +# PostgreSQL specific +################################## +### PostgreSQL connections ### +################################## + +variable "postgresql_connection_threshold_critical" { + default = 80 + description = "Maximum critical acceptable percent of connections" +} + +variable "postgresql_connection_threshold_warning" { + default = 70 + description = "Maximum warning acceptable percent of connections" +} + +variable "postgresql_connection_silenced" { + description = "Groups to mute PostgreSQL connection monitor" + type = "map" + default = {} +} + +variable "postgresql_connection_message" { + description = "Custom message for PostgreSQL connection monitor" + type = "string" + default = "" +} + +############################ +### PostgreSQL locks ### +############################ + +variable "postgresql_lock_threshold_critical" { + default = 99 + description = "Maximum critical acceptable number of locks" +} + +variable "postgresql_lock_threshold_warning" { + default = 70 + description = "Maximum warning acceptable number of locks" +} + +variable "posgresql_lock_silenced" { + description = "Groups to mute PostgreSQL lock monitor" + type = "map" + default = {} +} + +variable "postgresql_lock_message" { + description = "Custom message for PostgreSQL lock monitor" + type = "string" + default = "" +} diff --git a/databases/postgresql/mon-postgresql.tf b/databases/postgresql/mon-postgresql.tf new file mode 100644 index 0000000..d2778d7 --- /dev/null +++ b/databases/postgresql/mon-postgresql.tf @@ -0,0 +1,67 @@ +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_postgres:enabled,db_env:%s", var.environment) : "${var.filter_tags_custom}"}" + } +} + +resource "datadog_monitor" "postgresql_connection_too_high" { + name = "[${var.environment}] PostgreSQL Connections {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.postgresql_connection_message, var.message)}" + type = "metric alert" + + query = < ${var.postgresql_connection_threshold_critical} + EOF + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + thresholds { + warning = "${var.postgresql_connection_threshold_warning}" + critical = "${var.postgresql_connection_threshold_critical}" + } + + notify_no_data = true + renotify_interval = 0 + require_full_window = true + timeout_h = 0 + include_tags = true + + silenced = "${var.postgresql_connection_silenced}" + + tags = ["env:${var.environment}", "resource:postgresql"] +} + +resource "datadog_monitor" "postgresql_too_many_locks" { + name = "[${var.environment}] PostgreSQL too many locks {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" + message = "${coalesce(var.postgresql_lock_message, var.message)}" + type = "metric alert" + + query = < ${var.postgresql_lock_threshold_critical} + EOF + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + thresholds { + warning = "${var.postgresql_lock_threshold_warning}" + critical = "${var.postgresql_lock_threshold_critical}" + } + + notify_no_data = true + renotify_interval = 0 + require_full_window = true + timeout_h = 0 + include_tags = true + + silenced = "${var.posgresql_lock_silenced}" + + tags = ["env:${var.environment}", "resource:postgresql"] +} From 46e3fb1624d08a2849fd88579b23badef2138579 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 13:36:49 +0200 Subject: [PATCH 02/14] MON-105 rename terraform file according our convention --- .../postgresql/{mon-postgresql.tf => monitors-postgresql.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename databases/postgresql/{mon-postgresql.tf => monitors-postgresql.tf} (100%) diff --git a/databases/postgresql/mon-postgresql.tf b/databases/postgresql/monitors-postgresql.tf similarity index 100% rename from databases/postgresql/mon-postgresql.tf rename to databases/postgresql/monitors-postgresql.tf From c26741ebc9c203e3499f475d77668bbb215c9d45 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 13:37:47 +0200 Subject: [PATCH 03/14] MON-105 move postgresql to database directory --- {databases => database}/postgresql/README.md | 0 {databases => database}/postgresql/inputs.tf | 0 {databases => database}/postgresql/monitors-postgresql.tf | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {databases => database}/postgresql/README.md (100%) rename {databases => database}/postgresql/inputs.tf (100%) rename {databases => database}/postgresql/monitors-postgresql.tf (100%) diff --git a/databases/postgresql/README.md b/database/postgresql/README.md similarity index 100% rename from databases/postgresql/README.md rename to database/postgresql/README.md diff --git a/databases/postgresql/inputs.tf b/database/postgresql/inputs.tf similarity index 100% rename from databases/postgresql/inputs.tf rename to database/postgresql/inputs.tf diff --git a/databases/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf similarity index 100% rename from databases/postgresql/monitors-postgresql.tf rename to database/postgresql/monitors-postgresql.tf From abab6aa763550a0ef30186818c42de7672ba45d2 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:13:52 +0200 Subject: [PATCH 04/14] MON-105 add enabled feature --- database/postgresql/README.md | 4 ++-- database/postgresql/inputs.tf | 16 ++++++++++++++-- database/postgresql/monitors-postgresql.tf | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/database/postgresql/README.md b/database/postgresql/README.md index e96d886..d5228e0 100644 --- a/database/postgresql/README.md +++ b/database/postgresql/README.md @@ -31,11 +31,11 @@ Inputs | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | postgresql_connection_threshold_critical | Maximum critical acceptable percent of connections | string | `80` | no | | postgresql_connection_threshold_warning | Maximum warning acceptable percent of connections | string | `70` | no | -| postgresql_connection_silenced | Groups to mute PostgreSQL connection monitor | map | `` | no | +| postgresql_connection_silenced | Groups to mute for PostgreSQL connection monitor | map | `` | no | | postgresql_connection_message | Custom message for PostgreSQL connection monitor | string | `` | no | | postgresql_thread_threshold_critical | Maximum critical acceptable number of threads | string | `500` | no | | postgresql_thread_threshold_warning | Maximum warning acceptable number of threads | string | `400` | no | -| postgresql_thread_silenced | Groups to mute PostgreSQL threads monitor | map | `` | no | +| postgresql_thread_silenced | Groups to mute for PostgreSQL threads monitor | map | `` | no | | postgresql_thread_message | Custom message for PostgreSQL thread monitor | string | `` | no | | message | Message sent when a monitor is triggered | string | - | yes | diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index 9c884f5..a6eecbb 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -39,11 +39,17 @@ variable "postgresql_connection_threshold_warning" { } variable "postgresql_connection_silenced" { - description = "Groups to mute PostgreSQL connection monitor" + description = "Groups to mute for PostgreSQL connection monitor" type = "map" default = {} } +variable "postgresql_connection_enabled" { + description = "Flag to enable PostgreSQL connection monitor" + type = "string" + default = "true" +} + variable "postgresql_connection_message" { description = "Custom message for PostgreSQL connection monitor" type = "string" @@ -65,11 +71,17 @@ variable "postgresql_lock_threshold_warning" { } variable "posgresql_lock_silenced" { - description = "Groups to mute PostgreSQL lock monitor" + description = "Groups to mute for PostgreSQL lock monitor" type = "map" default = {} } +variable "postgresql_lock_enabled" { + description = "Flag to enable PostgreSQL lock monitor" + type = "string" + default = "true" +} + variable "postgresql_lock_message" { description = "Custom message for PostgreSQL lock monitor" type = "string" diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index d2778d7..090edfa 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -7,6 +7,7 @@ data "template_file" "filter" { } resource "datadog_monitor" "postgresql_connection_too_high" { + count = "${var.postgresql_connection_enabled ? 1 : 0}" name = "[${var.environment}] PostgreSQL Connections {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.postgresql_connection_message, var.message)}" type = "metric alert" @@ -37,6 +38,7 @@ resource "datadog_monitor" "postgresql_connection_too_high" { } resource "datadog_monitor" "postgresql_too_many_locks" { + count = "${var.postgresql_lock_enabled ? 1 : 0}" name = "[${var.environment}] PostgreSQL too many locks {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.postgresql_lock_message, var.message)}" type = "metric alert" From 2a64727e81291f419dec9d612348218b7f5aa077 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:15:15 +0200 Subject: [PATCH 05/14] MON-105 split delay variables --- database/postgresql/inputs.tf | 7 ++++++- database/postgresql/monitors-postgresql.tf | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index a6eecbb..8b44746 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -4,11 +4,16 @@ variable "environment" { } # Global DataDog -variable "delay" { +variable "evaluation_delay" { description = "Delay in seconds for the metric evaluation" default = 15 } +variable "new_host_delay" { + description = "Delay in seconds for the metric evaluation" + default = 300 +} + variable "message" { description = "Message sent when an alert is triggered" } diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index 090edfa..515c695 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -18,8 +18,8 @@ resource "datadog_monitor" "postgresql_connection_too_high" { ) * 100 > ${var.postgresql_connection_threshold_critical} EOF - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" thresholds { warning = "${var.postgresql_connection_threshold_warning}" @@ -49,8 +49,8 @@ resource "datadog_monitor" "postgresql_too_many_locks" { ) > ${var.postgresql_lock_threshold_critical} EOF - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" thresholds { warning = "${var.postgresql_lock_threshold_warning}" From c049dc07f221cfedad430e4c0aaa4f675af6c324 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:42:54 +0200 Subject: [PATCH 06/14] MON-105 use filter tags modules --- database/postgresql/modules.tf | 8 ++++++++ database/postgresql/monitors-postgresql.tf | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 database/postgresql/modules.tf diff --git a/database/postgresql/modules.tf b/database/postgresql/modules.tf new file mode 100644 index 0000000..8d42f36 --- /dev/null +++ b/database/postgresql/modules.tf @@ -0,0 +1,8 @@ +module "filter-tags" { + source = "../../common/filter-tags" + + environment = "${var.environment}" + resource = "postgresql" + filter_tags_use_defaults = "${var.filter_tags_use_defaults}" + filter_tags_custom = "${var.filter_tags_custom}" +} diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index 515c695..bcd92ec 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -14,7 +14,7 @@ resource "datadog_monitor" "postgresql_connection_too_high" { query = < ${var.postgresql_connection_threshold_critical} EOF @@ -45,7 +45,7 @@ resource "datadog_monitor" "postgresql_too_many_locks" { query = < ${var.postgresql_lock_threshold_critical} EOF From 60bde176e61d454d5e1555a0818fa8f97288eb47 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:54:21 +0200 Subject: [PATCH 07/14] MON-105 change grouping tag --- database/postgresql/monitors-postgresql.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index bcd92ec..46c770f 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -14,7 +14,7 @@ resource "datadog_monitor" "postgresql_connection_too_high" { query = < ${var.postgresql_connection_threshold_critical} EOF @@ -45,7 +45,7 @@ resource "datadog_monitor" "postgresql_too_many_locks" { query = < ${var.postgresql_lock_threshold_critical} EOF From 79de95f9556bfecc8db3236b2d85c2daa4305091 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:54:38 +0200 Subject: [PATCH 08/14] MON-105 delete useless template file --- database/postgresql/monitors-postgresql.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index 46c770f..ca54b47 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -1,11 +1,3 @@ -data "template_file" "filter" { - template = "$${filter}" - - vars { - filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_postgres:enabled,db_env:%s", var.environment) : "${var.filter_tags_custom}"}" - } -} - resource "datadog_monitor" "postgresql_connection_too_high" { count = "${var.postgresql_connection_enabled ? 1 : 0}" name = "[${var.environment}] PostgreSQL Connections {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" From 8bb14e6792b3d963a3382114e9d65a2476ff301e Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 14:56:13 +0200 Subject: [PATCH 09/14] MON-105 change notify no data to false --- database/postgresql/monitors-postgresql.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index ca54b47..e5bf424 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -18,7 +18,7 @@ resource "datadog_monitor" "postgresql_connection_too_high" { critical = "${var.postgresql_connection_threshold_critical}" } - notify_no_data = true + notify_no_data = false renotify_interval = 0 require_full_window = true timeout_h = 0 @@ -49,7 +49,7 @@ resource "datadog_monitor" "postgresql_too_many_locks" { critical = "${var.postgresql_lock_threshold_critical}" } - notify_no_data = true + notify_no_data = false renotify_interval = 0 require_full_window = true timeout_h = 0 From ee5a4cf236e9cbdd749ebd66a145ffa38fd19177 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 15:05:50 +0200 Subject: [PATCH 10/14] MON-105 variable for timeframe and time aggregator --- database/postgresql/inputs.tf | 27 ++++++++++++++++++++++ database/postgresql/monitors-postgresql.tf | 10 ++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index 8b44746..68f2a69 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -61,6 +61,19 @@ variable "postgresql_connection_message" { default = "" } +variable "postgresql_connection_time_aggregator" { + description = "Monitor time aggregator for PostgreSQL connection monitor [available values: min, max or avg]" + type = "string" + default = "avg" +} + +variable "postgresql_connection_timeframe" { + description = "Monitor timeframe for PostgreSQL connection monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_15m" +} + + ############################ ### PostgreSQL locks ### ############################ @@ -92,3 +105,17 @@ variable "postgresql_lock_message" { type = "string" default = "" } + +variable "postgresql_lock_time_aggregator" { + description = "Monitor time aggregator for PostgreSQL lock monitor [available values: min, max or avg]" + type = "string" + default = "min" +} + +variable "postgresql_lock_timeframe" { + description = "Monitor timeframe for PostgreSQL lock monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} + + diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index e5bf424..30fac1c 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -5,9 +5,9 @@ resource "datadog_monitor" "postgresql_connection_too_high" { type = "metric alert" query = < ${var.postgresql_connection_threshold_critical} + * 100 > ${var.postgresql_connection_threshold_critical} EOF evaluation_delay = "${var.evaluation_delay}" @@ -36,9 +36,9 @@ resource "datadog_monitor" "postgresql_too_many_locks" { type = "metric alert" query = < ${var.postgresql_lock_threshold_critical} + ${var.postgresql_lock_time_aggregator}(${var.postgresql_lock_timeframe}): + default(avg:postgresql.locks${module.filter-tags.query_alert} by {server}, 0) + > ${var.postgresql_lock_threshold_critical} EOF evaluation_delay = "${var.evaluation_delay}" From ff338dad609afb2725c2c51ce531f7b1bda97405 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 15:20:20 +0200 Subject: [PATCH 11/14] MON-105 update tagging convention and add extra tags --- database/postgresql/inputs.tf | 12 ++++++++++++ database/postgresql/monitors-postgresql.tf | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index 68f2a69..a875f3d 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -55,6 +55,12 @@ variable "postgresql_connection_enabled" { default = "true" } +variable "postgresql_connection_extra_tags" { + description = "Extra tags for PostgreSQL connection connects monitor" + type = "list" + default = [] +} + variable "postgresql_connection_message" { description = "Custom message for PostgreSQL connection monitor" type = "string" @@ -100,6 +106,12 @@ variable "postgresql_lock_enabled" { default = "true" } +variable "postgresql_lock_extra_tags" { + description = "Extra tags for PostgreSQL lock connects monitor" + type = "list" + default = [] +} + variable "postgresql_lock_message" { description = "Custom message for PostgreSQL lock monitor" type = "string" diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index 30fac1c..fb47486 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -26,7 +26,7 @@ resource "datadog_monitor" "postgresql_connection_too_high" { silenced = "${var.postgresql_connection_silenced}" - tags = ["env:${var.environment}", "resource:postgresql"] + tags = ["env:${var.environment}", "type:database", "provider:postgres", "resource:postgresql", "team:claranet", "created-by:terraform", "${var.postgresql_connection_extra_tags}"] } resource "datadog_monitor" "postgresql_too_many_locks" { @@ -57,5 +57,5 @@ resource "datadog_monitor" "postgresql_too_many_locks" { silenced = "${var.posgresql_lock_silenced}" - tags = ["env:${var.environment}", "resource:postgresql"] + tags = ["env:${var.environment}", "type:database", "provider:postgres", "resource:postgresql", "team:claranet", "created-by:terraform", "${var.postgresql_lock_extra_tags}"] } From 8b99e301327136b3dad449b5045a3184516c6d30 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 15:21:29 +0200 Subject: [PATCH 12/14] MON-105 add output and autoupdate --- README.md | 1 + database/postgresql/README.md | 60 +++++++++++++++++++++------------- database/postgresql/inputs.tf | 3 -- database/postgresql/outputs.tf | 9 +++++ 4 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 database/postgresql/outputs.tf diff --git a/README.md b/README.md index ef5bd6f..ed6b246 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi - [elasticsearch](https://bitbucket.org/morea/terraform.feature.datadog/src/master/database/elasticsearch/) - [mongodb](https://bitbucket.org/morea/terraform.feature.datadog/src/master/database/mongodb/) - [mysql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/database/mysql/) + - [postgresql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/database/postgresql/) - [redis](https://bitbucket.org/morea/terraform.feature.datadog/src/master/database/redis/) - [middleware](https://bitbucket.org/morea/terraform.feature.datadog/src/master/middleware/) - [apache](https://bitbucket.org/morea/terraform.feature.datadog/src/master/middleware/apache/) diff --git a/database/postgresql/README.md b/database/postgresql/README.md index d5228e0..63278b0 100644 --- a/database/postgresql/README.md +++ b/database/postgresql/README.md @@ -1,43 +1,57 @@ -PostgreSQL DataDog monitors -====================== +# DATABASE POSTGRESQL DataDog monitors -How to use this module ----------------------- +## How to use this module ``` -module "datadog-monitors-postgresql" { - source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//databases/postgresql?ref={revision}" +module "datadog-monitors-database-postgresql" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//database/postgresql?ref={revision}" environment = "${var.environment}" - message = "${module.datadog-message-alerting.alerting-message}" + message = "${module.datadog-message-alerting.alerting-message}" } ``` -Purpose -------- -Creates DataDog monitors with the following checks : -* PostgreSQL not enough available connections -* PostgreSQL too many locks +## Purpose -Inputs ------- +Creates DataDog monitors with the following checks: + +- PostgreSQL Connections +- PostgreSQL too many locks + +## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| delay | Delay in seconds for the metric evaluation | string | `15` | no | | environment | Environment | string | - | yes | -| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `15` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when an alert is triggered | string | - | yes | +| new_host_delay | Delay in seconds for the metric evaluation | string | `300` | no | +| posgresql_lock_silenced | Groups to mute for PostgreSQL lock monitor | map | `` | no | +| postgresql_connection_enabled | Flag to enable PostgreSQL connection monitor | string | `true` | no | +| postgresql_connection_extra_tags | Extra tags for PostgreSQL connection connects monitor | list | `` | no | +| postgresql_connection_message | Custom message for PostgreSQL connection monitor | string | `` | no | +| postgresql_connection_silenced | Groups to mute for PostgreSQL connection monitor | map | `` | no | | postgresql_connection_threshold_critical | Maximum critical acceptable percent of connections | string | `80` | no | | postgresql_connection_threshold_warning | Maximum warning acceptable percent of connections | string | `70` | no | -| postgresql_connection_silenced | Groups to mute for PostgreSQL connection monitor | map | `` | no | -| postgresql_connection_message | Custom message for PostgreSQL connection monitor | string | `` | no | -| postgresql_thread_threshold_critical | Maximum critical acceptable number of threads | string | `500` | no | -| postgresql_thread_threshold_warning | Maximum warning acceptable number of threads | string | `400` | no | -| postgresql_thread_silenced | Groups to mute for PostgreSQL threads monitor | map | `` | no | -| postgresql_thread_message | Custom message for PostgreSQL thread monitor | string | `` | no | -| message | Message sent when a monitor is triggered | string | - | yes | +| postgresql_connection_time_aggregator | Monitor time aggregator for PostgreSQL connection monitor [available values: min, max or avg] | string | `avg` | no | +| postgresql_connection_timeframe | Monitor timeframe for PostgreSQL connection monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_15m` | no | +| postgresql_lock_enabled | Flag to enable PostgreSQL lock monitor | string | `true` | no | +| postgresql_lock_extra_tags | Extra tags for PostgreSQL lock connects monitor | list | `` | no | +| postgresql_lock_message | Custom message for PostgreSQL lock monitor | string | `` | no | +| postgresql_lock_threshold_critical | Maximum critical acceptable number of locks | string | `99` | no | +| postgresql_lock_threshold_warning | Maximum warning acceptable number of locks | string | `70` | no | +| postgresql_lock_time_aggregator | Monitor time aggregator for PostgreSQL lock monitor [available values: min, max or avg] | string | `min` | no | +| postgresql_lock_timeframe | Monitor timeframe for PostgreSQL lock monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| postgresql_connection_too_high_id | id for monitor postgresql_connection_too_high | +| postgresql_too_many_locks_id | id for monitor postgresql_too_many_locks | Related documentation --------------------- diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index a875f3d..ca0dde6 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -79,7 +79,6 @@ variable "postgresql_connection_timeframe" { default = "last_15m" } - ############################ ### PostgreSQL locks ### ############################ @@ -129,5 +128,3 @@ variable "postgresql_lock_timeframe" { type = "string" default = "last_5m" } - - diff --git a/database/postgresql/outputs.tf b/database/postgresql/outputs.tf new file mode 100644 index 0000000..79532c6 --- /dev/null +++ b/database/postgresql/outputs.tf @@ -0,0 +1,9 @@ +output "postgresql_connection_too_high_id" { + description = "id for monitor postgresql_connection_too_high" + value = "${datadog_monitor.postgresql_connection_too_high.*.id}" +} + +output "postgresql_too_many_locks_id" { + description = "id for monitor postgresql_too_many_locks" + value = "${datadog_monitor.postgresql_too_many_locks.*.id}" +} From b2bcc58b003af4730c6ce27999c0761cdc64a500 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 16:36:40 +0200 Subject: [PATCH 13/14] MON-105 change tag to match existing --- database/postgresql/modules.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/postgresql/modules.tf b/database/postgresql/modules.tf index 8d42f36..19fcbdb 100644 --- a/database/postgresql/modules.tf +++ b/database/postgresql/modules.tf @@ -2,7 +2,7 @@ module "filter-tags" { source = "../../common/filter-tags" environment = "${var.environment}" - resource = "postgresql" + resource = "postgres" filter_tags_use_defaults = "${var.filter_tags_use_defaults}" filter_tags_custom = "${var.filter_tags_custom}" } From c2dfc0ed24710538f593ccd63c8898a95c48035e Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 18 Sep 2018 17:05:14 +0200 Subject: [PATCH 14/14] MON-105 add service check --- database/mysql/monitors-mysql.tf | 2 +- database/postgresql/README.md | 8 +++++ database/postgresql/inputs.tf | 40 ++++++++++++++++++++++ database/postgresql/monitors-postgresql.tf | 31 +++++++++++++++++ database/postgresql/outputs.tf | 5 +++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/database/mysql/monitors-mysql.tf b/database/mysql/monitors-mysql.tf index 14814b3..82133a1 100644 --- a/database/mysql/monitors-mysql.tf +++ b/database/mysql/monitors-mysql.tf @@ -26,7 +26,7 @@ resource "datadog_monitor" "mysql_availability" { silenced = "${var.mysql_availability_silenced}" - tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_connection_extra_tags}"] + tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_availability_extra_tags}"] } resource "datadog_monitor" "mysql_connection" { diff --git a/database/postgresql/README.md b/database/postgresql/README.md index 63278b0..21b4dc0 100644 --- a/database/postgresql/README.md +++ b/database/postgresql/README.md @@ -17,6 +17,7 @@ module "datadog-monitors-database-postgresql" { Creates DataDog monitors with the following checks: - PostgreSQL Connections +- PostgreSQL server does not respond - PostgreSQL too many locks ## Inputs @@ -30,6 +31,12 @@ Creates DataDog monitors with the following checks: | message | Message sent when an alert is triggered | string | - | yes | | new_host_delay | Delay in seconds for the metric evaluation | string | `300` | no | | posgresql_lock_silenced | Groups to mute for PostgreSQL lock monitor | map | `` | no | +| postgresql_availability_enabled | Flag to enable PostgreSQL availability monitor | string | `true` | no | +| postgresql_availability_extra_tags | Extra tags for PostgreSQL availability monitor | list | `` | no | +| postgresql_availability_message | Custom message for PostgreSQL availability monitor | string | `` | no | +| postgresql_availability_no_data_timeframe | PostgreSQL availability monitor no data timeframe | string | `10` | no | +| postgresql_availability_silenced | Groups to mute for PostgreSQL availability monitor | map | `` | no | +| postgresql_availability_threshold_warning | PostgreSQL availability monitor (warning threshold) | string | `3` | no | | postgresql_connection_enabled | Flag to enable PostgreSQL connection monitor | string | `true` | no | | postgresql_connection_extra_tags | Extra tags for PostgreSQL connection connects monitor | list | `` | no | | postgresql_connection_message | Custom message for PostgreSQL connection monitor | string | `` | no | @@ -50,6 +57,7 @@ Creates DataDog monitors with the following checks: | Name | Description | |------|-------------| +| postgresql_availability_id | id for monitor postgresql_availability | | postgresql_connection_too_high_id | id for monitor postgresql_connection_too_high | | postgresql_too_many_locks_id | id for monitor postgresql_too_many_locks | diff --git a/database/postgresql/inputs.tf b/database/postgresql/inputs.tf index ca0dde6..9386a5e 100644 --- a/database/postgresql/inputs.tf +++ b/database/postgresql/inputs.tf @@ -29,6 +29,46 @@ variable "filter_tags_custom" { } # PostgreSQL specific +################################## +### PostgreSQL availability ### +################################## + +variable "postgresql_availability_silenced" { + description = "Groups to mute for PostgreSQL availability monitor" + type = "map" + default = {} +} + +variable "postgresql_availability_enabled" { + description = "Flag to enable PostgreSQL availability monitor" + type = "string" + default = "true" +} + +variable "postgresql_availability_extra_tags" { + description = "Extra tags for PostgreSQL availability monitor" + type = "list" + default = [] +} + +variable "postgresql_availability_message" { + description = "Custom message for PostgreSQL availability monitor" + type = "string" + default = "" +} + +variable "postgresql_availability_threshold_warning" { + description = "PostgreSQL availability monitor (warning threshold)" + type = "string" + default = 3 +} + +variable "postgresql_availability_no_data_timeframe" { + description = "PostgreSQL availability monitor no data timeframe" + type = "string" + default = 10 +} + ################################## ### PostgreSQL connections ### ################################## diff --git a/database/postgresql/monitors-postgresql.tf b/database/postgresql/monitors-postgresql.tf index fb47486..a37ef7a 100644 --- a/database/postgresql/monitors-postgresql.tf +++ b/database/postgresql/monitors-postgresql.tf @@ -1,3 +1,34 @@ +resource "datadog_monitor" "postgresql_availability" { + count = "${var.postgresql_availability_enabled ? 1 : 0}" + name = "[${var.environment}] PostgreSQL server does not respond" + message = "${coalesce(var.postgresql_availability_message, var.message)}" + + type = "service check" + + query = <