From e6fc224d734f6fb679b6d0d1bc6ca97b9d5ff5e2 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Thu, 5 Jul 2018 16:00:14 +0200 Subject: [PATCH 1/7] MON-226 Basic README --- cloud/gcp/pubsub/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 cloud/gcp/pubsub/README.md diff --git a/cloud/gcp/pubsub/README.md b/cloud/gcp/pubsub/README.md new file mode 100644 index 0000000..a7cc282 --- /dev/null +++ b/cloud/gcp/pubsub/README.md @@ -0,0 +1,29 @@ +How to use this module +---------------------- + +``` +module "datadog-monitors-gcp-pubsub" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/gcp/pubsub?ref={revision}" + + project_id = "${var.gcp_project_id}" + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` + +Purpose +------- +Creates DataDog monitors with the following checks : + +* + +Inputs +------ + + +Related documentation +------------ + +* [GCP Pub/Sub Metrics](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub) +* [Datadog GCP Pub/Sub integration](https://docs.datadoghq.com/integrations/google_cloud_pubsub/) From 687b04876db90488c3913ab5f99e023d5cb9d3ad Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Wed, 1 Aug 2018 14:40:24 +0200 Subject: [PATCH 2/7] MON-226 Basic monitors --- cloud/gcp/pubsub/inputs.tf | 118 ++++++++++++++++++++++++++++ cloud/gcp/pubsub/monitors-pubsub.tf | 99 +++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 cloud/gcp/pubsub/inputs.tf create mode 100644 cloud/gcp/pubsub/monitors-pubsub.tf diff --git a/cloud/gcp/pubsub/inputs.tf b/cloud/gcp/pubsub/inputs.tf new file mode 100644 index 0000000..eeb6013 --- /dev/null +++ b/cloud/gcp/pubsub/inputs.tf @@ -0,0 +1,118 @@ +# +# Datadog global variables +# +variable "environment" { + description = "Architecture environment" + type = "string" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +variable "message" { + description = "Message sent when a monitor is triggered" +} + +variable "delay" { + description = "Delay in seconds for the metric evaluation" + default = 900 +} + +# +# Filter variables +# +variable "project_id" { + type = "string" + description = "ID of the GCP Project" +} + +# +# Sending Operations Count +# +variable "sending_operations_count_message" { + description = "Custom message for the GCP Pub/Sub Sending Operations Count monitor" + type = "string" + default = "" +} + +variable "sending_operations_count_time_aggregator" { + description = "Timeframe for the GCP Pub/Sub Sending Operations Count monitor" + type = "string" + default = "sum" +} + +variable "sending_operations_count_timeframe" { + description = "Timeframe for the GCP Pub/Sub Sending Operations Count monitor" + type = "string" + default = "last_30m" +} + +variable "sending_operations_count_threshold_critical" { + description = "Critical threshold" + type = "string" + default = 0 +} + +variable "sending_operations_count_silenced" { + description = "Groups to mute for GCP Pub/Sub Sending Operations Count monitor" + type = "map" + default = {} +} + +variable "sending_operations_count_extra_tags" { + description = "Extra tags for GCP Pub/Sub Sending Operations Count monitor" + type = "list" + default = [] +} + +# +# Unavailable Sending Operations Count +# +variable "unavailable_sending_operations_count_message" { + description = "Custom message for the GCP Pub/Sub Unavailable Sending Operations Count monitor" + type = "string" + default = "" +} + +variable "unavailable_sending_operations_count_time_aggregator" { + description = "Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor" + type = "string" + default = "sum" +} + +variable "unavailable_sending_operations_count_timeframe" { + description = "Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor" + type = "string" + default = "last_10m" +} + +variable "unavailable_sending_operations_count_threshold_warning" { + description = "Warning threshold" + type = "string" + default = 2 +} + +variable "unavailable_sending_operations_count_threshold_critical" { + description = "Critical threshold" + type = "string" + default = 4 +} + +variable "unavailable_sending_operations_count_silenced" { + description = "Groups to mute for GCP Pub/Sub Unavailable Sending Operations Count monitor" + type = "map" + default = {} +} + +variable "unavailable_sending_operations_count_extra_tags" { + description = "Extra tags for GCP Pub/Sub Unavailable Sending Operations Count monitor" + type = "list" + default = [] +} diff --git a/cloud/gcp/pubsub/monitors-pubsub.tf b/cloud/gcp/pubsub/monitors-pubsub.tf new file mode 100644 index 0000000..6ea5b34 --- /dev/null +++ b/cloud/gcp/pubsub/monitors-pubsub.tf @@ -0,0 +1,99 @@ +# +# FILTER +# +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? + format("project_id:%s", var.project_id) : + "${var.filter_tags_custom}"}" + } +} + +# +# Sending Operations Count +# +resource "datadog_monitor" "sending_operations_count" { + name = "[${var.environment}] GCP pubsub sending messages operations {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" + message = "${coalesce(var.sending_operations_count_message, var.message)}" + + type = "query alert" + + query = <= ${var.unavailable_sending_operations_count_threshold_critical} +EOF + + thresholds { + warning = "${var.unavailable_sending_operations_count_threshold_warning}" + critical = "${var.unavailable_sending_operations_count_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + no_data_timeframe = 25 + require_full_window = false + notify_no_data = true + renotify_interval = 0 + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.unavailable_sending_operations_count_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "resource:pubsub", + "env:${var.environment}", + "created_by:terraform", + "${var.unavailable_sending_operations_count_extra_tags}", + ] +} From cd0feccf5932a46670a552ab82fd3626805e5187 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Wed, 1 Aug 2018 14:42:23 +0200 Subject: [PATCH 3/7] MON-226 Auto readme and outputs for the basic monitors --- README.md | 1 + cloud/gcp/pubsub/README.md | 48 +++++++++++++++++++++++++++++-------- cloud/gcp/pubsub/outputs.tf | 9 +++++++ 3 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 cloud/gcp/pubsub/outputs.tf diff --git a/README.md b/README.md index 72455ee..db806cc 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi - [cloud-sql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/) - [common](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/common/) - [mysql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/mysql/) + - [pubsub](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/pubsub/) - [common](https://bitbucket.org/morea/terraform.feature.datadog/src/master/common/) - [alerting-message](https://bitbucket.org/morea/terraform.feature.datadog/src/master/common/alerting-message/) - [filter-tags](https://bitbucket.org/morea/terraform.feature.datadog/src/master/common/filter-tags/) diff --git a/cloud/gcp/pubsub/README.md b/cloud/gcp/pubsub/README.md index a7cc282..73e1076 100644 --- a/cloud/gcp/pubsub/README.md +++ b/cloud/gcp/pubsub/README.md @@ -1,26 +1,54 @@ -How to use this module ----------------------- +# CLOUD GCP PUBSUB DataDog monitors + +## How to use this module ``` -module "datadog-monitors-gcp-pubsub" { +module "datadog-monitors-cloud-gcp-pubsub" { source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/gcp/pubsub?ref={revision}" - project_id = "${var.gcp_project_id}" environment = "${var.environment}" message = "${module.datadog-message-alerting.alerting-message}" } ``` -Purpose -------- -Creates DataDog monitors with the following checks : +## Purpose -* +Creates DataDog monitors with the following checks: -Inputs ------- +- GCP pubsub sending messages operations +- GCP pubsub sending messages with result unavailable +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| delay | Delay in seconds for the metric evaluation | string | `900` | no | +| environment | Architecture environment | string | - | yes | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when a monitor is triggered | string | - | yes | +| project_id | ID of the GCP Project | string | - | yes | +| sending_operations_count_extra_tags | Extra tags for GCP Pub/Sub Sending Operations Count monitor | list | `` | no | +| sending_operations_count_message | Custom message for the GCP Pub/Sub Sending Operations Count monitor | string | `` | no | +| sending_operations_count_silenced | Groups to mute for GCP Pub/Sub Sending Operations Count monitor | map | `` | no | +| sending_operations_count_threshold_critical | Critical threshold | string | `0` | no | +| sending_operations_count_time_aggregator | Timeframe for the GCP Pub/Sub Sending Operations Count monitor | string | `sum` | no | +| sending_operations_count_timeframe | Timeframe for the GCP Pub/Sub Sending Operations Count monitor | string | `last_30m` | no | +| unavailable_sending_operations_count_extra_tags | Extra tags for GCP Pub/Sub Unavailable Sending Operations Count monitor | list | `` | no | +| unavailable_sending_operations_count_message | Custom message for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `` | no | +| unavailable_sending_operations_count_silenced | Groups to mute for GCP Pub/Sub Unavailable Sending Operations Count monitor | map | `` | no | +| unavailable_sending_operations_count_threshold_critical | Critical threshold | string | `4` | no | +| unavailable_sending_operations_count_threshold_warning | Warning threshold | string | `2` | no | +| unavailable_sending_operations_count_time_aggregator | Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `sum` | no | +| unavailable_sending_operations_count_timeframe | Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `last_10m` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| sending_operations_count_id | id for monitor sending_operations_count | +| unavailable_sending_operations_count_id | id for monitor unavailable_sending_operations_count | Related documentation ------------ diff --git a/cloud/gcp/pubsub/outputs.tf b/cloud/gcp/pubsub/outputs.tf new file mode 100644 index 0000000..869f4b8 --- /dev/null +++ b/cloud/gcp/pubsub/outputs.tf @@ -0,0 +1,9 @@ +output "sending_operations_count_id" { + description = "id for monitor sending_operations_count" + value = "${datadog_monitor.sending_operations_count.*.id}" +} + +output "unavailable_sending_operations_count_id" { + description = "id for monitor unavailable_sending_operations_count" + value = "${datadog_monitor.unavailable_sending_operations_count.*.id}" +} From 8b3f5f0cff585ba4c4e80fdc54187a9d5f945427 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Fri, 10 Aug 2018 17:28:56 +0200 Subject: [PATCH 4/7] MON-226 Removed no_data_timeframe from both monitors --- cloud/gcp/pubsub/monitors-pubsub.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloud/gcp/pubsub/monitors-pubsub.tf b/cloud/gcp/pubsub/monitors-pubsub.tf index 6ea5b34..b808452 100644 --- a/cloud/gcp/pubsub/monitors-pubsub.tf +++ b/cloud/gcp/pubsub/monitors-pubsub.tf @@ -34,7 +34,6 @@ EOF locked = false timeout_h = 0 include_tags = true - no_data_timeframe = 45 require_full_window = false notify_no_data = true renotify_interval = 0 @@ -78,7 +77,6 @@ EOF locked = false timeout_h = 0 include_tags = true - no_data_timeframe = 25 require_full_window = false notify_no_data = true renotify_interval = 0 From 509f4074f1342c7e45a971652f9d06095c1f587b Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Mon, 13 Aug 2018 16:46:51 +0200 Subject: [PATCH 5/7] MON-226 Standardize filter and split delays --- cloud/gcp/pubsub/README.md | 7 +++---- cloud/gcp/pubsub/inputs.tf | 20 ++++++-------------- cloud/gcp/pubsub/monitors-pubsub.tf | 25 ++++++------------------- 3 files changed, 15 insertions(+), 37 deletions(-) diff --git a/cloud/gcp/pubsub/README.md b/cloud/gcp/pubsub/README.md index 73e1076..2e49ece 100644 --- a/cloud/gcp/pubsub/README.md +++ b/cloud/gcp/pubsub/README.md @@ -23,12 +23,11 @@ Creates DataDog monitors with the following checks: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | -| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | -| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no | +| filter_tags | Tags used for filtering | string | `*` | no | | message | Message sent when a monitor is triggered | string | - | yes | -| project_id | ID of the GCP Project | string | - | yes | +| new_host_delay | Delay in seconds for the new host evaluation | string | `300` | no | | sending_operations_count_extra_tags | Extra tags for GCP Pub/Sub Sending Operations Count monitor | list | `` | no | | sending_operations_count_message | Custom message for the GCP Pub/Sub Sending Operations Count monitor | string | `` | no | | sending_operations_count_silenced | Groups to mute for GCP Pub/Sub Sending Operations Count monitor | map | `` | no | diff --git a/cloud/gcp/pubsub/inputs.tf b/cloud/gcp/pubsub/inputs.tf index eeb6013..626a596 100644 --- a/cloud/gcp/pubsub/inputs.tf +++ b/cloud/gcp/pubsub/inputs.tf @@ -6,13 +6,8 @@ variable "environment" { type = "string" } -variable "filter_tags_use_defaults" { - description = "Use default filter tags convention" - default = "true" -} - -variable "filter_tags_custom" { - description = "Tags used for custom filtering when filter_tags_use_defaults is false" +variable "filter_tags" { + description = "Tags used for filtering" default = "*" } @@ -20,17 +15,14 @@ variable "message" { description = "Message sent when a monitor is triggered" } -variable "delay" { +variable "evaluation_delay" { description = "Delay in seconds for the metric evaluation" default = 900 } -# -# Filter variables -# -variable "project_id" { - type = "string" - description = "ID of the GCP Project" +variable "new_host_delay" { + description = "Delay in seconds for the new host evaluation" + default = 300 } # diff --git a/cloud/gcp/pubsub/monitors-pubsub.tf b/cloud/gcp/pubsub/monitors-pubsub.tf index b808452..7ef4d35 100644 --- a/cloud/gcp/pubsub/monitors-pubsub.tf +++ b/cloud/gcp/pubsub/monitors-pubsub.tf @@ -1,16 +1,3 @@ -# -# FILTER -# -data "template_file" "filter" { - template = "$${filter}" - - vars { - filter = "${var.filter_tags_use_defaults == "true" ? - format("project_id:%s", var.project_id) : - "${var.filter_tags_custom}"}" - } -} - # # Sending Operations Count # @@ -22,7 +9,7 @@ resource "datadog_monitor" "sending_operations_count" { query = <= ${var.unavailable_sending_operations_count_threshold_critical} EOF @@ -81,8 +68,8 @@ EOF notify_no_data = true renotify_interval = 0 - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.unavailable_sending_operations_count_silenced}" From 57e3316eeb83845ef6f5e81abd58084e666052fc Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 21 Aug 2018 16:24:43 +0200 Subject: [PATCH 6/7] MON-226 Tags migrated to the new standard --- cloud/gcp/pubsub/monitors-pubsub.tf | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cloud/gcp/pubsub/monitors-pubsub.tf b/cloud/gcp/pubsub/monitors-pubsub.tf index 7ef4d35..6189794 100644 --- a/cloud/gcp/pubsub/monitors-pubsub.tf +++ b/cloud/gcp/pubsub/monitors-pubsub.tf @@ -30,14 +30,7 @@ EOF silenced = "${var.sending_operations_count_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "resource:pubsub", - "env:${var.environment}", - "created_by:terraform", - "${var.sending_operations_count_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:pubsub", "team:claranet", "created-by:terraform", "${var.sending_operations_count_extra_tags}"] } # @@ -73,12 +66,5 @@ EOF silenced = "${var.unavailable_sending_operations_count_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "resource:pubsub", - "env:${var.environment}", - "created_by:terraform", - "${var.unavailable_sending_operations_count_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:pubsub", "team:claranet", "created-by:terraform", "${var.unavailable_sending_operations_count_extra_tags}"] } From 0d20764e7844085a0f74627ab85adcfb0c846703 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Fri, 24 Aug 2018 17:44:54 +0200 Subject: [PATCH 7/7] MON-226 Descriptions fixed --- cloud/gcp/pubsub/README.md | 6 +++--- cloud/gcp/pubsub/inputs.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cloud/gcp/pubsub/README.md b/cloud/gcp/pubsub/README.md index 2e49ece..3ca819c 100644 --- a/cloud/gcp/pubsub/README.md +++ b/cloud/gcp/pubsub/README.md @@ -31,14 +31,14 @@ Creates DataDog monitors with the following checks: | sending_operations_count_extra_tags | Extra tags for GCP Pub/Sub Sending Operations Count monitor | list | `` | no | | sending_operations_count_message | Custom message for the GCP Pub/Sub Sending Operations Count monitor | string | `` | no | | sending_operations_count_silenced | Groups to mute for GCP Pub/Sub Sending Operations Count monitor | map | `` | no | -| sending_operations_count_threshold_critical | Critical threshold | string | `0` | no | +| sending_operations_count_threshold_critical | Critical threshold for the number of sending operations. | string | `0` | no | | sending_operations_count_time_aggregator | Timeframe for the GCP Pub/Sub Sending Operations Count monitor | string | `sum` | no | | sending_operations_count_timeframe | Timeframe for the GCP Pub/Sub Sending Operations Count monitor | string | `last_30m` | no | | unavailable_sending_operations_count_extra_tags | Extra tags for GCP Pub/Sub Unavailable Sending Operations Count monitor | list | `` | no | | unavailable_sending_operations_count_message | Custom message for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `` | no | | unavailable_sending_operations_count_silenced | Groups to mute for GCP Pub/Sub Unavailable Sending Operations Count monitor | map | `` | no | -| unavailable_sending_operations_count_threshold_critical | Critical threshold | string | `4` | no | -| unavailable_sending_operations_count_threshold_warning | Warning threshold | string | `2` | no | +| unavailable_sending_operations_count_threshold_critical | Critical threshold for the number of unavailable sending operations | string | `4` | no | +| unavailable_sending_operations_count_threshold_warning | Warning threshold for the number of unavailable sending operations | string | `2` | no | | unavailable_sending_operations_count_time_aggregator | Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `sum` | no | | unavailable_sending_operations_count_timeframe | Timeframe for the GCP Pub/Sub Unavailable Sending Operations Count monitor | string | `last_10m` | no | diff --git a/cloud/gcp/pubsub/inputs.tf b/cloud/gcp/pubsub/inputs.tf index 626a596..eb86ff4 100644 --- a/cloud/gcp/pubsub/inputs.tf +++ b/cloud/gcp/pubsub/inputs.tf @@ -47,7 +47,7 @@ variable "sending_operations_count_timeframe" { } variable "sending_operations_count_threshold_critical" { - description = "Critical threshold" + description = "Critical threshold for the number of sending operations." type = "string" default = 0 } @@ -86,13 +86,13 @@ variable "unavailable_sending_operations_count_timeframe" { } variable "unavailable_sending_operations_count_threshold_warning" { - description = "Warning threshold" + description = "Warning threshold for the number of unavailable sending operations" type = "string" default = 2 } variable "unavailable_sending_operations_count_threshold_critical" { - description = "Critical threshold" + description = "Critical threshold for the number of unavailable sending operations" type = "string" default = 4 }