From f168c64e8c7052d10407285bc159686370367433 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Thu, 28 Jun 2018 19:21:25 +0200 Subject: [PATCH 1/9] MON-230 First version of monitors and Readme --- cloud/gcp/big-query/README.md | 98 +++++++++++++ cloud/gcp/big-query/inputs.tf | 34 +++++ cloud/gcp/big-query/monitors-big-query.tf | 12 ++ cloud/gcp/big-query/query.tf | 138 +++++++++++++++++++ cloud/gcp/big-query/scanned_bytes.tf | 70 ++++++++++ cloud/gcp/big-query/scanned_bytes_billed.tf | 70 ++++++++++ cloud/gcp/big-query/slots.tf | 67 +++++++++ cloud/gcp/big-query/storage.tf | 71 ++++++++++ cloud/gcp/big-query/table_count.tf | 71 ++++++++++ cloud/gcp/big-query/uploaded_bytes.tf | 71 ++++++++++ cloud/gcp/big-query/uploaded_bytes_billed.tf | 71 ++++++++++ 11 files changed, 773 insertions(+) create mode 100644 cloud/gcp/big-query/README.md create mode 100644 cloud/gcp/big-query/inputs.tf create mode 100644 cloud/gcp/big-query/monitors-big-query.tf create mode 100644 cloud/gcp/big-query/query.tf create mode 100644 cloud/gcp/big-query/scanned_bytes.tf create mode 100644 cloud/gcp/big-query/scanned_bytes_billed.tf create mode 100644 cloud/gcp/big-query/slots.tf create mode 100644 cloud/gcp/big-query/storage.tf create mode 100644 cloud/gcp/big-query/table_count.tf create mode 100644 cloud/gcp/big-query/uploaded_bytes.tf create mode 100644 cloud/gcp/big-query/uploaded_bytes_billed.tf diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md new file mode 100644 index 0000000..ba831fb --- /dev/null +++ b/cloud/gcp/big-query/README.md @@ -0,0 +1,98 @@ +GCP Big Query +============================== + +How to use this module +---------------------- + +``` +module "datadog-monitors-gcp-big-query" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/gcp/big-query?ref={revision}" + + project_id = "${var.gcp_project_id}" + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` + +Purpose +------- +Creates DataDog monitors with the following checks : + +* Query Concurrent Queries +* Query Execution Times (muted by default) +* Query Scanned Bytes (muted by default) +* Query Scanned Bytes Billed (muted by default) + +* Slots Available + +* Storage Stored Bytes (muted by default) +* Storage Table Count per Dataset (muted by default) +* Storage Uploaded Bytes (muted by default) +* Storage Uploaded Bytes Billed (muted by default) + + +Useful links +------------ + +* [GCP Big Query monitoring](https://cloud.google.com/bigquery/docs/monitoring) +* [Datadog Integration for GCP Big Query](https://docs.datadoghq.com/integrations/google_cloud_big_query/) +* [GCP Big Query Quotas and Limits](https://cloud.google.com/bigquery/quotas) + +Inputs +------ + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| concurrent_queries_message | Custom message for the Concurrent Queries monitor | string | `` | no | +| concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | +| concurrent_queries_threshold_critical | Concurrent Queries (critical threshold) (hard limit 50) | string | `45` | no | +| concurrent_queries_threshold_warning | Concurrent Queries (warning threshold) (hard limit 50) | string | `40` | no | +| concurrent_queries_timeframe | Timeframe for the Concurrent Queries monitor | string | `last_5m` | no | +| delay | Delay in seconds for the metric evaluation | string | `900` | no | +| environment | Architecture environment | string | - | yes | +| execution_time_message | Custom message for the Execution Time monitor | string | `` | no | +| execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | +| execution_time_threshold_critical | Average Execution Time in seconds (critical threshold) | string | `150` | no | +| execution_time_threshold_warning | Average Execution Time in seconds (warning threshold) | string | `100` | no | +| execution_time_timeframe | Timeframe for the Execution Time monitor | string | `last_5m` | no | +| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | +| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| message | Message sent when a monitor is triggered | string | - | yes | +| project_id | ID of the GCP Project | string | - | yes | +| scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | +| scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | +| scanned_bytes_billed_threshold_critical | Scanned Bytes Billed Bytes (critical threshold) | string | `1` | no | +| scanned_bytes_billed_threshold_warning | Scanned Bytes Billed Bytes (warning threshold) | string | `0` | no | +| scanned_bytes_billed_timeframe | Timeframe for the Scanned Bytes Billed monitor | string | `last_4h` | no | +| scanned_bytes_message | Custom message for the Scanned Bytes monitor | string | `` | no | +| scanned_bytes_silenced | Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default. | map | `` | no | +| scanned_bytes_threshold_critical | Scanned Bytes Bytes (critical threshold) | string | `1` | no | +| scanned_bytes_threshold_warning | Scanned Bytes Bytes (warning threshold) | string | `0` | no | +| scanned_bytes_timeframe | Timeframe for the Scanned Bytes monitor | string | `last_4h` | no | +| slots_available_message | Custom message for the Available Slots monitor | string | `` | no | +| slots_available_silenced | Groups to mute for GCP Big Query Available Slots monitor. | map | `` | no | +| slots_available_threshold_critical | Available Slots (critical threshold) | string | `200` | no | +| slots_available_threshold_warning | Available Slots (warning threshold) | string | `300` | no | +| slots_available_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | +| stored_bytes_message | Custom message for the Stored Bytes monitor | string | `` | no | +| stored_bytes_silenced | Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default. | map | `` | no | +| stored_bytes_threshold_critical | Stored Bytes in fraction (critical threshold) | string | `1` | no | +| stored_bytes_threshold_warning | Stored Bytes in fraction (warning threshold) | string | `0` | no | +| stored_bytes_timeframe | Timeframe for the Stored Bytes monitor | string | `last_5m` | no | +| table_count_message | Custom message for the Table Count monitor | string | `` | no | +| table_count_silenced | Groups to mute for GCP Big Query Table Count monitor. Muted by default. | map | `` | no | +| table_count_threshold_critical | Table Count (critical threshold) | string | `1` | no | +| table_count_threshold_warning | Table Count (warning threshold) | string | `0` | no | +| table_count_timeframe | Timeframe for the Table Count monitor | string | `last_4h` | no | +| uploaded_bytes_billed_message | Custom message for the Uploaded Bytes Billed monitor | string | `` | no | +| uploaded_bytes_billed_silenced | Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default. | map | `` | no | +| uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed in Bytes (critical threshold) | string | `1` | no | +| uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed in Bytes (warning threshold) | string | `0` | no | +| uploaded_bytes_billed_timeframe | Timeframe for the Uploaded Bytes Billed monitor | string | `last_4h` | no | +| uploaded_bytes_message | Custom message for the Uploaded Bytes monitor | string | `` | no | +| uploaded_bytes_silenced | Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default. | map | `` | no | +| uploaded_bytes_threshold_critical | Uploaded Bytes in Bytes (critical threshold) | string | `1` | no | +| uploaded_bytes_threshold_warning | Uploaded Bytes in Bytes (warning threshold) | string | `0` | no | +| uploaded_bytes_timeframe | Timeframe for the Uploaded Bytes monitor | string | `last_4h` | no | + diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf new file mode 100644 index 0000000..97fc013 --- /dev/null +++ b/cloud/gcp/big-query/inputs.tf @@ -0,0 +1,34 @@ +# +# Datadog global variables +# +variable "environment" { + description = "Architecture environment" + type = "string" +} + +variable "filter_tags_use_defaults" { + description = "Use default filter tags convention" + default = "true" +} + +variable "filter_tags_custom" { + description = "Tags used for custom filtering when filter_tags_use_defaults is false" + default = "*" +} + +variable "message" { + description = "Message sent when a monitor is triggered" +} + +variable "delay" { + description = "Delay in seconds for the metric evaluation" + default = 900 +} + +# +# Filter variables +# +variable "project_id" { + type = "string" + description = "ID of the GCP Project" +} diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf new file mode 100644 index 0000000..a04a935 --- /dev/null +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -0,0 +1,12 @@ +# +# FILTER +# +data "template_file" "filter" { + template = "$${filter}" + + vars { + filter = "${var.filter_tags_use_defaults == "true" ? + format("project_id:%s", var.project_id) : + "${var.filter_tags_custom}"}" + } +} diff --git a/cloud/gcp/big-query/query.tf b/cloud/gcp/big-query/query.tf new file mode 100644 index 0000000..4153ea8 --- /dev/null +++ b/cloud/gcp/big-query/query.tf @@ -0,0 +1,138 @@ +# +# Concurrent queries +# +variable "concurrent_queries_message" { + description = "Custom message for the Concurrent Queries monitor" + type = "string" + default = "" +} + +variable "concurrent_queries_timeframe" { + description = "Timeframe for the Concurrent Queries monitor" + type = "string" + default = "last_5m" +} + +variable "concurrent_queries_threshold_warning" { + description = "Concurrent Queries (warning threshold) (hard limit 50)" + type = "string" + default = 40 +} + +variable "concurrent_queries_threshold_critical" { + description = "Concurrent Queries (critical threshold) (hard limit 50)" + type = "string" + default = 45 +} + +variable "concurrent_queries_silenced" { + description = "Groups to mute for GCP Big Query Concurrent Queries monitor" + type = "map" + default = {} +} + +resource "datadog_monitor" "concurrent_queries" { + name = "[${var.environment}] GCP Big Query Concurrent Queries close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.concurrent_queries_message, var.message)}" + + type = "metric alert" + + query = < ${var.concurrent_queries_threshold_critical} +EOF + + thresholds { + warning = "${var.concurrent_queries_threshold_warning}" + critical = "${var.concurrent_queries_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.concurrent_queries_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} + +# +# Execution Time +# +variable "execution_time_message" { + description = "Custom message for the Execution Time monitor" + type = "string" + default = "" +} + +variable "execution_time_timeframe" { + description = "Timeframe for the Execution Time monitor" + type = "string" + default = "last_5m" +} + +variable "execution_time_threshold_warning" { + description = "Average Execution Time in seconds (warning threshold)" + type = "string" + default = 100 +} + +variable "execution_time_threshold_critical" { + description = "Average Execution Time in seconds (critical threshold)" + type = "string" + default = 150 +} + +variable "execution_time_silenced" { + description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "execution_time" { + name = "[${var.environment}] GCP Big Query Execution Time too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.execution_time_message, var.message)}" + + type = "metric alert" + + query = < ${var.execution_time_threshold_critical} +EOF + + thresholds { + warning = "${var.execution_time_threshold_warning}" + critical = "${var.execution_time_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.execution_time_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/scanned_bytes.tf b/cloud/gcp/big-query/scanned_bytes.tf new file mode 100644 index 0000000..f660142 --- /dev/null +++ b/cloud/gcp/big-query/scanned_bytes.tf @@ -0,0 +1,70 @@ +# +# Scanned Bytes +# +variable "scanned_bytes_message" { + description = "Custom message for the Scanned Bytes monitor" + type = "string" + default = "" +} + +variable "scanned_bytes_timeframe" { + description = "Timeframe for the Scanned Bytes monitor" + type = "string" + default = "last_4h" +} + +variable "scanned_bytes_threshold_warning" { + description = "Scanned Bytes Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "scanned_bytes_threshold_critical" { + description = "Scanned Bytes Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "scanned_bytes_silenced" { + description = "Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "scanned_bytes" { + name = "[${var.environment}] GCP Big Query Scanned Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.scanned_bytes_message, var.message)}" + + type = "metric alert" + + query = < ${var.scanned_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.scanned_bytes_threshold_warning}" + critical = "${var.scanned_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.scanned_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/scanned_bytes_billed.tf b/cloud/gcp/big-query/scanned_bytes_billed.tf new file mode 100644 index 0000000..8e7fd1a --- /dev/null +++ b/cloud/gcp/big-query/scanned_bytes_billed.tf @@ -0,0 +1,70 @@ +# +# Scanned Bytes Billed +# +variable "scanned_bytes_billed_message" { + description = "Custom message for the Scanned Bytes Billed monitor" + type = "string" + default = "" +} + +variable "scanned_bytes_billed_timeframe" { + description = "Timeframe for the Scanned Bytes Billed monitor" + type = "string" + default = "last_4h" +} + +variable "scanned_bytes_billed_threshold_warning" { + description = "Scanned Bytes Billed Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "scanned_bytes_billed_threshold_critical" { + description = "Scanned Bytes Billed Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "scanned_bytes_billed_silenced" { + description = "Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "scanned_bytes_billed" { + name = "[${var.environment}] GCP Big Query Scanned Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" + + type = "metric alert" + + query = < ${var.scanned_bytes_billed_threshold_critical} +EOF + + thresholds { + warning = "${var.scanned_bytes_billed_threshold_warning}" + critical = "${var.scanned_bytes_billed_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.scanned_bytes_billed_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/slots.tf b/cloud/gcp/big-query/slots.tf new file mode 100644 index 0000000..e41ba80 --- /dev/null +++ b/cloud/gcp/big-query/slots.tf @@ -0,0 +1,67 @@ +# +# Available Slots +# +variable "slots_available_message" { + description = "Custom message for the Available Slots monitor" + type = "string" + default = "" +} + +variable "slots_available_timeframe" { + description = "Timeframe for the Available Slots monitor" + type = "string" + default = "last_5m" +} + +variable "slots_available_threshold_warning" { + description = "Available Slots (warning threshold)" + type = "string" + default = 300 +} + +variable "slots_available_threshold_critical" { + description = "Available Slots (critical threshold)" + type = "string" + default = 200 +} + +variable "slots_available_silenced" { + description = "Groups to mute for GCP Big Query Available Slots monitor." + type = "map" + default = {} +} + +resource "datadog_monitor" "available_slots" { + name = "[${var.environment}] GCP Big Query Available Slots close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.slots_available_message, var.message)}" + + type = "metric alert" + + query = < ${var.stored_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.stored_bytes_threshold_warning}" + critical = "${var.stored_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.stored_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/table_count.tf b/cloud/gcp/big-query/table_count.tf new file mode 100644 index 0000000..97a3565 --- /dev/null +++ b/cloud/gcp/big-query/table_count.tf @@ -0,0 +1,71 @@ +# +# Table Count +# +variable "table_count_message" { + description = "Custom message for the Table Count monitor" + type = "string" + default = "" +} + +variable "table_count_timeframe" { + description = "Timeframe for the Table Count monitor" + type = "string" + default = "last_4h" +} + +variable "table_count_threshold_warning" { + description = "Table Count (warning threshold)" + type = "string" + default = 0 +} + +variable "table_count_threshold_critical" { + description = "Table Count (critical threshold)" + type = "string" + default = 1 +} + +variable "table_count_silenced" { + description = "Groups to mute for GCP Big Query Table Count monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "table_count" { + name = "[${var.environment}] GCP Big Query Table Count too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.table_count_message, var.message)}" + + type = "metric alert" + + query = < ${var.table_count_threshold_critical} +EOF + + thresholds { + warning = "${var.table_count_threshold_warning}" + critical = "${var.table_count_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.table_count_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/uploaded_bytes.tf b/cloud/gcp/big-query/uploaded_bytes.tf new file mode 100644 index 0000000..a68716e --- /dev/null +++ b/cloud/gcp/big-query/uploaded_bytes.tf @@ -0,0 +1,71 @@ +# +# Uploaded Bytes +# +variable "uploaded_bytes_message" { + description = "Custom message for the Uploaded Bytes monitor" + type = "string" + default = "" +} + +variable "uploaded_bytes_timeframe" { + description = "Timeframe for the Uploaded Bytes monitor" + type = "string" + default = "last_4h" +} + +variable "uploaded_bytes_threshold_warning" { + description = "Uploaded Bytes in Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "uploaded_bytes_threshold_critical" { + description = "Uploaded Bytes in Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "uploaded_bytes_silenced" { + description = "Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "uploaded_bytes" { + name = "[${var.environment}] GCP Big Query Uploaded Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.uploaded_bytes_message, var.message)}" + + type = "metric alert" + + query = < ${var.uploaded_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.uploaded_bytes_threshold_warning}" + critical = "${var.uploaded_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.uploaded_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} diff --git a/cloud/gcp/big-query/uploaded_bytes_billed.tf b/cloud/gcp/big-query/uploaded_bytes_billed.tf new file mode 100644 index 0000000..d15f52e --- /dev/null +++ b/cloud/gcp/big-query/uploaded_bytes_billed.tf @@ -0,0 +1,71 @@ +# +# Uploaded Bytes Billed +# +variable "uploaded_bytes_billed_message" { + description = "Custom message for the Uploaded Bytes Billed monitor" + type = "string" + default = "" +} + +variable "uploaded_bytes_billed_timeframe" { + description = "Timeframe for the Uploaded Bytes Billed monitor" + type = "string" + default = "last_4h" +} + +variable "uploaded_bytes_billed_threshold_warning" { + description = "Uploaded Bytes Billed in Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "uploaded_bytes_billed_threshold_critical" { + description = "Uploaded Bytes Billed in Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "uploaded_bytes_billed_silenced" { + description = "Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +resource "datadog_monitor" "upload_bytes_billed" { + name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.uploaded_bytes_billed_message, var.message)}" + + type = "metric alert" + + query = < ${var.uploaded_bytes_billed_threshold_critical} +EOF + + thresholds { + warning = "${var.uploaded_bytes_billed_threshold_warning}" + critical = "${var.uploaded_bytes_billed_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + silenced = "${var.uploaded_bytes_billed_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + ] +} From aa57c19f0384ac1e4e2ea6497b2398741d684381 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Thu, 5 Jul 2018 15:49:09 +0200 Subject: [PATCH 2/9] MON-230 Extra tags added for all monitors --- cloud/gcp/big-query/README.md | 17 ++++++++---- cloud/gcp/big-query/scanned_bytes.tf | 7 +++++ cloud/gcp/big-query/scanned_bytes_billed.tf | 7 +++++ cloud/gcp/big-query/slots.tf | 29 ++++++++++++-------- cloud/gcp/big-query/storage.tf | 7 +++++ cloud/gcp/big-query/table_count.tf | 7 +++++ cloud/gcp/big-query/uploaded_bytes.tf | 7 +++++ cloud/gcp/big-query/uploaded_bytes_billed.tf | 9 +++++- 8 files changed, 73 insertions(+), 17 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index ba831fb..b949786 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -44,6 +44,12 @@ Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| available_slots_extra_tags | Extra tags for GCP Big Query Available Slots monitor | list | `` | no | +| available_slots_message | Custom message for the Available Slots monitor | string | `` | no | +| available_slots_silenced | Groups to mute for GCP Big Query Available Slots monitor. | map | `` | no | +| available_slots_threshold_critical | Available Slots (critical threshold) | string | `200` | no | +| available_slots_threshold_warning | Available Slots (warning threshold) | string | `300` | no | +| available_slots_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | | concurrent_queries_message | Custom message for the Concurrent Queries monitor | string | `` | no | | concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | | concurrent_queries_threshold_critical | Concurrent Queries (critical threshold) (hard limit 50) | string | `45` | no | @@ -60,36 +66,37 @@ Inputs | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when a monitor is triggered | string | - | yes | | project_id | ID of the GCP Project | string | - | yes | +| scanned_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes Billed monitor | list | `` | no | | scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | | scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | | scanned_bytes_billed_threshold_critical | Scanned Bytes Billed Bytes (critical threshold) | string | `1` | no | | scanned_bytes_billed_threshold_warning | Scanned Bytes Billed Bytes (warning threshold) | string | `0` | no | | scanned_bytes_billed_timeframe | Timeframe for the Scanned Bytes Billed monitor | string | `last_4h` | no | +| scanned_bytes_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | scanned_bytes_message | Custom message for the Scanned Bytes monitor | string | `` | no | | scanned_bytes_silenced | Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default. | map | `` | no | | scanned_bytes_threshold_critical | Scanned Bytes Bytes (critical threshold) | string | `1` | no | | scanned_bytes_threshold_warning | Scanned Bytes Bytes (warning threshold) | string | `0` | no | | scanned_bytes_timeframe | Timeframe for the Scanned Bytes monitor | string | `last_4h` | no | -| slots_available_message | Custom message for the Available Slots monitor | string | `` | no | -| slots_available_silenced | Groups to mute for GCP Big Query Available Slots monitor. | map | `` | no | -| slots_available_threshold_critical | Available Slots (critical threshold) | string | `200` | no | -| slots_available_threshold_warning | Available Slots (warning threshold) | string | `300` | no | -| slots_available_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | +| stored_bytes_extra_tags | Extra tags for GCP Big Query Stored Bytes monitor | list | `` | no | | stored_bytes_message | Custom message for the Stored Bytes monitor | string | `` | no | | stored_bytes_silenced | Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default. | map | `` | no | | stored_bytes_threshold_critical | Stored Bytes in fraction (critical threshold) | string | `1` | no | | stored_bytes_threshold_warning | Stored Bytes in fraction (warning threshold) | string | `0` | no | | stored_bytes_timeframe | Timeframe for the Stored Bytes monitor | string | `last_5m` | no | +| table_count_extra_tags | Extra tags for GCP Big Query Table Count monitor | list | `` | no | | table_count_message | Custom message for the Table Count monitor | string | `` | no | | table_count_silenced | Groups to mute for GCP Big Query Table Count monitor. Muted by default. | map | `` | no | | table_count_threshold_critical | Table Count (critical threshold) | string | `1` | no | | table_count_threshold_warning | Table Count (warning threshold) | string | `0` | no | | table_count_timeframe | Timeframe for the Table Count monitor | string | `last_4h` | no | +| uploaded_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | uploaded_bytes_billed_message | Custom message for the Uploaded Bytes Billed monitor | string | `` | no | | uploaded_bytes_billed_silenced | Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default. | map | `` | no | | uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed in Bytes (critical threshold) | string | `1` | no | | uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed in Bytes (warning threshold) | string | `0` | no | | uploaded_bytes_billed_timeframe | Timeframe for the Uploaded Bytes Billed monitor | string | `last_4h` | no | +| uploaded_bytes_extra_tags | Extra tags for GCP Big Query Uploaded Bytes monitor | list | `` | no | | uploaded_bytes_message | Custom message for the Uploaded Bytes monitor | string | `` | no | | uploaded_bytes_silenced | Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default. | map | `` | no | | uploaded_bytes_threshold_critical | Uploaded Bytes in Bytes (critical threshold) | string | `1` | no | diff --git a/cloud/gcp/big-query/scanned_bytes.tf b/cloud/gcp/big-query/scanned_bytes.tf index f660142..00e284e 100644 --- a/cloud/gcp/big-query/scanned_bytes.tf +++ b/cloud/gcp/big-query/scanned_bytes.tf @@ -34,6 +34,12 @@ variable "scanned_bytes_silenced" { } } +variable "scanned_bytes_extra_tags" { + description = "Extra tags for GCP Big Query Scanned Bytes monitor" + type = "list" + default = [] +} + resource "datadog_monitor" "scanned_bytes" { name = "[${var.environment}] GCP Big Query Scanned Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.scanned_bytes_message, var.message)}" @@ -66,5 +72,6 @@ EOF "provider:gcp", "env:${var.environment}", "resource:big-query", + "${var.scanned_bytes_extra_tags}", ] } diff --git a/cloud/gcp/big-query/scanned_bytes_billed.tf b/cloud/gcp/big-query/scanned_bytes_billed.tf index 8e7fd1a..749999d 100644 --- a/cloud/gcp/big-query/scanned_bytes_billed.tf +++ b/cloud/gcp/big-query/scanned_bytes_billed.tf @@ -34,6 +34,12 @@ variable "scanned_bytes_billed_silenced" { } } +variable "scanned_bytes_billed_extra_tags" { + description = "Extra tags for GCP Big Query Scanned Bytes Billed monitor" + type = "list" + default = [] +} + resource "datadog_monitor" "scanned_bytes_billed" { name = "[${var.environment}] GCP Big Query Scanned Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" @@ -66,5 +72,6 @@ EOF "provider:gcp", "env:${var.environment}", "resource:big-query", + "${var.scanned_bytes_billed_extra_tags}", ] } diff --git a/cloud/gcp/big-query/slots.tf b/cloud/gcp/big-query/slots.tf index e41ba80..0a8c22d 100644 --- a/cloud/gcp/big-query/slots.tf +++ b/cloud/gcp/big-query/slots.tf @@ -1,50 +1,56 @@ # # Available Slots # -variable "slots_available_message" { +variable "available_slots_message" { description = "Custom message for the Available Slots monitor" type = "string" default = "" } -variable "slots_available_timeframe" { +variable "available_slots_timeframe" { description = "Timeframe for the Available Slots monitor" type = "string" default = "last_5m" } -variable "slots_available_threshold_warning" { +variable "available_slots_threshold_warning" { description = "Available Slots (warning threshold)" type = "string" default = 300 } -variable "slots_available_threshold_critical" { +variable "available_slots_threshold_critical" { description = "Available Slots (critical threshold)" type = "string" default = 200 } -variable "slots_available_silenced" { +variable "available_slots_silenced" { description = "Groups to mute for GCP Big Query Available Slots monitor." type = "map" default = {} } +variable "available_slots_extra_tags" { + description = "Extra tags for GCP Big Query Available Slots monitor" + type = "list" + default = [] +} + resource "datadog_monitor" "available_slots" { name = "[${var.environment}] GCP Big Query Available Slots close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.slots_available_message, var.message)}" + message = "${coalesce(var.available_slots_message, var.message)}" type = "metric alert" query = < Date: Fri, 6 Jul 2018 15:48:40 +0200 Subject: [PATCH 3/9] MON-230 Added extra tags on the monitors where they were missing --- cloud/gcp/big-query/README.md | 6 ++++-- cloud/gcp/big-query/query.tf | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index b949786..c7a9d32 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -50,15 +50,17 @@ Inputs | available_slots_threshold_critical | Available Slots (critical threshold) | string | `200` | no | | available_slots_threshold_warning | Available Slots (warning threshold) | string | `300` | no | | available_slots_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | +| concurrent_queries_extra_tags | Extra tags for GCP Big Query Concurrent Queries monitor | list | `` | no | | concurrent_queries_message | Custom message for the Concurrent Queries monitor | string | `` | no | -| concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | +| concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | | concurrent_queries_threshold_critical | Concurrent Queries (critical threshold) (hard limit 50) | string | `45` | no | | concurrent_queries_threshold_warning | Concurrent Queries (warning threshold) (hard limit 50) | string | `40` | no | | concurrent_queries_timeframe | Timeframe for the Concurrent Queries monitor | string | `last_5m` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | +| execution_time_extra_tags | Extra tags for GCP Big Query Execution Time monitor | list | `` | no | | execution_time_message | Custom message for the Execution Time monitor | string | `` | no | -| execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | +| execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | | execution_time_threshold_critical | Average Execution Time in seconds (critical threshold) | string | `150` | no | | execution_time_threshold_warning | Average Execution Time in seconds (warning threshold) | string | `100` | no | | execution_time_timeframe | Timeframe for the Execution Time monitor | string | `last_5m` | no | diff --git a/cloud/gcp/big-query/query.tf b/cloud/gcp/big-query/query.tf index 4153ea8..ea218c6 100644 --- a/cloud/gcp/big-query/query.tf +++ b/cloud/gcp/big-query/query.tf @@ -26,11 +26,17 @@ variable "concurrent_queries_threshold_critical" { } variable "concurrent_queries_silenced" { - description = "Groups to mute for GCP Big Query Concurrent Queries monitor" + description = "Groups to mute for GCP Big Query Concurrent Queries monitor" type = "map" default = {} } +variable "concurrent_queries_extra_tags" { + description = "Extra tags for GCP Big Query Concurrent Queries monitor" + type = "list" + default = [] +} + resource "datadog_monitor" "concurrent_queries" { name = "[${var.environment}] GCP Big Query Concurrent Queries close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.concurrent_queries_message, var.message)}" @@ -63,6 +69,7 @@ EOF "provider:gcp", "env:${var.environment}", "resource:big-query", + "${var.concurrent_queries_extra_tags}", ] } @@ -94,7 +101,7 @@ variable "execution_time_threshold_critical" { } variable "execution_time_silenced" { - description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." + description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." type = "map" default = { @@ -102,6 +109,12 @@ variable "execution_time_silenced" { } } +variable "execution_time_extra_tags" { + description = "Extra tags for GCP Big Query Execution Time monitor" + type = "list" + default = [] +} + resource "datadog_monitor" "execution_time" { name = "[${var.environment}] GCP Big Query Execution Time too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" message = "${coalesce(var.execution_time_message, var.message)}" @@ -134,5 +147,6 @@ EOF "provider:gcp", "env:${var.environment}", "resource:big-query", + "${var.execution_time_extra_tags}", ] } From 63857e57d14f3d8f9f53f8c80646c7c7237115db Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 31 Jul 2018 14:47:21 +0200 Subject: [PATCH 4/9] MON-230 Big Query monitors generalized and outputs under control with variable --- README.md | 1 + cloud/gcp/big-query/README.md | 72 ++-- cloud/gcp/big-query/inputs.tf | 426 +++++++++++++++++++ cloud/gcp/big-query/monitors-big-query.tf | 400 +++++++++++++++++ cloud/gcp/big-query/outputs.tf | 44 ++ cloud/gcp/big-query/query.tf | 152 ------- cloud/gcp/big-query/scanned_bytes.tf | 77 ---- cloud/gcp/big-query/scanned_bytes_billed.tf | 77 ---- cloud/gcp/big-query/slots.tf | 74 ---- cloud/gcp/big-query/storage.tf | 78 ---- cloud/gcp/big-query/table_count.tf | 78 ---- cloud/gcp/big-query/uploaded_bytes.tf | 78 ---- cloud/gcp/big-query/uploaded_bytes_billed.tf | 78 ---- 13 files changed, 914 insertions(+), 721 deletions(-) create mode 100644 cloud/gcp/big-query/outputs.tf delete mode 100644 cloud/gcp/big-query/query.tf delete mode 100644 cloud/gcp/big-query/scanned_bytes.tf delete mode 100644 cloud/gcp/big-query/scanned_bytes_billed.tf delete mode 100644 cloud/gcp/big-query/slots.tf delete mode 100644 cloud/gcp/big-query/storage.tf delete mode 100644 cloud/gcp/big-query/table_count.tf delete mode 100644 cloud/gcp/big-query/uploaded_bytes.tf delete mode 100644 cloud/gcp/big-query/uploaded_bytes_billed.tf diff --git a/README.md b/README.md index 4439901..72455ee 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi - [storage](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/azure/storage/) - [stream-analytics](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/azure/stream-analytics/) - [gcp](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/) + - [big-query](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/big-query/) - [cloud-sql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/) - [common](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/common/) - [mysql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/mysql/) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index c7a9d32..7964f96 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -1,55 +1,43 @@ -GCP Big Query -============================== +# CLOUD GCP BIG-QUERY DataDog monitors -How to use this module ----------------------- +## How to use this module ``` -module "datadog-monitors-gcp-big-query" { +module "datadog-monitors-cloud-gcp-big-query" { source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/gcp/big-query?ref={revision}" - project_id = "${var.gcp_project_id}" environment = "${var.environment}" message = "${module.datadog-message-alerting.alerting-message}" } ``` -Purpose -------- -Creates DataDog monitors with the following checks : +## Purpose -* Query Concurrent Queries -* Query Execution Times (muted by default) -* Query Scanned Bytes (muted by default) -* Query Scanned Bytes Billed (muted by default) +Creates DataDog monitors with the following checks: -* Slots Available +- GCP Big Query Available Slots close to the limit +- GCP Big Query Concurrent Queries close to the limit +- GCP Big Query Execution Time too high +- GCP Big Query Scanned Bytes Billed too high +- GCP Big Query Scanned Bytes too high +- GCP Big Query Stored Bytes too high +- GCP Big Query Table Count too high +- GCP Big Query Uploaded Bytes Billed too high +- GCP Big Query Uploaded Bytes too high -* Storage Stored Bytes (muted by default) -* Storage Table Count per Dataset (muted by default) -* Storage Uploaded Bytes (muted by default) -* Storage Uploaded Bytes Billed (muted by default) - - -Useful links ------------- - -* [GCP Big Query monitoring](https://cloud.google.com/bigquery/docs/monitoring) -* [Datadog Integration for GCP Big Query](https://docs.datadoghq.com/integrations/google_cloud_big_query/) -* [GCP Big Query Quotas and Limits](https://cloud.google.com/bigquery/quotas) - -Inputs ------- +## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| available_slots_enabled | Whether or not to create the monitor | string | `true` | no | | available_slots_extra_tags | Extra tags for GCP Big Query Available Slots monitor | list | `` | no | | available_slots_message | Custom message for the Available Slots monitor | string | `` | no | | available_slots_silenced | Groups to mute for GCP Big Query Available Slots monitor. | map | `` | no | | available_slots_threshold_critical | Available Slots (critical threshold) | string | `200` | no | | available_slots_threshold_warning | Available Slots (warning threshold) | string | `300` | no | | available_slots_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | +| concurrent_queries_enabled | Whether or not to create the monitor | string | `true` | no | | concurrent_queries_extra_tags | Extra tags for GCP Big Query Concurrent Queries monitor | list | `` | no | | concurrent_queries_message | Custom message for the Concurrent Queries monitor | string | `` | no | | concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | @@ -58,6 +46,7 @@ Inputs | concurrent_queries_timeframe | Timeframe for the Concurrent Queries monitor | string | `last_5m` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | +| execution_time_enabled | Whether or not to create the monitor | string | `true` | no | | execution_time_extra_tags | Extra tags for GCP Big Query Execution Time monitor | list | `` | no | | execution_time_message | Custom message for the Execution Time monitor | string | `` | no | | execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | @@ -68,36 +57,42 @@ Inputs | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when a monitor is triggered | string | - | yes | | project_id | ID of the GCP Project | string | - | yes | +| scanned_bytes_billed_enabled | Whether or not to create the monitor | string | `true` | no | | scanned_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes Billed monitor | list | `` | no | | scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | | scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | | scanned_bytes_billed_threshold_critical | Scanned Bytes Billed Bytes (critical threshold) | string | `1` | no | | scanned_bytes_billed_threshold_warning | Scanned Bytes Billed Bytes (warning threshold) | string | `0` | no | | scanned_bytes_billed_timeframe | Timeframe for the Scanned Bytes Billed monitor | string | `last_4h` | no | +| scanned_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | scanned_bytes_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | scanned_bytes_message | Custom message for the Scanned Bytes monitor | string | `` | no | | scanned_bytes_silenced | Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default. | map | `` | no | | scanned_bytes_threshold_critical | Scanned Bytes Bytes (critical threshold) | string | `1` | no | | scanned_bytes_threshold_warning | Scanned Bytes Bytes (warning threshold) | string | `0` | no | | scanned_bytes_timeframe | Timeframe for the Scanned Bytes monitor | string | `last_4h` | no | +| stored_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | stored_bytes_extra_tags | Extra tags for GCP Big Query Stored Bytes monitor | list | `` | no | | stored_bytes_message | Custom message for the Stored Bytes monitor | string | `` | no | | stored_bytes_silenced | Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default. | map | `` | no | | stored_bytes_threshold_critical | Stored Bytes in fraction (critical threshold) | string | `1` | no | | stored_bytes_threshold_warning | Stored Bytes in fraction (warning threshold) | string | `0` | no | | stored_bytes_timeframe | Timeframe for the Stored Bytes monitor | string | `last_5m` | no | +| table_count_enabled | Whether or not to create the monitor | string | `true` | no | | table_count_extra_tags | Extra tags for GCP Big Query Table Count monitor | list | `` | no | | table_count_message | Custom message for the Table Count monitor | string | `` | no | | table_count_silenced | Groups to mute for GCP Big Query Table Count monitor. Muted by default. | map | `` | no | | table_count_threshold_critical | Table Count (critical threshold) | string | `1` | no | | table_count_threshold_warning | Table Count (warning threshold) | string | `0` | no | | table_count_timeframe | Timeframe for the Table Count monitor | string | `last_4h` | no | +| uploaded_bytes_billed_enabled | Whether or not to create the monitor | string | `true` | no | | uploaded_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | uploaded_bytes_billed_message | Custom message for the Uploaded Bytes Billed monitor | string | `` | no | | uploaded_bytes_billed_silenced | Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default. | map | `` | no | | uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed in Bytes (critical threshold) | string | `1` | no | | uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed in Bytes (warning threshold) | string | `0` | no | | uploaded_bytes_billed_timeframe | Timeframe for the Uploaded Bytes Billed monitor | string | `last_4h` | no | +| uploaded_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | uploaded_bytes_extra_tags | Extra tags for GCP Big Query Uploaded Bytes monitor | list | `` | no | | uploaded_bytes_message | Custom message for the Uploaded Bytes monitor | string | `` | no | | uploaded_bytes_silenced | Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default. | map | `` | no | @@ -105,3 +100,22 @@ Inputs | uploaded_bytes_threshold_warning | Uploaded Bytes in Bytes (warning threshold) | string | `0` | no | | uploaded_bytes_timeframe | Timeframe for the Uploaded Bytes monitor | string | `last_4h` | no | +## Outputs + +| Name | Description | +|------|-------------| +| available_slots_id | id for monitor available_slots | +| concurrent_queries_id | id for monitor concurrent_queries | +| execution_time_id | id for monitor execution_time | +| scanned_bytes_billed_id | id for monitor scanned_bytes_billed | +| scanned_bytes_id | id for monitor scanned_bytes | +| stored_bytes_id | id for monitor stored_bytes | +| table_count_id | id for monitor table_count | +| uploaded_bytes_billed_id | id for monitor uploaded_bytes_billed | +| uploaded_bytes_id | id for monitor uploaded_bytes | + +## Related documentation + +* [GCP Big Query monitoring](https://cloud.google.com/bigquery/docs/monitoring) +* [Datadog Integration for GCP Big Query](https://docs.datadoghq.com/integrations/google_cloud_big_query/) +* [GCP Big Query Quotas and Limits](https://cloud.google.com/bigquery/quotas) diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf index 97fc013..c390b22 100644 --- a/cloud/gcp/big-query/inputs.tf +++ b/cloud/gcp/big-query/inputs.tf @@ -32,3 +32,429 @@ variable "project_id" { type = "string" description = "ID of the GCP Project" } + +# +# Concurrent queries +# +variable "concurrent_queries_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "concurrent_queries_message" { + description = "Custom message for the Concurrent Queries monitor" + type = "string" + default = "" +} + +variable "concurrent_queries_timeframe" { + description = "Timeframe for the Concurrent Queries monitor" + type = "string" + default = "last_5m" +} + +variable "concurrent_queries_threshold_warning" { + description = "Concurrent Queries (warning threshold) (hard limit 50)" + type = "string" + default = 40 +} + +variable "concurrent_queries_threshold_critical" { + description = "Concurrent Queries (critical threshold) (hard limit 50)" + type = "string" + default = 45 +} + +variable "concurrent_queries_silenced" { + description = "Groups to mute for GCP Big Query Concurrent Queries monitor" + type = "map" + default = {} +} + +variable "concurrent_queries_extra_tags" { + description = "Extra tags for GCP Big Query Concurrent Queries monitor" + type = "list" + default = [] +} + +# +# Execution Time +# +variable "execution_time_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "execution_time_message" { + description = "Custom message for the Execution Time monitor" + type = "string" + default = "" +} + +variable "execution_time_timeframe" { + description = "Timeframe for the Execution Time monitor" + type = "string" + default = "last_5m" +} + +variable "execution_time_threshold_warning" { + description = "Average Execution Time in seconds (warning threshold)" + type = "string" + default = 100 +} + +variable "execution_time_threshold_critical" { + description = "Average Execution Time in seconds (critical threshold)" + type = "string" + default = 150 +} + +variable "execution_time_silenced" { + description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "execution_time_extra_tags" { + description = "Extra tags for GCP Big Query Execution Time monitor" + type = "list" + default = [] +} + +# +# Scanned Bytes +# +variable "scanned_bytes_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "scanned_bytes_message" { + description = "Custom message for the Scanned Bytes monitor" + type = "string" + default = "" +} + +variable "scanned_bytes_timeframe" { + description = "Timeframe for the Scanned Bytes monitor" + type = "string" + default = "last_4h" +} + +variable "scanned_bytes_threshold_warning" { + description = "Scanned Bytes Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "scanned_bytes_threshold_critical" { + description = "Scanned Bytes Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "scanned_bytes_silenced" { + description = "Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "scanned_bytes_extra_tags" { + description = "Extra tags for GCP Big Query Scanned Bytes monitor" + type = "list" + default = [] +} + +# +# Scanned Bytes Billed +# +variable "scanned_bytes_billed_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "scanned_bytes_billed_message" { + description = "Custom message for the Scanned Bytes Billed monitor" + type = "string" + default = "" +} + +variable "scanned_bytes_billed_timeframe" { + description = "Timeframe for the Scanned Bytes Billed monitor" + type = "string" + default = "last_4h" +} + +variable "scanned_bytes_billed_threshold_warning" { + description = "Scanned Bytes Billed Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "scanned_bytes_billed_threshold_critical" { + description = "Scanned Bytes Billed Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "scanned_bytes_billed_silenced" { + description = "Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "scanned_bytes_billed_extra_tags" { + description = "Extra tags for GCP Big Query Scanned Bytes Billed monitor" + type = "list" + default = [] +} + +# +# Available Slots +# +variable "available_slots_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "available_slots_message" { + description = "Custom message for the Available Slots monitor" + type = "string" + default = "" +} + +variable "available_slots_timeframe" { + description = "Timeframe for the Available Slots monitor" + type = "string" + default = "last_5m" +} + +variable "available_slots_threshold_warning" { + description = "Available Slots (warning threshold)" + type = "string" + default = 300 +} + +variable "available_slots_threshold_critical" { + description = "Available Slots (critical threshold)" + type = "string" + default = 200 +} + +variable "available_slots_silenced" { + description = "Groups to mute for GCP Big Query Available Slots monitor." + type = "map" + default = {} +} + +variable "available_slots_extra_tags" { + description = "Extra tags for GCP Big Query Available Slots monitor" + type = "list" + default = [] +} + +# +# Stored Bytes +# +variable "stored_bytes_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "stored_bytes_message" { + description = "Custom message for the Stored Bytes monitor" + type = "string" + default = "" +} + +variable "stored_bytes_timeframe" { + description = "Timeframe for the Stored Bytes monitor" + type = "string" + default = "last_5m" +} + +variable "stored_bytes_threshold_warning" { + description = "Stored Bytes in fraction (warning threshold)" + type = "string" + default = 0 +} + +variable "stored_bytes_threshold_critical" { + description = "Stored Bytes in fraction (critical threshold)" + type = "string" + default = 1 +} + +variable "stored_bytes_silenced" { + description = "Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "stored_bytes_extra_tags" { + description = "Extra tags for GCP Big Query Stored Bytes monitor" + type = "list" + default = [] +} + +# +# Table Count +# +variable "table_count_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "table_count_message" { + description = "Custom message for the Table Count monitor" + type = "string" + default = "" +} + +variable "table_count_timeframe" { + description = "Timeframe for the Table Count monitor" + type = "string" + default = "last_4h" +} + +variable "table_count_threshold_warning" { + description = "Table Count (warning threshold)" + type = "string" + default = 0 +} + +variable "table_count_threshold_critical" { + description = "Table Count (critical threshold)" + type = "string" + default = 1 +} + +variable "table_count_silenced" { + description = "Groups to mute for GCP Big Query Table Count monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "table_count_extra_tags" { + description = "Extra tags for GCP Big Query Table Count monitor" + type = "list" + default = [] +} + +# +# Uploaded Bytes +# +variable "uploaded_bytes_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "uploaded_bytes_message" { + description = "Custom message for the Uploaded Bytes monitor" + type = "string" + default = "" +} + +variable "uploaded_bytes_timeframe" { + description = "Timeframe for the Uploaded Bytes monitor" + type = "string" + default = "last_4h" +} + +variable "uploaded_bytes_threshold_warning" { + description = "Uploaded Bytes in Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "uploaded_bytes_threshold_critical" { + description = "Uploaded Bytes in Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "uploaded_bytes_silenced" { + description = "Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "uploaded_bytes_extra_tags" { + description = "Extra tags for GCP Big Query Uploaded Bytes monitor" + type = "list" + default = [] +} + +# +# Uploaded Bytes Billed +# +variable "uploaded_bytes_billed_enabled" { + description = "Whether or not to create the monitor" + type = "string" + default = "true" +} + +variable "uploaded_bytes_billed_message" { + description = "Custom message for the Uploaded Bytes Billed monitor" + type = "string" + default = "" +} + +variable "uploaded_bytes_billed_timeframe" { + description = "Timeframe for the Uploaded Bytes Billed monitor" + type = "string" + default = "last_4h" +} + +variable "uploaded_bytes_billed_threshold_warning" { + description = "Uploaded Bytes Billed in Bytes (warning threshold)" + type = "string" + default = 0 +} + +variable "uploaded_bytes_billed_threshold_critical" { + description = "Uploaded Bytes Billed in Bytes (critical threshold)" + type = "string" + default = 1 +} + +variable "uploaded_bytes_billed_silenced" { + description = "Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default." + type = "map" + + default = { + "*" = 0 + } +} + +variable "uploaded_bytes_billed_extra_tags" { + description = "Extra tags for GCP Big Query Scanned Bytes monitor" + type = "list" + default = [] +} diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf index a04a935..0a87403 100644 --- a/cloud/gcp/big-query/monitors-big-query.tf +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -10,3 +10,403 @@ data "template_file" "filter" { "${var.filter_tags_custom}"}" } } + +# +# Concurrent queries +# +resource "datadog_monitor" "concurrent_queries" { + count = "${var.concurrent_queries_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Concurrent Queries close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.concurrent_queries_message, var.message)}" + + type = "metric alert" + + query = < ${var.concurrent_queries_threshold_critical} +EOF + + thresholds { + warning = "${var.concurrent_queries_threshold_warning}" + critical = "${var.concurrent_queries_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.concurrent_queries_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.concurrent_queries_extra_tags}", + ] +} + +# +# Execution Time +# +resource "datadog_monitor" "execution_time" { + count = "${var.execution_time_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Execution Time too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.execution_time_message, var.message)}" + + type = "metric alert" + + query = < ${var.execution_time_threshold_critical} +EOF + + thresholds { + warning = "${var.execution_time_threshold_warning}" + critical = "${var.execution_time_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.execution_time_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.execution_time_extra_tags}", + ] +} + +# +# Scanned Bytes +# +resource "datadog_monitor" "scanned_bytes" { + count = "${var.scanned_bytes_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Scanned Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.scanned_bytes_message, var.message)}" + + type = "metric alert" + + query = < ${var.scanned_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.scanned_bytes_threshold_warning}" + critical = "${var.scanned_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.scanned_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.scanned_bytes_extra_tags}", + ] +} + +# +# Scanned Bytes Billed +# +resource "datadog_monitor" "scanned_bytes_billed" { + count = "${var.scanned_bytes_billed_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Scanned Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" + + type = "metric alert" + + query = < ${var.scanned_bytes_billed_threshold_critical} +EOF + + thresholds { + warning = "${var.scanned_bytes_billed_threshold_warning}" + critical = "${var.scanned_bytes_billed_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.scanned_bytes_billed_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.scanned_bytes_billed_extra_tags}", + ] +} + +# +# Available Slots +# +resource "datadog_monitor" "available_slots" { + count = "${var.available_slots_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Available Slots close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.available_slots_message, var.message)}" + + type = "metric alert" + + query = < ${var.stored_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.stored_bytes_threshold_warning}" + critical = "${var.stored_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.stored_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.stored_bytes_extra_tags}", + ] +} + +# +# Table Count +# +resource "datadog_monitor" "table_count" { + count = "${var.table_count_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Table Count too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.table_count_message, var.message)}" + + type = "metric alert" + + query = < ${var.table_count_threshold_critical} +EOF + + thresholds { + warning = "${var.table_count_threshold_warning}" + critical = "${var.table_count_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.table_count_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.table_count_extra_tags}", + ] +} + +# +# Uploaded Bytes +# +resource "datadog_monitor" "uploaded_bytes" { + count = "${var.uploaded_bytes_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Uploaded Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.uploaded_bytes_message, var.message)}" + + type = "metric alert" + + query = < ${var.uploaded_bytes_threshold_critical} +EOF + + thresholds { + warning = "${var.uploaded_bytes_threshold_warning}" + critical = "${var.uploaded_bytes_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.uploaded_bytes_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.uploaded_bytes_extra_tags}", + ] +} + +# +# Uploaded Bytes Billed +# +resource "datadog_monitor" "uploaded_bytes_billed" { + count = "${var.uploaded_bytes_billed_enabled == "true" ? 1 : 0}" + + name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.uploaded_bytes_billed_message, var.message)}" + + type = "metric alert" + + query = < ${var.uploaded_bytes_billed_threshold_critical} +EOF + + thresholds { + warning = "${var.uploaded_bytes_billed_threshold_warning}" + critical = "${var.uploaded_bytes_billed_threshold_critical}" + } + + include_tags = true + notify_no_data = false + require_full_window = false + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + locked = false + + evaluation_delay = "${var.delay}" + new_host_delay = "${var.delay}" + + silenced = "${var.uploaded_bytes_billed_silenced}" + + tags = [ + "team:gcp", + "provider:gcp", + "env:${var.environment}", + "resource:big-query", + "created_by:terraform", + "${var.uploaded_bytes_billed_extra_tags}", + ] +} diff --git a/cloud/gcp/big-query/outputs.tf b/cloud/gcp/big-query/outputs.tf new file mode 100644 index 0000000..be9de5f --- /dev/null +++ b/cloud/gcp/big-query/outputs.tf @@ -0,0 +1,44 @@ +output "concurrent_queries_id" { + description = "id for monitor concurrent_queries" + value = "${datadog_monitor.concurrent_queries.*.id}" +} + +output "execution_time_id" { + description = "id for monitor execution_time" + value = "${datadog_monitor.execution_time.*.id}" +} + +output "scanned_bytes_id" { + description = "id for monitor scanned_bytes" + value = "${datadog_monitor.scanned_bytes.*.id}" +} + +output "scanned_bytes_billed_id" { + description = "id for monitor scanned_bytes_billed" + value = "${datadog_monitor.scanned_bytes_billed.*.id}" +} + +output "available_slots_id" { + description = "id for monitor available_slots" + value = "${datadog_monitor.available_slots.*.id}" +} + +output "stored_bytes_id" { + description = "id for monitor stored_bytes" + value = "${datadog_monitor.stored_bytes.*.id}" +} + +output "table_count_id" { + description = "id for monitor table_count" + value = "${datadog_monitor.table_count.*.id}" +} + +output "uploaded_bytes_id" { + description = "id for monitor uploaded_bytes" + value = "${datadog_monitor.uploaded_bytes.*.id}" +} + +output "uploaded_bytes_billed_id" { + description = "id for monitor uploaded_bytes_billed" + value = "${datadog_monitor.uploaded_bytes_billed.*.id}" +} diff --git a/cloud/gcp/big-query/query.tf b/cloud/gcp/big-query/query.tf deleted file mode 100644 index ea218c6..0000000 --- a/cloud/gcp/big-query/query.tf +++ /dev/null @@ -1,152 +0,0 @@ -# -# Concurrent queries -# -variable "concurrent_queries_message" { - description = "Custom message for the Concurrent Queries monitor" - type = "string" - default = "" -} - -variable "concurrent_queries_timeframe" { - description = "Timeframe for the Concurrent Queries monitor" - type = "string" - default = "last_5m" -} - -variable "concurrent_queries_threshold_warning" { - description = "Concurrent Queries (warning threshold) (hard limit 50)" - type = "string" - default = 40 -} - -variable "concurrent_queries_threshold_critical" { - description = "Concurrent Queries (critical threshold) (hard limit 50)" - type = "string" - default = 45 -} - -variable "concurrent_queries_silenced" { - description = "Groups to mute for GCP Big Query Concurrent Queries monitor" - type = "map" - default = {} -} - -variable "concurrent_queries_extra_tags" { - description = "Extra tags for GCP Big Query Concurrent Queries monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "concurrent_queries" { - name = "[${var.environment}] GCP Big Query Concurrent Queries close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.concurrent_queries_message, var.message)}" - - type = "metric alert" - - query = < ${var.concurrent_queries_threshold_critical} -EOF - - thresholds { - warning = "${var.concurrent_queries_threshold_warning}" - critical = "${var.concurrent_queries_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.concurrent_queries_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.concurrent_queries_extra_tags}", - ] -} - -# -# Execution Time -# -variable "execution_time_message" { - description = "Custom message for the Execution Time monitor" - type = "string" - default = "" -} - -variable "execution_time_timeframe" { - description = "Timeframe for the Execution Time monitor" - type = "string" - default = "last_5m" -} - -variable "execution_time_threshold_warning" { - description = "Average Execution Time in seconds (warning threshold)" - type = "string" - default = 100 -} - -variable "execution_time_threshold_critical" { - description = "Average Execution Time in seconds (critical threshold)" - type = "string" - default = 150 -} - -variable "execution_time_silenced" { - description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "execution_time_extra_tags" { - description = "Extra tags for GCP Big Query Execution Time monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "execution_time" { - name = "[${var.environment}] GCP Big Query Execution Time too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.execution_time_message, var.message)}" - - type = "metric alert" - - query = < ${var.execution_time_threshold_critical} -EOF - - thresholds { - warning = "${var.execution_time_threshold_warning}" - critical = "${var.execution_time_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.execution_time_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.execution_time_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/scanned_bytes.tf b/cloud/gcp/big-query/scanned_bytes.tf deleted file mode 100644 index 00e284e..0000000 --- a/cloud/gcp/big-query/scanned_bytes.tf +++ /dev/null @@ -1,77 +0,0 @@ -# -# Scanned Bytes -# -variable "scanned_bytes_message" { - description = "Custom message for the Scanned Bytes monitor" - type = "string" - default = "" -} - -variable "scanned_bytes_timeframe" { - description = "Timeframe for the Scanned Bytes monitor" - type = "string" - default = "last_4h" -} - -variable "scanned_bytes_threshold_warning" { - description = "Scanned Bytes Bytes (warning threshold)" - type = "string" - default = 0 -} - -variable "scanned_bytes_threshold_critical" { - description = "Scanned Bytes Bytes (critical threshold)" - type = "string" - default = 1 -} - -variable "scanned_bytes_silenced" { - description = "Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "scanned_bytes_extra_tags" { - description = "Extra tags for GCP Big Query Scanned Bytes monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "scanned_bytes" { - name = "[${var.environment}] GCP Big Query Scanned Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.scanned_bytes_message, var.message)}" - - type = "metric alert" - - query = < ${var.scanned_bytes_threshold_critical} -EOF - - thresholds { - warning = "${var.scanned_bytes_threshold_warning}" - critical = "${var.scanned_bytes_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.scanned_bytes_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.scanned_bytes_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/scanned_bytes_billed.tf b/cloud/gcp/big-query/scanned_bytes_billed.tf deleted file mode 100644 index 749999d..0000000 --- a/cloud/gcp/big-query/scanned_bytes_billed.tf +++ /dev/null @@ -1,77 +0,0 @@ -# -# Scanned Bytes Billed -# -variable "scanned_bytes_billed_message" { - description = "Custom message for the Scanned Bytes Billed monitor" - type = "string" - default = "" -} - -variable "scanned_bytes_billed_timeframe" { - description = "Timeframe for the Scanned Bytes Billed monitor" - type = "string" - default = "last_4h" -} - -variable "scanned_bytes_billed_threshold_warning" { - description = "Scanned Bytes Billed Bytes (warning threshold)" - type = "string" - default = 0 -} - -variable "scanned_bytes_billed_threshold_critical" { - description = "Scanned Bytes Billed Bytes (critical threshold)" - type = "string" - default = 1 -} - -variable "scanned_bytes_billed_silenced" { - description = "Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "scanned_bytes_billed_extra_tags" { - description = "Extra tags for GCP Big Query Scanned Bytes Billed monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "scanned_bytes_billed" { - name = "[${var.environment}] GCP Big Query Scanned Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" - - type = "metric alert" - - query = < ${var.scanned_bytes_billed_threshold_critical} -EOF - - thresholds { - warning = "${var.scanned_bytes_billed_threshold_warning}" - critical = "${var.scanned_bytes_billed_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.scanned_bytes_billed_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.scanned_bytes_billed_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/slots.tf b/cloud/gcp/big-query/slots.tf deleted file mode 100644 index 0a8c22d..0000000 --- a/cloud/gcp/big-query/slots.tf +++ /dev/null @@ -1,74 +0,0 @@ -# -# Available Slots -# -variable "available_slots_message" { - description = "Custom message for the Available Slots monitor" - type = "string" - default = "" -} - -variable "available_slots_timeframe" { - description = "Timeframe for the Available Slots monitor" - type = "string" - default = "last_5m" -} - -variable "available_slots_threshold_warning" { - description = "Available Slots (warning threshold)" - type = "string" - default = 300 -} - -variable "available_slots_threshold_critical" { - description = "Available Slots (critical threshold)" - type = "string" - default = 200 -} - -variable "available_slots_silenced" { - description = "Groups to mute for GCP Big Query Available Slots monitor." - type = "map" - default = {} -} - -variable "available_slots_extra_tags" { - description = "Extra tags for GCP Big Query Available Slots monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "available_slots" { - name = "[${var.environment}] GCP Big Query Available Slots close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.available_slots_message, var.message)}" - - type = "metric alert" - - query = < ${var.stored_bytes_threshold_critical} -EOF - - thresholds { - warning = "${var.stored_bytes_threshold_warning}" - critical = "${var.stored_bytes_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.stored_bytes_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.stored_bytes_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/table_count.tf b/cloud/gcp/big-query/table_count.tf deleted file mode 100644 index 4b8f7a3..0000000 --- a/cloud/gcp/big-query/table_count.tf +++ /dev/null @@ -1,78 +0,0 @@ -# -# Table Count -# -variable "table_count_message" { - description = "Custom message for the Table Count monitor" - type = "string" - default = "" -} - -variable "table_count_timeframe" { - description = "Timeframe for the Table Count monitor" - type = "string" - default = "last_4h" -} - -variable "table_count_threshold_warning" { - description = "Table Count (warning threshold)" - type = "string" - default = 0 -} - -variable "table_count_threshold_critical" { - description = "Table Count (critical threshold)" - type = "string" - default = 1 -} - -variable "table_count_silenced" { - description = "Groups to mute for GCP Big Query Table Count monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "table_count_extra_tags" { - description = "Extra tags for GCP Big Query Table Count monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "table_count" { - name = "[${var.environment}] GCP Big Query Table Count too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.table_count_message, var.message)}" - - type = "metric alert" - - query = < ${var.table_count_threshold_critical} -EOF - - thresholds { - warning = "${var.table_count_threshold_warning}" - critical = "${var.table_count_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.table_count_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.table_count_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/uploaded_bytes.tf b/cloud/gcp/big-query/uploaded_bytes.tf deleted file mode 100644 index bc48ff4..0000000 --- a/cloud/gcp/big-query/uploaded_bytes.tf +++ /dev/null @@ -1,78 +0,0 @@ -# -# Uploaded Bytes -# -variable "uploaded_bytes_message" { - description = "Custom message for the Uploaded Bytes monitor" - type = "string" - default = "" -} - -variable "uploaded_bytes_timeframe" { - description = "Timeframe for the Uploaded Bytes monitor" - type = "string" - default = "last_4h" -} - -variable "uploaded_bytes_threshold_warning" { - description = "Uploaded Bytes in Bytes (warning threshold)" - type = "string" - default = 0 -} - -variable "uploaded_bytes_threshold_critical" { - description = "Uploaded Bytes in Bytes (critical threshold)" - type = "string" - default = 1 -} - -variable "uploaded_bytes_silenced" { - description = "Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "uploaded_bytes_extra_tags" { - description = "Extra tags for GCP Big Query Uploaded Bytes monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "uploaded_bytes" { - name = "[${var.environment}] GCP Big Query Uploaded Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.uploaded_bytes_message, var.message)}" - - type = "metric alert" - - query = < ${var.uploaded_bytes_threshold_critical} -EOF - - thresholds { - warning = "${var.uploaded_bytes_threshold_warning}" - critical = "${var.uploaded_bytes_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.uploaded_bytes_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.uploaded_bytes_extra_tags}", - ] -} diff --git a/cloud/gcp/big-query/uploaded_bytes_billed.tf b/cloud/gcp/big-query/uploaded_bytes_billed.tf deleted file mode 100644 index c28e14e..0000000 --- a/cloud/gcp/big-query/uploaded_bytes_billed.tf +++ /dev/null @@ -1,78 +0,0 @@ -# -# Uploaded Bytes Billed -# -variable "uploaded_bytes_billed_message" { - description = "Custom message for the Uploaded Bytes Billed monitor" - type = "string" - default = "" -} - -variable "uploaded_bytes_billed_timeframe" { - description = "Timeframe for the Uploaded Bytes Billed monitor" - type = "string" - default = "last_4h" -} - -variable "uploaded_bytes_billed_threshold_warning" { - description = "Uploaded Bytes Billed in Bytes (warning threshold)" - type = "string" - default = 0 -} - -variable "uploaded_bytes_billed_threshold_critical" { - description = "Uploaded Bytes Billed in Bytes (critical threshold)" - type = "string" - default = 1 -} - -variable "uploaded_bytes_billed_silenced" { - description = "Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default." - type = "map" - - default = { - "*" = 0 - } -} - -variable "uploaded_bytes_billed_extra_tags" { - description = "Extra tags for GCP Big Query Scanned Bytes monitor" - type = "list" - default = [] -} - -resource "datadog_monitor" "uploaded_bytes_billed" { - name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" - message = "${coalesce(var.uploaded_bytes_billed_message, var.message)}" - - type = "metric alert" - - query = < ${var.uploaded_bytes_billed_threshold_critical} -EOF - - thresholds { - warning = "${var.uploaded_bytes_billed_threshold_warning}" - critical = "${var.uploaded_bytes_billed_threshold_critical}" - } - - include_tags = true - notify_no_data = false - require_full_window = false - renotify_interval = 0 - notify_audit = false - timeout_h = 0 - locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" - silenced = "${var.uploaded_bytes_billed_silenced}" - - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "${var.uploaded_bytes_billed_extra_tags}", - ] -} From 3ed551d2a18d32c6884b3ffb3371f4f9aed0417d Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 31 Jul 2018 15:30:11 +0200 Subject: [PATCH 5/9] MON-230 Monitors fixed acording to what is deployed on Hubside's Datadog --- cloud/gcp/big-query/README.md | 18 +++++++-------- cloud/gcp/big-query/inputs.tf | 28 ++++++----------------- cloud/gcp/big-query/monitors-big-query.tf | 18 +++++++-------- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index 7964f96..b3f985c 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -16,15 +16,15 @@ module "datadog-monitors-cloud-gcp-big-query" { Creates DataDog monitors with the following checks: -- GCP Big Query Available Slots close to the limit -- GCP Big Query Concurrent Queries close to the limit -- GCP Big Query Execution Time too high -- GCP Big Query Scanned Bytes Billed too high -- GCP Big Query Scanned Bytes too high -- GCP Big Query Stored Bytes too high -- GCP Big Query Table Count too high -- GCP Big Query Uploaded Bytes Billed too high -- GCP Big Query Uploaded Bytes too high +- GCP Big Query Available Slots +- GCP Big Query Concurrent Queries +- GCP Big Query Execution Time +- GCP Big Query Scanned Bytes Billed +- GCP Big Query Scanned Bytes +- GCP Big Query Stored Bytes +- GCP Big Query Table Count +- GCP Big Query Uploaded Bytes Billed +- GCP Big Query Uploaded Bytes ## Inputs diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf index c390b22..c1ab176 100644 --- a/cloud/gcp/big-query/inputs.tf +++ b/cloud/gcp/big-query/inputs.tf @@ -115,9 +115,7 @@ variable "execution_time_silenced" { description = "Groups to mute for GCP Big Query Execution Time monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "execution_time_extra_tags" { @@ -163,9 +161,7 @@ variable "scanned_bytes_silenced" { description = "Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "scanned_bytes_extra_tags" { @@ -211,9 +207,7 @@ variable "scanned_bytes_billed_silenced" { description = "Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "scanned_bytes_billed_extra_tags" { @@ -304,9 +298,7 @@ variable "stored_bytes_silenced" { description = "Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "stored_bytes_extra_tags" { @@ -352,9 +344,7 @@ variable "table_count_silenced" { description = "Groups to mute for GCP Big Query Table Count monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "table_count_extra_tags" { @@ -400,9 +390,7 @@ variable "uploaded_bytes_silenced" { description = "Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "uploaded_bytes_extra_tags" { @@ -448,9 +436,7 @@ variable "uploaded_bytes_billed_silenced" { description = "Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default." type = "map" - default = { - "*" = 0 - } + default = {} } variable "uploaded_bytes_billed_extra_tags" { diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf index 0a87403..4e16a84 100644 --- a/cloud/gcp/big-query/monitors-big-query.tf +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -17,7 +17,7 @@ data "template_file" "filter" { resource "datadog_monitor" "concurrent_queries" { count = "${var.concurrent_queries_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Concurrent Queries close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Concurrent Queries {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.concurrent_queries_message, var.message)}" type = "metric alert" @@ -61,7 +61,7 @@ EOF resource "datadog_monitor" "execution_time" { count = "${var.execution_time_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Execution Time too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Execution Time {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" message = "${coalesce(var.execution_time_message, var.message)}" type = "metric alert" @@ -105,7 +105,7 @@ EOF resource "datadog_monitor" "scanned_bytes" { count = "${var.scanned_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Scanned Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Scanned Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.scanned_bytes_message, var.message)}" type = "metric alert" @@ -149,7 +149,7 @@ EOF resource "datadog_monitor" "scanned_bytes_billed" { count = "${var.scanned_bytes_billed_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Scanned Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Scanned Bytes Billed {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" type = "metric alert" @@ -193,7 +193,7 @@ EOF resource "datadog_monitor" "available_slots" { count = "${var.available_slots_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Available Slots close to the limit {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Available Slots {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.available_slots_message, var.message)}" type = "metric alert" @@ -237,7 +237,7 @@ EOF resource "datadog_monitor" "stored_bytes" { count = "${var.stored_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Stored Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Stored Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B ({{value}}B){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B ({{value}}B){{/is_warning}}" message = "${coalesce(var.stored_bytes_message, var.message)}" type = "metric alert" @@ -282,7 +282,7 @@ EOF resource "datadog_monitor" "table_count" { count = "${var.table_count_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Table Count too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Table Count {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.table_count_message, var.message)}" type = "metric alert" @@ -327,7 +327,7 @@ EOF resource "datadog_monitor" "uploaded_bytes" { count = "${var.uploaded_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Uploaded Bytes too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Uploaded Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.uploaded_bytes_message, var.message)}" type = "metric alert" @@ -372,7 +372,7 @@ EOF resource "datadog_monitor" "uploaded_bytes_billed" { count = "${var.uploaded_bytes_billed_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.uploaded_bytes_billed_message, var.message)}" type = "metric alert" From 0177d37686a178fbeb876fc81f162e4345fe1b39 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 31 Jul 2018 15:42:48 +0200 Subject: [PATCH 6/9] MON-230 Removing the control of the monitors creation. It must be done in a different PR --- cloud/gcp/big-query/README.md | 9 ----- cloud/gcp/big-query/inputs.tf | 47 +---------------------- cloud/gcp/big-query/monitors-big-query.tf | 18 --------- 3 files changed, 2 insertions(+), 72 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index b3f985c..06ef809 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -30,14 +30,12 @@ Creates DataDog monitors with the following checks: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| available_slots_enabled | Whether or not to create the monitor | string | `true` | no | | available_slots_extra_tags | Extra tags for GCP Big Query Available Slots monitor | list | `` | no | | available_slots_message | Custom message for the Available Slots monitor | string | `` | no | | available_slots_silenced | Groups to mute for GCP Big Query Available Slots monitor. | map | `` | no | | available_slots_threshold_critical | Available Slots (critical threshold) | string | `200` | no | | available_slots_threshold_warning | Available Slots (warning threshold) | string | `300` | no | | available_slots_timeframe | Timeframe for the Available Slots monitor | string | `last_5m` | no | -| concurrent_queries_enabled | Whether or not to create the monitor | string | `true` | no | | concurrent_queries_extra_tags | Extra tags for GCP Big Query Concurrent Queries monitor | list | `` | no | | concurrent_queries_message | Custom message for the Concurrent Queries monitor | string | `` | no | | concurrent_queries_silenced | Groups to mute for GCP Big Query Concurrent Queries monitor | map | `` | no | @@ -46,7 +44,6 @@ Creates DataDog monitors with the following checks: | concurrent_queries_timeframe | Timeframe for the Concurrent Queries monitor | string | `last_5m` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | -| execution_time_enabled | Whether or not to create the monitor | string | `true` | no | | execution_time_extra_tags | Extra tags for GCP Big Query Execution Time monitor | list | `` | no | | execution_time_message | Custom message for the Execution Time monitor | string | `` | no | | execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | @@ -57,42 +54,36 @@ Creates DataDog monitors with the following checks: | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | message | Message sent when a monitor is triggered | string | - | yes | | project_id | ID of the GCP Project | string | - | yes | -| scanned_bytes_billed_enabled | Whether or not to create the monitor | string | `true` | no | | scanned_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes Billed monitor | list | `` | no | | scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | | scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | | scanned_bytes_billed_threshold_critical | Scanned Bytes Billed Bytes (critical threshold) | string | `1` | no | | scanned_bytes_billed_threshold_warning | Scanned Bytes Billed Bytes (warning threshold) | string | `0` | no | | scanned_bytes_billed_timeframe | Timeframe for the Scanned Bytes Billed monitor | string | `last_4h` | no | -| scanned_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | scanned_bytes_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | scanned_bytes_message | Custom message for the Scanned Bytes monitor | string | `` | no | | scanned_bytes_silenced | Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default. | map | `` | no | | scanned_bytes_threshold_critical | Scanned Bytes Bytes (critical threshold) | string | `1` | no | | scanned_bytes_threshold_warning | Scanned Bytes Bytes (warning threshold) | string | `0` | no | | scanned_bytes_timeframe | Timeframe for the Scanned Bytes monitor | string | `last_4h` | no | -| stored_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | stored_bytes_extra_tags | Extra tags for GCP Big Query Stored Bytes monitor | list | `` | no | | stored_bytes_message | Custom message for the Stored Bytes monitor | string | `` | no | | stored_bytes_silenced | Groups to mute for GCP Big Query Stored Bytes monitor. Muted by default. | map | `` | no | | stored_bytes_threshold_critical | Stored Bytes in fraction (critical threshold) | string | `1` | no | | stored_bytes_threshold_warning | Stored Bytes in fraction (warning threshold) | string | `0` | no | | stored_bytes_timeframe | Timeframe for the Stored Bytes monitor | string | `last_5m` | no | -| table_count_enabled | Whether or not to create the monitor | string | `true` | no | | table_count_extra_tags | Extra tags for GCP Big Query Table Count monitor | list | `` | no | | table_count_message | Custom message for the Table Count monitor | string | `` | no | | table_count_silenced | Groups to mute for GCP Big Query Table Count monitor. Muted by default. | map | `` | no | | table_count_threshold_critical | Table Count (critical threshold) | string | `1` | no | | table_count_threshold_warning | Table Count (warning threshold) | string | `0` | no | | table_count_timeframe | Timeframe for the Table Count monitor | string | `last_4h` | no | -| uploaded_bytes_billed_enabled | Whether or not to create the monitor | string | `true` | no | | uploaded_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | uploaded_bytes_billed_message | Custom message for the Uploaded Bytes Billed monitor | string | `` | no | | uploaded_bytes_billed_silenced | Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default. | map | `` | no | | uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed in Bytes (critical threshold) | string | `1` | no | | uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed in Bytes (warning threshold) | string | `0` | no | | uploaded_bytes_billed_timeframe | Timeframe for the Uploaded Bytes Billed monitor | string | `last_4h` | no | -| uploaded_bytes_enabled | Whether or not to create the monitor | string | `true` | no | | uploaded_bytes_extra_tags | Extra tags for GCP Big Query Uploaded Bytes monitor | list | `` | no | | uploaded_bytes_message | Custom message for the Uploaded Bytes monitor | string | `` | no | | uploaded_bytes_silenced | Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default. | map | `` | no | diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf index c1ab176..70ec912 100644 --- a/cloud/gcp/big-query/inputs.tf +++ b/cloud/gcp/big-query/inputs.tf @@ -1,6 +1,7 @@ # # Datadog global variables # + variable "environment" { description = "Architecture environment" type = "string" @@ -28,6 +29,7 @@ variable "delay" { # # Filter variables # + variable "project_id" { type = "string" description = "ID of the GCP Project" @@ -36,11 +38,6 @@ variable "project_id" { # # Concurrent queries # -variable "concurrent_queries_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "concurrent_queries_message" { description = "Custom message for the Concurrent Queries monitor" @@ -81,11 +78,6 @@ variable "concurrent_queries_extra_tags" { # # Execution Time # -variable "execution_time_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "execution_time_message" { description = "Custom message for the Execution Time monitor" @@ -127,11 +119,6 @@ variable "execution_time_extra_tags" { # # Scanned Bytes # -variable "scanned_bytes_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "scanned_bytes_message" { description = "Custom message for the Scanned Bytes monitor" @@ -173,11 +160,6 @@ variable "scanned_bytes_extra_tags" { # # Scanned Bytes Billed # -variable "scanned_bytes_billed_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "scanned_bytes_billed_message" { description = "Custom message for the Scanned Bytes Billed monitor" @@ -219,11 +201,6 @@ variable "scanned_bytes_billed_extra_tags" { # # Available Slots # -variable "available_slots_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "available_slots_message" { description = "Custom message for the Available Slots monitor" @@ -264,11 +241,6 @@ variable "available_slots_extra_tags" { # # Stored Bytes # -variable "stored_bytes_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "stored_bytes_message" { description = "Custom message for the Stored Bytes monitor" @@ -310,11 +282,6 @@ variable "stored_bytes_extra_tags" { # # Table Count # -variable "table_count_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "table_count_message" { description = "Custom message for the Table Count monitor" @@ -356,11 +323,6 @@ variable "table_count_extra_tags" { # # Uploaded Bytes # -variable "uploaded_bytes_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "uploaded_bytes_message" { description = "Custom message for the Uploaded Bytes monitor" @@ -402,11 +364,6 @@ variable "uploaded_bytes_extra_tags" { # # Uploaded Bytes Billed # -variable "uploaded_bytes_billed_enabled" { - description = "Whether or not to create the monitor" - type = "string" - default = "true" -} variable "uploaded_bytes_billed_message" { description = "Custom message for the Uploaded Bytes Billed monitor" diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf index 4e16a84..a851151 100644 --- a/cloud/gcp/big-query/monitors-big-query.tf +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -15,8 +15,6 @@ data "template_file" "filter" { # Concurrent queries # resource "datadog_monitor" "concurrent_queries" { - count = "${var.concurrent_queries_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Concurrent Queries {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.concurrent_queries_message, var.message)}" @@ -59,8 +57,6 @@ EOF # Execution Time # resource "datadog_monitor" "execution_time" { - count = "${var.execution_time_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Execution Time {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" message = "${coalesce(var.execution_time_message, var.message)}" @@ -103,8 +99,6 @@ EOF # Scanned Bytes # resource "datadog_monitor" "scanned_bytes" { - count = "${var.scanned_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Scanned Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.scanned_bytes_message, var.message)}" @@ -147,8 +141,6 @@ EOF # Scanned Bytes Billed # resource "datadog_monitor" "scanned_bytes_billed" { - count = "${var.scanned_bytes_billed_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Scanned Bytes Billed {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.scanned_bytes_billed_message, var.message)}" @@ -191,8 +183,6 @@ EOF # Available Slots # resource "datadog_monitor" "available_slots" { - count = "${var.available_slots_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Available Slots {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.available_slots_message, var.message)}" @@ -235,8 +225,6 @@ EOF # Stored Bytes # resource "datadog_monitor" "stored_bytes" { - count = "${var.stored_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Stored Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B ({{value}}B){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B ({{value}}B){{/is_warning}}" message = "${coalesce(var.stored_bytes_message, var.message)}" @@ -280,8 +268,6 @@ EOF # Table Count # resource "datadog_monitor" "table_count" { - count = "${var.table_count_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Table Count {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" message = "${coalesce(var.table_count_message, var.message)}" @@ -325,8 +311,6 @@ EOF # Uploaded Bytes # resource "datadog_monitor" "uploaded_bytes" { - count = "${var.uploaded_bytes_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Uploaded Bytes {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.uploaded_bytes_message, var.message)}" @@ -370,8 +354,6 @@ EOF # Uploaded Bytes Billed # resource "datadog_monitor" "uploaded_bytes_billed" { - count = "${var.uploaded_bytes_billed_enabled == "true" ? 1 : 0}" - name = "[${var.environment}] GCP Big Query Uploaded Bytes Billed {{#is_alert}}{{{comparator}}} {{threshold}}B/mn ({{value}}B/mn){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}B/mn ({{value}}B/mn){{/is_warning}}" message = "${coalesce(var.uploaded_bytes_billed_message, var.message)}" From cb7dd538f33422bc0c5a8af1a0c72bc8944d06d4 Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Mon, 13 Aug 2018 16:50:33 +0200 Subject: [PATCH 7/9] MON-230 Standardize filter and split delays --- cloud/gcp/big-query/README.md | 7 +-- cloud/gcp/big-query/inputs.tf | 21 ++----- cloud/gcp/big-query/monitors-big-query.tf | 67 +++++++++-------------- 3 files changed, 36 insertions(+), 59 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index 06ef809..7de7482 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -42,18 +42,17 @@ Creates DataDog monitors with the following checks: | concurrent_queries_threshold_critical | Concurrent Queries (critical threshold) (hard limit 50) | string | `45` | no | | concurrent_queries_threshold_warning | Concurrent Queries (warning threshold) (hard limit 50) | string | `40` | no | | concurrent_queries_timeframe | Timeframe for the Concurrent Queries monitor | string | `last_5m` | no | -| delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no | | execution_time_extra_tags | Extra tags for GCP Big Query Execution Time monitor | list | `` | no | | execution_time_message | Custom message for the Execution Time monitor | string | `` | no | | execution_time_silenced | Groups to mute for GCP Big Query Execution Time monitor. Muted by default. | map | `` | no | | execution_time_threshold_critical | Average Execution Time in seconds (critical threshold) | string | `150` | no | | execution_time_threshold_warning | Average Execution Time in seconds (warning threshold) | string | `100` | no | | execution_time_timeframe | Timeframe for the Execution Time monitor | string | `last_5m` | no | -| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | -| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | +| filter_tags | Tags used for filtering | string | `*` | no | | message | Message sent when a monitor is triggered | string | - | yes | -| project_id | ID of the GCP Project | string | - | yes | +| new_host_delay | Delay in seconds for the new host evaluation | string | `300` | no | | scanned_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes Billed monitor | list | `` | no | | scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | | scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf index 70ec912..92dcb77 100644 --- a/cloud/gcp/big-query/inputs.tf +++ b/cloud/gcp/big-query/inputs.tf @@ -7,13 +7,8 @@ variable "environment" { type = "string" } -variable "filter_tags_use_defaults" { - description = "Use default filter tags convention" - default = "true" -} - -variable "filter_tags_custom" { - description = "Tags used for custom filtering when filter_tags_use_defaults is false" +variable "filter_tags" { + description = "Tags used for filtering" default = "*" } @@ -21,18 +16,14 @@ variable "message" { description = "Message sent when a monitor is triggered" } -variable "delay" { +variable "evaluation_delay" { description = "Delay in seconds for the metric evaluation" default = 900 } -# -# Filter variables -# - -variable "project_id" { - type = "string" - description = "ID of the GCP Project" +variable "new_host_delay" { + description = "Delay in seconds for the new host evaluation" + default = 300 } # diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf index a851151..ca290f8 100644 --- a/cloud/gcp/big-query/monitors-big-query.tf +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -1,16 +1,3 @@ -# -# FILTER -# -data "template_file" "filter" { - template = "$${filter}" - - vars { - filter = "${var.filter_tags_use_defaults == "true" ? - format("project_id:%s", var.project_id) : - "${var.filter_tags_custom}"}" - } -} - # # Concurrent queries # @@ -21,7 +8,7 @@ resource "datadog_monitor" "concurrent_queries" { type = "metric alert" query = < ${var.concurrent_queries_threshold_critical} EOF @@ -38,8 +25,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.concurrent_queries_silenced}" @@ -63,7 +50,7 @@ resource "datadog_monitor" "execution_time" { type = "metric alert" query = < ${var.execution_time_threshold_critical} EOF @@ -80,8 +67,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.execution_time_silenced}" @@ -105,7 +92,7 @@ resource "datadog_monitor" "scanned_bytes" { type = "metric alert" query = < ${var.scanned_bytes_threshold_critical} EOF @@ -122,8 +109,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.scanned_bytes_silenced}" @@ -147,7 +134,7 @@ resource "datadog_monitor" "scanned_bytes_billed" { type = "metric alert" query = < ${var.scanned_bytes_billed_threshold_critical} EOF @@ -164,8 +151,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.scanned_bytes_billed_silenced}" @@ -189,7 +176,7 @@ resource "datadog_monitor" "available_slots" { type = "metric alert" query = < ${var.stored_bytes_threshold_critical} EOF @@ -249,8 +236,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.stored_bytes_silenced}" @@ -274,7 +261,7 @@ resource "datadog_monitor" "table_count" { type = "metric alert" query = < ${var.table_count_threshold_critical} EOF @@ -292,8 +279,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.table_count_silenced}" @@ -317,7 +304,7 @@ resource "datadog_monitor" "uploaded_bytes" { type = "metric alert" query = < ${var.uploaded_bytes_threshold_critical} EOF @@ -335,8 +322,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.uploaded_bytes_silenced}" @@ -360,7 +347,7 @@ resource "datadog_monitor" "uploaded_bytes_billed" { type = "metric alert" query = < ${var.uploaded_bytes_billed_threshold_critical} EOF @@ -378,8 +365,8 @@ EOF timeout_h = 0 locked = false - evaluation_delay = "${var.delay}" - new_host_delay = "${var.delay}" + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" silenced = "${var.uploaded_bytes_billed_silenced}" From 083e9c239ca0c85f97456449d163af462cb280fb Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Tue, 21 Aug 2018 16:26:30 +0200 Subject: [PATCH 8/9] MON-230 Tags migrated to the new standard --- cloud/gcp/big-query/monitors-big-query.tf | 81 +++-------------------- 1 file changed, 9 insertions(+), 72 deletions(-) diff --git a/cloud/gcp/big-query/monitors-big-query.tf b/cloud/gcp/big-query/monitors-big-query.tf index ca290f8..5046055 100644 --- a/cloud/gcp/big-query/monitors-big-query.tf +++ b/cloud/gcp/big-query/monitors-big-query.tf @@ -30,14 +30,7 @@ EOF silenced = "${var.concurrent_queries_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.concurrent_queries_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.concurrent_queries_extra_tags}"] } # @@ -72,14 +65,7 @@ EOF silenced = "${var.execution_time_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.execution_time_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.execution_time_extra_tags}"] } # @@ -114,14 +100,7 @@ EOF silenced = "${var.scanned_bytes_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.scanned_bytes_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.scanned_bytes_extra_tags}"] } # @@ -156,14 +135,7 @@ EOF silenced = "${var.scanned_bytes_billed_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.scanned_bytes_billed_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.scanned_bytes_billed_extra_tags}"] } # @@ -198,14 +170,7 @@ EOF silenced = "${var.available_slots_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.available_slots_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.available_slots_extra_tags}"] } # @@ -241,14 +206,7 @@ EOF silenced = "${var.stored_bytes_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.stored_bytes_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.stored_bytes_extra_tags}"] } # @@ -284,14 +242,7 @@ EOF silenced = "${var.table_count_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.table_count_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.table_count_extra_tags}"] } # @@ -327,14 +278,7 @@ EOF silenced = "${var.uploaded_bytes_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.uploaded_bytes_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.uploaded_bytes_extra_tags}"] } # @@ -370,12 +314,5 @@ EOF silenced = "${var.uploaded_bytes_billed_silenced}" - tags = [ - "team:gcp", - "provider:gcp", - "env:${var.environment}", - "resource:big-query", - "created_by:terraform", - "${var.uploaded_bytes_billed_extra_tags}", - ] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:big-query", "team:claranet", "created-by:terraform", "${var.uploaded_bytes_billed_extra_tags}"] } From 27f1c1f03ebc3988b899aa461614641d96c2e08f Mon Sep 17 00:00:00 2001 From: Rafael Romero Carmona Date: Fri, 24 Aug 2018 18:00:29 +0200 Subject: [PATCH 9/9] MON-230 Descriptions fixed --- cloud/gcp/big-query/README.md | 16 ++++++++-------- cloud/gcp/big-query/inputs.tf | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cloud/gcp/big-query/README.md b/cloud/gcp/big-query/README.md index 7de7482..1d1c088 100644 --- a/cloud/gcp/big-query/README.md +++ b/cloud/gcp/big-query/README.md @@ -56,14 +56,14 @@ Creates DataDog monitors with the following checks: | scanned_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes Billed monitor | list | `` | no | | scanned_bytes_billed_message | Custom message for the Scanned Bytes Billed monitor | string | `` | no | | scanned_bytes_billed_silenced | Groups to mute for GCP Big Query Scanned Bytes Billed monitor. Muted by default. | map | `` | no | -| scanned_bytes_billed_threshold_critical | Scanned Bytes Billed Bytes (critical threshold) | string | `1` | no | -| scanned_bytes_billed_threshold_warning | Scanned Bytes Billed Bytes (warning threshold) | string | `0` | no | +| scanned_bytes_billed_threshold_critical | Scanned Bytes Billed (critical threshold) | string | `1` | no | +| scanned_bytes_billed_threshold_warning | Scanned Bytes Billed (warning threshold) | string | `0` | no | | scanned_bytes_billed_timeframe | Timeframe for the Scanned Bytes Billed monitor | string | `last_4h` | no | | scanned_bytes_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | scanned_bytes_message | Custom message for the Scanned Bytes monitor | string | `` | no | | scanned_bytes_silenced | Groups to mute for GCP Big Query Scanned Bytes monitor. Muted by default. | map | `` | no | -| scanned_bytes_threshold_critical | Scanned Bytes Bytes (critical threshold) | string | `1` | no | -| scanned_bytes_threshold_warning | Scanned Bytes Bytes (warning threshold) | string | `0` | no | +| scanned_bytes_threshold_critical | Scanned Bytes (critical threshold) | string | `1` | no | +| scanned_bytes_threshold_warning | Scanned Bytes (warning threshold) | string | `0` | no | | scanned_bytes_timeframe | Timeframe for the Scanned Bytes monitor | string | `last_4h` | no | | stored_bytes_extra_tags | Extra tags for GCP Big Query Stored Bytes monitor | list | `` | no | | stored_bytes_message | Custom message for the Stored Bytes monitor | string | `` | no | @@ -80,14 +80,14 @@ Creates DataDog monitors with the following checks: | uploaded_bytes_billed_extra_tags | Extra tags for GCP Big Query Scanned Bytes monitor | list | `` | no | | uploaded_bytes_billed_message | Custom message for the Uploaded Bytes Billed monitor | string | `` | no | | uploaded_bytes_billed_silenced | Groups to mute for GCP Big Query Uploaded Bytes Billed monitor. Muted by default. | map | `` | no | -| uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed in Bytes (critical threshold) | string | `1` | no | -| uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed in Bytes (warning threshold) | string | `0` | no | +| uploaded_bytes_billed_threshold_critical | Uploaded Bytes Billed (critical threshold) | string | `1` | no | +| uploaded_bytes_billed_threshold_warning | Uploaded Bytes Billed (warning threshold) | string | `0` | no | | uploaded_bytes_billed_timeframe | Timeframe for the Uploaded Bytes Billed monitor | string | `last_4h` | no | | uploaded_bytes_extra_tags | Extra tags for GCP Big Query Uploaded Bytes monitor | list | `` | no | | uploaded_bytes_message | Custom message for the Uploaded Bytes monitor | string | `` | no | | uploaded_bytes_silenced | Groups to mute for GCP Big Query Uploaded Bytes monitor. Muted by default. | map | `` | no | -| uploaded_bytes_threshold_critical | Uploaded Bytes in Bytes (critical threshold) | string | `1` | no | -| uploaded_bytes_threshold_warning | Uploaded Bytes in Bytes (warning threshold) | string | `0` | no | +| uploaded_bytes_threshold_critical | Uploaded Bytes (critical threshold) | string | `1` | no | +| uploaded_bytes_threshold_warning | Uploaded Bytes (warning threshold) | string | `0` | no | | uploaded_bytes_timeframe | Timeframe for the Uploaded Bytes monitor | string | `last_4h` | no | ## Outputs diff --git a/cloud/gcp/big-query/inputs.tf b/cloud/gcp/big-query/inputs.tf index 92dcb77..e6ad0bb 100644 --- a/cloud/gcp/big-query/inputs.tf +++ b/cloud/gcp/big-query/inputs.tf @@ -124,13 +124,13 @@ variable "scanned_bytes_timeframe" { } variable "scanned_bytes_threshold_warning" { - description = "Scanned Bytes Bytes (warning threshold)" + description = "Scanned Bytes (warning threshold)" type = "string" default = 0 } variable "scanned_bytes_threshold_critical" { - description = "Scanned Bytes Bytes (critical threshold)" + description = "Scanned Bytes (critical threshold)" type = "string" default = 1 } @@ -165,13 +165,13 @@ variable "scanned_bytes_billed_timeframe" { } variable "scanned_bytes_billed_threshold_warning" { - description = "Scanned Bytes Billed Bytes (warning threshold)" + description = "Scanned Bytes Billed (warning threshold)" type = "string" default = 0 } variable "scanned_bytes_billed_threshold_critical" { - description = "Scanned Bytes Billed Bytes (critical threshold)" + description = "Scanned Bytes Billed (critical threshold)" type = "string" default = 1 } @@ -328,13 +328,13 @@ variable "uploaded_bytes_timeframe" { } variable "uploaded_bytes_threshold_warning" { - description = "Uploaded Bytes in Bytes (warning threshold)" + description = "Uploaded Bytes (warning threshold)" type = "string" default = 0 } variable "uploaded_bytes_threshold_critical" { - description = "Uploaded Bytes in Bytes (critical threshold)" + description = "Uploaded Bytes (critical threshold)" type = "string" default = 1 } @@ -369,13 +369,13 @@ variable "uploaded_bytes_billed_timeframe" { } variable "uploaded_bytes_billed_threshold_warning" { - description = "Uploaded Bytes Billed in Bytes (warning threshold)" + description = "Uploaded Bytes Billed (warning threshold)" type = "string" default = 0 } variable "uploaded_bytes_billed_threshold_critical" { - description = "Uploaded Bytes Billed in Bytes (critical threshold)" + description = "Uploaded Bytes Billed (critical threshold)" type = "string" default = 1 }