diff --git a/README.md b/README.md index 7090482..a8706f7 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ The `//` is very important, it's a terraform specific syntax used to separate gi - [cloud-sql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/) - [common](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/common/) - [mysql](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/cloud-sql/mysql/) + - [lb](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/lb/) - [pubsub](https://bitbucket.org/morea/terraform.feature.datadog/src/master/cloud/gcp/pubsub/) - [common](https://bitbucket.org/morea/terraform.feature.datadog/src/master/common/) - [alerting-message](https://bitbucket.org/morea/terraform.feature.datadog/src/master/common/alerting-message/) diff --git a/cloud/gcp/lb/README.md b/cloud/gcp/lb/README.md new file mode 100644 index 0000000..cc707e7 --- /dev/null +++ b/cloud/gcp/lb/README.md @@ -0,0 +1,84 @@ +# CLOUD GCP LB DataDog monitors + +## How to use this module + +``` +module "datadog-monitors-cloud-gcp-lb" { + source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/gcp/lb?ref={revision}" + + environment = "${var.environment}" + message = "${module.datadog-message-alerting.alerting-message}" +} + +``` + +## Purpose + +Creates DataDog monitors with the following checks: + +- GCP LB 4xx errors +- GCP LB 5xx errors +- GCP LB bucket backend latency +- GCP LB Requests count increased abruptly +- GCP LB service backend latency + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| backend_latency_bucket_extra_tags | Extra tags for GCP LB Backend Latency monitor | list | `` | no | +| backend_latency_bucket_message | Custom message for the GCP LB Backend Latency monitor | string | `` | no | +| backend_latency_bucket_silenced | Groups to mute for GCP LB Backend Latency monitor | map | `` | no | +| backend_latency_bucket_threshold_critical | Latency in milliseconds (critical threshold) | string | `8000` | no | +| backend_latency_bucket_threshold_warning | Latency in milliseconds (warning threshold) | string | `4000` | no | +| backend_latency_bucket_time_aggregator | Timeframe for the GCP LB Backend Latency monitor | string | `min` | no | +| backend_latency_bucket_timeframe | Timeframe for the GCP LB Backend Latency monitor | string | `last_10m` | no | +| backend_latency_service_extra_tags | Extra tags for GCP LB Backend Latency monitor | list | `` | no | +| backend_latency_service_message | Custom message for the GCP LB Backend Latency monitor | string | `` | no | +| backend_latency_service_silenced | Groups to mute for GCP LB Backend Latency monitor | map | `` | no | +| backend_latency_service_threshold_critical | Latency in milliseconds (critical threshold) | string | `1500` | no | +| backend_latency_service_threshold_warning | Latency in milliseconds (warning threshold) | string | `1000` | no | +| backend_latency_service_time_aggregator | Timeframe for the GCP LB Backend Latency monitor | string | `min` | no | +| backend_latency_service_timeframe | Timeframe for the GCP LB Backend Latency monitor | string | `last_10m` | no | +| environment | Architecture environment | string | - | yes | +| error_rate_4xx_artificial_request | Divisor Delta for the GCP LB 4XX Errors monitor | string | `5` | no | +| error_rate_4xx_extra_tags | Extra tags for GCP LB 4XX Errors monitor | list | `` | no | +| error_rate_4xx_message | Custom message for the GCP LB 4XX Errors monitor | string | `` | no | +| error_rate_4xx_silenced | Groups to mute for GCP LB 4XX Errors monitor | map | `` | no | +| error_rate_4xx_threshold_critical | Rate error in percentage (critical threshold) | string | `50` | no | +| error_rate_4xx_time_aggregator | Timeframe for the GCP LB 4XX Errors monitor | string | `sum` | no | +| error_rate_4xx_timeframe | Timeframe for the GCP LB 4XX Errors monitor | string | `last_5m` | no | +| error_rate_5xx_artificial_request | Divisor Delta for the GCP LB 5XX Errors monitor | string | `5` | no | +| error_rate_5xx_extra_tags | Extra tags for GCP LB 5XX Errors monitor | list | `` | no | +| error_rate_5xx_message | Custom message for the GCP LB 5XX Errors monitor | string | `` | no | +| error_rate_5xx_silenced | Groups to mute for GCP LB 5XX Errors monitor | map | `` | no | +| error_rate_5xx_threshold_critical | Rate error in percentage (critical threshold) | string | `50` | no | +| error_rate_5xx_time_aggregator | Timeframe for the GCP LB 5XX Errors monitor | string | `sum` | no | +| error_rate_5xx_timeframe | Timeframe for the GCP LB 5XX Errors monitor | string | `last_5m` | no | +| evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no | +| filter_tags | Tags used for filtering | string | `*` | no | +| message | Message sent when a monitor is triggered | string | - | yes | +| new_host_delay | Delay in seconds for the new host evaluation | string | `300` | no | +| request_count_extra_tags | Extra tags for GCP LB Request Count monitor | list | `` | no | +| request_count_message | Custom message for the GCP LB Request Count monitor | string | `` | no | +| request_count_silenced | Groups to mute for GCP LB Request Count monitor | map | `` | no | +| request_count_threshold_critical | Desviation in percentage (critical threshold) | string | `500` | no | +| request_count_threshold_warning | Desviation in percentage (warning threshold) | string | `250` | no | +| request_count_time_aggregator | Timeframe for the GCP LB Request Count monitor | string | `sum` | no | +| request_count_timeframe | Timeframe for the GCP LB Request Count monitor | string | `last_5m` | no | +| request_count_timeshift | Timeshift for the GCP LB Request Count monitor | string | `last_5m` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| backend_latency_bucket_id | id for monitor backend_latency_bucket | +| backend_latency_service_id | id for monitor backend_latency_service | +| error_rate_4xx_id | id for monitor error_rate_4xx | +| error_rate_5xx_id | id for monitor error_rate_5xx | +| request_count_id | id for monitor request_count | + +## Related documentation + +* [GCP LB Metrics](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-loadbalancing) +* [Datadog GCP integration](https://docs.datadoghq.com/integrations/google_cloud_platform/) diff --git a/cloud/gcp/lb/inputs.tf b/cloud/gcp/lb/inputs.tf new file mode 100644 index 0000000..69f3375 --- /dev/null +++ b/cloud/gcp/lb/inputs.tf @@ -0,0 +1,257 @@ +# +# Datadog global variables +# +variable "environment" { + description = "Architecture environment" + type = "string" +} + +variable "filter_tags" { + description = "Tags used for filtering" + default = "*" +} + +variable "message" { + description = "Message sent when a monitor is triggered" +} + +variable "evaluation_delay" { + description = "Delay in seconds for the metric evaluation" + default = 900 +} + +variable "new_host_delay" { + description = "Delay in seconds for the new host evaluation" + default = 300 +} + +# +# 4XX Errors +# +variable "error_rate_4xx_message" { + description = "Custom message for the GCP LB 4XX Errors monitor" + type = "string" + default = "" +} + +variable "error_rate_4xx_time_aggregator" { + description = "Timeframe for the GCP LB 4XX Errors monitor" + type = "string" + default = "sum" +} + +variable "error_rate_4xx_timeframe" { + description = "Timeframe for the GCP LB 4XX Errors monitor" + type = "string" + default = "last_5m" +} + +variable "error_rate_4xx_artificial_request" { + description = "Divisor Delta for the GCP LB 4XX Errors monitor" + type = "string" + default = 5 +} + +variable "error_rate_4xx_threshold_critical" { + description = "Rate error in percentage (critical threshold)" + type = "string" + default = 50 +} + +variable "error_rate_4xx_silenced" { + description = "Groups to mute for GCP LB 4XX Errors monitor" + type = "map" + default = {} +} + +variable "error_rate_4xx_extra_tags" { + description = "Extra tags for GCP LB 4XX Errors monitor" + type = "list" + default = [] +} + +# +# 5XX Errors +# +variable "error_rate_5xx_message" { + description = "Custom message for the GCP LB 5XX Errors monitor" + type = "string" + default = "" +} + +variable "error_rate_5xx_time_aggregator" { + description = "Timeframe for the GCP LB 5XX Errors monitor" + type = "string" + default = "sum" +} + +variable "error_rate_5xx_timeframe" { + description = "Timeframe for the GCP LB 5XX Errors monitor" + type = "string" + default = "last_5m" +} + +variable "error_rate_5xx_artificial_request" { + description = "Divisor Delta for the GCP LB 5XX Errors monitor" + type = "string" + default = 5 +} + +variable "error_rate_5xx_threshold_critical" { + description = "Rate error in percentage (critical threshold)" + type = "string" + default = 50 +} + +variable "error_rate_5xx_silenced" { + description = "Groups to mute for GCP LB 5XX Errors monitor" + type = "map" + default = {} +} + +variable "error_rate_5xx_extra_tags" { + description = "Extra tags for GCP LB 5XX Errors monitor" + type = "list" + default = [] +} + +# +# Latency Backend service +# +variable "backend_latency_service_message" { + description = "Custom message for the GCP LB Backend Latency monitor" + type = "string" + default = "" +} + +variable "backend_latency_service_time_aggregator" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "min" +} + +variable "backend_latency_service_timeframe" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "last_10m" +} + +variable "backend_latency_service_threshold_warning" { + description = "Latency in milliseconds (warning threshold)" + type = "string" + default = 1000 +} + +variable "backend_latency_service_threshold_critical" { + description = "Latency in milliseconds (critical threshold)" + type = "string" + default = 1500 +} + +variable "backend_latency_service_silenced" { + description = "Groups to mute for GCP LB Backend Latency monitor" + type = "map" + default = {} +} + +variable "backend_latency_service_extra_tags" { + description = "Extra tags for GCP LB Backend Latency monitor" + type = "list" + default = [] +} + +# +# Latency Backend bucket +# +variable "backend_latency_bucket_message" { + description = "Custom message for the GCP LB Backend Latency monitor" + type = "string" + default = "" +} + +variable "backend_latency_bucket_time_aggregator" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "min" +} + +variable "backend_latency_bucket_timeframe" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "last_10m" +} + +variable "backend_latency_bucket_threshold_warning" { + description = "Latency in milliseconds (warning threshold)" + type = "string" + default = 4000 +} + +variable "backend_latency_bucket_threshold_critical" { + description = "Latency in milliseconds (critical threshold)" + type = "string" + default = 8000 +} + +variable "backend_latency_bucket_silenced" { + description = "Groups to mute for GCP LB Backend Latency monitor" + type = "map" + default = {} +} + +variable "backend_latency_bucket_extra_tags" { + description = "Extra tags for GCP LB Backend Latency monitor" + type = "list" + default = [] +} + +# +# Request Count +# +variable "request_count_message" { + description = "Custom message for the GCP LB Request Count monitor" + type = "string" + default = "" +} + +variable "request_count_time_aggregator" { + description = "Timeframe for the GCP LB Request Count monitor" + type = "string" + default = "sum" +} + +variable "request_count_timeframe" { + description = "Timeframe for the GCP LB Request Count monitor" + type = "string" + default = "last_5m" +} + +variable "request_count_timeshift" { + description = "Timeshift for the GCP LB Request Count monitor" + type = "string" + default = "last_5m" +} + +variable "request_count_threshold_warning" { + description = "Desviation in percentage (warning threshold)" + type = "string" + default = 250 +} + +variable "request_count_threshold_critical" { + description = "Desviation in percentage (critical threshold)" + type = "string" + default = 500 +} + +variable "request_count_silenced" { + description = "Groups to mute for GCP LB Request Count monitor" + type = "map" + default = {} +} + +variable "request_count_extra_tags" { + description = "Extra tags for GCP LB Request Count monitor" + type = "list" + default = [] +} diff --git a/cloud/gcp/lb/monitors-lb.tf b/cloud/gcp/lb/monitors-lb.tf new file mode 100644 index 0000000..8982222 --- /dev/null +++ b/cloud/gcp/lb/monitors-lb.tf @@ -0,0 +1,181 @@ +# +# 4XX Errors +# +resource "datadog_monitor" "error_rate_4xx" { + name = "[${var.environment}] GCP LB 4xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.error_rate_4xx_message, var.message)}" + + type = "metric alert" + + query = < ${var.error_rate_4xx_threshold_critical} +EOF + + thresholds { + critical = "${var.error_rate_4xx_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.error_rate_4xx_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.error_rate_4xx_extra_tags}"] +} + +# +# 5XX Errors +# +resource "datadog_monitor" "error_rate_5xx" { + name = "[${var.environment}] GCP LB 5xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.error_rate_5xx_message, var.message)}" + + type = "metric alert" + + query = < ${var.error_rate_4xx_threshold_critical} +EOF + + thresholds { + critical = "${var.error_rate_5xx_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.error_rate_5xx_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.error_rate_5xx_extra_tags}"] +} + +# +# Backend Latency for service +# +resource "datadog_monitor" "backend_latency_service" { + name = "[${var.environment}] GCP LB service backend latency {{#is_alert}}{{{comparator}}} {{threshold}}ms ({{value}}ms){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}ms ({{value}}ms){{/is_warning}}" + message = "${coalesce(var.backend_latency_service_message, var.message)}" + + type = "metric alert" + + query = < ${var.backend_latency_service_threshold_critical} +EOF + + thresholds { + warning = "${var.backend_latency_service_threshold_warning}" + critical = "${var.backend_latency_service_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.backend_latency_service_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.backend_latency_service_extra_tags}"] +} + +# +# Backend Latency for bucket +# +resource "datadog_monitor" "backend_latency_bucket" { + name = "[${var.environment}] GCP LB bucket backend latency {{#is_alert}}{{{comparator}}} {{threshold}}ms ({{value}}ms){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}ms ({{value}}ms){{/is_warning}}" + message = "${coalesce(var.backend_latency_bucket_message, var.message)}" + + type = "metric alert" + + query = < ${var.backend_latency_bucket_threshold_critical} +EOF + + thresholds { + warning = "${var.backend_latency_bucket_threshold_warning}" + critical = "${var.backend_latency_bucket_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.backend_latency_bucket_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.backend_latency_bucket_extra_tags}"] +} + +# +# Request Count +# +resource "datadog_monitor" "request_count" { + name = "[${var.environment}] GCP LB Requests count increased abruptly {{#is_alert}}{{value}}%{{/is_alert}}{{#is_warning}}{{value}}%{{/is_warning}}" + message = "${coalesce(var.request_count_message, var.message)}" + + type = "query alert" + + query = < ${var.request_count_threshold_critical} +EOF + + thresholds { + warning = "${var.request_count_threshold_warning}" + critical = "${var.request_count_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.request_count_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.request_count_extra_tags}"] +} diff --git a/cloud/gcp/lb/outputs.tf b/cloud/gcp/lb/outputs.tf new file mode 100644 index 0000000..4718769 --- /dev/null +++ b/cloud/gcp/lb/outputs.tf @@ -0,0 +1,24 @@ +output "error_rate_4xx_id" { + description = "id for monitor error_rate_4xx" + value = "${datadog_monitor.error_rate_4xx.*.id}" +} + +output "error_rate_5xx_id" { + description = "id for monitor error_rate_5xx" + value = "${datadog_monitor.error_rate_5xx.*.id}" +} + +output "backend_latency_service_id" { + description = "id for monitor backend_latency_service" + value = "${datadog_monitor.backend_latency_service.*.id}" +} + +output "backend_latency_bucket_id" { + description = "id for monitor backend_latency_bucket" + value = "${datadog_monitor.backend_latency_bucket.*.id}" +} + +output "request_count_id" { + description = "id for monitor request_count" + value = "${datadog_monitor.request_count.*.id}" +}