From a1440f20b07ec854ba3bc124887ba06d7e69a874 Mon Sep 17 00:00:00 2001 From: Laurent Piroelle Date: Thu, 30 Aug 2018 14:42:50 +0200 Subject: [PATCH 1/3] MON-298 Cosmos DB scaling monitor --- cloud/azure/cosmosdb/README.md | 10 +++++ cloud/azure/cosmosdb/inputs.tf | 46 +++++++++++++++++++++++ cloud/azure/cosmosdb/monitors-cosmosdb.tf | 42 +++++++++++++++++++++ cloud/azure/cosmosdb/outputs.tf | 5 +++ 4 files changed, 103 insertions(+) diff --git a/cloud/azure/cosmosdb/README.md b/cloud/azure/cosmosdb/README.md index 6de3209..adbe728 100644 --- a/cloud/azure/cosmosdb/README.md +++ b/cloud/azure/cosmosdb/README.md @@ -19,6 +19,7 @@ Creates DataDog monitors with the following checks: - Cosmos DB 4xx requests rate is high - Cosmos DB 5xx requests rate is high - Cosmos DB is down +- Cosmos DB max scaling reached for collection ## Inputs @@ -40,6 +41,14 @@ Creates DataDog monitors with the following checks: | cosmos_db_5xx_requests_enabled | Flag to enable Cosmos DB 5xx requests monitor | string | `true` | no | | cosmos_db_5xx_requests_message | Custom message for Cosmos DB 5xx requests monitor | string | `` | no | | cosmos_db_5xx_requests_silenced | Groups to mute for Cosmos DB 5xx requests monitor | map | `` | no | +| cosmos_db_scaling_enabled | Flag to enable Cosmos DB scaling monitor | string | `true` | no | +| cosmos_db_scaling_error_rate_threshold_critical | Critical threshold for Cosmos DB scaling monitor | string | `10` | no | +| cosmos_db_scaling_error_rate_threshold_warning | Warning threshold for Cosmos DB scaling monitor | string | `5` | no | +| cosmos_db_scaling_extra_tags | Extra tags for Cosmos DB scaling monitor | list | `` | no | +| cosmos_db_scaling_message | Custom message for Cosmos DB scaling monitor | string | `` | no | +| cosmos_db_scaling_silenced | Groups to mute for Cosmos DB scaling monitor | map | `` | no | +| cosmos_db_scaling_time_aggregator | Monitor aggregator for Cosmos DB scaling [available values: min, max or avg] | string | `sum` | no | +| cosmos_db_scaling_timeframe | Monitor timeframe for Cosmos DB scaling [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | environment | Architecture environment | string | - | yes | | evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | @@ -59,6 +68,7 @@ Creates DataDog monitors with the following checks: |------|-------------| | cosmos_db_4xx_requests_id | id for monitor cosmos_db_4xx_requests | | cosmos_db_5xx_requests_id | id for monitor cosmos_db_5xx_requests | +| cosmos_db_scaling_id | id for monitor cosmos_db_scaling | | cosmos_db_status_id | id for monitor cosmos_db_status | Related documentation diff --git a/cloud/azure/cosmosdb/inputs.tf b/cloud/azure/cosmosdb/inputs.tf index 4d6cd55..beaaad9 100644 --- a/cloud/azure/cosmosdb/inputs.tf +++ b/cloud/azure/cosmosdb/inputs.tf @@ -155,3 +155,49 @@ variable "cosmos_db_5xx_request_timeframe" { type = "string" default = "last_5m" } + +variable "cosmos_db_scaling_message" { + description = "Custom message for Cosmos DB scaling monitor" + type = "string" + default = "" +} + +variable "cosmos_db_scaling_enabled" { + description = "Flag to enable Cosmos DB scaling monitor" + type = "string" + default = "true" +} + +variable "cosmos_db_scaling_silenced" { + description = "Groups to mute for Cosmos DB scaling monitor" + type = "map" + default = {} +} + +variable "cosmos_db_scaling_error_rate_threshold_critical" { + description = "Critical threshold for Cosmos DB scaling monitor" + default = 10 +} + +variable "cosmos_db_scaling_error_rate_threshold_warning" { + description = "Warning threshold for Cosmos DB scaling monitor" + default = 5 +} + +variable "cosmos_db_scaling_extra_tags" { + description = "Extra tags for Cosmos DB scaling monitor" + type = "list" + default = [] +} + +variable "cosmos_db_scaling_time_aggregator" { + description = "Monitor aggregator for Cosmos DB scaling [available values: min, max or avg]" + type = "string" + default = "sum" +} + +variable "cosmos_db_scaling_timeframe" { + description = "Monitor timeframe for Cosmos DB scaling [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} diff --git a/cloud/azure/cosmosdb/monitors-cosmosdb.tf b/cloud/azure/cosmosdb/monitors-cosmosdb.tf index f4aa6bb..39e2b92 100644 --- a/cloud/azure/cosmosdb/monitors-cosmosdb.tf +++ b/cloud/azure/cosmosdb/monitors-cosmosdb.tf @@ -134,3 +134,45 @@ resource "datadog_monitor" "cosmos_db_5xx_requests" { tags = ["env:${var.environment}", "type:cloud", "provider:azure", "resource:cosmos_db", "team:claranet", "created-by:terraform", "${var.cosmos_db_5xx_request_rate_extra_tags}"] } + +resource "datadog_monitor" "cosmos_db_scaling" { + count = "${var.cosmos_db_scaling_enabled ? 1 : 0}" + + name = "[${var.environment}] Cosmos DB max scaling reached for collection {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + message = "${coalesce(var.cosmos_db_scaling_message, var.message)}" + + # List of available status codes : https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb + query = < ${var.cosmos_db_scaling_error_rate_threshold_critical} + EOF + + type = "metric alert" + + thresholds { + critical = "${var.cosmos_db_scaling_error_rate_threshold_critical}" + warning = "${var.cosmos_db_scaling_error_rate_threshold_warning}" + } + + silenced = "${var.cosmos_db_scaling_silenced}" + + notify_no_data = false + evaluation_delay = "${var.evaluation_delay}" + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + new_host_delay = "${var.new_host_delay}" + + tags = ["env:${var.environment}", "type:cloud", "provider:azure", "resource:cosmos_db", "team:claranet", "created-by:terraform", "${var.cosmos_db_scaling_extra_tags}"] +} diff --git a/cloud/azure/cosmosdb/outputs.tf b/cloud/azure/cosmosdb/outputs.tf index 6901b39..41bc5ec 100644 --- a/cloud/azure/cosmosdb/outputs.tf +++ b/cloud/azure/cosmosdb/outputs.tf @@ -12,3 +12,8 @@ output "cosmos_db_5xx_requests_id" { description = "id for monitor cosmos_db_5xx_requests" value = "${datadog_monitor.cosmos_db_5xx_requests.*.id}" } + +output "cosmos_db_scaling_id" { + description = "id for monitor cosmos_db_scaling" + value = "${datadog_monitor.cosmos_db_scaling.*.id}" +} From 3561a0ef92d34a9c5df50d1f01206d6e87fa47a2 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Fri, 14 Sep 2018 16:45:27 +0200 Subject: [PATCH 2/3] MON-298 fix require full window --- cloud/azure/cosmosdb/monitors-cosmosdb.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/azure/cosmosdb/monitors-cosmosdb.tf b/cloud/azure/cosmosdb/monitors-cosmosdb.tf index 39e2b92..7525e39 100644 --- a/cloud/azure/cosmosdb/monitors-cosmosdb.tf +++ b/cloud/azure/cosmosdb/monitors-cosmosdb.tf @@ -171,7 +171,7 @@ resource "datadog_monitor" "cosmos_db_scaling" { timeout_h = 0 include_tags = true locked = false - require_full_window = true + require_full_window = false new_host_delay = "${var.new_host_delay}" tags = ["env:${var.environment}", "type:cloud", "provider:azure", "resource:cosmos_db", "team:claranet", "created-by:terraform", "${var.cosmos_db_scaling_extra_tags}"] From c1d2397d03cddde316e4067c70122db87ea0dc96 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Fri, 14 Sep 2018 16:48:14 +0200 Subject: [PATCH 3/3] MON-298 add 429 status code doc --- cloud/azure/cosmosdb/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud/azure/cosmosdb/README.md b/cloud/azure/cosmosdb/README.md index adbe728..7ac3826 100644 --- a/cloud/azure/cosmosdb/README.md +++ b/cloud/azure/cosmosdb/README.md @@ -79,3 +79,4 @@ You must search `cosmosdb`, there is no integration for now. Azure metrics documentation : [https://docs.microsoft.com/fr-fr/azure/monitoring-and-diagnostics/monitoring-supported-metrics#microsoftdocumentdbdatabaseaccounts](https://docs.microsoft.com/fr-fr/azure/monitoring-and-diagnostics/monitoring-supported-metrics#microsoftdocumentdbdatabaseaccounts) +429 status code : [https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb](https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb)