Merged in MON-298-cosmosdb-scaling-monitor (pull request #145)

MON-298 Cosmos DB scaling monitor

Approved-by: Laurent Piroelle <laurent.piroelle@fr.clara.net>
Approved-by: Quentin Manfroi <quentin.manfroi@yahoo.fr>

cloud/azure/cosmosdb/README.md

@@ -19,6 +19,7 @@ Creates DataDog monitors with the following checks:
 - Cosmos DB 4xx requests rate is high
 - Cosmos DB 5xx requests rate is high
 - Cosmos DB is down
+- Cosmos DB max scaling reached for collection
 
 ## Inputs
 
@@ -40,6 +41,14 @@ Creates DataDog monitors with the following checks:
 | cosmos_db_5xx_requests_enabled | Flag to enable Cosmos DB 5xx requests monitor | string | `true` | no |
 | cosmos_db_5xx_requests_message | Custom message for Cosmos DB 5xx requests monitor | string | `` | no |
 | cosmos_db_5xx_requests_silenced | Groups to mute for Cosmos DB 5xx requests monitor | map | `<map>` | no |
+| cosmos_db_scaling_enabled | Flag to enable Cosmos DB scaling monitor | string | `true` | no |
+| cosmos_db_scaling_error_rate_threshold_critical | Critical threshold for Cosmos DB scaling monitor | string | `10` | no |
+| cosmos_db_scaling_error_rate_threshold_warning | Warning threshold for Cosmos DB scaling monitor | string | `5` | no |
+| cosmos_db_scaling_extra_tags | Extra tags for Cosmos DB scaling monitor | list | `<list>` | no |
+| cosmos_db_scaling_message | Custom message for Cosmos DB scaling monitor | string | `` | no |
+| cosmos_db_scaling_silenced | Groups to mute for Cosmos DB scaling monitor | map | `<map>` | no |
+| cosmos_db_scaling_time_aggregator | Monitor aggregator for Cosmos DB scaling [available values: min, max or avg] | string | `sum` | no |
+| cosmos_db_scaling_timeframe | Monitor timeframe for Cosmos DB scaling [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no |
 | environment | Architecture environment | string | - | yes |
 | evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no |
 | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
@@ -59,6 +68,7 @@ Creates DataDog monitors with the following checks:
 |------|-------------|
 | cosmos_db_4xx_requests_id | id for monitor cosmos_db_4xx_requests |
 | cosmos_db_5xx_requests_id | id for monitor cosmos_db_5xx_requests |
+| cosmos_db_scaling_id | id for monitor cosmos_db_scaling |
 | cosmos_db_status_id | id for monitor cosmos_db_status |
 
 Related documentation
@@ -69,3 +79,4 @@ You must search `cosmosdb`, there is no integration for now.
 
 Azure metrics documentation : [https://docs.microsoft.com/fr-fr/azure/monitoring-and-diagnostics/monitoring-supported-metrics#microsoftdocumentdbdatabaseaccounts](https://docs.microsoft.com/fr-fr/azure/monitoring-and-diagnostics/monitoring-supported-metrics#microsoftdocumentdbdatabaseaccounts)
 
+429 status code : [https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb](https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb)

cloud/azure/cosmosdb/inputs.tf

@@ -155,3 +155,49 @@ variable "cosmos_db_5xx_request_timeframe" {
   type        = "string"
   default     = "last_5m"
 }
+
+variable "cosmos_db_scaling_message" {
+  description = "Custom message for Cosmos DB scaling monitor"
+  type        = "string"
+  default     = ""
+}
+
+variable "cosmos_db_scaling_enabled" {
+  description = "Flag to enable Cosmos DB scaling monitor"
+  type        = "string"
+  default     = "true"
+}
+
+variable "cosmos_db_scaling_silenced" {
+  description = "Groups to mute for Cosmos DB scaling monitor"
+  type        = "map"
+  default     = {}
+}
+
+variable "cosmos_db_scaling_error_rate_threshold_critical" {
+  description = "Critical threshold for Cosmos DB scaling monitor"
+  default     = 10
+}
+
+variable "cosmos_db_scaling_error_rate_threshold_warning" {
+  description = "Warning threshold for Cosmos DB scaling monitor"
+  default     = 5
+}
+
+variable "cosmos_db_scaling_extra_tags" {
+  description = "Extra tags for Cosmos DB scaling monitor"
+  type        = "list"
+  default     = []
+}
+
+variable "cosmos_db_scaling_time_aggregator" {
+  description = "Monitor aggregator for Cosmos DB scaling [available values: min, max or avg]"
+  type        = "string"
+  default     = "sum"
+}
+
+variable "cosmos_db_scaling_timeframe" {
+  description = "Monitor timeframe for Cosmos DB scaling [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = "string"
+  default     = "last_5m"
+}

cloud/azure/cosmosdb/monitors-cosmosdb.tf

@@ -134,3 +134,45 @@ resource "datadog_monitor" "cosmos_db_5xx_requests" {
 
   tags = ["env:${var.environment}", "type:cloud", "provider:azure", "resource:cosmos_db", "team:claranet", "created-by:terraform", "${var.cosmos_db_5xx_request_rate_extra_tags}"]
 }
+
+resource "datadog_monitor" "cosmos_db_scaling" {
+  count = "${var.cosmos_db_scaling_enabled ? 1 : 0}"
+
+  name    = "[${var.environment}] Cosmos DB max scaling reached for collection {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = "${coalesce(var.cosmos_db_scaling_message, var.message)}"
+
+  # List of available status codes : https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb
+  query = <<EOF
+      ${var.cosmos_db_scaling_time_aggregator}(${var.cosmos_db_scaling_timeframe}): (default(
+        (
+          sum:azure.cosmosdb.total_requests${format(module.filter-tags-statuscode.query_alert, "429")} by {resource_group,region,name,collectionname}.as_count() +
+          sum:azure.documentdb_databaseaccounts.total_requests${format(module.filter-tags-statuscode.query_alert, "429")} by {resource_group,region,name,collectionname}.as_count()
+        ) / (
+          sum:azure.cosmosdb.total_requests${module.filter-tags.query_alert} by {resource_group,region,name,collectionname}.as_count() +
+          sum:azure.documentdb_databaseaccounts.total_requests${module.filter-tags.query_alert} by {resource_group,region,name,collectionname}.as_count()
+        )
+        * 100, 0)
+      ) > ${var.cosmos_db_scaling_error_rate_threshold_critical}
+      EOF
+
+  type = "metric alert"
+
+  thresholds {
+    critical = "${var.cosmos_db_scaling_error_rate_threshold_critical}"
+    warning  = "${var.cosmos_db_scaling_error_rate_threshold_warning}"
+  }
+
+  silenced = "${var.cosmos_db_scaling_silenced}"
+
+  notify_no_data      = false
+  evaluation_delay    = "${var.evaluation_delay}"
+  renotify_interval   = 0
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = false
+  new_host_delay      = "${var.new_host_delay}"
+
+  tags = ["env:${var.environment}", "type:cloud", "provider:azure", "resource:cosmos_db", "team:claranet", "created-by:terraform", "${var.cosmos_db_scaling_extra_tags}"]
+}

cloud/azure/cosmosdb/outputs.tf

@@ -12,3 +12,8 @@ output "cosmos_db_5xx_requests_id" {
   description = "id for monitor cosmos_db_5xx_requests"
   value       = "${datadog_monitor.cosmos_db_5xx_requests.*.id}"
 }
+
+output "cosmos_db_scaling_id" {
+  description = "id for monitor cosmos_db_scaling"
+  value       = "${datadog_monitor.cosmos_db_scaling.*.id}"
+}