MON-366 Add multiple monitors

cloud/azure/app-gateway/README.md

@@ -16,22 +16,46 @@ module "datadog-monitors-cloud-azure-app-gateway" {
 
 Creates DataDog monitors with the following checks:
 
-- App Gateway failed requests (disabled by default)
+- App Gateway backend connect time
-- App Gateway HTTP 4xx errors too high
+- App Gateway Backend HTTP 4xx errors
-- App Gateway HTTP 5xx errors too high
+- App Gateway failed requests
+- App Gateway HTTP 4xx errors
+- App Gateway HTTP 5xx errors
+- App Gateway HTTP Backend 5xx errors
 - App Gateway is down
-- App Gateway no connection (disabled by default)
+- App Gateway no connection
 - App Gateway no healthy host
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| appgateway\_failed\_requests\_enabled | Flag to enable App Gateway failed requests monitor | string | `"false"` | no |
+| appgateway\_backend\_connect\_time\_enabled | Flag to enable App Gateway backend_connect_time monitor | string | `"true"` | no |
+| appgateway\_backend\_connect\_time\_extra\_tags | Extra tags for App Gateway backend_connect_time monitor | list(string) | `[]` | no |
+| appgateway\_backend\_connect\_time\_message | Custom message for App Gateway backend_connect_time monitor | string | `""` | no |
+| appgateway\_backend\_connect\_time\_threshold\_critical | Maximum critical backend_connect_time errors | string | `"50"` | no |
+| appgateway\_backend\_connect\_time\_threshold\_warning | Warning regarding backend_connect_time errors | string | `"40"` | no |
+| appgateway\_backend\_connect\_time\_time\_aggregator | Monitor aggregator for App Gateway backend_connect_time [available values: min, max or avg] | string | `"min"` | no |
+| appgateway\_backend\_connect\_time\_timeframe | Monitor timeframe for App Gateway backend_connect_time [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
+| appgateway\_backend\_http\_4xx\_errors\_enabled | Flag to enable App Gateway http 4xx errors monitor | string | `"true"` | no |
+| appgateway\_backend\_http\_4xx\_errors\_extra\_tags | Extra tags for App Gateway http 4xx errors monitor | list(string) | `[]` | no |
+| appgateway\_backend\_http\_4xx\_errors\_message | Custom message for App Gateway http 4xx errors monitor | string | `""` | no |
+| appgateway\_backend\_http\_4xx\_errors\_threshold\_critical | Minimum critical acceptable percent of 4xx error | string | `"30"` | no |
+| appgateway\_backend\_http\_4xx\_errors\_threshold\_warning | Warning regarding acceptable percent of 4xx error | string | `"20"` | no |
+| appgateway\_backend\_http\_4xx\_errors\_time\_aggregator | Monitor aggregator for App Gateway http 4xx errors [available values: min, max or avg] | string | `"max"` | no |
+| appgateway\_backend\_http\_4xx\_errors\_timeframe | Monitor timeframe for App Gateway http 4xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
+| appgateway\_backend\_http\_5xx\_errors\_enabled | Flag to enable App Gateway http 5xx errors monitor | string | `"true"` | no |
+| appgateway\_backend\_http\_5xx\_errors\_extra\_tags | Extra tags for App Gateway http 5xx errors monitor | list(string) | `[]` | no |
+| appgateway\_backend\_http\_5xx\_errors\_message | Custom message for App Gateway http 5xx errors monitor | string | `""` | no |
+| appgateway\_backend\_http\_5xx\_errors\_threshold\_critical | Minimum critical acceptable percent of 5xx error | string | `"30"` | no |
+| appgateway\_backend\_http\_5xx\_errors\_threshold\_warning | Warning regarding acceptable percent of 5xx error | string | `"20"` | no |
+| appgateway\_backend\_http\_5xx\_errors\_time\_aggregator | Monitor aggregator for App Gateway http 5xx errors [available values: min, max or avg] | string | `"max"` | no |
+| appgateway\_backend\_http\_5xx\_errors\_timeframe | Monitor timeframe for App Gateway http 5xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
+| appgateway\_failed\_requests\_enabled | Flag to enable App Gateway failed requests monitor | string | `"true"` | no |
 | appgateway\_failed\_requests\_extra\_tags | Extra tags for App Gateway failed requests monitor | list(string) | `[]` | no |
 | appgateway\_failed\_requests\_message | Custom message for App Gateway failed requests monitor | string | `""` | no |
-| appgateway\_failed\_requests\_threshold\_critical | Maximum critical acceptable percent of failed errors | string | `"100"` | no |
+| appgateway\_failed\_requests\_threshold\_critical | Maximum critical acceptable percent of failed errors | string | `"50"` | no |
-| appgateway\_failed\_requests\_threshold\_warning | Warning regarding acceptable percent of failed errors | string | `"200"` | no |
+| appgateway\_failed\_requests\_threshold\_warning | Warning regarding acceptable percent of failed errors | string | `"40"` | no |
 | appgateway\_failed\_requests\_time\_aggregator | Monitor aggregator for App Gateway failed requests [available values: min, max or avg] | string | `"min"` | no |
 | appgateway\_failed\_requests\_timeframe | Monitor timeframe for App Gateway failed requests [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
 | appgateway\_healthy\_host\_count\_enabled | Flag to enable App Gateway healthy host monitor | string | `"true"` | no |
@@ -53,7 +77,7 @@ Creates DataDog monitors with the following checks:
 | appgateway\_http\_5xx\_errors\_threshold\_warning | Warning regarding acceptable percent of 5xx error | string | `"20"` | no |
 | appgateway\_http\_5xx\_errors\_time\_aggregator | Monitor aggregator for App Gateway http 5xx errors [available values: min, max or avg] | string | `"max"` | no |
 | appgateway\_http\_5xx\_errors\_timeframe | Monitor timeframe for App Gateway http 5xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
-| current\_connection\_enabled | Flag to enable App Gateway current connections monitor | string | `"false"` | no |
+| current\_connection\_enabled | Flag to enable App Gateway current connections monitor | string | `"true"` | no |
 | current\_connection\_extra\_tags | Extra tags for App Gateway current connections monitor | list(string) | `[]` | no |
 | current\_connection\_message | Custom message for App Gateway current connections monitor | string | `""` | no |
 | current\_connection\_time\_aggregator | Monitor aggregator for App Gateway current connections [available values: min, max or avg] | string | `"min"` | no |
@@ -76,6 +100,9 @@ Creates DataDog monitors with the following checks:
 
 | Name | Description |
 |------|-------------|
+| appgateway\_backend\_connect\_time\_id | id for monitor appgateway_backend_connect_time |
+| appgateway\_backend\_http\_4xx\_errors\_id | id for monitor appgateway_backend_http_4xx_errors |
+| appgateway\_backend\_http\_5xx\_errors\_id | id for monitor appgateway_backend_http_5xx_errors |
 | appgateway\_failed\_requests\_id | id for monitor appgateway_failed_requests |
 | appgateway\_healthy\_host\_count\_id | id for monitor appgateway_healthy_host_count |
 | appgateway\_http\_4xx\_errors\_id | id for monitor appgateway_http_4xx_errors |

cloud/azure/app-gateway/inputs.tf

@@ -1,3 +1,4 @@
+# Azure App Gateway global variables
 variable "environment" {
   description = "Architecture environment"
   type        = string
@@ -69,11 +70,11 @@ variable "status_timeframe" {
   default     = "last_5m"
 }
 
-# Monitoring App Gateway current_connections (count)
+# Monitoring App Gateway current_connections
 variable "current_connection_enabled" {
   description = "Flag to enable App Gateway current connections monitor"
   type        = string
-  default     = "false"
+  default     = "true"
 }
 
 variable "current_connection_extra_tags" {
@@ -100,11 +101,52 @@ variable "current_connection_timeframe" {
   default     = "last_5m"
 }
 
-# Monitoring App Gateway failed_requests (count)
+# Monitoring App Gateway failed_requests
+variable "appgateway_backend_connect_time_enabled" {
+  description = "Flag to enable App Gateway backend_connect_time monitor"
+  type        = string
+  default     = "true"
+}
+
+variable "appgateway_backend_connect_time_extra_tags" {
+  description = "Extra tags for App Gateway backend_connect_time monitor"
+  type        = list(string)
+  default     = []
+}
+
+variable "appgateway_backend_connect_time_message" {
+  description = "Custom message for App Gateway backend_connect_time monitor"
+  type        = string
+  default     = ""
+}
+
+variable "appgateway_backend_connect_time_time_aggregator" {
+  description = "Monitor aggregator for App Gateway backend_connect_time [available values: min, max or avg]"
+  type        = string
+  default     = "min"
+}
+
+variable "appgateway_backend_connect_time_timeframe" {
+  description = "Monitor timeframe for App Gateway backend_connect_time [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = string
+  default     = "last_5m"
+}
+
+variable "appgateway_backend_connect_time_threshold_critical" {
+  default     = 50
+  description = "Maximum critical backend_connect_time errors"
+}
+
+variable "appgateway_backend_connect_time_threshold_warning" {
+  default     = 40
+  description = "Warning regarding backend_connect_time errors"
+}
+
+# Monitoring App Gateway failed_requests
 variable "appgateway_failed_requests_enabled" {
   description = "Flag to enable App Gateway failed requests monitor"
   type        = string
-  default     = "false"
+  default     = "true"
 }
 
 variable "appgateway_failed_requests_extra_tags" {
@@ -132,16 +174,16 @@ variable "appgateway_failed_requests_timeframe" {
 }
 
 variable "appgateway_failed_requests_threshold_critical" {
-  default     = 100
+  default     = 50
   description = "Maximum critical acceptable percent of failed errors"
 }
 
 variable "appgateway_failed_requests_threshold_warning" {
-  default     = 200
+  default     = 40
   description = "Warning regarding acceptable percent of failed errors"
 }
 
-# Monitoring App Gateway healthy_host_count (count)
+# Monitoring App Gateway healthy_host_count
 variable "appgateway_healthy_host_count_enabled" {
   description = "Flag to enable App Gateway healthy host monitor"
   type        = string
@@ -172,7 +214,7 @@ variable "appgateway_healthy_host_count_timeframe" {
   default     = "last_5m"
 }
 
-# Monitoring App Gateway response_status 4xx (count)
+# Monitoring App Gateway response_status 4xx
 variable "appgateway_http_4xx_errors_enabled" {
   description = "Flag to enable App Gateway http 4xx errors monitor"
   type        = string
@@ -213,7 +255,7 @@ variable "appgateway_http_4xx_errors_threshold_warning" {
   description = "Warning regarding acceptable percent of 4xx error"
 }
 
-# Monitoring App Gateway response_status 5xx (count)
+# Monitoring App Gateway response_status 5xx
 variable "appgateway_http_5xx_errors_enabled" {
   description = "Flag to enable App Gateway http 5xx errors monitor"
   type        = string
@@ -253,3 +295,85 @@ variable "appgateway_http_5xx_errors_threshold_warning" {
   default     = 20
   description = "Warning regarding acceptable percent of 5xx error"
 }
+
+# Monitoring App Gateway Backend response_status 4xx
+variable "appgateway_backend_http_4xx_errors_enabled" {
+  description = "Flag to enable App Gateway http 4xx errors monitor"
+  type        = string
+  default     = "true"
+}
+
+variable "appgateway_backend_http_4xx_errors_extra_tags" {
+  description = "Extra tags for App Gateway http 4xx errors monitor"
+  type        = list(string)
+  default     = []
+}
+
+variable "appgateway_backend_http_4xx_errors_message" {
+  description = "Custom message for App Gateway http 4xx errors monitor"
+  type        = string
+  default     = ""
+}
+
+variable "appgateway_backend_http_4xx_errors_time_aggregator" {
+  description = "Monitor aggregator for App Gateway http 4xx errors [available values: min, max or avg]"
+  type        = string
+  default     = "max"
+}
+
+variable "appgateway_backend_http_4xx_errors_timeframe" {
+  description = "Monitor timeframe for App Gateway http 4xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = string
+  default     = "last_5m"
+}
+
+variable "appgateway_backend_http_4xx_errors_threshold_critical" {
+  default     = 30
+  description = "Minimum critical acceptable percent of 4xx error"
+}
+
+variable "appgateway_backend_http_4xx_errors_threshold_warning" {
+  default     = 20
+  description = "Warning regarding acceptable percent of 4xx error"
+}
+
+# Monitoring App Gateway Backend response_status 5xx
+variable "appgateway_backend_http_5xx_errors_enabled" {
+  description = "Flag to enable App Gateway http 5xx errors monitor"
+  type        = string
+  default     = "true"
+}
+
+variable "appgateway_backend_http_5xx_errors_extra_tags" {
+  description = "Extra tags for App Gateway http 5xx errors monitor"
+  type        = list(string)
+  default     = []
+}
+
+variable "appgateway_backend_http_5xx_errors_message" {
+  description = "Custom message for App Gateway http 5xx errors monitor"
+  type        = string
+  default     = ""
+}
+
+variable "appgateway_backend_http_5xx_errors_time_aggregator" {
+  description = "Monitor aggregator for App Gateway http 5xx errors [available values: min, max or avg]"
+  type        = string
+  default     = "max"
+}
+
+variable "appgateway_backend_http_5xx_errors_timeframe" {
+  description = "Monitor timeframe for App Gateway http 5xx errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = string
+  default     = "last_5m"
+}
+
+variable "appgateway_backend_http_5xx_errors_threshold_critical" {
+  default     = 30
+  description = "Minimum critical acceptable percent of 5xx error"
+}
+
+variable "appgateway_backend_http_5xx_errors_threshold_warning" {
+  default     = 20
+  description = "Warning regarding acceptable percent of 5xx error"
+}

cloud/azure/app-gateway/modules.tf

@@ -29,3 +29,25 @@ module "filter-tags-5xx-error" {
   filter_tags_custom_excluded = var.filter_tags_custom_excluded
   extra_tags                  = ["httpstatusgroup:5xx"]
 }
+
+module "filter-tags-backend-4xx-error" {
+  source = "../../../common/filter-tags"
+
+  environment                 = var.environment
+  resource                    = "azure_app-gateway"
+  filter_tags_use_defaults    = var.filter_tags_use_defaults
+  filter_tags_custom          = var.filter_tags_custom
+  filter_tags_custom_excluded = var.filter_tags_custom_excluded
+  extra_tags                  = ["httpstatusgroup:4xx"]
+}
+
+module "filter-tags-backend-5xx-error" {
+  source = "../../../common/filter-tags"
+
+  environment                 = var.environment
+  resource                    = "azure_app-gateway"
+  filter_tags_use_defaults    = var.filter_tags_use_defaults
+  filter_tags_custom          = var.filter_tags_custom
+  filter_tags_custom_excluded = var.filter_tags_custom_excluded
+  extra_tags                  = ["httpstatusgroup:5xx"]
+}

cloud/azure/app-gateway/monitors-app_services.tf

@@ -28,7 +28,7 @@ EOQ
   }
 }
 
-# Monitoring App Gateway current_connections (count)
+# Monitoring App Gateway current_connections
 resource "datadog_monitor" "current_connection" {
   count   = var.current_connection_enabled == "true" ? 1 : 0
   name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway no connection {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
@@ -57,16 +57,52 @@ EOQ
   }
 }
 
-# Monitoring App Gateway failed_requests (count)
+# Monitoring App Gateway backend_connect_time
+resource "datadog_monitor" "appgateway_backend_connect_time" {
+  count   = var.appgateway_backend_connect_time_enabled == "true" ? 1 : 0
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway backend connect time {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = coalesce(var.appgateway_backend_connect_time_message, var.message)
+  type    = "query alert"
+
+  query = <<EOQ
+    ${var.appgateway_backend_connect_time_time_aggregator}(${var.appgateway_backend_connect_time_timeframe}):
+      sum:azure.network_applicationgateways.backend_connect_time${module.filter-tags.query_alert} by {resource_group,region,name,backendserver} > ${var.appgateway_backend_connect_time_threshold_critical}
+EOQ
+
+  thresholds = {
+    critical = var.appgateway_backend_connect_time_threshold_critical
+    warning  = var.appgateway_backend_connect_time_threshold_warning
+  }
+
+  evaluation_delay    = var.evaluation_delay
+  new_host_delay      = var.new_host_delay
+  notify_no_data      = false
+  renotify_interval   = 0
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = false
+
+  tags = concat(["env:${var.environment}", "type:cloud", "provider:azure", "resource:app-gateway", "team:claranet", "created-by:terraform"], var.appgateway_backend_connect_time_extra_tags)
+
+  lifecycle {
+    ignore_changes = ["silenced"]
+  }
+}
+
+# Monitoring App Gateway failed_requests
 resource "datadog_monitor" "appgateway_failed_requests" {
   count   = var.appgateway_failed_requests_enabled == "true" ? 1 : 0
   name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway failed requests {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
   message = coalesce(var.appgateway_failed_requests_message, var.message)
-  type    = "query alert"
+  type    = "metric alert"
 
   query = <<EOQ
-    ${var.appgateway_failed_requests_time_aggregator}(${var.appgateway_failed_requests_timeframe}): (
+    ${var.appgateway_failed_requests_time_aggregator}(${var.appgateway_failed_requests_timeframe}):
-      avg:azure.network_applicationgateways.failed_requests${module.filter-tags.query_alert} by {resource_group,region,name}.as_count() > ${var.appgateway_failed_requests_threshold_critical}
+      default((default(avg:azure.network_applicationgateways.failed_requests${module.filter-tags.query_alert} by {resource_group,region,name}.as_rate(), 0) /
+      default(avg:azure.network_applicationgateways.total_requests${module.filter-tags.query_alert} by {resource_group,region,name}.as_rate(),0)
+      * 100),0) > ${var.appgateway_failed_requests_threshold_critical}
 EOQ
 
   thresholds = {
@@ -91,7 +127,7 @@ EOQ
   }
 }
 
-# Monitoring App Gateway healthy_host_count (count)
+# Monitoring App Gateway healthy_host_count
 resource "datadog_monitor" "appgateway_healthy_host_count" {
   count   = var.appgateway_healthy_host_count_enabled == "true" ? 1 : 0
   name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway no healthy host {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
@@ -120,10 +156,10 @@ EOQ
   }
 }
 
-# Monitoring App Gateway response_status 4xx (count)
+# Monitoring App Gateway response_status 4xx
 resource "datadog_monitor" "appgateway_http_4xx_errors" {
   count   = var.appgateway_http_4xx_errors_enabled == "true" ? 1 : 0
-  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway HTTP 4xx errors too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway HTTP 4xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
   message = coalesce(var.appgateway_http_4xx_errors_message, var.message)
   type    = "metric alert"
 
@@ -155,10 +191,10 @@ EOQ
   }
 }
 
-# Monitoring App Gateway response_status 5xx (count)
+# Monitoring App Gateway response_status 5xx
 resource "datadog_monitor" "appgateway_http_5xx_errors" {
   count   = var.appgateway_http_5xx_errors_enabled == "true" ? 1 : 0
-  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway HTTP 5xx errors too high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway HTTP 5xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
   message = coalesce(var.appgateway_http_5xx_errors_message, var.message)
   type    = "query alert"
 
@@ -188,3 +224,72 @@ EOQ
     ignore_changes = ["silenced"]
   }
 }
+
+# Monitoring App Gateway Backend response_status 4xx
+resource "datadog_monitor" "appgateway_backend_http_4xx_errors" {
+  count   = var.appgateway_backend_http_4xx_errors_enabled == "true" ? 1 : 0
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway Backend HTTP 4xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = coalesce(var.appgateway_backend_http_4xx_errors_message, var.message)
+  type    = "metric alert"
+
+  query = <<EOQ
+    ${var.appgateway_backend_http_4xx_errors_time_aggregator}(${var.appgateway_backend_http_4xx_errors_timeframe}):
+      default((default(sum:azure.network_applicationgateways.backend_response_status${module.filter-tags-backend-4xx-error.query_alert} by {resource_group,region,name}.as_rate(), 0) /
+      default(sum:azure.network_applicationgateways.backend_response_status${module.filter-tags.query_alert} by {resource_group,region,name}.as_rate(),0)
+      * 100),0) > ${var.appgateway_backend_http_4xx_errors_threshold_critical}
+EOQ
+
+
+  thresholds = {
+    warning  = var.appgateway_backend_http_4xx_errors_threshold_warning
+    critical = var.appgateway_backend_http_4xx_errors_threshold_critical
+  }
+
+  evaluation_delay    = var.evaluation_delay
+  new_host_delay      = var.new_host_delay
+  notify_no_data      = false
+  renotify_interval   = 0
+  require_full_window = false
+  timeout_h           = 1
+  include_tags        = true
+
+  tags = concat(["env:${var.environment}", "type:cloud", "provider:azure", "resource:app-gateway", "team:claranet", "created-by:terraform"], var.appgateway_backend_http_4xx_errors_extra_tags)
+
+  lifecycle {
+    ignore_changes = ["silenced"]
+  }
+}
+
+# Monitoring App Gateway Backend response_status 5xx
+resource "datadog_monitor" "appgateway_backend_http_5xx_errors" {
+  count   = var.appgateway_backend_http_5xx_errors_enabled == "true" ? 1 : 0
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] App Gateway HTTP Backend 5xx errors {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = coalesce(var.appgateway_backend_http_5xx_errors_message, var.message)
+  type    = "query alert"
+
+  query = <<EOQ
+    ${var.appgateway_backend_http_5xx_errors_time_aggregator}(${var.appgateway_backend_http_5xx_errors_timeframe}):
+      default((default(sum:azure.network_applicationgateways.backend_response_status${module.filter-tags-backend-5xx-error.query_alert} by {resource_group,region,name}.as_rate(), 0) /
+      default(sum:azure.network_applicationgateways.backend_response_status${module.filter-tags.query_alert} by {resource_group,region,name}.as_rate(),0)
+      * 100),0) > ${var.appgateway_backend_http_5xx_errors_threshold_critical}
+EOQ
+
+  thresholds = {
+    warning  = var.appgateway_backend_http_5xx_errors_threshold_warning
+    critical = var.appgateway_backend_http_5xx_errors_threshold_critical
+  }
+
+  evaluation_delay    = var.evaluation_delay
+  new_host_delay      = var.new_host_delay
+  notify_no_data      = false
+  renotify_interval   = 0
+  require_full_window = false
+  timeout_h           = 1
+  include_tags        = true
+
+  tags = concat(["env:${var.environment}", "type:cloud", "provider:azure", "resource:app-gateway", "team:claranet", "created-by:terraform"], var.appgateway_backend_http_5xx_errors_extra_tags)
+
+  lifecycle {
+    ignore_changes = ["silenced"]
+  }
+}

cloud/azure/app-gateway/outputs.tf

@@ -1,3 +1,18 @@
+output "appgateway_backend_connect_time_id" {
+  description = "id for monitor appgateway_backend_connect_time"
+  value       = datadog_monitor.appgateway_backend_connect_time.*.id
+}
+
+output "appgateway_backend_http_4xx_errors_id" {
+  description = "id for monitor appgateway_backend_http_4xx_errors"
+  value       = datadog_monitor.appgateway_backend_http_4xx_errors.*.id
+}
+
+output "appgateway_backend_http_5xx_errors_id" {
+  description = "id for monitor appgateway_backend_http_5xx_errors"
+  value       = datadog_monitor.appgateway_backend_http_5xx_errors.*.id
+}
+
 output "appgateway_failed_requests_id" {
   description = "id for monitor appgateway_failed_requests"
   value       = datadog_monitor.appgateway_failed_requests.*.id