MON-122 Add Aurora replica lag monitor (mysql and postgresql) and update rds free space monitor

cloud/aws/rds/README.md

@@ -1,59 +1,70 @@
-# CLOUD AWS RDS DataDog monitors
+AWS RDS Instance DataDog monitors
+=================================
 
-## How to use this module
+How to use this module
+----------------------
+
+For a single RDS instance :
 
 ```
-module "datadog-monitors-cloud-aws-rds" {
+module "datadog-monitors-aws-rds" {
   source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/aws/rds?ref={revision}"
 
-  environment = "${var.environment}"
   message     = "${module.datadog-message-alerting.alerting-message}"
+  environment = "${var.environment}"
 }
-
 ```
 
-## Purpose
+For a RDS Aurora cluster :
 
-Creates DataDog monitors with the following checks:
+```
+module "datadog-monitors-aws-rds-aurora" {
+  source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/aws/rds?ref={revision}"
 
-- RDS instance CPU high
+  message     = "${module.datadog-message-alerting.alerting-message}"
-- RDS instance free space
+  environment = "${var.environment}"
 
-## Inputs
+  aurora_cluster_type = "mysql" # if Aurora cluster Mysql. Else "postgresql" if Aurora cluster PostgreSQL.
+}
+```
+
+Purpose
+-------
+Creates a DataDog monitors with the following checks :
+
+* CPU High
+* Free disk space low (only for single RDS instances)
+* Aurora Replica Lag (only for Aurora Cluster Mysql or PostgreSQL)
+
+Inputs
+------
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| cpu_enabled | Flag to enable RDS CPU usage monitor | string | `true` | no |
+| aurora_cluster_type | RDS Aurora cluster monitoring.  If empty, single RDS instance monitoring. [available values: `mysql`, `postgresql`] | string | `` | no |
-| cpu_extra_tags | Extra tags for RDS CPU usage monitor | list | `<list>` | no |
+| aurora_replicalag_message | Custom message for RDS Aurora replica lag monitor | string | `` | no |
+| aurora_replicalag_silenced | Groups to mute for RDS Aurora replica lag monitor | map | `<map>` | no |
+| aurora_replicalag_threshold_critical | Aurora replica lag in milliseconds (critical threshold) | string | `200` | no |
+| aurora_replicalag_threshold_warning | Aurora replica lag in milliseconds (warning threshold) | string | `100` | no |
+| aurora_replicalag_timeframe | Monitor timeframe for RDS Aurora replica lag monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no |
 | cpu_message | Custom message for RDS CPU usage monitor | string | `` | no |
 | cpu_silenced | Groups to mute for RDS CPU usage monitor | map | `<map>` | no |
 | cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no |
 | cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no |
-| cpu_time_aggregator | Monitor aggregator for RDS CPU usage [available values: min, max or avg] | string | `min` | no |
 | cpu_timeframe | Monitor timeframe for RDS CPU usage [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_15m` | no |
-| diskspace_enabled | Flag to enable RDS free diskspace monitor | string | `true` | no |
+| delay | Delay in seconds for the metric evaluation | string | `900` | no |
-| diskspace_extra_tags | Extra tags for RDS free diskspace monitor | list | `<list>` | no |
 | diskspace_message | Custom message for RDS free diskspace monitor | string | `` | no |
 | diskspace_silenced | Groups to mute for RDS free diskspace monitor | map | `<map>` | no |
 | diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no |
 | diskspace_threshold_warning | Disk free space in percent (warning threshold) | string | `20` | no |
-| diskspace_time_aggregator | Monitor aggregator for RDS free diskspace [available values: min, max or avg] | string | `min` | no |
 | diskspace_timeframe | Monitor timeframe for RDS free diskspace [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_15m` | no |
 | environment | Architecture Environment | string | - | yes |
-| evaluation_delay | Delay in seconds for the metric evaluation | string | `900` | no |
 | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
 | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no |
 | message | Message sent when an alert is triggered | string | - | yes |
-| new_host_delay | Delay in seconds before monitor new resource | string | `300` | no |
 
-## Outputs
+Related documentation
-
+---------------------
-| Name | Description |
-|------|-------------|
-| rds_cpu_90_15min_id | id for monitor rds_cpu_90_15min |
-| rds_free_space_low_id | id for monitor rds_free_space_low |
-
-## Related documentation
 
 DataDog documentation: [https://docs.datadoghq.com/integrations/amazon_rds/](https://docs.datadoghq.com/integrations/amazon_rds/)
 

cloud/aws/rds/inputs.tf

@@ -5,16 +5,11 @@ variable "environment" {
 }
 
 # Global DataDog
-variable "evaluation_delay" {
+variable "delay" {
   description = "Delay in seconds for the metric evaluation"
   default     = 900
 }
 
-variable "new_host_delay" {
-  description = "Delay in seconds before monitor new resource"
-  default     = 300
-}
-
 variable "message" {
   description = "Message sent when an alert is triggered"
 }
@@ -29,6 +24,12 @@ variable "filter_tags_custom" {
   default     = "*"
 }
 
+variable "aurora_cluster_type" {
+  description = "RDS Aurora cluster monitoring.  If empty, single RDS instance monitoring. [available values: `mysql`, `postgresql`]"
+  type        = "string"
+  default     = ""
+}
+
 # AWS RDS instance specific
 
 variable "cpu_silenced" {
@@ -37,30 +38,12 @@ variable "cpu_silenced" {
   default     = {}
 }
 
-variable "cpu_enabled" {
-  description = "Flag to enable RDS CPU usage monitor"
-  type        = "string"
-  default     = "true"
-}
-
-variable "cpu_extra_tags" {
-  description = "Extra tags for RDS CPU usage monitor"
-  type        = "list"
-  default     = []
-}
-
 variable "cpu_message" {
   description = "Custom message for RDS CPU usage monitor"
   type        = "string"
   default     = ""
 }
 
-variable "cpu_time_aggregator" {
-  description = "Monitor aggregator for RDS CPU usage [available values: min, max or avg]"
-  type        = "string"
-  default     = "min"
-}
-
 variable "cpu_timeframe" {
   description = "Monitor timeframe for RDS CPU usage [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
   type        = "string"
@@ -83,30 +66,12 @@ variable "diskspace_silenced" {
   default     = {}
 }
 
-variable "diskspace_enabled" {
-  description = "Flag to enable RDS free diskspace monitor"
-  type        = "string"
-  default     = "true"
-}
-
-variable "diskspace_extra_tags" {
-  description = "Extra tags for RDS free diskspace monitor"
-  type        = "list"
-  default     = []
-}
-
 variable "diskspace_message" {
   description = "Custom message for RDS free diskspace monitor"
   type        = "string"
   default     = ""
 }
 
-variable "diskspace_time_aggregator" {
-  description = "Monitor aggregator for RDS free diskspace [available values: min, max or avg]"
-  type        = "string"
-  default     = "min"
-}
-
 variable "diskspace_timeframe" {
   description = "Monitor timeframe for RDS free diskspace [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
   type        = "string"
@@ -122,3 +87,31 @@ variable "diskspace_threshold_critical" {
   description = "Disk free space in percent (critical threshold)"
   default     = "10"
 }
+
+variable "aurora_replicalag_silenced" {
+  description = "Groups to mute for RDS Aurora replica lag monitor"
+  type        = "map"
+  default     = {}
+}
+
+variable "aurora_replicalag_message" {
+  description = "Custom message for RDS Aurora replica lag monitor"
+  type        = "string"
+  default     = ""
+}
+
+variable "aurora_replicalag_timeframe" {
+  description = "Monitor timeframe for RDS Aurora replica lag monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = "string"
+  default     = "last_5m"
+}
+
+variable "aurora_replicalag_threshold_warning" {
+  description = "Aurora replica lag in milliseconds (warning threshold)"
+  default     = "100"
+}
+
+variable "aurora_replicalag_threshold_critical" {
+  description = "Aurora replica lag in milliseconds (critical threshold)"
+  default     = "200"
+}

cloud/aws/rds/monitors-rds.tf

@@ -1,14 +1,21 @@
+data "template_file" "filter" {
+  template = "$${filter}"
+
+  vars {
+    filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,dd_aws_rds:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}"
+  }
+}
+
 ### RDS instance CPU monitor ###
 resource "datadog_monitor" "rds_cpu_90_15min" {
-  count   = "${var.cpu_enabled ? 1 : 0}"
   name    = "[${var.environment}] RDS instance CPU high {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
   message = "${coalesce(var.cpu_message, var.message)}"
 
   type = "metric alert"
 
   query = <<EOF
-    ${var.cpu_time_aggregator}(${var.cpu_timeframe}): (
+    avg(${var.cpu_timeframe}): (
-      avg:aws.rds.cpuutilization${module.filter-tags.query_alert} by {region,name}
+      avg:aws.rds.cpuutilization{${data.template_file.filter.rendered}} by {region,name}
     ) > ${var.cpu_threshold_critical}
 EOF
 
@@ -18,31 +25,30 @@ EOF
   }
 
   notify_no_data      = true
-  evaluation_delay    = "${var.evaluation_delay}"
+  evaluation_delay    = "${var.delay}"
   notify_audit        = false
   timeout_h           = 0
   include_tags        = true
   locked              = false
   require_full_window = false
-  new_host_delay      = "${var.new_host_delay}"
+  new_host_delay      = "${var.delay}"
 
   silenced = "${var.cpu_silenced}"
 
-  tags = ["env:${var.environment}", "type:cloud", "provider:aws", "resource:rds", "team:claranet", "created-by:terraform", "${var.cpu_extra_tags}"]
+  tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"]
 }
 
 ### RDS instance free space monitor ###
 resource "datadog_monitor" "rds_free_space_low" {
-  count   = "${var.diskspace_enabled ? 1 : 0}"
   name    = "[${var.environment}] RDS instance free space {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
   message = "${coalesce(var.diskspace_message, var.message)}"
 
   type = "metric alert"
 
   query = <<EOF
-  ${var.diskspace_time_aggregator}(${var.diskspace_timeframe}): (
+  avg(${var.diskspace_timeframe}): (
-    avg:aws.rds.free_storage_space${module.filter-tags.query_alert} by {region,name} /
+    avg:aws.rds.free_storage_space{${data.template_file.filter.rendered}} by {region,name} /
-    avg:aws.rds.total_storage_space${module.filter-tags.query_alert} by {region,name} * 100
+    avg:aws.rds.total_storage_space{${data.template_file.filter.rendered}} by {region,name} * 100
   ) < ${var.diskspace_threshold_critical}
 EOF
 
@@ -52,15 +58,85 @@ EOF
   }
 
   notify_no_data      = true
-  evaluation_delay    = "${var.evaluation_delay}"
+  evaluation_delay    = "${var.delay}"
   notify_audit        = false
   timeout_h           = 0
   include_tags        = true
   locked              = false
   require_full_window = false
-  new_host_delay      = "${var.new_host_delay}"
+  new_host_delay      = "${var.delay}"
 
   silenced = "${var.diskspace_silenced}"
 
-  tags = ["env:${var.environment}", "type:cloud", "provider:aws", "resource:rds", "team:claranet", "created-by:terraform", "${var.diskspace_extra_tags}"]
+  tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"]
+
+  count = "${var.aurora_cluster_type == "" ? 1 : 0}"
+}
+
+### RDS Aurora Postgresql Replica Lag monitor ###
+resource "datadog_monitor" "rds_aurora_postgresql_replica_lag" {
+  name    = "[${var.environment}] RDS Aurora PostgreSQL replica lag {{#is_alert}}{{{comparator}}} {{threshold}} ms ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ms ({{value}}%){{/is_warning}}"
+  message = "${coalesce(var.aurora_replicalag_message, var.message)}"
+
+  type = "metric alert"
+
+  query = <<EOF
+  avg(${var.aurora_replicalag_timeframe}): (
+    avg:aws.rds.rdsto_aurora_postgre_sqlreplica_lag{${data.template_file.filter.rendered}} by {region,name}
+  ) > ${var.aurora_replicalag_threshold_critical}
+EOF
+
+  thresholds {
+    warning  = "${var.aurora_replicalag_threshold_warning}"
+    critical = "${var.aurora_replicalag_threshold_critical}"
+  }
+
+  notify_no_data      = true
+  evaluation_delay    = "${var.delay}"
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = false
+  new_host_delay      = "${var.delay}"
+
+  silenced = "${var.aurora_replicalag_silenced}"
+
+  tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"]
+
+  count = "${var.aurora_cluster_type == "postgresql" ? 1 : 0}"
+}
+
+### RDS Aurora Mysql Replica Lag monitor ###
+resource "datadog_monitor" "rds_aurora_mysql_replica_lag" {
+  name    = "[${var.environment}] RDS Aurora Mysql replica lag {{#is_alert}}{{{comparator}}} {{threshold}} ms ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ms ({{value}}%){{/is_warning}}"
+  message = "${coalesce(var.aurora_replicalag_message, var.message)}"
+
+  type = "metric alert"
+
+  query = <<EOF
+  avg(${var.aurora_replicalag_timeframe}): (
+    avg:aws.rds.aws.rds.aurora_replica_lag{${data.template_file.filter.rendered}} by {region,name}
+  ) > ${var.aurora_replicalag_threshold_critical}
+EOF
+
+  thresholds {
+    warning  = "${var.aurora_replicalag_threshold_warning}"
+    critical = "${var.aurora_replicalag_threshold_critical}"
+  }
+
+  notify_no_data      = true
+  evaluation_delay    = "${var.delay}"
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = false
+  new_host_delay      = "${var.delay}"
+
+  silenced = "${var.aurora_replicalag_silenced}"
+
+  tags = ["env:${var.environment}", "resource:rds", "team:aws", "provider:aws"]
+
+  count = "${var.aurora_cluster_type == "mysql" ? 1 : 0}"
 }