diff --git a/middleware/redis/README.md b/middleware/redis/README.md index 5b6919e..d3300db 100644 --- a/middleware/redis/README.md +++ b/middleware/redis/README.md @@ -16,7 +16,7 @@ module "datadog-monitors-middleware-redis" { Creates DataDog monitors with the following checks: -- Redis too many evictedkeys +- Redis suddenly too many evicted keys - Redis too many expired keys - Redis too many blocked clients - Redis keyspace seems full @@ -32,62 +32,62 @@ Creates DataDog monitors with the following checks: |------|-------------|:----:|:-----:|:-----:| | blocked_clients_message | Custom message for Redis Blocked clients monitor | string | `` | no | | blocked_clients_silenced | Groups to mute for Redis Blocked clients monitor | map | `` | no | -| blocked_clients_threshold_critical | Blocked clients rate (critical threshold) | string | `90` | no | -| blocked_clients_threshold_warning | Blocked clients rate (warning threshold) | string | `70` | no | +| blocked_clients_threshold_critical | Blocked clients rate (critical threshold) | string | `30` | no | +| blocked_clients_threshold_warning | Blocked clients rate (warning threshold) | string | `10` | no | | blocked_clients_time_aggregator | Monitor aggregator for Redis Blocked clients [available values: min, max or avg] | string | `min` | no | | blocked_clients_timeframe | Monitor timeframe for Redis Blocked clients [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | delay | Delay in seconds for the metric evaluation | string | `900` | no | | environment | Architecture environment | string | - | yes | -| evictedkeys_limit_message | Custom message for Redis evicted keys monitor | string | `` | no | -| evictedkeys_limit_silenced | Groups to mute for Redis evicted keys monitor | map | `` | no | -| evictedkeys_limit_threshold_critical | Evicted keys limit (critical threshold) | string | `100` | no | -| evictedkeys_limit_threshold_warning | Evicted keys limit (warning threshold) | string | `0` | no | -| evictedkeys_limit_time_aggregator | Monitor aggregator for Redis evicted keys [available values: min, max or avg] | string | `avg` | no | -| evictedkeys_limit_timeframe | Monitor timeframe for Redis evicted keys [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | -| expirations_limit_message | Custom message for Redis keys expirations monitor | string | `` | no | -| expirations_limit_threshold_critical | Expirations percent (critical threshold) | string | `80` | no | -| expirations_limit_threshold_warning | Expirations percent (warning threshold) | string | `60` | no | -| expirations_limit_time_aggregator | Monitor aggregator for Redis keys expirations [available values: min, max or avg] | string | `min` | no | -| expirations_limit_timeframe | Monitor timeframe for Redis keys expirations [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | +| evictedkeys_change_message | Custom message for Redis evicted keys monitor | string | `` | no | +| evictedkeys_change_silenced | Groups to mute for Redis evicted keys monitor | map | `` | no | +| evictedkeys_change_threshold_critical | Evicted keys change (critical threshold) | string | `100` | no | +| evictedkeys_change_threshold_warning | Evicted keys change (warning threshold) | string | `20` | no | +| evictedkeys_change_time_aggregator | Monitor aggregator for Redis evicted keys [available values: min, max or avg] | string | `avg` | no | +| evictedkeys_change_timeframe | Monitor timeframe for Redis evicted keys [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | +| expirations_rate_message | Custom message for Redis keys expirations monitor | string | `` | no | +| expirations_rate_threshold_critical | Expirations percent (critical threshold) | string | `80` | no | +| expirations_rate_threshold_warning | Expirations percent (warning threshold) | string | `60` | no | +| expirations_rate_time_aggregator | Monitor aggregator for Redis keys expirations [available values: min, max or avg] | string | `min` | no | +| expirations_rate_timeframe | Monitor timeframe for Redis keys expirations [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | expirations_silenced | Groups to mute for Redis keys expirations monitor | map | `` | no | | filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no | | filter_tags_use_defaults | Use default filter tags convention | string | `true` | no | | hitrate_message | Custom message for Redis hitrate monitor | string | `` | no | | hitrate_silenced | Groups to mute for Redis hitrate monitor | map | `` | no | -| hitrate_threshold_critical | hitrate limit (critical threshold) | string | `90` | no | -| hitrate_threshold_warning | hitrate limit (warning threshold) | string | `70` | no | +| hitrate_threshold_critical | hitrate limit (critical threshold) | string | `10` | no | +| hitrate_threshold_warning | hitrate limit (warning threshold) | string | `30` | no | | hitrate_time_aggregator | Monitor aggregator for Redis hitrate [available values: min, max or avg] | string | `min` | no | | hitrate_timeframe | Monitor timeframe for Redis hitrate [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | keyspace_message | Custom message for Redis keyspace monitor | string | `` | no | | keyspace_silenced | Groups to mute for Redis keyspace monitor | map | `` | no | -| keyspace_threshold_critical | Keyspace changement (critical threshold) | string | `90` | no | -| keyspace_threshold_warning | Keyspace changement (warning threshold) | string | `70` | no | +| keyspace_threshold_critical | Keyspace no changement (critical threshold) | string | `0` | no | +| keyspace_threshold_warning | Keyspace no changement (warning threshold) | string | `1` | no | | keyspace_time_aggregator | Monitor aggregator for Redis keyspace [available values: min, max or avg] | string | `min` | no | | keyspace_timeframe | Monitor timeframe for Redis keyspace [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | latency_message | Custom message for Redis latency monitor | string | `` | no | | latency_silenced | Groups to mute for Redis latency monitor | map | `` | no | -| latency_threshold_critical | latency limit (critical threshold) | string | `90` | no | -| latency_threshold_warning | latency limit (warning threshold) | string | `70` | no | +| latency_threshold_critical | latency limit (critical threshold) | string | `100` | no | +| latency_threshold_warning | latency limit (warning threshold) | string | `50` | no | | latency_time_aggregator | Monitor aggregator for Redis latency [available values: min, max or avg] | string | `min` | no | | latency_timeframe | Monitor timeframe for Redis latency [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | mem_frag_message | Custom message for Redis memory RAM fragmentation monitor | string | `` | no | | mem_frag_silenced | Groups to mute for Redis memory RAM fragmentation monitor | map | `` | no | -| mem_frag_threshold_critical | memory RAM fragmentation limit (critical threshold) | string | `90` | no | -| mem_frag_threshold_warning | memory RAM fragmentation limit (warning threshold) | string | `70` | no | +| mem_frag_threshold_critical | memory RAM fragmentation limit (critical threshold) | string | `130` | no | +| mem_frag_threshold_warning | memory RAM fragmentation limit (warning threshold) | string | `100` | no | | mem_frag_time_aggregator | Monitor aggregator for Redis memory RAM fragmentation [available values: min, max or avg] | string | `min` | no | | mem_frag_timeframe | Monitor timeframe for Redis memory RAM fragmentation [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | mem_used_message | Custom message for Redis RAM memory used monitor | string | `` | no | | mem_used_silenced | Groups to mute for Redis RAM memory used monitor | map | `` | no | -| mem_used_threshold_critical | RAM memory used limit (critical threshold) | string | `90` | no | -| mem_used_threshold_warning | RAM memory used limit (warning threshold) | string | `70` | no | +| mem_used_threshold_critical | RAM memory used limit (critical threshold) | string | `95` | no | +| mem_used_threshold_warning | RAM memory used limit (warning threshold) | string | `85` | no | | mem_used_time_aggregator | Monitor aggregator for Redis RAM memory used [available values: min, max or avg] | string | `min` | no | | mem_used_timeframe | Monitor timeframe for Redis RAM memory used [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | | message | Message sent when a Redis monitor is triggered | string | - | yes | | redis_silenced | Groups to mute for Redis monitors | map | `` | no | | rejected_con_message | Custom message for Redis rejected connections errors monitor | string | `` | no | | rejected_con_silenced | Groups to mute for Redis rejected connections errors monitor | map | `` | no | -| rejected_con_threshold_critical | rejected connections errors limit (critical threshold) | string | `90` | no | -| rejected_con_threshold_warning | rejected connections errors limit (warning threshold) | string | `70` | no | +| rejected_con_threshold_critical | rejected connections errors limit (critical threshold) | string | `50` | no | +| rejected_con_threshold_warning | rejected connections errors limit (warning threshold) | string | `10` | no | | rejected_con_time_aggregator | Monitor aggregator for Redis rejected connections errors [available values: min, max or avg] | string | `min` | no | | rejected_con_timeframe | Monitor timeframe for Redis rejected connections errors [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `last_5m` | no | diff --git a/middleware/redis/inputs.tf b/middleware/redis/inputs.tf index cabf2ab..527b5d4 100644 --- a/middleware/redis/inputs.tf +++ b/middleware/redis/inputs.tf @@ -31,37 +31,37 @@ variable "redis_silenced" { default = {} } -variable "evictedkeys_limit_silenced" { +variable "evictedkeys_change_silenced" { description = "Groups to mute for Redis evicted keys monitor" type = "map" default = {} } -variable "evictedkeys_limit_message" { +variable "evictedkeys_change_message" { description = "Custom message for Redis evicted keys monitor" type = "string" default = "" } -variable "evictedkeys_limit_time_aggregator" { +variable "evictedkeys_change_time_aggregator" { description = "Monitor aggregator for Redis evicted keys [available values: min, max or avg]" type = "string" default = "avg" } -variable "evictedkeys_limit_timeframe" { +variable "evictedkeys_change_timeframe" { description = "Monitor timeframe for Redis evicted keys [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" type = "string" default = "last_5m" } -variable "evictedkeys_limit_threshold_warning" { - description = "Evicted keys limit (warning threshold)" - default = 0 +variable "evictedkeys_change_threshold_warning" { + description = "Evicted keys change (warning threshold)" + default = 20 } -variable "evictedkeys_limit_threshold_critical" { - description = "Evicted keys limit (critical threshold)" +variable "evictedkeys_change_threshold_critical" { + description = "Evicted keys change (critical threshold)" default = 100 } @@ -71,30 +71,30 @@ variable "expirations_silenced" { default = {} } -variable "expirations_limit_message" { +variable "expirations_rate_message" { description = "Custom message for Redis keys expirations monitor" type = "string" default = "" } -variable "expirations_limit_time_aggregator" { +variable "expirations_rate_time_aggregator" { description = "Monitor aggregator for Redis keys expirations [available values: min, max or avg]" type = "string" default = "min" } -variable "expirations_limit_timeframe" { +variable "expirations_rate_timeframe" { description = "Monitor timeframe for Redis keys expirations [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" type = "string" default = "last_5m" } -variable "expirations_limit_threshold_critical" { +variable "expirations_rate_threshold_critical" { description = "Expirations percent (critical threshold)" default = 80 } -variable "expirations_limit_threshold_warning" { +variable "expirations_rate_threshold_warning" { description = "Expirations percent (warning threshold)" default = 60 } @@ -125,18 +125,21 @@ variable "blocked_clients_timeframe" { variable "blocked_clients_threshold_critical" { description = "Blocked clients rate (critical threshold)" - default = 90 + default = 30 } variable "blocked_clients_threshold_warning" { description = "Blocked clients rate (warning threshold)" - default = 70 + default = 10 } variable "keyspace_silenced" { description = "Groups to mute for Redis keyspace monitor" type = "map" - default = {} + + default = { + "*" = 0 # Mute all for now by default + } } variable "keyspace_message" { @@ -158,13 +161,13 @@ variable "keyspace_timeframe" { } variable "keyspace_threshold_critical" { - description = "Keyspace changement (critical threshold)" - default = 90 + description = "Keyspace no changement (critical threshold)" + default = 0 } variable "keyspace_threshold_warning" { - description = "Keyspace changement (warning threshold)" - default = 70 + description = "Keyspace no changement (warning threshold)" + default = 1 } variable "mem_used_silenced" { @@ -193,12 +196,12 @@ variable "mem_used_timeframe" { variable "mem_used_threshold_critical" { description = "RAM memory used limit (critical threshold)" - default = 90 + default = 95 } variable "mem_used_threshold_warning" { description = "RAM memory used limit (warning threshold)" - default = 70 + default = 85 } variable "mem_frag_silenced" { @@ -227,12 +230,12 @@ variable "mem_frag_timeframe" { variable "mem_frag_threshold_critical" { description = "memory RAM fragmentation limit (critical threshold)" - default = 90 + default = 130 } variable "mem_frag_threshold_warning" { description = "memory RAM fragmentation limit (warning threshold)" - default = 70 + default = 100 } variable "rejected_con_silenced" { @@ -261,12 +264,12 @@ variable "rejected_con_timeframe" { variable "rejected_con_threshold_critical" { description = "rejected connections errors limit (critical threshold)" - default = 90 + default = 50 } variable "rejected_con_threshold_warning" { description = "rejected connections errors limit (warning threshold)" - default = 70 + default = 10 } variable "latency_silenced" { @@ -295,12 +298,12 @@ variable "latency_timeframe" { variable "latency_threshold_critical" { description = "latency limit (critical threshold)" - default = 90 + default = 100 } variable "latency_threshold_warning" { description = "latency limit (warning threshold)" - default = 70 + default = 50 } variable "hitrate_silenced" { @@ -329,10 +332,10 @@ variable "hitrate_timeframe" { variable "hitrate_threshold_critical" { description = "hitrate limit (critical threshold)" - default = 90 + default = 10 } variable "hitrate_threshold_warning" { description = "hitrate limit (warning threshold)" - default = 70 + default = 30 } diff --git a/middleware/redis/monitors-redis.tf b/middleware/redis/monitors-redis.tf index 98eac73..0389fea 100644 --- a/middleware/redis/monitors-redis.tf +++ b/middleware/redis/monitors-redis.tf @@ -7,23 +7,23 @@ data "template_file" "filter" { } resource "datadog_monitor" "redis_evictedkeys" { - name = "[${var.environment}] Redis too many evictedkeys {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" - message = "${coalesce(var.evictedkeys_limit_message, var.message)}" + name = "[${var.environment}] Redis suddenly too many evicted keys {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" + message = "${coalesce(var.evictedkeys_change_message, var.message)}" query = < ${var.evictedkeys_limit_threshold_critical} + ) > ${var.evictedkeys_change_threshold_critical} EOL type = "metric alert" thresholds { - warning = "${var.evictedkeys_limit_threshold_warning}" - critical = "${var.evictedkeys_limit_threshold_critical}" + warning = "${var.evictedkeys_change_threshold_warning}" + critical = "${var.evictedkeys_change_threshold_critical}" } - silenced = "${var.evictedkeys_limit_silenced}" + silenced = "${var.evictedkeys_change_silenced}" notify_no_data = false evaluation_delay = "${var.delay}" @@ -40,19 +40,19 @@ EOL resource "datadog_monitor" "redis_expirations" { name = "[${var.environment}] Redis too many expired keys {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}" - message = "${coalesce(var.expirations_limit_message, var.message)}" + message = "${coalesce(var.expirations_rate_message, var.message)}" query = < ${var.expirations_limit_threshold_critical} + ) > ${var.expirations_rate_threshold_critical} EOL type = "metric alert" thresholds { - warning = "${var.expirations_limit_threshold_warning}" - critical = "${var.expirations_limit_threshold_critical}" + warning = "${var.expirations_rate_threshold_warning}" + critical = "${var.expirations_rate_threshold_critical}" } silenced = "${var.expirations_silenced}" @@ -76,8 +76,9 @@ resource "datadog_monitor" "redis_blocked_clients" { query = < ${var.blocked_clients_threshold_critical} + sum:redis.clients.blocked{${data.template_file.filter.rendered}} by {name,host} + / sum:redis.net.clients{${data.template_file.filter.rendered}} by {name,host} + ) * 100 > ${var.blocked_clients_threshold_critical} EOL type = "metric alert" @@ -108,8 +109,8 @@ resource "datadog_monitor" "redis_keyspace" { query = < ${var.keyspace_threshold_critical} + abs(diff(avg:redis.keys{${data.template_file.filter.rendered}} by {name,host})) + ) == ${var.keyspace_threshold_critical} EOL type = "metric alert" @@ -141,7 +142,8 @@ resource "datadog_monitor" "redis_mem_used" { query = < ${var.mem_used_threshold_critical} + / max:redis.mem.maxmemory{${data.template_file.filter.rendered}} by {name,host} + ) * 100 > ${var.mem_used_threshold_critical} EOL type = "metric alert" @@ -173,7 +175,7 @@ resource "datadog_monitor" "redis_mem_frag" { query = < ${var.mem_frag_threshold_critical} + ) * 100 > ${var.mem_frag_threshold_critical} EOL type = "metric alert" @@ -203,7 +205,7 @@ resource "datadog_monitor" "redis_rejected_con" { message = "${coalesce(var.rejected_con_message, var.message)}" query = < ${var.rejected_con_threshold_critical} EOL @@ -235,7 +237,7 @@ resource "datadog_monitor" "redis_latency" { message = "${coalesce(var.latency_message, var.message)}" query = < ${var.latency_threshold_critical} EOL @@ -268,10 +270,10 @@ resource "datadog_monitor" "redis_hitrate" { query = <