From 17d0650b363c36d91e2957aee360d5264f847c08 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Tue, 7 May 2019 18:04:44 +0200 Subject: [PATCH] MON-453 add inodes and space monitors to kubernetes node --- caas/kubernetes/node/README.md | 20 +++++ caas/kubernetes/node/inputs.tf | 92 +++++++++++++++++++++++ caas/kubernetes/node/monitors-k8s-node.tf | 66 ++++++++++++++++ caas/kubernetes/node/outputs.tf | 10 +++ 4 files changed, 188 insertions(+) diff --git a/caas/kubernetes/node/README.md b/caas/kubernetes/node/README.md index 7f7f90d..379135a 100644 --- a/caas/kubernetes/node/README.md +++ b/caas/kubernetes/node/README.md @@ -24,6 +24,8 @@ Creates DataDog monitors with the following checks: - Kubernetes Node not ready - Kubernetes Node Out of disk - Kubernetes Node unschedulable +- Kubernetes Node volume inodes usage +- Kubernetes Node volume space usage ## Inputs @@ -80,6 +82,22 @@ Creates DataDog monitors with the following checks: | unregister\_net\_device\_threshold\_critical | Unregister net device critical threshold | string | `"3"` | no | | unregister\_net\_device\_time\_aggregator | Monitor aggregator for Unregister net device [available values: min, max or avg] | string | `"min"` | no | | unregister\_net\_device\_timeframe | Monitor timeframe for Unregister net device [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"15m"` | no | +| volume\_inodes\_enabled | Flag to enable Volume inodes monitor | string | `"true"` | no | +| volume\_inodes\_extra\_tags | Extra tags for Volume inodes monitor | list | `[]` | no | +| volume\_inodes\_message | Custom message for Volume inodes monitor | string | `""` | no | +| volume\_inodes\_silenced | Groups to mute for Volume inodes monitor | map | `{}` | no | +| volume\_inodes\_threshold\_critical | Volume inodes critical threshold | string | `"95"` | no | +| volume\_inodes\_threshold\_warning | Volume inodes warning threshold | string | `"90"` | no | +| volume\_inodes\_time\_aggregator | Monitor aggregator for Volume inodes [available values: min, max or avg] | string | `"min"` | no | +| volume\_inodes\_timeframe | Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no | +| volume\_space\_enabled | Flag to enable Volume space monitor | string | `"true"` | no | +| volume\_space\_extra\_tags | Extra tags for Volume space monitor | list | `[]` | no | +| volume\_space\_message | Custom message for Volume space monitor | string | `""` | no | +| volume\_space\_silenced | Groups to mute for Volume space monitor | map | `{}` | no | +| volume\_space\_threshold\_critical | Volume space critical threshold | string | `"95"` | no | +| volume\_space\_threshold\_warning | Volume space warning threshold | string | `"90"` | no | +| volume\_space\_time\_aggregator | Monitor aggregator for Volume space [available values: min, max or avg] | string | `"min"` | no | +| volume\_space\_timeframe | Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no | ## Outputs @@ -93,6 +111,8 @@ Creates DataDog monitors with the following checks: | node\_unschedulable\_id | id for monitor node_unschedulable | | ready\_id | id for monitor ready | | unregister\_net\_device\_id | id for monitor unregister_net_device | +| volume\_inodes\_id | id for monitor volume_inodes | +| volume\_space\_id | id for monitor volume_space | ## Related documentation diff --git a/caas/kubernetes/node/inputs.tf b/caas/kubernetes/node/inputs.tf index 7f75dbc..6084ab8 100644 --- a/caas/kubernetes/node/inputs.tf +++ b/caas/kubernetes/node/inputs.tf @@ -296,3 +296,95 @@ variable "node_unschedulable_timeframe" { type = "string" default = "last_1h" } + +variable "volume_space_silenced" { + description = "Groups to mute for Volume space monitor" + type = "map" + default = {} +} + +variable "volume_space_enabled" { + description = "Flag to enable Volume space monitor" + type = "string" + default = "true" +} + +variable "volume_space_extra_tags" { + description = "Extra tags for Volume space monitor" + type = "list" + default = [] +} + +variable "volume_space_message" { + description = "Custom message for Volume space monitor" + type = "string" + default = "" +} + +variable "volume_space_time_aggregator" { + description = "Monitor aggregator for Volume space [available values: min, max or avg]" + type = "string" + default = "min" +} + +variable "volume_space_timeframe" { + description = "Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} + +variable "volume_space_threshold_critical" { + default = 95 + description = "Volume space critical threshold" +} + +variable "volume_space_threshold_warning" { + default = 90 + description = "Volume space warning threshold" +} + +variable "volume_inodes_silenced" { + description = "Groups to mute for Volume inodes monitor" + type = "map" + default = {} +} + +variable "volume_inodes_enabled" { + description = "Flag to enable Volume inodes monitor" + type = "string" + default = "true" +} + +variable "volume_inodes_extra_tags" { + description = "Extra tags for Volume inodes monitor" + type = "list" + default = [] +} + +variable "volume_inodes_message" { + description = "Custom message for Volume inodes monitor" + type = "string" + default = "" +} + +variable "volume_inodes_time_aggregator" { + description = "Monitor aggregator for Volume inodes [available values: min, max or avg]" + type = "string" + default = "min" +} + +variable "volume_inodes_timeframe" { + description = "Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]" + type = "string" + default = "last_5m" +} + +variable "volume_inodes_threshold_critical" { + default = 95 + description = "Volume inodes critical threshold" +} + +variable "volume_inodes_threshold_warning" { + default = 90 + description = "Volume inodes warning threshold" +} diff --git a/caas/kubernetes/node/monitors-k8s-node.tf b/caas/kubernetes/node/monitors-k8s-node.tf index b719c88..997ee5d 100644 --- a/caas/kubernetes/node/monitors-k8s-node.tf +++ b/caas/kubernetes/node/monitors-k8s-node.tf @@ -231,3 +231,69 @@ resource "datadog_monitor" "node_unschedulable" { silenced = "${var.node_unschedulable_silenced}" tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.node_unschedulable_extra_tags}"] } + +resource "datadog_monitor" "volume_space" { + count = "${var.volume_space_enabled == "true" ? 1 : 0}" + name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume space usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + type = "metric alert" + message = "${coalesce(var.volume_space_message, var.message)}" + + query = < ${var.volume_space_threshold_critical} + EOQ + + thresholds { + critical = "${var.volume_space_threshold_critical}" + warning = "${var.volume_space_threshold_warning}" + } + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + notify_no_data = true + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + + silenced = "${var.volume_space_silenced}" + tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_space_extra_tags}"] +} + +resource "datadog_monitor" "volume_inodes" { + count = "${var.volume_inodes_enabled == "true" ? 1 : 0}" + name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume inodes usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}" + type = "metric alert" + message = "${coalesce(var.volume_inodes_message, var.message)}" + + query = < ${var.volume_space_threshold_critical} + EOQ + + thresholds { + critical = "${var.volume_inodes_threshold_critical}" + warning = "${var.volume_inodes_threshold_warning}" + } + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + notify_no_data = true + renotify_interval = 0 + notify_audit = false + timeout_h = 0 + include_tags = true + locked = false + require_full_window = true + + silenced = "${var.volume_inodes_silenced}" + tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_inodes_extra_tags}"] +} diff --git a/caas/kubernetes/node/outputs.tf b/caas/kubernetes/node/outputs.tf index 0d0ffa3..b88a1d3 100644 --- a/caas/kubernetes/node/outputs.tf +++ b/caas/kubernetes/node/outputs.tf @@ -37,3 +37,13 @@ output "node_unschedulable_id" { description = "id for monitor node_unschedulable" value = "${datadog_monitor.node_unschedulable.*.id}" } + +output "volume_space_id" { + description = "id for monitor volume_space" + value = "${datadog_monitor.volume_space.*.id}" +} + +output "volume_inodes_id" { + description = "id for monitor volume_inodes" + value = "${datadog_monitor.volume_inodes.*.id}" +}