MON-453 add inodes and space monitors to kubernetes node

caas/kubernetes/node/README.md

@@ -24,6 +24,8 @@ Creates DataDog monitors with the following checks:
 - Kubernetes Node not ready
 - Kubernetes Node Out of disk
 - Kubernetes Node unschedulable
+- Kubernetes Node volume inodes usage
+- Kubernetes Node volume space usage
 
 ## Inputs
 
@@ -80,6 +82,22 @@ Creates DataDog monitors with the following checks:
 | unregister\_net\_device\_threshold\_critical | Unregister net device critical threshold | string | `"3"` | no |
 | unregister\_net\_device\_time\_aggregator | Monitor aggregator for Unregister net device [available values: min, max or avg] | string | `"min"` | no |
 | unregister\_net\_device\_timeframe | Monitor timeframe for Unregister net device [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"15m"` | no |
+| volume\_inodes\_enabled | Flag to enable Volume inodes monitor | string | `"true"` | no |
+| volume\_inodes\_extra\_tags | Extra tags for Volume inodes monitor | list | `[]` | no |
+| volume\_inodes\_message | Custom message for Volume inodes monitor | string | `""` | no |
+| volume\_inodes\_silenced | Groups to mute for Volume inodes monitor | map | `{}` | no |
+| volume\_inodes\_threshold\_critical | Volume inodes critical threshold | string | `"95"` | no |
+| volume\_inodes\_threshold\_warning | Volume inodes warning threshold | string | `"90"` | no |
+| volume\_inodes\_time\_aggregator | Monitor aggregator for Volume inodes [available values: min, max or avg] | string | `"min"` | no |
+| volume\_inodes\_timeframe | Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
+| volume\_space\_enabled | Flag to enable Volume space monitor | string | `"true"` | no |
+| volume\_space\_extra\_tags | Extra tags for Volume space monitor | list | `[]` | no |
+| volume\_space\_message | Custom message for Volume space monitor | string | `""` | no |
+| volume\_space\_silenced | Groups to mute for Volume space monitor | map | `{}` | no |
+| volume\_space\_threshold\_critical | Volume space critical threshold | string | `"95"` | no |
+| volume\_space\_threshold\_warning | Volume space warning threshold | string | `"90"` | no |
+| volume\_space\_time\_aggregator | Monitor aggregator for Volume space [available values: min, max or avg] | string | `"min"` | no |
+| volume\_space\_timeframe | Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
 
 ## Outputs
 
@@ -93,6 +111,8 @@ Creates DataDog monitors with the following checks:
 | node\_unschedulable\_id | id for monitor node_unschedulable |
 | ready\_id | id for monitor ready |
 | unregister\_net\_device\_id | id for monitor unregister_net_device |
+| volume\_inodes\_id | id for monitor volume_inodes |
+| volume\_space\_id | id for monitor volume_space |
 
 ## Related documentation
 

caas/kubernetes/node/inputs.tf

@@ -296,3 +296,95 @@ variable "node_unschedulable_timeframe" {
   type        = "string"
   default     = "last_1h"
 }
+
+variable "volume_space_silenced" {
+  description = "Groups to mute for Volume space monitor"
+  type        = "map"
+  default     = {}
+}
+
+variable "volume_space_enabled" {
+  description = "Flag to enable Volume space monitor"
+  type        = "string"
+  default     = "true"
+}
+
+variable "volume_space_extra_tags" {
+  description = "Extra tags for Volume space monitor"
+  type        = "list"
+  default     = []
+}
+
+variable "volume_space_message" {
+  description = "Custom message for Volume space monitor"
+  type        = "string"
+  default     = ""
+}
+
+variable "volume_space_time_aggregator" {
+  description = "Monitor aggregator for Volume space [available values: min, max or avg]"
+  type        = "string"
+  default     = "min"
+}
+
+variable "volume_space_timeframe" {
+  description = "Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = "string"
+  default     = "last_5m"
+}
+
+variable "volume_space_threshold_critical" {
+  default     = 95
+  description = "Volume space critical threshold"
+}
+
+variable "volume_space_threshold_warning" {
+  default     = 90
+  description = "Volume space warning threshold"
+}
+
+variable "volume_inodes_silenced" {
+  description = "Groups to mute for Volume inodes monitor"
+  type        = "map"
+  default     = {}
+}
+
+variable "volume_inodes_enabled" {
+  description = "Flag to enable Volume inodes monitor"
+  type        = "string"
+  default     = "true"
+}
+
+variable "volume_inodes_extra_tags" {
+  description = "Extra tags for Volume inodes monitor"
+  type        = "list"
+  default     = []
+}
+
+variable "volume_inodes_message" {
+  description = "Custom message for Volume inodes monitor"
+  type        = "string"
+  default     = ""
+}
+
+variable "volume_inodes_time_aggregator" {
+  description = "Monitor aggregator for Volume inodes [available values: min, max or avg]"
+  type        = "string"
+  default     = "min"
+}
+
+variable "volume_inodes_timeframe" {
+  description = "Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
+  type        = "string"
+  default     = "last_5m"
+}
+
+variable "volume_inodes_threshold_critical" {
+  default     = 95
+  description = "Volume inodes critical threshold"
+}
+
+variable "volume_inodes_threshold_warning" {
+  default     = 90
+  description = "Volume inodes warning threshold"
+}

caas/kubernetes/node/monitors-k8s-node.tf

@@ -231,3 +231,69 @@ resource "datadog_monitor" "node_unschedulable" {
   silenced = "${var.node_unschedulable_silenced}"
   tags     = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.node_unschedulable_extra_tags}"]
 }
+
+resource "datadog_monitor" "volume_space" {
+  count   = "${var.volume_space_enabled == "true" ? 1 : 0}"
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume space usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  type    = "metric alert"
+  message = "${coalesce(var.volume_space_message, var.message)}"
+
+  query = <<EOQ
+    ${var.volume_space_time_aggregator}(${var.volume_space_timeframe}):
+      avg:kubernetes.kubelet.volume.stats.used_bytes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim} /
+      avg:kubernetes.kubelet.volume.stats.capacity_bytes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim}
+    * 100 > ${var.volume_space_threshold_critical}
+  EOQ
+
+  thresholds {
+    critical = "${var.volume_space_threshold_critical}"
+    warning  = "${var.volume_space_threshold_warning}"
+  }
+
+  evaluation_delay = "${var.evaluation_delay}"
+  new_host_delay   = "${var.new_host_delay}"
+
+  notify_no_data      = true
+  renotify_interval   = 0
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+
+  silenced = "${var.volume_space_silenced}"
+  tags     = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_space_extra_tags}"]
+}
+
+resource "datadog_monitor" "volume_inodes" {
+  count   = "${var.volume_inodes_enabled == "true" ? 1 : 0}"
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume inodes usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  type    = "metric alert"
+  message = "${coalesce(var.volume_inodes_message, var.message)}"
+
+  query = <<EOQ
+    ${var.volume_space_time_aggregator}(${var.volume_space_timeframe}):
+      avg:kubernetes.kubelet.volume.stats.inodes_used${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim} /
+      avg:kubernetes.kubelet.volume.stats.inodes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim}
+    * 100 > ${var.volume_space_threshold_critical}
+  EOQ
+
+  thresholds {
+    critical = "${var.volume_inodes_threshold_critical}"
+    warning  = "${var.volume_inodes_threshold_warning}"
+  }
+
+  evaluation_delay = "${var.evaluation_delay}"
+  new_host_delay   = "${var.new_host_delay}"
+
+  notify_no_data      = true
+  renotify_interval   = 0
+  notify_audit        = false
+  timeout_h           = 0
+  include_tags        = true
+  locked              = false
+  require_full_window = true
+
+  silenced = "${var.volume_inodes_silenced}"
+  tags     = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_inodes_extra_tags}"]
+}

caas/kubernetes/node/outputs.tf

@@ -37,3 +37,13 @@ output "node_unschedulable_id" {
   description = "id for monitor node_unschedulable"
   value       = "${datadog_monitor.node_unschedulable.*.id}"
 }
+
+output "volume_space_id" {
+  description = "id for monitor volume_space"
+  value       = "${datadog_monitor.volume_space.*.id}"
+}
+
+output "volume_inodes_id" {
+  description = "id for monitor volume_inodes"
+  value       = "${datadog_monitor.volume_inodes.*.id}"
+}