MON-453 add inodes and space monitors to kubernetes node
This commit is contained in:
Quentin Manfroi
parent
4ae24a8a4e
commit
17d0650b36
@ -24,6 +24,8 @@ Creates DataDog monitors with the following checks:
|
||||
- Kubernetes Node not ready
|
||||
- Kubernetes Node Out of disk
|
||||
- Kubernetes Node unschedulable
|
||||
- Kubernetes Node volume inodes usage
|
||||
- Kubernetes Node volume space usage
|
||||
|
||||
## Inputs
|
||||
|
||||
@ -80,6 +82,22 @@ Creates DataDog monitors with the following checks:
|
||||
| unregister\_net\_device\_threshold\_critical | Unregister net device critical threshold | string | `"3"` | no |
|
||||
| unregister\_net\_device\_time\_aggregator | Monitor aggregator for Unregister net device [available values: min, max or avg] | string | `"min"` | no |
|
||||
| unregister\_net\_device\_timeframe | Monitor timeframe for Unregister net device [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"15m"` | no |
|
||||
| volume\_inodes\_enabled | Flag to enable Volume inodes monitor | string | `"true"` | no |
|
||||
| volume\_inodes\_extra\_tags | Extra tags for Volume inodes monitor | list | `[]` | no |
|
||||
| volume\_inodes\_message | Custom message for Volume inodes monitor | string | `""` | no |
|
||||
| volume\_inodes\_silenced | Groups to mute for Volume inodes monitor | map | `{}` | no |
|
||||
| volume\_inodes\_threshold\_critical | Volume inodes critical threshold | string | `"95"` | no |
|
||||
| volume\_inodes\_threshold\_warning | Volume inodes warning threshold | string | `"90"` | no |
|
||||
| volume\_inodes\_time\_aggregator | Monitor aggregator for Volume inodes [available values: min, max or avg] | string | `"min"` | no |
|
||||
| volume\_inodes\_timeframe | Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
|
||||
| volume\_space\_enabled | Flag to enable Volume space monitor | string | `"true"` | no |
|
||||
| volume\_space\_extra\_tags | Extra tags for Volume space monitor | list | `[]` | no |
|
||||
| volume\_space\_message | Custom message for Volume space monitor | string | `""` | no |
|
||||
| volume\_space\_silenced | Groups to mute for Volume space monitor | map | `{}` | no |
|
||||
| volume\_space\_threshold\_critical | Volume space critical threshold | string | `"95"` | no |
|
||||
| volume\_space\_threshold\_warning | Volume space warning threshold | string | `"90"` | no |
|
||||
| volume\_space\_time\_aggregator | Monitor aggregator for Volume space [available values: min, max or avg] | string | `"min"` | no |
|
||||
| volume\_space\_timeframe | Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`] | string | `"last_5m"` | no |
|
||||
|
||||
## Outputs
|
||||
|
||||
@ -93,6 +111,8 @@ Creates DataDog monitors with the following checks:
|
||||
| node\_unschedulable\_id | id for monitor node_unschedulable |
|
||||
| ready\_id | id for monitor ready |
|
||||
| unregister\_net\_device\_id | id for monitor unregister_net_device |
|
||||
| volume\_inodes\_id | id for monitor volume_inodes |
|
||||
| volume\_space\_id | id for monitor volume_space |
|
||||
|
||||
## Related documentation
|
||||
|
||||
|
||||
@ -296,3 +296,95 @@ variable "node_unschedulable_timeframe" {
|
||||
type = "string"
|
||||
default = "last_1h"
|
||||
}
|
||||
|
||||
variable "volume_space_silenced" {
|
||||
description = "Groups to mute for Volume space monitor"
|
||||
type = "map"
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "volume_space_enabled" {
|
||||
description = "Flag to enable Volume space monitor"
|
||||
type = "string"
|
||||
default = "true"
|
||||
}
|
||||
|
||||
variable "volume_space_extra_tags" {
|
||||
description = "Extra tags for Volume space monitor"
|
||||
type = "list"
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "volume_space_message" {
|
||||
description = "Custom message for Volume space monitor"
|
||||
type = "string"
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "volume_space_time_aggregator" {
|
||||
description = "Monitor aggregator for Volume space [available values: min, max or avg]"
|
||||
type = "string"
|
||||
default = "min"
|
||||
}
|
||||
|
||||
variable "volume_space_timeframe" {
|
||||
description = "Monitor timeframe for Volume space [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
|
||||
type = "string"
|
||||
default = "last_5m"
|
||||
}
|
||||
|
||||
variable "volume_space_threshold_critical" {
|
||||
default = 95
|
||||
description = "Volume space critical threshold"
|
||||
}
|
||||
|
||||
variable "volume_space_threshold_warning" {
|
||||
default = 90
|
||||
description = "Volume space warning threshold"
|
||||
}
|
||||
|
||||
variable "volume_inodes_silenced" {
|
||||
description = "Groups to mute for Volume inodes monitor"
|
||||
type = "map"
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "volume_inodes_enabled" {
|
||||
description = "Flag to enable Volume inodes monitor"
|
||||
type = "string"
|
||||
default = "true"
|
||||
}
|
||||
|
||||
variable "volume_inodes_extra_tags" {
|
||||
description = "Extra tags for Volume inodes monitor"
|
||||
type = "list"
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "volume_inodes_message" {
|
||||
description = "Custom message for Volume inodes monitor"
|
||||
type = "string"
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "volume_inodes_time_aggregator" {
|
||||
description = "Monitor aggregator for Volume inodes [available values: min, max or avg]"
|
||||
type = "string"
|
||||
default = "min"
|
||||
}
|
||||
|
||||
variable "volume_inodes_timeframe" {
|
||||
description = "Monitor timeframe for Volume inodes [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
|
||||
type = "string"
|
||||
default = "last_5m"
|
||||
}
|
||||
|
||||
variable "volume_inodes_threshold_critical" {
|
||||
default = 95
|
||||
description = "Volume inodes critical threshold"
|
||||
}
|
||||
|
||||
variable "volume_inodes_threshold_warning" {
|
||||
default = 90
|
||||
description = "Volume inodes warning threshold"
|
||||
}
|
||||
|
||||
@ -231,3 +231,69 @@ resource "datadog_monitor" "node_unschedulable" {
|
||||
silenced = "${var.node_unschedulable_silenced}"
|
||||
tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.node_unschedulable_extra_tags}"]
|
||||
}
|
||||
|
||||
resource "datadog_monitor" "volume_space" {
|
||||
count = "${var.volume_space_enabled == "true" ? 1 : 0}"
|
||||
name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume space usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
|
||||
type = "metric alert"
|
||||
message = "${coalesce(var.volume_space_message, var.message)}"
|
||||
|
||||
query = <<EOQ
|
||||
${var.volume_space_time_aggregator}(${var.volume_space_timeframe}):
|
||||
avg:kubernetes.kubelet.volume.stats.used_bytes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim} /
|
||||
avg:kubernetes.kubelet.volume.stats.capacity_bytes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim}
|
||||
* 100 > ${var.volume_space_threshold_critical}
|
||||
EOQ
|
||||
|
||||
thresholds {
|
||||
critical = "${var.volume_space_threshold_critical}"
|
||||
warning = "${var.volume_space_threshold_warning}"
|
||||
}
|
||||
|
||||
evaluation_delay = "${var.evaluation_delay}"
|
||||
new_host_delay = "${var.new_host_delay}"
|
||||
|
||||
notify_no_data = true
|
||||
renotify_interval = 0
|
||||
notify_audit = false
|
||||
timeout_h = 0
|
||||
include_tags = true
|
||||
locked = false
|
||||
require_full_window = true
|
||||
|
||||
silenced = "${var.volume_space_silenced}"
|
||||
tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_space_extra_tags}"]
|
||||
}
|
||||
|
||||
resource "datadog_monitor" "volume_inodes" {
|
||||
count = "${var.volume_inodes_enabled == "true" ? 1 : 0}"
|
||||
name = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Kubernetes Node volume inodes usage {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
|
||||
type = "metric alert"
|
||||
message = "${coalesce(var.volume_inodes_message, var.message)}"
|
||||
|
||||
query = <<EOQ
|
||||
${var.volume_space_time_aggregator}(${var.volume_space_timeframe}):
|
||||
avg:kubernetes.kubelet.volume.stats.inodes_used${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim} /
|
||||
avg:kubernetes.kubelet.volume.stats.inodes${module.filter-tags.query_alert} by {kubernetescluster,name,persistentvolumeclaim}
|
||||
* 100 > ${var.volume_space_threshold_critical}
|
||||
EOQ
|
||||
|
||||
thresholds {
|
||||
critical = "${var.volume_inodes_threshold_critical}"
|
||||
warning = "${var.volume_inodes_threshold_warning}"
|
||||
}
|
||||
|
||||
evaluation_delay = "${var.evaluation_delay}"
|
||||
new_host_delay = "${var.new_host_delay}"
|
||||
|
||||
notify_no_data = true
|
||||
renotify_interval = 0
|
||||
notify_audit = false
|
||||
timeout_h = 0
|
||||
include_tags = true
|
||||
locked = false
|
||||
require_full_window = true
|
||||
|
||||
silenced = "${var.volume_inodes_silenced}"
|
||||
tags = ["env:${var.environment}", "type:caas", "provider:kubernetes", "resource:kubernetes-node", "team:claranet", "created-by:terraform", "${var.volume_inodes_extra_tags}"]
|
||||
}
|
||||
|
||||
@ -37,3 +37,13 @@ output "node_unschedulable_id" {
|
||||
description = "id for monitor node_unschedulable"
|
||||
value = "${datadog_monitor.node_unschedulable.*.id}"
|
||||
}
|
||||
|
||||
output "volume_space_id" {
|
||||
description = "id for monitor volume_space"
|
||||
value = "${datadog_monitor.volume_space.*.id}"
|
||||
}
|
||||
|
||||
output "volume_inodes_id" {
|
||||
description = "id for monitor volume_inodes"
|
||||
value = "${datadog_monitor.volume_inodes.*.id}"
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user