wbyc/terraform-datadog-old-monitors
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MON-113 fix error in es_cluster_status monitor's query and apply latest best practices

Browse Source
This commit is contained in:
Guillaume Kerivel 2018-03-20 10:33:54 +01:00
parent d8b8aef2ac
commit fcf6b32393
3 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cloud/aws/elasticsearch/README.md
View File

@ -33,6 +33,7 @@ Inputs
| cpu_silenced | Groups to mute for ES cluster cpu monitor | map | `<map>` | no |
| cpu_threshold_critical | CPU usage in percent (critical threshold) | string | `90` | no |
| cpu_threshold_warning | CPU usage in percent (warning threshold) | string | `80` | no |
| delay | Delay in seconds for the metric evaluation | string | `600` | no |
| diskspace_message | Custom message for ES cluster diskspace monitor | string | `` | no |
| diskspace_silenced | Groups to mute for ES cluster diskspace monitor | map | `<map>` | no |
| diskspace_threshold_critical | Disk free space in percent (critical threshold) | string | `10` | no |
@ -41,7 +42,6 @@ Inputs
| es_cluster_status_message | Custom message for ES cluster status monitor | string | `` | no |
| es_cluster_status_silenced | Groups to mute for ES cluster status monitor | map | `<map>` | no |
| es_cluster_volume_size | ElasticSearch Domain volume size (in GB) | string | - | yes |
| evaluation_delay | Delay in seconds for the metric evaluation | string | `600` | no |
| filter_tags_custom | Tags used for custom filtering when filter_tags_use_defaults is false | string | `*` | no |
| filter_tags_use_defaults | Use default filter tags convention | string | `true` | no |
| message | Message sent when an alert is triggered | string | - | yes |

2
cloud/aws/elasticsearch/inputs.tf
View File

@ -5,7 +5,7 @@ variable "environment" {
}
# Global DataDog
variable "evaluation_delay" {
variable "delay" {
description = "Delay in seconds for the metric evaluation"
default = 600
}

27
cloud/aws/elasticsearch/monitors-elasticsearch.tf
View File

@ -7,34 +7,37 @@ data "template_file" "filter" {
}
### Elasticsearch cluster status monitor ###
/* Note about the query
- If aws.es.cluster_statusred is 1 --> query value (= 2.1) > 2 : critical
- If aws.es.cluster_statusyellow is 1 --> 1 < query value (=1.1) < 2 : warning
Workaround : in the query, we add "0.1" to the result and we use the comparator ">=". No alert was triggered without that. */
resource "datadog_monitor" "es_cluster_status" {
name = "[${var.environment}] ElasticSearch cluster status is not green"
message = "${coalesce(var.es_cluster_status_message, var.message)}"
type = "query alert"
type = "metric alert"
query = <<EOF
max(last_30m): (
avg:aws.es.cluster_statusred{${data.template_file.filter.rendered}} by {region,name} * 2 +
avg:aws.es.cluster_statusyel{${data.template_file.filter.rendered}} by {region,name}
) > 2
(avg:aws.es.cluster_statusyellow{${data.template_file.filter.rendered}} by {region,name} + 0.1)
) >= 2
EOF
thresholds {
ok = 0
warning = 1
critical = 2
}
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
evaluation_delay = "${var.delay}"
renotify_interval = 0
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = false
new_host_delay = "${var.evaluation_delay}"
new_host_delay = "${var.delay}"
no_data_timeframe = 20
silenced = "${var.es_cluster_status_silenced}"
@ -47,7 +50,7 @@ resource "datadog_monitor" "es_free_space_low" {
name = "[${var.environment}] ElasticSearch cluster free storage space {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
message = "${coalesce(var.diskspace_message, var.message)}"
type = "query alert"
type = "metric alert"
query = <<EOF
avg(last_15m): (
@ -62,14 +65,14 @@ EOF
}
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
evaluation_delay = "${var.delay}"
renotify_interval = 0
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = false
new_host_delay = "${var.evaluation_delay}"
new_host_delay = "${var.delay}"
no_data_timeframe = 20
silenced = "${var.diskspace_silenced}"
@ -82,7 +85,7 @@ resource "datadog_monitor" "es_cpu_90_15min" {
name = "[${var.environment}] ElasticSearch cluster CPU high {{#is_alert}}{{comparator}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{comparator}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
message = "${coalesce(var.cpu_message, var.message)}"
type = "query alert"
type = "metric alert"
query = <<EOF
avg(last_15m): (
@ -96,14 +99,14 @@ EOF
}
notify_no_data = true
evaluation_delay = "${var.evaluation_delay}"
evaluation_delay = "${var.delay}"
renotify_interval = 0
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = false
new_host_delay = "${var.evaluation_delay}"
new_host_delay = "${var.delay}"
no_data_timeframe = 20
silenced = "${var.cpu_silenced}"