wbyc/terraform-datadog-old-monitors
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MON-228 use filter tags module

Browse Source
This commit is contained in:
Quentin Manfroi 2018-08-30 14:35:47 +02:00
parent b2d16a8524
commit dc969f9649
2 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
database/elasticsearch/modules.tf Normal file
View File

@ -0,0 +1,8 @@
module "filter-tags" {
source = "../../common/filter-tags"
environment = "${var.environment}"
resource = "elasticsearch"
filter_tags_use_defaults = "${var.filter_tags_use_defaults}"
filter_tags_custom = "${var.filter_tags_custom}"
}

54
database/elasticsearch/monitors-elasticsearch.tf
View File

@ -1,11 +1,3 @@
data "template_file" "filter" {
template = "$${filter}"
vars {
filter = "${var.filter_tags_use_defaults == "true" ? format("dd_monitoring:enabled,env:%s", var.environment) : "${var.filter_tags_custom}"}"
}
}
#
# Cluster Status Not Green
#
@ -17,7 +9,7 @@ resource "datadog_monitor" "cluster_status_not_green" {
query = <<EOF
${var.cluster_status_not_green_time_aggregator}(${var.cluster_status_not_green_timeframe}):
min:elasticsearch.cluster_status{${data.template_file.filter.rendered}} by {cluster_name}
min:elasticsearch.cluster_status${module.filter-tags.query_alert} by {cluster_name}
<= ${var.cluster_status_not_green_threshold_critical}
EOF
@ -59,7 +51,7 @@ resource "datadog_monitor" "cluster_initializing_shards" {
query = <<EOF
${var.cluster_initializing_shards_time_aggregator}(${var.cluster_initializing_shards_timeframe}):
avg:elasticsearch.initializing_shards{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.initializing_shards${module.filter-tags.query_alert} by {cluster_name}
> ${var.cluster_initializing_shards_threshold_critical}
EOF
@ -100,7 +92,7 @@ resource "datadog_monitor" "cluster_relocating_shards" {
query = <<EOF
${var.cluster_relocating_shards_time_aggregator}(${var.cluster_relocating_shards_timeframe}):
avg:elasticsearch.relocating_shards{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.relocating_shards${module.filter-tags.query_alert} by {cluster_name}
> ${var.cluster_relocating_shards_threshold_critical}
EOF
@ -141,7 +133,7 @@ resource "datadog_monitor" "cluster_unassigned_shards" {
query = <<EOF
${var.cluster_unassigned_shards_time_aggregator}(${var.cluster_unassigned_shards_timeframe}):
avg:elasticsearch.unassigned_shards{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.unassigned_shards${module.filter-tags.query_alert} by {cluster_name}
> ${var.cluster_unassigned_shards_threshold_critical}
EOF
@ -182,9 +174,9 @@ resource "datadog_monitor" "node_free_space" {
query = <<EOF
${var.node_free_space_time_aggregator}(${var.node_free_space_timeframe}):
(min:elasticsearch.fs.total.available_in_bytes{${data.template_file.filter.rendered}} by {node_name}
(min:elasticsearch.fs.total.available_in_bytes${module.filter-tags.query_alert} by {node_name}
/
min:elasticsearch.fs.total.total_in_bytes{${data.template_file.filter.rendered}} by {node_name}
min:elasticsearch.fs.total.total_in_bytes${module.filter-tags.query_alert} by {node_name}
) * 100
< ${var.node_free_space_threshold_critical}
EOF
@ -226,7 +218,7 @@ resource "datadog_monitor" "jvm_heap_memory_usage" {
query = <<EOF
${var.jvm_heap_memory_usage_time_aggregator}(${var.jvm_heap_memory_usage_timeframe}):
avg:jvm.mem.heap_in_use{${data.template_file.filter.rendered}} by {node_name}
avg:jvm.mem.heap_in_use${module.filter-tags.query_alert} by {node_name}
> ${var.jvm_heap_memory_usage_threshold_critical}
EOF
@ -267,7 +259,7 @@ resource "datadog_monitor" "jvm_memory_young_usage" {
query = <<EOF
${var.jvm_memory_young_usage_time_aggregator}(${var.jvm_memory_young_usage_timeframe}):
avg:jvm.mem.pools.young.used{${data.template_file.filter.rendered}} by {node_name} / avg:jvm.mem.pools.young.max{${data.template_file.filter.rendered}} by {node_name} * 100
avg:jvm.mem.pools.young.used${module.filter-tags.query_alert} by {node_name} / avg:jvm.mem.pools.young.max${module.filter-tags.query_alert} by {node_name} * 100
> ${var.jvm_memory_young_usage_threshold_critical}
EOF
@ -308,7 +300,7 @@ resource "datadog_monitor" "jvm_memory_old_usage" {
query = <<EOF
${var.jvm_memory_old_usage_time_aggregator}(${var.jvm_memory_old_usage_timeframe}):
avg:jvm.mem.pools.old.used{${data.template_file.filter.rendered}} by {node_name} / avg:jvm.mem.pools.old.max{${data.template_file.filter.rendered}} by {node_name} * 100
avg:jvm.mem.pools.old.used${module.filter-tags.query_alert} by {node_name} / avg:jvm.mem.pools.old.max${module.filter-tags.query_alert} by {node_name} * 100
> ${var.jvm_memory_old_usage_threshold_critical}
EOF
@ -349,7 +341,7 @@ resource "datadog_monitor" "jvm_gc_old_collection_latency" {
query = <<EOF
${var.jvm_gc_old_collection_latency_time_aggregator}(${var.jvm_gc_old_collection_latency_timeframe}):
avg:jvm.gc.collectors.old.collection_time{${data.template_file.filter.rendered}} by {node_name} / avg:jvm.gc.collectors.old.count{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:jvm.gc.collectors.old.collection_time${module.filter-tags.query_alert} by {node_name} / avg:jvm.gc.collectors.old.count${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.jvm_gc_old_collection_latency_threshold_critical}
EOF
@ -390,7 +382,7 @@ resource "datadog_monitor" "jvm_gc_young_collection_latency" {
query = <<EOF
${var.jvm_gc_young_collection_latency_time_aggregator}(${var.jvm_gc_young_collection_latency_timeframe}):
avg:jvm.gc.collectors.young.collection_time{${data.template_file.filter.rendered}} by {node_name} / avg:jvm.gc.collectors.young.count{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:jvm.gc.collectors.young.collection_time${module.filter-tags.query_alert} by {node_name} / avg:jvm.gc.collectors.young.count${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.jvm_gc_young_collection_latency_threshold_critical}
EOF
@ -432,7 +424,7 @@ resource "datadog_monitor" "indexing_latency" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
${var.indexing_latency_time_aggregator}(${var.indexing_latency_timeframe}):
avg:elasticsearch.indexing.index.time{${data.template_file.filter.rendered}} by {node_name}/ avg:elasticsearch.indexing.index.total{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:elasticsearch.indexing.index.time${module.filter-tags.query_alert} by {node_name}/ avg:elasticsearch.indexing.index.total${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.indexing_latency_threshold_critical}
EOF
@ -474,7 +466,7 @@ resource "datadog_monitor" "flush_latency" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
${var.flush_latency_time_aggregator}(${var.flush_latency_timeframe}):
avg:elasticsearch.flush.total.time{${data.template_file.filter.rendered}} by {node_name} / avg:elasticsearch.flush.total{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:elasticsearch.flush.total.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.flush.total${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.flush_latency_threshold_critical}
EOF
@ -515,7 +507,7 @@ resource "datadog_monitor" "http_connections_anomaly" {
query = <<EOF
${var.http_connections_anomaly_time_aggregator}(${var.http_connections_anomaly_timeframe}):
anomalies(avg:elasticsearch.http.current_open{${data.template_file.filter.rendered}} by {node_name},
anomalies(avg:elasticsearch.http.current_open${module.filter-tags.query_alert} by {node_name},
'${var.http_connections_anomaly_detection_algorithm}',
${var.http_connections_anomaly_deviations},
direction='${var.http_connections_anomaly_direction}',
@ -565,7 +557,7 @@ resource "datadog_monitor" "search_query_latency" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
${var.search_query_latency_time_aggregator}(${var.search_query_latency_timeframe}):
avg:elasticsearch.search.query.time{${data.template_file.filter.rendered}} by {node_name} / avg:elasticsearch.search.query.total{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:elasticsearch.search.query.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.search.query.total${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.search_query_latency_threshold_critical}
EOF
@ -607,7 +599,7 @@ resource "datadog_monitor" "fetch_latency" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
${var.fetch_latency_time_aggregator}(${var.fetch_latency_timeframe}):
avg:elasticsearch.search.fetch.time{${data.template_file.filter.rendered}} by {node_name} / avg:elasticsearch.search.fetch.total{${data.template_file.filter.rendered}} by {node_name} * 1000
avg:elasticsearch.search.fetch.time${module.filter-tags.query_alert} by {node_name} / avg:elasticsearch.search.fetch.total${module.filter-tags.query_alert} by {node_name} * 1000
> ${var.fetch_latency_threshold_critical}
EOF
@ -648,7 +640,7 @@ resource "datadog_monitor" "search_query_change" {
query = <<EOF
pct_change(${var.search_query_change_time_aggregator}(${var.search_query_change_timeframe}),${var.search_query_change_timeshift}):
avg:elasticsearch.search.query.current{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.search.query.current${module.filter-tags.query_alert} by {cluster_name}
>= ${var.search_query_change_threshold_critical}
EOF
@ -689,7 +681,7 @@ resource "datadog_monitor" "fetch_change" {
query = <<EOF
pct_change(${var.fetch_change_time_aggregator}(${var.fetch_change_timeframe}),${var.fetch_change_timeshift}):
avg:elasticsearch.search.fetch.current{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.search.fetch.current${module.filter-tags.query_alert} by {cluster_name}
>= ${var.fetch_change_threshold_critical}
EOF
@ -731,7 +723,7 @@ resource "datadog_monitor" "field_data_evictions_change" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
change(${var.field_data_evictions_change_time_aggregator}(${var.field_data_evictions_change_timeframe}),${var.field_data_evictions_change_timeshift}):
avg:elasticsearch.fielddata.evictions{${data.template_file.filter.rendered}} by {node_name}
avg:elasticsearch.fielddata.evictions${module.filter-tags.query_alert} by {node_name}
> ${var.field_data_evictions_change_threshold_critical}
EOF
@ -773,7 +765,7 @@ resource "datadog_monitor" "query_cache_evictions_change" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
change(${var.query_cache_evictions_change_time_aggregator}(${var.query_cache_evictions_change_timeframe}),${var.query_cache_evictions_change_timeshift}):
avg:elasticsearch.indices.query_cache.evictions{${data.template_file.filter.rendered}} by {node_name}
avg:elasticsearch.indices.query_cache.evictions${module.filter-tags.query_alert} by {node_name}
> ${var.query_cache_evictions_change_threshold_critical}
EOF
@ -815,7 +807,7 @@ resource "datadog_monitor" "request_cache_evictions_change" {
// TODO add tags to filter by node type and do not apply this monitor on non-data nodes
query = <<EOF
change(${var.request_cache_evictions_change_time_aggregator}(${var.request_cache_evictions_change_timeframe}),${var.request_cache_evictions_change_timeshift}):
avg:elasticsearch.indices.request_cache.evictions{${data.template_file.filter.rendered}} by {node_name}
avg:elasticsearch.indices.request_cache.evictions${module.filter-tags.query_alert} by {node_name}
> ${var.request_cache_evictions_change_threshold_critical}
EOF
@ -856,7 +848,7 @@ resource "datadog_monitor" "task_time_in_queue_change" {
query = <<EOF
change(${var.task_time_in_queue_change_time_aggregator}(${var.task_time_in_queue_change_timeframe}),${var.task_time_in_queue_change_timeshift}):
avg:elasticsearch.pending_tasks_time_in_queue{${data.template_file.filter.rendered}} by {cluster_name}
avg:elasticsearch.pending_tasks_time_in_queue${module.filter-tags.query_alert} by {cluster_name}
> ${var.task_time_in_queue_change_threshold_critical}
EOF
@ -894,7 +886,7 @@ resource "datadog_monitor" "not_responding" {
message = "${coalesce(var.not_responding_message, var.message)}"
query = <<EOL
"elasticsearch.can_connect".over("${replace(data.template_file.filter.rendered, ",", "\",\"")}").by(${var.not_responding_by}).last(${var.not_responding_last}).pct_by_status()
"elasticsearch.can_connect".over${module.filter-tags.service_check}.by(${var.not_responding_by}).last(${var.not_responding_last}).pct_by_status()
EOL
type = "service check"