MON-142 add some monitors for mysql

This commit is contained in:
Quentin Manfroi 2018-08-27 16:50:49 +02:00
parent f8ea23eaf4
commit 0e22a05489
2 changed files with 574 additions and 23 deletions

View File

@ -28,6 +28,36 @@ variable "filter_tags_custom" {
}
# MySQL specific
#################################
### MySQL availability ###
#################################
variable "mysql_availability_silenced" {
description = "Groups to mute for Mysql availability monitor"
type = "map"
default = {}
}
variable "mysql_availability_extra_tags" {
description = "Extra tags for Mysql availability monitor"
type = "list"
default = []
}
variable "mysql_availability_message" {
description = "Custom message for Mysql availability monitor"
type = "string"
default = ""
}
variable "mysql_availability_threshold_critical" {
description = "Nginx status monitor (critical threshold)"
type = "string"
default = 1.1754943508222875e-38
}
#################################
### MySQL connections ###
#################################
@ -42,8 +72,20 @@ variable "mysql_connection_threshold_warning" {
description = "Maximum warning acceptable percent of connections"
}
variable "mysql_connection_time_aggregator" {
description = "Monitor time aggregator for MySQL connection monitor [available values: min, max or avg]"
type = "string"
default = "avg"
}
variable "mysql_connection_timeframe" {
description = "Monitor timeframe for MySQL connection monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_10m"
}
variable "mysql_connection_silenced" {
description = "Groups to mute mysql connection monitor"
description = "Groups to mute MySQL connection monitor"
type = "map"
default = {}
}
@ -60,34 +102,340 @@ variable "mysql_connection_extra_tags" {
default = []
}
#################################
### MySQL aborted connects ###
#################################
variable "mysql_aborted_threshold_critical" {
default = 10
description = "Maximum critical acceptable percent of aborted connects"
}
variable "mysql_aborted_threshold_warning" {
default = 5
description = "Maximum warning acceptable percent of aborted connects"
}
variable "mysql_aborted_time_aggregator" {
description = "Monitor time aggregator for MySQL aborted connects monitor [available values: min, max or avg]"
type = "string"
default = "avg"
}
variable "mysql_aborted_timeframe" {
description = "Monitor timeframe for MySQL aborted connects monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_10m"
}
variable "mysql_aborted_silenced" {
description = "Groups to mute MySQL aborted connects monitor"
type = "map"
default = {}
}
variable "mysql_aborted_message" {
description = "Custom message for MySQL aborted connects monitor"
type = "string"
default = ""
}
variable "mysql_aborted_extra_tags" {
description = "Extra tags for MySQL aborted connects monitor"
type = "list"
default = []
}
#################################
### MySQL slow queries ###
#################################
variable "mysql_slow_threshold_critical" {
default = 20
description = "Maximum critical acceptable percent of slow queries"
}
variable "mysql_slow_threshold_warning" {
default = 5
description = "Maximum warning acceptable percent of slow queries"
}
variable "mysql_slow_time_aggregator" {
description = "Monitor time aggregator for MySQL slow queries monitor [available values: min, max or avg]"
type = "string"
default = "avg"
}
variable "mysql_slow_timeframe" {
description = "Monitor timeframe for MySQL slow queries monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_15m"
}
variable "mysql_slow_silenced" {
description = "Groups to mute MySQL slow queries monitor"
type = "map"
default = {}
}
variable "mysql_slow_message" {
description = "Custom message for MySQL slow queries monitor"
type = "string"
default = ""
}
variable "mysql_slow_extra_tags" {
description = "Extra tags for MySQL slow queries monitor"
type = "list"
default = []
}
#################################
# MySQL innodb pool efficiency #
#################################
variable "mysql_pool_efficiency_threshold_critical" {
default = 20
description = "Maximum critical acceptable percent of innodb buffer pool efficiency"
}
variable "mysql_pool_efficiency_threshold_warning" {
default = 1
description = "Maximum warning acceptable percent of innodb buffer pool efficiency"
}
variable "mysql_pool_efficiency_time_aggregator" {
description = "Monitor time aggregator for MySQL innodb buffer pool efficiency monitor [available values: min, max or avg]"
type = "string"
default = "sum"
}
variable "mysql_pool_efficiency_timeframe" {
description = "Monitor timeframe for MySQL innodb buffer pool efficiency monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_1h"
}
variable "mysql_pool_efficiency_silenced" {
description = "Groups to mute MySQL innodb buffer pool efficiency monitor"
type = "map"
default = {}
}
variable "mysql_pool_efficiency_message" {
description = "Custom message for MySQL innodb buffer pool efficiency monitor"
type = "string"
default = ""
}
variable "mysql_pool_efficiency_extra_tags" {
description = "Extra tags for MySQL innodb buffer pool efficiency monitor"
type = "list"
default = []
}
#################################
# MySQL innodb pool utilization #
#################################
variable "mysql_pool_utilization_threshold_critical" {
default = 90
description = "Maximum critical acceptable percent of innodb buffer pool utilization"
}
variable "mysql_pool_utilization_threshold_warning" {
default = 75
description = "Maximum warning acceptable percent of innodb buffer pool utilization"
}
variable "mysql_pool_utilization_time_aggregator" {
description = "Monitor time aggregator for MySQL innodb buffer pool utilization monitor [available values: min, max or avg]"
type = "string"
default = "min"
}
variable "mysql_pool_utilization_timeframe" {
description = "Monitor timeframe for MySQL innodb buffer pool utilization monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_4h"
}
variable "mysql_pool_utilization_silenced" {
description = "Groups to mute MySQL innodb buffer pool utilization monitor"
type = "map"
default = {}
}
variable "mysql_pool_utilization_message" {
description = "Custom message for MySQL innodb buffer pool utilization monitor"
type = "string"
default = ""
}
variable "mysql_pool_utilization_extra_tags" {
description = "Extra tags for MySQL innodb buffer pool utilization monitor"
type = "list"
default = []
}
#################################
### MySQL threads ###
#################################
variable "mysql_thread_threshold_critical" {
default = 500
variable "mysql_threads_threshold_critical" {
default = 1
description = "Maximum critical acceptable number of threads"
}
variable "mysql_thread_threshold_warning" {
default = 400
description = "Maximum warning acceptable number of threads"
variable "mysql_threads_detection_algorithm" {
description = "Anomaly Detection Algorithm used"
type = "string"
default = "basic"
}
variable "mysql_thread_silenced" {
variable "mysql_threads_deviations" {
description = "Deviations to detect the anomaly"
type = "string"
default = 2
}
variable "mysql_threads_direction" {
description = "Direction of the anomaly. It can be both, below or above."
type = "string"
default = "above"
}
variable "mysql_threads_alert_window" {
description = "Alert window."
type = "string"
default = "last_15m"
}
variable "mysql_threads_interval" {
description = "Interval."
type = "string"
default = 60
}
variable "mysql_threads_count_default_zero" {
description = "Count default zero."
type = "string"
default = "true"
}
variable "mysql_threads_seasonality" {
description = "Seasonality of the algorithm"
type = "string"
default = "daily"
}
variable "mysql_threads_time_aggregator" {
description = "Monitor time aggregator for MySQL threads monitor [available values: min, max or avg]"
type = "string"
default = "avg"
}
variable "mysql_threads_timeframe" {
description = "Monitor timeframe for MySQL threads monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_4h"
}
variable "mysql_threads_silenced" {
description = "Groups to mute mysql threads monitor"
type = "map"
default = {}
}
variable "mysql_thread_message" {
description = "Custom message for MySQL thread monitor"
variable "mysql_threads_message" {
description = "Custom message for MySQL threads monitor"
type = "string"
default = ""
}
variable "mysql_thread_extra_tags" {
description = "Extra tags for MySQL thread monitor"
variable "mysql_threads_extra_tags" {
description = "Extra tags for MySQL threads monitor"
type = "list"
default = []
}
#################################
### MySQL queries ###
#################################
variable "mysql_queries_threshold_critical" {
default = 1
description = "Maximum critical acceptable number of queries"
}
variable "mysql_queries_detection_algorithm" {
description = "Anomaly Detection Algorithm used"
type = "string"
default = "agile"
}
variable "mysql_queries_deviations" {
description = "Deviations to detect the anomaly"
type = "string"
default = 2
}
variable "mysql_queries_direction" {
description = "Direction of the anomaly. It can be both, below or above."
type = "string"
default = "both"
}
variable "mysql_queries_alert_window" {
description = "Alert window."
type = "string"
default = "last_15m"
}
variable "mysql_queries_interval" {
description = "Interval."
type = "string"
default = 60
}
variable "mysql_queries_count_default_zero" {
description = "Count default zero."
type = "string"
default = "true"
}
variable "mysql_queries_seasonality" {
description = "Seasonality of the algorithm"
type = "string"
default = "daily"
}
variable "mysql_queries_time_aggregator" {
description = "Monitor time aggregator for MySQL queries monitor [available values: min, max or avg]"
type = "string"
default = "avg"
}
variable "mysql_queries_timeframe" {
description = "Monitor timeframe for MySQL queries monitor [available values: `last_#m` (1, 5, 10, 15, or 30), `last_#h` (1, 2, or 4), or `last_1d`]"
type = "string"
default = "last_4h"
}
variable "mysql_queries_silenced" {
description = "Groups to mute mysql queries monitor"
type = "map"
default = {}
}
variable "mysql_queries_message" {
description = "Custom message for MySQL queries monitor"
type = "string"
default = ""
}
variable "mysql_queries_extra_tags" {
description = "Extra tags for MySQL queries monitor"
type = "list"
default = []
}

View File

@ -1,10 +1,40 @@
resource "datadog_monitor" "mysql_connection_too_high" {
resource "datadog_monitor" "mysql_availability" {
name = "[${var.environment}] Mysql server does not respond"
message = "${coalesce(var.mysql_availability_message, var.message)}"
type = "service check"
query = <<EOF
"mysql.can_connect".over${module.filter-tags.service_check}.by("host","port","server").last(1).pct_by_status()
EOF
thresholds = {
warning = 0
critical = "${var.mysql_availability_threshold_critical}"
}
notify_no_data = true
new_host_delay = "${var.new_host_delay}"
renotify_interval = 0
notify_audit = false
timeout_h = 0
include_tags = true
locked = false
require_full_window = true
silenced = "${var.mysql_availability_silenced}"
tags = ["env:${var.environment}", "type:middleware", "provider:php-fpm", "resource:php-fpm", "team:claranet", "created-by:terraform", "${var.mysql_availability_extra_tags}"]
}
resource "datadog_monitor" "mysql_connection" {
name = "[${var.environment}] Mysql Connections {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
message = "${coalesce(var.mysql_connection_message, var.message)}"
type = "metric alert"
query = <<EOF
avg(last_15m): (
${var.mysql_connection_time_aggregator}(${var.mysql_connection_timeframe}): (
avg:mysql.net.connections${module.filter-tags.query_alert} by {server} /
avg:mysql.net.max_connections_available${module.filter-tags.query_alert} by {server}
) * 100 > ${var.mysql_connection_threshold_critical}
@ -29,23 +59,24 @@ resource "datadog_monitor" "mysql_connection_too_high" {
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_connection_extra_tags}"]
}
resource "datadog_monitor" "mysql_thread_too_high" {
name = "[${var.environment}] Mysql threads {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}"
message = "${coalesce(var.mysql_thread_message, var.message)}"
resource "datadog_monitor" "mysql_aborted" {
name = "[${var.environment}] Mysql Aborted connects {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
message = "${coalesce(var.mysql_aborted_message, var.message)}"
type = "metric alert"
query = <<EOF
avg(last_5m): (
avg:mysql.performance.threads_running${module.filter-tags.query_alert} by {server}
) > ${var.mysql_thread_threshold_critical}
${var.mysql_aborted_time_aggregator}(${var.mysql_aborted_timeframe}): (
avg:mysql.net.aborted_connects${module.filter-tags.query_alert} by {server} /
avg:mysql.performance.queries_connected${module.filter-tags.query_alert} by {server}
) * 100 > ${var.mysql_aborted_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
warning = "${var.mysql_thread_threshold_warning}"
critical = "${var.mysql_thread_threshold_critical}"
warning = "${var.mysql_aborted_threshold_warning}"
critical = "${var.mysql_aborted_threshold_critical}"
}
notify_no_data = false
@ -54,7 +85,179 @@ resource "datadog_monitor" "mysql_thread_too_high" {
timeout_h = 0
include_tags = true
silenced = "${var.mysql_thread_silenced}"
silenced = "${var.mysql_aborted_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_thread_extra_tags}"]
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_aborted_extra_tags}"]
}
resource "datadog_monitor" "mysql_slow" {
name = "[${var.environment}] Mysql Slow queries {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
message = "${coalesce(var.mysql_slow_message, var.message)}"
type = "metric alert"
query = <<EOF
${var.mysql_slow_time_aggregator}(${var.mysql_slow_timeframe}): (
avg:mysql.performance.slow_queries${module.filter-tags.query_alert} by {server} /
avg:mysql.performance.queries${module.filter-tags.query_alert} by {server}
) * 100 > ${var.mysql_slow_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
warning = "${var.mysql_slow_threshold_warning}"
critical = "${var.mysql_slow_threshold_critical}"
}
notify_no_data = false
renotify_interval = 0
require_full_window = true
timeout_h = 0
include_tags = true
silenced = "${var.mysql_slow_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_slow_extra_tags}"]
}
resource "datadog_monitor" "mysql_pool_efficiency" {
name = "[${var.environment}] Mysql Innodb buffer pool efficiency {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}"
message = "${coalesce(var.mysql_pool_efficiency_message, var.message)}"
type = "metric alert"
query = <<EOF
${var.mysql_pool_efficiency_time_aggregator}(${var.mysql_pool_efficiency_timeframe}): (
avg:mysql.innodb.buffer_pool_reads${module.filter-tags.query_alert} by {server} /
avg:mysql.innodb.buffer_pool_read_requests${module.filter-tags.query_alert} by {server}
) * 100 > ${var.mysql_pool_efficiency_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
warning = "${var.mysql_pool_efficiency_threshold_warning}"
critical = "${var.mysql_pool_efficiency_threshold_critical}"
}
notify_no_data = false
renotify_interval = 0
require_full_window = true
timeout_h = 0
include_tags = true
silenced = "${var.mysql_pool_efficiency_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_pool_efficiency_extra_tags}"]
}
resource "datadog_monitor" "mysql_pool_utilization" {
name = "[${var.environment}] Mysql Innodb buffer pool utilization {{#is_alert}}{{{comparator}}} {{threshold}} ({{value}}){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}} ({{value}}){{/is_warning}}"
message = "${coalesce(var.mysql_pool_utilization_message, var.message)}"
type = "metric alert"
query = <<EOF
${var.mysql_pool_utilization_time_aggregator}(${var.mysql_pool_utilization_timeframe}):
( avg:mysql.innodb.buffer_pool_total${module.filter-tags.query_alert} by {server} -
avg:mysql.innodb.buffer_pool_free${module.filter-tags.query_alert} by {server} ) /
avg:mysql.innodb.buffer_pool_total${module.filter-tags.query_alert} by {server}
* 100 > ${var.mysql_pool_utilization_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
warning = "${var.mysql_pool_utilization_threshold_warning}"
critical = "${var.mysql_pool_utilization_threshold_critical}"
}
notify_no_data = false
renotify_interval = 0
require_full_window = true
timeout_h = 0
include_tags = true
silenced = "${var.mysql_pool_utilization_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_pool_utilization_extra_tags}"]
}
resource "datadog_monitor" "mysql_threads_anomaly" {
name = "[${var.environment}] Mysql threads changed abnormally"
message = "${coalesce(var.mysql_threads_message, var.message)}"
type = "metric alert"
query = <<EOF
${var.mysql_threads_time_aggregator}(${var.mysql_threads_timeframe}):
anomalies(
avg:mysql.performance.threads_running${module.filter-tags.query_alert} by {server},
'${var.mysql_threads_detection_algorithm}',
${var.mysql_threads_deviations},
direction='${var.mysql_threads_direction}',
alert_window='${var.mysql_threads_alert_window}',
interval=${var.mysql_threads_interval},
count_default_zero='${var.mysql_threads_count_default_zero}'
${var.mysql_threads_seasonality == "agile" ? format(",seasonality='%s'", var.mysql_queries_seasonality): ""}
)
>= ${var.mysql_threads_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
critical = "${var.mysql_threads_threshold_critical}"
critical_recovery = 0
}
notify_no_data = false
renotify_interval = 0
require_full_window = true
timeout_h = 0
include_tags = true
silenced = "${var.mysql_threads_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_threads_extra_tags}"]
}
resource "datadog_monitor" "mysql_queries_anomaly" {
name = "[${var.environment}] Mysql queries changed abnormally"
message = "${coalesce(var.mysql_queries_message, var.message)}"
type = "metric alert"
query = <<EOF
${var.mysql_queries_time_aggregator}(${var.mysql_queries_timeframe}):
anomalies(
avg:mysql.performance.queries${module.filter-tags.query_alert} by {server},
'${var.mysql_queries_detection_algorithm}',
${var.mysql_queries_deviations},
direction='${var.mysql_queries_direction}',
alert_window='${var.mysql_queries_alert_window}',
interval=${var.mysql_queries_interval},
count_default_zero='${var.mysql_queries_count_default_zero}'
${var.mysql_queries_detection_algorithm == "agile" ? format(",seasonality='%s'", var.mysql_queries_seasonality): ""}
)
>= ${var.mysql_queries_threshold_critical}
EOF
evaluation_delay = "${var.evaluation_delay}"
new_host_delay = "${var.new_host_delay}"
thresholds {
critical = "${var.mysql_queries_threshold_critical}"
critical_recovery = 0
}
notify_no_data = false
renotify_interval = 0
require_full_window = true
timeout_h = 0
include_tags = true
silenced = "${var.mysql_queries_silenced}"
tags = ["env:${var.environment}", "type:database", "provider:mysql", "resource:mysql", "team:claranet", "created-by:terraform", "${var.mysql_queries_extra_tags}"]
}