Merge branch 'MON-560-solr-monitors' into 'master'

MON-560 "Solr monitors"

Closes MON-560

See merge request claranet/pt-monitoring/projects/datadog/terraform/monitors!161

README.md

@@ -201,6 +201,7 @@ module "datadog-monitors-system-generic" {
 	- [mysql](https://github.com/claranet/terraform-datadog-monitors/tree/master/database/mysql/)
 	- [postgresql](https://github.com/claranet/terraform-datadog-monitors/tree/master/database/postgresql/)
 	- [redis](https://github.com/claranet/terraform-datadog-monitors/tree/master/database/redis/)
+	- [solr](https://github.com/claranet/terraform-datadog-monitors/tree/master/database/solr/)
 	- [sqlserver](https://github.com/claranet/terraform-datadog-monitors/tree/master/database/sqlserver/)
 - [middleware](https://github.com/claranet/terraform-datadog-monitors/tree/master/middleware/)
 	- [apache](https://github.com/claranet/terraform-datadog-monitors/tree/master/middleware/apache/)

database/solr/README.md

@@ -0,0 +1,67 @@
+# DATABASE SOLR DataDog monitors
+
+## How to use this module
+
+```hcl
+module "datadog-monitors-database-solr" {
+  source      = "claranet/monitors/datadog//database/solr"
+  version     = "{revision}"
+
+  environment = var.environment
+  message     = module.datadog-message-alerting.alerting-message
+}
+
+```
+
+## Purpose
+
+Creates DataDog monitors with the following checks:
+
+- Solr does not respond
+- Solr searcher warmup time too high
+- Too many errors on Solr search handler
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:-----:|
+| environment | Architecture environment | `string` | n/a | yes |
+| evaluation\_delay | Delay in seconds for the metric evaluation | `number` | `15` | no |
+| filter\_tags\_custom | Tags used for custom filtering when filter\_tags\_use\_defaults is false | `string` | `"*"` | no |
+| filter\_tags\_custom\_excluded | Tags excluded for custom filtering when filter\_tags\_use\_defaults is false | `string` | `""` | no |
+| filter\_tags\_use\_defaults | Use default filter tags convention | `string` | `"true"` | no |
+| message | Message sent when a monitor is triggered | `any` | n/a | yes |
+| new\_host\_delay | Delay in seconds before begin to monitor new host | `number` | `300` | no |
+| not\_responding\_enabled | Flag to enable Solr does not repsond monitor | `bool` | `true` | no |
+| not\_responding\_extra\_tags | Extra tags for solr does not respond monitor | `list(string)` | `[]` | no |
+| not\_responding\_message | Custom message for Solr does not respond monitor | `string` | `""` | no |
+| not\_responding\_no\_data\_timeframe | Solr not responding monitor no data timeframe | `number` | `10` | no |
+| not\_responding\_threshold\_warning | Solr not responding limit (warning threshold) | `number` | `3` | no |
+| notify\_no\_data | Will raise no data alert if set to true | `bool` | `true` | no |
+| prefix\_slug | Prefix string to prepend between brackets on every monitors names | `string` | `""` | no |
+| search\_handler\_errors\_enabled | Flag to enable Solr search handler errors monitor | `bool` | `true` | no |
+| search\_handler\_errors\_extra\_tags | Extra tags for Search handler errors monitor | `list(string)` | `[]` | no |
+| search\_handler\_errors\_message | Custom message for Solr search handler errors monitor | `string` | `""` | no |
+| search\_handler\_errors\_rate\_threshold\_critical | Handler errors rate critical threshold | `number` | `50` | no |
+| search\_handler\_errors\_rate\_threshold\_warning | Handler errors rate warning threshold | `number` | `10` | no |
+| search\_handler\_errors\_time\_aggregator | Time aggregator for the Handler errors monitor | `string` | `"min"` | no |
+| search\_handler\_errors\_timeframe | Timeframe for the search handler errors monitor | `string` | `"last_5m"` | no |
+| searcher\_warmup\_time\_aggregator | Time aggregator for the searcher warmup time monitor | `string` | `"max"` | no |
+| searcher\_warmup\_time\_enabled | Flag to enable Solr searcher warmup time monitor | `bool` | `true` | no |
+| searcher\_warmup\_time\_extra\_tags | Extra tags for searcher warmum time monitor | `list(string)` | `[]` | no |
+| searcher\_warmup\_time\_message | Custom message for Solr searcher warmup time monitor | `string` | `""` | no |
+| searcher\_warmup\_time\_threshold\_critical | Searcher warmup time critical threshold in ms | `number` | `5000` | no |
+| searcher\_warmup\_time\_threshold\_warning | Searcher warmup time warning threshold in ms | `number` | `2000` | no |
+| searcher\_warmup\_time\_timeframe | Timeframe for the searcher warmup time monitor | `string` | `"last_5m"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| not\_responding\_id | id for monitor not\_responding |
+| search\_handler\_errors\_id | id for monitor search\_handler\_errors |
+| searcher\_warmup\_time\_id | id for monitor searcher\_warmup\_time |
+
+## Related documentation
+
+ - [Integration Datadog & Solr](https://docs.datadoghq.com/integrations/solr/)

database/solr/inputs.tf

@@ -0,0 +1,171 @@
+#
+# Datadog global variables
+#
+variable "environment" {
+  description = "Architecture environment"
+  type        = string
+}
+
+variable "filter_tags_use_defaults" {
+  description = "Use default filter tags convention"
+  default     = "true"
+}
+
+variable "filter_tags_custom" {
+  description = "Tags used for custom filtering when filter_tags_use_defaults is false"
+  default     = "*"
+}
+
+variable "filter_tags_custom_excluded" {
+  description = "Tags excluded for custom filtering when filter_tags_use_defaults is false"
+  default     = ""
+}
+
+variable "message" {
+  description = "Message sent when a monitor is triggered"
+}
+
+variable "evaluation_delay" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 15
+}
+
+variable "new_host_delay" {
+  description = "Delay in seconds before begin to monitor new host"
+  default     = 300
+}
+
+variable "prefix_slug" {
+  description = "Prefix string to prepend between brackets on every monitors names"
+  default     = ""
+}
+
+variable "notify_no_data" {
+  description = "Will raise no data alert if set to true"
+  default     = true
+}
+
+
+#
+# Service Check
+#
+variable "not_responding_enabled" {
+  type        = bool
+  description = "Flag to enable Solr does not repsond monitor"
+  default     = true
+}
+
+variable "not_responding_message" {
+  description = "Custom message for Solr does not respond monitor"
+  type        = string
+  default     = ""
+}
+
+variable "not_responding_threshold_warning" {
+  type        = number
+  default     = 3
+  description = "Solr not responding limit (warning threshold)"
+}
+
+variable "not_responding_no_data_timeframe" {
+  description = "Solr not responding monitor no data timeframe"
+  type        = number
+  default     = 10
+}
+
+variable "not_responding_extra_tags" {
+  description = "Extra tags for solr does not respond monitor"
+  type        = list(string)
+  default     = []
+}
+
+#
+# Handler errors
+#
+variable "search_handler_errors_enabled" {
+  description = "Flag to enable Solr search handler errors monitor"
+  type        = bool
+  default     = true
+}
+
+variable "search_handler_errors_message" {
+  description = "Custom message for Solr search handler errors monitor"
+  type        = string
+  default     = ""
+}
+variable "search_handler_errors_rate_threshold_critical" {
+  description = "Handler errors rate critical threshold"
+  default     = 50
+  type        = number
+}
+
+variable "search_handler_errors_rate_threshold_warning" {
+  description = "Handler errors rate warning threshold"
+  default     = 10
+  type        = number
+}
+
+variable "search_handler_errors_time_aggregator" {
+  description = "Time aggregator for the Handler errors monitor"
+  type        = string
+  default     = "min"
+}
+
+variable "search_handler_errors_timeframe" {
+  description = "Timeframe for the search handler errors monitor"
+  type        = string
+  default     = "last_5m"
+}
+
+variable "search_handler_errors_extra_tags" {
+  description = "Extra tags for Search handler errors monitor"
+  default     = []
+  type        = list(string)
+}
+
+#
+# Searcher warmup time
+#
+variable "searcher_warmup_time_enabled" {
+  default     = true
+  description = "Flag to enable Solr searcher warmup time monitor"
+  type        = bool
+}
+
+variable "searcher_warmup_time_message" {
+  description = "Custom message for Solr searcher warmup time monitor"
+  default     = ""
+  type        = string
+}
+
+variable "searcher_warmup_time_threshold_warning" {
+  description = "Searcher warmup time warning threshold in ms"
+  # 2sec
+  default = 2000
+  type    = number
+}
+
+variable "searcher_warmup_time_threshold_critical" {
+  description = "Searcher warmup time critical threshold in ms"
+  # 5 sec
+  default = 5000
+  type    = number
+}
+
+variable "searcher_warmup_time_extra_tags" {
+  description = "Extra tags for searcher warmum time monitor"
+  default     = []
+  type        = list(string)
+}
+
+variable "searcher_warmup_time_aggregator" {
+  description = "Time aggregator for the searcher warmup time monitor"
+  default     = "max"
+  type        = string
+}
+
+variable "searcher_warmup_time_timeframe" {
+  description = "Timeframe for the searcher warmup time monitor"
+  default     = "last_5m"
+  type        = string
+}

database/solr/modules.tf

@@ -0,0 +1,9 @@
+module "filter-tags" {
+  source = "../../common/filter-tags"
+
+  environment                 = var.environment
+  resource                    = "solr"
+  filter_tags_use_defaults    = var.filter_tags_use_defaults
+  filter_tags_custom          = var.filter_tags_custom
+  filter_tags_custom_excluded = var.filter_tags_custom_excluded
+}

database/solr/monitors-solr.tf

@@ -0,0 +1,103 @@
+#
+# Service Check
+#
+
+resource "datadog_monitor" "not_responding" {
+  count   = var.not_responding_enabled ? 1 : 0
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Solr does not respond {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = coalesce(var.not_responding_message, var.message)
+  type    = "service check"
+
+  query = <<EOQ
+    "solr.can_connect"${module.filter-tags.service_check}.by("instance").last(6).count_by_status()
+
+EOQ
+
+  thresholds = {
+    warning  = var.not_responding_threshold_warning
+    critical = 5
+  }
+
+  no_data_timeframe   = var.not_responding_no_data_timeframe
+  new_host_delay      = var.new_host_delay
+  notify_no_data      = var.notify_no_data
+  notify_audit        = false
+  locked              = false
+  timeout_h           = 0
+  include_tags        = true
+  require_full_window = true
+  renotify_interval   = 0
+
+  tags = concat(["env:${var.environment}", "type:database", "provider:solr", "resource:solr", "team:claranet",
+  "created-by:terraform"], var.not_responding_extra_tags)
+
+  lifecycle {
+    ignore_changes = [silenced]
+  }
+}
+
+#
+# Handler errors
+#
+
+resource "datadog_monitor" "search_handler_errors" {
+  count   = var.search_handler_errors_enabled ? 1 : 0
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Too many errors on Solr search handler {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  message = coalesce(var.search_handler_errors_message, var.message)
+  type    = "metric alert"
+
+  query = <<EOQ
+  ${var.search_handler_errors_time_aggregator}(${var.search_handler_errors_timeframe}):
+    default(sum:solr.search_handler.errors${module.filter-tags.query_alert} by {instance}.as_rate(), 0) / default(sum:solr.search_handler.requests${module.filter-tags.query_alert} by {instance}.as_rate(), 1) * 100
+    > ${var.search_handler_errors_rate_threshold_critical}
+EOQ
+
+  thresholds = {
+    warning  = var.search_handler_errors_rate_threshold_warning
+    critical = var.search_handler_errors_rate_threshold_critical
+  }
+
+  evaluation_delay    = var.evaluation_delay
+  new_host_delay      = var.new_host_delay
+  notify_audit        = false
+  locked              = false
+  include_tags        = true
+  require_full_window = true
+  notify_no_data      = false
+
+  tags = concat(["env:${var.environment}", "type:database", "provider:solr", "resource:solr", "team:claranet",
+  "created-by:terraform"], var.search_handler_errors_extra_tags)
+
+}
+
+#
+# Solr Warmup time
+#
+
+resource "datadog_monitor" "searcher_warmup_time" {
+  count   = var.searcher_warmup_time_enabled ? 1 : 0
+  message = coalesce(var.searcher_warmup_time_message, var.message)
+  name    = "${var.prefix_slug == "" ? "" : "[${var.prefix_slug}]"}[${var.environment}] Solr searcher warmup time too high  {{#is_alert}}{{{comparator}}} {{threshold}}% ({{value}}%){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}% ({{value}}%){{/is_warning}}"
+  type    = "metric alert"
+  query   = <<EOQ
+  ${var.searcher_warmup_time_aggregator}(${var.searcher_warmup_time_timeframe}):
+    min:solr.searcher.warmup${module.filter-tags.query_alert} by {instance}
+    >= ${var.searcher_warmup_time_threshold_critical}
+EOQ
+
+  thresholds = {
+    warning  = var.searcher_warmup_time_threshold_warning
+    critical = var.searcher_warmup_time_threshold_critical
+  }
+
+  evaluation_delay    = var.evaluation_delay
+  new_host_delay      = var.new_host_delay
+  notify_audit        = false
+  locked              = false
+  include_tags        = true
+  require_full_window = true
+  notify_no_data      = false
+
+  tags = concat(["env:${var.environment}", "type:database", "provider:solr", "resource:solr", "team:claranet",
+  "created-by:terraform"], var.searcher_warmup_time_extra_tags)
+}

database/solr/outputs.tf

@@ -0,0 +1,15 @@
+output "not_responding_id" {
+  description = "id for monitor not_responding"
+  value       = datadog_monitor.not_responding.*.id
+}
+
+output "search_handler_errors_id" {
+  description = "id for monitor search_handler_errors"
+  value       = datadog_monitor.search_handler_errors.*.id
+}
+
+output "searcher_warmup_time_id" {
+  description = "id for monitor searcher_warmup_time"
+  value       = datadog_monitor.searcher_warmup_time.*.id
+}
+