From bc7211ba5f5eed33576c7496d0a7c93cbe2cfc33 Mon Sep 17 00:00:00 2001 From: Quentin Manfroi Date: Thu, 30 Aug 2018 15:34:15 +0200 Subject: [PATCH] MON-227 delete total latencies and split backend latency into bucket and service --- cloud/gcp/lb/inputs.tf | 110 ++++++++++++++++++------------------ cloud/gcp/lb/monitors-lb.tf | 94 +++++++++++++++--------------- 2 files changed, 102 insertions(+), 102 deletions(-) diff --git a/cloud/gcp/lb/inputs.tf b/cloud/gcp/lb/inputs.tf index 3c3d758..8644ce9 100644 --- a/cloud/gcp/lb/inputs.tf +++ b/cloud/gcp/lb/inputs.tf @@ -116,90 +116,90 @@ variable "error_rate_5xx_extra_tags" { } # -# Latency +# Latency Backend service # -variable "latency_message" { - description = "Custom message for the GCP LB Latency monitor" - type = "string" - default = "" -} - -variable "latency_time_aggregator" { - description = "Timeframe for the GCP LB Latency monitor" - type = "string" - default = "min" -} - -variable "latency_timeframe" { - description = "Timeframe for the GCP LB Latency monitor" - type = "string" - default = "last_10m" -} - -variable "latency_threshold_warning" { - description = "Latency in seconds (warning threshold)" - type = "string" - default = 3000 -} - -variable "latency_threshold_critical" { - description = "Latency in seconds (critical threshold)" - type = "string" - default = 5000 -} - -variable "latency_silenced" { - description = "Groups to mute for GCP LB Latency monitor" - type = "map" - default = {} -} - -variable "latency_extra_tags" { - description = "Extra tags for GCP LB Latency monitor" - type = "list" - default = [] -} - -# -# Latency Backend -# -variable "backend_latency_message" { +variable "backend_latency_service_message" { description = "Custom message for the GCP LB Backend Latency monitor" type = "string" default = "" } -variable "backend_latency_time_aggregator" { +variable "backend_latency_service_time_aggregator" { description = "Timeframe for the GCP LB Backend Latency monitor" type = "string" default = "min" } -variable "backend_latency_timeframe" { +variable "backend_latency_service_timeframe" { description = "Timeframe for the GCP LB Backend Latency monitor" type = "string" default = "last_10m" } -variable "backend_latency_threshold_warning" { +variable "backend_latency_service_threshold_warning" { description = "Latency in seconds (warning threshold)" type = "string" - default = 2000 + default = 1000 } -variable "backend_latency_threshold_critical" { +variable "backend_latency_service_threshold_critical" { description = "Latency in seconds (critical threshold)" type = "string" - default = 4000 + default = 1500 } -variable "backend_latency_silenced" { +variable "backend_latency_service_silenced" { description = "Groups to mute for GCP LB Backend Latency monitor" type = "map" default = {} } -variable "backend_latency_extra_tags" { +variable "backend_latency_service_extra_tags" { + description = "Extra tags for GCP LB Backend Latency monitor" + type = "list" + default = [] +} + +# +# Latency Backend bucket +# +variable "backend_latency_bucket_message" { + description = "Custom message for the GCP LB Backend Latency monitor" + type = "string" + default = "" +} + +variable "backend_latency_bucket_time_aggregator" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "min" +} + +variable "backend_latency_bucket_timeframe" { + description = "Timeframe for the GCP LB Backend Latency monitor" + type = "string" + default = "last_10m" +} + +variable "backend_latency_bucket_threshold_warning" { + description = "Latency in seconds (warning threshold)" + type = "string" + default = 4000 +} + +variable "backend_latency_bucket_threshold_critical" { + description = "Latency in seconds (critical threshold)" + type = "string" + default = 8000 +} + +variable "backend_latency_bucket_silenced" { + description = "Groups to mute for GCP LB Backend Latency monitor" + type = "map" + default = {} +} + +variable "backend_latency_bucket_extra_tags" { description = "Extra tags for GCP LB Backend Latency monitor" type = "list" default = [] diff --git a/cloud/gcp/lb/monitors-lb.tf b/cloud/gcp/lb/monitors-lb.tf index 3d34cb4..141e229 100644 --- a/cloud/gcp/lb/monitors-lb.tf +++ b/cloud/gcp/lb/monitors-lb.tf @@ -73,59 +73,23 @@ EOF } # -# Latency +# Backend Latency for service # -resource "datadog_monitor" "latency" { - name = "[${var.environment}] GCP LB latency {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" - message = "${coalesce(var.latency_message, var.message)}" - - type = "query alert" - - query = < ${var.latency_threshold_critical} -EOF - - thresholds { - warning = "${var.latency_threshold_warning}" - critical = "${var.latency_threshold_critical}" - } - - notify_audit = false - locked = false - timeout_h = 0 - include_tags = true - require_full_window = false - notify_no_data = false - renotify_interval = 0 - - evaluation_delay = "${var.evaluation_delay}" - new_host_delay = "${var.new_host_delay}" - - silenced = "${var.latency_silenced}" - - tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.latency_extra_tags}"] -} - -# -# Backend Latency -# -resource "datadog_monitor" "backend_latency" { - name = "[${var.environment}] GCP LB backend latency {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" - message = "${coalesce(var.backend_latency_message, var.message)}" +resource "datadog_monitor" "backend_latency_service" { + name = "[${var.environment}] GCP LB service backend latency {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" + message = "${coalesce(var.backend_latency_service_message, var.message)}" type = "metric alert" query = < ${var.backend_latency_threshold_critical} + ${var.backend_latency_service_time_aggregator}(${var.backend_latency_service_timeframe}): + min:gcp.loadbalancing.https.backend_latencies.avg{${var.filter_tags},backend_target_type:backend_service} by {backend_target_name,forwarding_rule_name} + > ${var.backend_latency_service_threshold_critical} EOF thresholds { - warning = "${var.backend_latency_threshold_warning}" - critical = "${var.backend_latency_threshold_critical}" + warning = "${var.backend_latency_service_threshold_warning}" + critical = "${var.backend_latency_service_threshold_critical}" } notify_audit = false @@ -139,9 +103,45 @@ EOF evaluation_delay = "${var.evaluation_delay}" new_host_delay = "${var.new_host_delay}" - silenced = "${var.backend_latency_silenced}" + silenced = "${var.backend_latency_service_silenced}" - tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.backend_latency_extra_tags}"] + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.backend_latency_service_extra_tags}"] +} + +# +# Backend Latency for bucket +# +resource "datadog_monitor" "backend_latency_bucket" { + name = "[${var.environment}] GCP LB bucket backend latency {{#is_alert}}{{{comparator}}} {{threshold}}s ({{value}}s){{/is_alert}}{{#is_warning}}{{{comparator}}} {{warn_threshold}}s ({{value}}s){{/is_warning}}" + message = "${coalesce(var.backend_latency_bucket_message, var.message)}" + + type = "metric alert" + + query = < ${var.backend_latency_bucket_threshold_critical} +EOF + + thresholds { + warning = "${var.backend_latency_bucket_threshold_warning}" + critical = "${var.backend_latency_bucket_threshold_critical}" + } + + notify_audit = false + locked = false + timeout_h = 0 + include_tags = true + require_full_window = false + notify_no_data = false + renotify_interval = 0 + + evaluation_delay = "${var.evaluation_delay}" + new_host_delay = "${var.new_host_delay}" + + silenced = "${var.backend_latency_bucket_silenced}" + + tags = ["env:${var.environment}", "type:cloud", "provider:gcp", "resource:lb", "team:claranet", "created-by:terraform", "${var.backend_latency_bucket_extra_tags}"] } #