Merged in MON-48-monitors-for-api-gateway (pull request #41)

MON-48 monitors for api gateway

Approved-by: Boris Rousseau <boris.rousseau@morea.fr>
Approved-by: Alex Lemaresquier <alex+bitbucket@lemaresquier.org>
Approved-by: Kevin Pecquet <kevin.pecquet@fr.clara.net>
Approved-by: Adrien Broyere <adrien.broyere@fr.clara.net>
Approved-by: Alexandre Gaillet <alexandre.gaillet@fr.clara.net>
Approved-by: Quentin Manfroi <quentin.manfroi@yahoo.fr>

cloud/aws/apigateway/README.md

@@ -0,0 +1,47 @@
+AWS API Gateway DataDog monitors
+==========================================
+
+How to use this module
+----------------------
+
+```
+module "datadog-monitors-aws-api-gateway" {
+  source = "git::ssh://git@bitbucket.org/morea/terraform.feature.datadog.git//cloud/aws/apigateway?ref={revision}"
+
+  environment = "${var.environment}"
+  message = "${module.datadog-message-alerting.alerting-message}"
+}
+
+```
+
+Purpose
+-------
+Creates DataDog monitors with the following checks :
+
+* API Gateway too much 5xx errors
+* API Gateway too much 4xx errors
+* API Gateway latency to high
+
+Inputs
+------
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| delay | Delay in seconds for the metric evaluation | string | `900` | no |
+| environment | Environment | string | - | yes |
+| filter_tags | Tags used for custom filtering | string | `*` | no |
+| http_4xx_requests_threshold_critical | Maximum critical acceptable percent of 4xx errors | string | `30` | no |
+| http_4xx_requests_threshold_warning | Maximum warning acceptable percent of 4xx errors | string | `15` | no |
+| http_5xx_requests_threshold_critical | Maximum critical acceptable percent of 5xx errors | string | `20` | no |
+| http_5xx_requests_threshold_warning | Maximum warning acceptable percent of 5xx errors | string | `10` | no |
+| message | Message sent when a monitor is triggered | string | - | yes |
+| latency_threshold_critical | Alerting threshold in miliseconds | string | `800` | no |
+| latency_threshold_warning | Warning threshold in miliseconds | string | `400` | no |
+| artificial_requests_count | Number of false requests used to mitigate false positive in case of low trafic | string | `0` | no |
+
+Related documentation
+---------------------
+
+DataDog documentation: [https://docs.datadoghq.com/integrations/amazon_api_gateway/](https://docs.datadoghq.com/integrations/amazon_api_gateway/)
+
+AWS API Gateway metrics documentation: [https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/api-gateway-metrics-dimensions.html](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/api-gateway-metrics-dimensions.html)

cloud/aws/apigateway/inputs.tf

@@ -0,0 +1,65 @@
+variable "environment" {
+  description = "Environment"
+  type        = "string"
+}
+
+variable "filter_tags" {
+  description = "Tags used for filtering"
+  default     = "*"
+}
+
+variable "message" {
+  description = "Message sent when a monitor is triggered"
+}
+
+variable "delay" {
+  description = "Delay in seconds for the metric evaluation"
+  default     = 900
+}
+
+###################################
+###   LATENCY VARIABLES   ###
+###################################
+
+variable "latency_threshold_critical" {
+  default     = 800
+  description = "Alerting threshold in milliseconds"
+}
+
+variable "latency_threshold_warning" {
+  default     = 400
+  description = "Warning threshold in milliseconds"
+}
+
+#################################
+###   HTTP 5xx status pages   ###
+#################################
+
+variable "http_5xx_requests_threshold_critical" {
+  default     = 20
+  description = "Maximum critical acceptable percent of 5xx errors"
+}
+
+variable "http_5xx_requests_threshold_warning" {
+  default     = 10
+  description = "Maximum warning acceptable percent of 5xx errors"
+}
+
+#################################
+###   HTTP 4xx status pages   ###
+#################################
+
+variable "http_4xx_requests_threshold_critical" {
+  default     = 30
+  description = "Maximum critical acceptable percent of 4xx errors"
+}
+
+variable "http_4xx_requests_threshold_warning" {
+  default     = 15
+  description = "Maximum warning acceptable percent of 4xx errors"
+}
+
+variable "artificial_requests_count" {
+  default     = 5
+  description = "Number of false requests used to mitigate false positive in case of low trafic"
+}

cloud/aws/apigateway/monitors-api.tf

@@ -0,0 +1,88 @@
+# Monitoring Api Gateway latency
+resource "datadog_monitor" "API_Gateway_latency" {
+  name    = "[${var.environment}] API Gateway latency {{comparator}} {{#is_alert}}{{threshold}}ms{{/is_alert}}{{#is_warning}}{{warn_threshold}}ms{{/is_warning}} ({{value}}ms)"
+  type    = "metric alert"
+  message = "${var.message}"
+
+  query = <<EOF
+    avg(last_5m): (
+      avg:aws.apigateway.latency{${var.filter_tags}} by {region,apiname}
+    ) > ${var.latency_threshold_critical}
+  EOF
+
+  evaluation_delay = "${var.delay}"
+  new_host_delay   = "${var.delay}"
+
+  thresholds {
+    warning  = "${var.latency_threshold_warning}"
+    critical = "${var.latency_threshold_critical}"
+  }
+
+  notify_no_data      = true  # Will notify when no data is received
+  renotify_interval   = 0
+  require_full_window = false
+  timeout_h           = 0
+  include_tags        = true
+
+  tags = ["env:${var.environment}", "resource:apigateway", "team:aws", "provider:aws"]
+}
+
+# Monitoring API Gateway 5xx errors percent
+resource "datadog_monitor" "API_http_5xx_errors_count" {
+  name    = "[${var.environment}] API Gateway HTTP 5xx errors {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
+  type    = "metric alert"
+  message = "${var.message}"
+
+  query = <<EOF
+    sum(last_5m): (
+      avg:aws.apigateway.5xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
+      (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count})
+    ) * 100 > ${var.http_5xx_requests_threshold_critical}
+  EOF
+
+  evaluation_delay = "${var.delay}"
+  new_host_delay   = "${var.delay}"
+
+  thresholds {
+    warning  = "${var.http_5xx_requests_threshold_warning}"
+    critical = "${var.http_5xx_requests_threshold_critical}"
+  }
+
+  notify_no_data      = false # Will NOT notify when no data is received
+  renotify_interval   = 0
+  require_full_window = false
+  timeout_h           = 1
+  include_tags        = true
+
+  tags = ["env:${var.environment}", "resource:apigateway", "team:aws", "provider:aws"]
+}
+
+# Monitoring API Gateway 4xx errors percent
+resource "datadog_monitor" "API_http_4xx_errors_count" {
+  name    = "[${var.environment}] API Gateway HTTP 4xx errors {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
+  type    = "metric alert"
+  message = "${var.message}"
+
+  query = <<EOF
+    sum(last_5m): (
+      avg:aws.apigateway.4xxerror{${var.filter_tags}} by {region,apiname}.as_count() /
+      (avg:aws.apigateway.count{${var.filter_tags}} by {region,apiname}.as_count() + ${var.artificial_requests_count})
+    ) * 100 > ${var.http_4xx_requests_threshold_critical}
+  EOF
+
+  evaluation_delay = "${var.delay}"
+  new_host_delay   = "${var.delay}"
+
+  thresholds {
+    warning  = "${var.http_4xx_requests_threshold_warning}"
+    critical = "${var.http_4xx_requests_threshold_critical}"
+  }
+
+  notify_no_data      = false # Will NOT notify when no data is received
+  renotify_interval   = 0
+  require_full_window = false
+  timeout_h           = 1
+  include_tags        = true
+
+  tags = ["env:${var.environment}", "resource:apigateway", "team:aws", "provider:aws"]
+}

cloud/aws/elb/monitors-elb.tf

@@ -161,7 +161,7 @@ resource "datadog_monitor" "ELB_too_much_5xx_backend" {
 }
 
 resource "datadog_monitor" "ELB_backend_latency" {
-  name    = "[${var.environment}] ELB latency too high {{comparator}} {{#is_alert}}{{threshold}}%{{/is_alert}}{{#is_warning}}{{warn_threshold}}%{{/is_warning}} ({{value}}%)"
+  name    = "[${var.environment}] ELB latency too high {{comparator}} {{#is_alert}}{{threshold}}s{{/is_alert}}{{#is_warning}}{{warn_threshold}}s{{/is_warning}} ({{value}}s)"
   message = "${var.message}"
 
   query = <<EOF