MON-289 make nginx check more tolerant

middleware/nginx/inputs.tf

@@ -55,10 +55,16 @@ variable "nginx_connect_message" {
   default     = ""
 }
 
-variable "nginx_connect_threshold_critical" {
+variable "nginx_connect_threshold_warning" {
-  description = "Nginx status monitor (critical threshold)"
+  description = "Nginx status monitor (warning threshold)"
   type        = "string"
-  default     = "1.1754943508222875e-38"
+  default     = 3
+}
+
+variable "nginx_connect_no_data_timeframe" {
+  description = "Nginx status monitor no data timeframe"
+  type        = "string"
+  default     = 10
 }
 
 variable "nginx_dropped_silenced" {

middleware/nginx/monitors-nginx.tf

@@ -6,15 +6,16 @@ resource "datadog_monitor" "datadog_nginx_process" {
   type = "service check"
 
   query = <<EOF
-    "nginx.can_connect".over${module.filter-tags.service_check}.by("host","port","server").last(1).pct_by_status()
+    "nginx.can_connect".over${module.filter-tags.service_check}.by("server","port").last(6).count_by_status()
   EOF
 
   thresholds = {
-    warning  = 0
+    warning  = "${var.nginx_connect_threshold_warning}"
-    critical = "${var.nginx_connect_threshold_critical}"
+    critical = 5
   }
 
   notify_no_data      = true
+  no_data_timeframe   = "${var.nginx_connect_no_data_timeframe}"
   new_host_delay      = "${var.new_host_delay}"
   renotify_interval   = 0
   notify_audit        = false