commit ffb6c18b692657c2d2246ea05a38e5c341c43c05 Author: Patrick de Ruiter Date: Tue Aug 24 11:09:06 2021 +0200 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4eb1cb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# Compiled files +*.tfstate +*.tfstate.backup +.terraform.tfstate.lock.info +terraform.tfvars +# Module directory +.terraform/ +jq diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..46bcbc7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,38 @@ +# changelog +v0.1.17 +- bugfix: `dashboard_b` should also use local.scope +v0.1.16 +- Add support for template variables. Define `var.template_variable_{name,prefix,default}` + to filter across all graphs in the dashboard. +v0.1.15 +- Allow user to specify a list of metrics to graph rather than querying them from datadog. + If you specify `metrics_list` directly, then it will use this list of metrics to generate + the dashboard instead of using the specified prefix to query it from datadog. +v0.1.14 +- sort widgets + produce output that is ordered in a consistent manner to avoid idempotency issues +v0.1.13 +- fix validation warnings + Warning: Interpolation-only expressions are deprecated + Warning: Quoted type constraints are deprecated +v0.1.12 +- Introduced optional var.title for dashboard so it wouldn't have to be tied to var.prefix +- Parametrized `space_aggregation` and `scope` which are part of the query that defines + a particular timeseries graph. Previously `space_aggregation` was hardcoded to `avg`, + now you can choose from `avg, max, min, sum` + +v0.1.11 +- Fixed example + +v0.1.10 +- Moved logic to download jq, if needed, into `list_metrics.py` where it belongs + +v0.1.8 +- Added resource to download jq since terraform cloud does not have it +- Modified `list_metrics.py` to use locally "installed" jq + +v0.1.7 +- Minor fix + +v0.1.6 +- Refactored `list_metrics.py` to remove dependency on python3 and datadog python module diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2df2f2a --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# terraform module for autogenerating dashboards based on prefixed metrics + +The Terraform Datadog provider does not offer a way to autodiscover metrics, however, +Datadog API can do this via [GET /v1/metrics](https://docs.datadoghq.com/api/?lang=python#get-list-of-active-metrics) + +This terraform module uses an external data source [list_metrics.sh](https://github.com/borgified/terraform-datadog-dashboard/blob/master/list_metrics.sh) to fetch a list of +all available metrics filtered by a prefix so that this subset of metrics can be used as input to the [datadog_dashboard](https://www.terraform.io/docs/providers/datadog/r/dashboard.html) +resource. + +For example, if you have these metrics in Datadog: +``` + +app.dev.memstats.alloc +app.dev.memstats.frees +app.dev.memstats.heap_released +... +``` +then specify prefix = "app.dev.memstats" as your input to get a dashboard of all metrics +starting with that prefix. diff --git a/examples/main.tf b/examples/main.tf new file mode 100644 index 0000000..3849377 --- /dev/null +++ b/examples/main.tf @@ -0,0 +1,20 @@ +module "dashboard_a" { + source = "borgified/dashboard/datadog" + version = "0.1.17" + prefix = "aws.applicationelb" + api_key = "your datadog api key" + app_key = "your datadog app key" + space_aggregation = "sum" + scope = "*" +} + +module "dashboard_b" { + source = "borgified/dashboard/datadog" + version = "0.1.17" + prefix = "aws.applicationelb" + api_key = "your datadog api key" + app_key = "your datadog app key" + metrics_list = "active_connection_count,client_tlsnegotiation_error_count" + space_aggregation = "sum" + scope = "*" +} diff --git a/examples/outputs.tf b/examples/outputs.tf new file mode 100644 index 0000000..a6d488f --- /dev/null +++ b/examples/outputs.tf @@ -0,0 +1,4 @@ +output "url" { + description = "dashboard url" + value = module.dashboard.url +} diff --git a/external.tf b/external.tf new file mode 100644 index 0000000..0f6fe55 --- /dev/null +++ b/external.tf @@ -0,0 +1,8 @@ +data "external" "list_metrics" { + program = ["bash", "${path.module}/list_metrics.sh"] + query = { + api_key = var.api_key + app_key = var.app_key + prefix = var.prefix + } +} diff --git a/list_metrics.sh b/list_metrics.sh new file mode 100755 index 0000000..eee638f --- /dev/null +++ b/list_metrics.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# based on https://www.terraform.io/docs/providers/external/data_source.html#processing-json-in-shell-scripts + +set -e + +if ! [ -x "$(command -v jq)" ]; then + curl -sL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq && chmod u+x jq + JQ="./jq" +else + JQ=$(command -v jq) +fi + +eval "$($JQ -r '@sh "API_KEY=\(.api_key) APP_KEY=\(.app_key) PREFIX=\(.prefix)"')" + +RESULT=$(curl -s -X GET \ +-H "DD-API-KEY: ${API_KEY}" \ +-H "DD-APPLICATION-KEY: ${APP_KEY}" \ +-d "api_key=${API_KEY}" \ +-d "application_key=${APP_KEY}" \ +"https://api.datadoghq.com/api/v1/search?q=metrics:${PREFIX}" \ +| $JQ -r --arg prefix "$PREFIX." '.results.metrics|sort|map(sub($prefix;""))|join(",")') + +$JQ -n --arg result "$RESULT" '{"metrics":$result}' diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..f8775e7 --- /dev/null +++ b/main.tf @@ -0,0 +1,63 @@ +locals { + scope = format("%s,$%s", var.scope, var.template_variable_name) +} + +resource "datadog_dashboard" "dashboard_a" { + + count = var.metrics_list == "" ? 1 : 0 + + title = var.title + description = var.description + layout_type = "ordered" + + dynamic "widget" { + for_each = sort(split(",", data.external.list_metrics.result.metrics)) + content { + timeseries_definition { + title = widget.value + request { + q = "${var.space_aggregation}:${var.prefix}.${widget.value}{${local.scope}}" + metadata { + expression = "${var.space_aggregation}:${var.prefix}.${widget.value}{${local.scope}}" + alias_name = widget.value + } + } + } + } + } + template_variable { + name = var.template_variable_name + prefix = var.template_variable_prefix + default = var.template_variable_default + } +} + +resource "datadog_dashboard" "dashboard_b" { + + count = var.metrics_list != "" ? 1 : 0 + + title = var.title + description = var.description + layout_type = "ordered" + + dynamic "widget" { + for_each = sort(split(",", var.metrics_list)) + content { + timeseries_definition { + title = widget.value + request { + q = "${var.space_aggregation}:${var.prefix}.${widget.value}{${local.scope}}" + metadata { + expression = "${var.space_aggregation}:${var.prefix}.${widget.value}{${local.scope}}" + alias_name = widget.value + } + } + } + } + } + template_variable { + name = var.template_variable_name + prefix = var.template_variable_prefix + default = var.template_variable_default + } +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..afc4f35 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,9 @@ +output "url" { + description = "visit this url to go to your new dashboard" + value = var.metrics_list == "" ? "https://app.datadoghq.com/dashboard/${datadog_dashboard.dashboard_a[0].id}" : "https://app.datadoghq.com/dashboard/${datadog_dashboard.dashboard_b[0].id}" +} + +output "external_metrics" { + description = "queryable metrics from datadog discovered" + value = data.external.list_metrics.result.metrics +} diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..34f1f45 --- /dev/null +++ b/providers.tf @@ -0,0 +1,9 @@ +provider "datadog" { + version = "~> 2.5" + api_key = var.api_key + app_key = var.app_key +} + +provider "external" { + version = "~> 1.2" +} diff --git a/terraform-cloud-ci/main.tf b/terraform-cloud-ci/main.tf new file mode 100644 index 0000000..de8cd5b --- /dev/null +++ b/terraform-cloud-ci/main.tf @@ -0,0 +1,8 @@ +module "dashboard" { + source = "../" + prefix = var.prefix + api_key = var.api_key + app_key = var.app_key + space_aggregation = var.space_aggregation + scope = var.scope +} diff --git a/terraform-cloud-ci/outputs.tf b/terraform-cloud-ci/outputs.tf new file mode 100644 index 0000000..a6d488f --- /dev/null +++ b/terraform-cloud-ci/outputs.tf @@ -0,0 +1,4 @@ +output "url" { + description = "dashboard url" + value = module.dashboard.url +} diff --git a/terraform-cloud-ci/variables.tf b/terraform-cloud-ci/variables.tf new file mode 100644 index 0000000..b7820c8 --- /dev/null +++ b/terraform-cloud-ci/variables.tf @@ -0,0 +1,38 @@ +variable "title" { + description = "title of datadog dashboard" + type = "string" + default = "my dashboard" +} + +variable "api_key" { + description = "datadog api key" + type = "string" +} + +variable "app_key" { + description = "datadog app key" + type = "string" +} + +variable "prefix" { + description = "metric prefix to pattern match against available metrics in datadog also used for dashboard title" + type = "string" +} + +# https://docs.datadoghq.com/graphing/metrics/introduction/#breaking-down-the-metric-query + +variable "space_aggregation" { + description = "choices: avg, max, min, sum" + type = "string" +} + +variable "scope" { + description = "the set of tags used to choose time series for the query. eg. \"environment:production,service:myservice\"" + type = "string" +} + +variable "description" { + description = "datadog dashboard description" + type = "string" + default = "generated by Terraform" +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..0999145 --- /dev/null +++ b/variables.tf @@ -0,0 +1,62 @@ +variable "title" { + description = "title of datadog dashboard" + type = string + default = "my dashboard" +} + +variable "api_key" { + description = "datadog api key" + type = string +} + +variable "app_key" { + description = "datadog app key" + type = string +} + +variable "prefix" { + description = "metric prefix to pattern match against available metrics in datadog also used for dashboard title" + type = string +} + +# https://docs.datadoghq.com/graphing/metrics/introduction/#breaking-down-the-metric-query + +variable "space_aggregation" { + description = "choices: avg, max, min, sum" + type = string +} + +variable "scope" { + description = "the set of tags used to choose time series for the query. eg. \"environment:production,service:myservice\"" + type = string +} + +variable "description" { + description = "datadog dashboard description" + type = string + default = "generated by Terraform" +} + +variable "metrics_list" { + description = "list of metrics to provide if you dont want to use var.prefix to query them from datadog" + type = string + default = "" +} + +variable "template_variable_name" { + description = "name of template variable" + type = string + default = "variable1" +} + +variable "template_variable_prefix" { + description = "prefix of template variable" + type = string + default = "" +} + +variable "template_variable_default" { + description = "default value for template variable" + type = string + default = "*" +}