From 1404088f3c1486abf16d849f4e1fbc0ac84bee42 Mon Sep 17 00:00:00 2001
From: Martin Polreich <polreichmartin@gmail.com>
Date: Tue, 25 Apr 2017 14:37:44 +0200
Subject: [PATCH] Added ability to disable endpoints, alerts and recording
 rules and override alerts/recording rules from reclass

Change-Id: Ie79409f350f4e7d223955c177718350c38ed8302
---
 prometheus/collector.sls        |  2 +-
 prometheus/files/alerts.yml     | 61 ++++++++++++++-------------------
 prometheus/files/prometheus.yml | 12 ++++---
 3 files changed, 34 insertions(+), 41 deletions(-)

diff --git a/prometheus/collector.sls b/prometheus/collector.sls
index e5fb0d4..1da6ab9 100644
--- a/prometheus/collector.sls
+++ b/prometheus/collector.sls
@@ -1,4 +1,4 @@
-{%- set service_grains = {'prometheus': {'server': {'alert': {}, 'recording': [], 'target': {}}}} %}
+{%- set service_grains = {'prometheus': {'server': {'alert': {}, 'recording': {}, 'target': {}}}} %}
 {%- for service_name, service in pillar.items() %}
   {%- if service.get('_support', {}).get('prometheus', {}).get('enabled', False) %}
     {%- set grains_fragment_file = service_name+'/meta/prometheus.yml' %}
diff --git a/prometheus/files/alerts.yml b/prometheus/files/alerts.yml
index d9ebc7e..97d0dde 100644
--- a/prometheus/files/alerts.yml
+++ b/prometheus/files/alerts.yml
@@ -3,57 +3,48 @@
 {%- set alerts = {} %}
 {%- set recordings = {} %}
 
-
-{%- for recording_rule in server.get('recording', []) %}
-  {%- if recording_rule.name not in recordings %}
-    {%- do recordings.update({recording_rule.name: recording_rule.query}) %}
-  {%- endif %}
-{%- endfor %}
-
 {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
   {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
-  {%- for recording_rule in server_grain.get('recording', []) %}
-    {%- if recording_rule.name not in recordings %}
-      {%- do recordings.update({recording_rule.name: recording_rule.query}) %}
-    {%- endif %}
+  {%- for recordingname, recording in server_grain.get('recording', {}).iteritems() %}
+    {%- do recordings.update({recordingname: recording}) %}
   {%- endfor %}
 {%- endfor %}
 
-{%- for recording_name, query in recordings.iteritems() %}
-{{ recording_name }} = {{ query }}
-{%- endfor %}
+{%- set recordings = salt['grains.filter_by']({'default': recordings}, merge=server.get('recording', {})) %}
 
-{%- for alertname, alert in server.get('alert', {}).iteritems() %}
-  {%- if alertname not in alerts %}
-    {%- do alerts.update({alertname: alert}) %}
+{%- for recordingname, recording in recordings.iteritems() %}
+  {%- if recording.get('enabled', True) %}
+    {{ recordingname }} = {{ recording.query }}
   {%- endif %}
 {%- endfor %}
 
 {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
   {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
   {%- for alertname, alert in server_grain.get('alert', {}).iteritems() %}
-    {%- if alertname not in alerts %}
       {%- do alerts.update({alertname: alert}) %}
-    {%- endif %}
   {%- endfor %}
 {%- endfor %}
 
+{%- set alerts = salt['grains.filter_by']({'default': alerts}, merge=server.get('alert', {})) %}
+
 {%- for alertname, alert in alerts.iteritems() %}
-ALERT {{ alertname }}
-  IF {{ alert.if }}
-  {%- if alert.for is defined %}FOR {{ alert.for }}{%- endif %}
-  {%- if alert.labels is defined %}
-  LABELS {
-    {%- for name, value in alert.labels.iteritems() %}
-    {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %}
-    {%- endfor %}
-  }
-  {%- endif %}
-  {%- if alert.annotations is defined %}
-  ANNOTATIONS {
-    {%- for name, value in alert.annotations.iteritems() %}
-    {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %}
-    {%- endfor %}
-  }
+  {%- if alert.get('enabled', True) %}
+    ALERT {{ alertname }}
+      IF {{ alert.if }}
+      {%- if alert.for is defined %}FOR {{ alert.for }}{%- endif %}
+      {%- if alert.labels is defined %}
+      LABELS {
+        {%- for name, value in alert.labels.iteritems() %}
+        {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %}
+        {%- endfor %}
+      }
+      {%- endif %}
+      {%- if alert.annotations is defined %}
+      ANNOTATIONS {
+        {%- for name, value in alert.annotations.iteritems() %}
+        {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %}
+        {%- endfor %}
+      }
+      {%- endif %}
   {%- endif %}
 {%- endfor %}
diff --git a/prometheus/files/prometheus.yml b/prometheus/files/prometheus.yml
index 9b0d394..594bdba 100644
--- a/prometheus/files/prometheus.yml
+++ b/prometheus/files/prometheus.yml
@@ -25,11 +25,11 @@ scrape_configs:
 {%- for job_name, targets in static_target.iteritems() %}
   {%- set job = server.get('target', {}).get('static', {}).get(job_name, {}) %}
   {%- set nodes = [] %}
-  {%- for target in targets %}
+  {%- for target in targets + job.get('endpoint', []) %}
     {%- set address = "'%s:%d'" | format(target.address, target.port) %}
     {%- do nodes.append(address) %}
   {%- endfor %}
-  {%- if nodes|length > 0 %}
+  {%- if nodes|length > 0 and job.get('enabled', True) %}
   - job_name: {{ job_name }}
     {% if job.get('scheme') %}scheme: {{ job.scheme }}{%- endif %}
     {%- if job.get('tls_config') %}
@@ -43,7 +43,7 @@ scrape_configs:
   {%- endif %}
 {%- endfor %}
 
-{% if server.get('target', {}).get('kubernetes') %}
+{%- if server.get('target', {}).get('kubernetes', {}).get('enabled', True) %}
   {%- set kubernetes_target = server.target.kubernetes %}
   - job_name: 'kubernetes-api'
     scheme: https
@@ -164,7 +164,8 @@ scrape_configs:
       target_label: kubernetes_pod_name
 {%- endif %}
 
-{%- for target in server.get('target', {}).get('dns', []) %}
+{%- if server.get('target', {}).get('dns', {}).get('enabled', True) %}
+  {%- for target in server.get('target', {}).get('dns', {}).get('endpoint', []) %}
   - job_name: {{ target.name }}
     dns_sd_configs:
     - names:
@@ -173,4 +174,5 @@ scrape_configs:
   {%- endfor %}
       type: {{ target.type }}
       port: {{ target.port }}
-{%- endfor %}
+  {%- endfor %}
+{%- endif %}