From 1404088f3c1486abf16d849f4e1fbc0ac84bee42 Mon Sep 17 00:00:00 2001 From: Martin Polreich Date: Tue, 25 Apr 2017 14:37:44 +0200 Subject: [PATCH] Added ability to disable endpoints, alerts and recording rules and override alerts/recording rules from reclass Change-Id: Ie79409f350f4e7d223955c177718350c38ed8302 --- prometheus/collector.sls | 2 +- prometheus/files/alerts.yml | 61 ++++++++++++++------------------- prometheus/files/prometheus.yml | 12 ++++--- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/prometheus/collector.sls b/prometheus/collector.sls index e5fb0d4..1da6ab9 100644 --- a/prometheus/collector.sls +++ b/prometheus/collector.sls @@ -1,4 +1,4 @@ -{%- set service_grains = {'prometheus': {'server': {'alert': {}, 'recording': [], 'target': {}}}} %} +{%- set service_grains = {'prometheus': {'server': {'alert': {}, 'recording': {}, 'target': {}}}} %} {%- for service_name, service in pillar.items() %} {%- if service.get('_support', {}).get('prometheus', {}).get('enabled', False) %} {%- set grains_fragment_file = service_name+'/meta/prometheus.yml' %} diff --git a/prometheus/files/alerts.yml b/prometheus/files/alerts.yml index d9ebc7e..97d0dde 100644 --- a/prometheus/files/alerts.yml +++ b/prometheus/files/alerts.yml @@ -3,57 +3,48 @@ {%- set alerts = {} %} {%- set recordings = {} %} - -{%- for recording_rule in server.get('recording', []) %} - {%- if recording_rule.name not in recordings %} - {%- do recordings.update({recording_rule.name: recording_rule.query}) %} - {%- endif %} -{%- endfor %} - {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %} {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %} - {%- for recording_rule in server_grain.get('recording', []) %} - {%- if recording_rule.name not in recordings %} - {%- do recordings.update({recording_rule.name: recording_rule.query}) %} - {%- endif %} + {%- for recordingname, recording in server_grain.get('recording', {}).iteritems() %} + {%- do recordings.update({recordingname: recording}) %} {%- endfor %} {%- endfor %} -{%- for recording_name, query in recordings.iteritems() %} -{{ recording_name }} = {{ query }} -{%- endfor %} +{%- set recordings = salt['grains.filter_by']({'default': recordings}, merge=server.get('recording', {})) %} -{%- for alertname, alert in server.get('alert', {}).iteritems() %} - {%- if alertname not in alerts %} - {%- do alerts.update({alertname: alert}) %} +{%- for recordingname, recording in recordings.iteritems() %} + {%- if recording.get('enabled', True) %} + {{ recordingname }} = {{ recording.query }} {%- endif %} {%- endfor %} {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %} {%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %} {%- for alertname, alert in server_grain.get('alert', {}).iteritems() %} - {%- if alertname not in alerts %} {%- do alerts.update({alertname: alert}) %} - {%- endif %} {%- endfor %} {%- endfor %} +{%- set alerts = salt['grains.filter_by']({'default': alerts}, merge=server.get('alert', {})) %} + {%- for alertname, alert in alerts.iteritems() %} -ALERT {{ alertname }} - IF {{ alert.if }} - {%- if alert.for is defined %}FOR {{ alert.for }}{%- endif %} - {%- if alert.labels is defined %} - LABELS { - {%- for name, value in alert.labels.iteritems() %} - {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %} - {%- endfor %} - } - {%- endif %} - {%- if alert.annotations is defined %} - ANNOTATIONS { - {%- for name, value in alert.annotations.iteritems() %} - {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %} - {%- endfor %} - } + {%- if alert.get('enabled', True) %} + ALERT {{ alertname }} + IF {{ alert.if }} + {%- if alert.for is defined %}FOR {{ alert.for }}{%- endif %} + {%- if alert.labels is defined %} + LABELS { + {%- for name, value in alert.labels.iteritems() %} + {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %} + {%- endfor %} + } + {%- endif %} + {%- if alert.annotations is defined %} + ANNOTATIONS { + {%- for name, value in alert.annotations.iteritems() %} + {{ name }} = "{{ value }}"{%- if not loop.last %},{%- endif %} + {%- endfor %} + } + {%- endif %} {%- endif %} {%- endfor %} diff --git a/prometheus/files/prometheus.yml b/prometheus/files/prometheus.yml index 9b0d394..594bdba 100644 --- a/prometheus/files/prometheus.yml +++ b/prometheus/files/prometheus.yml @@ -25,11 +25,11 @@ scrape_configs: {%- for job_name, targets in static_target.iteritems() %} {%- set job = server.get('target', {}).get('static', {}).get(job_name, {}) %} {%- set nodes = [] %} - {%- for target in targets %} + {%- for target in targets + job.get('endpoint', []) %} {%- set address = "'%s:%d'" | format(target.address, target.port) %} {%- do nodes.append(address) %} {%- endfor %} - {%- if nodes|length > 0 %} + {%- if nodes|length > 0 and job.get('enabled', True) %} - job_name: {{ job_name }} {% if job.get('scheme') %}scheme: {{ job.scheme }}{%- endif %} {%- if job.get('tls_config') %} @@ -43,7 +43,7 @@ scrape_configs: {%- endif %} {%- endfor %} -{% if server.get('target', {}).get('kubernetes') %} +{%- if server.get('target', {}).get('kubernetes', {}).get('enabled', True) %} {%- set kubernetes_target = server.target.kubernetes %} - job_name: 'kubernetes-api' scheme: https @@ -164,7 +164,8 @@ scrape_configs: target_label: kubernetes_pod_name {%- endif %} -{%- for target in server.get('target', {}).get('dns', []) %} +{%- if server.get('target', {}).get('dns', {}).get('enabled', True) %} + {%- for target in server.get('target', {}).get('dns', {}).get('endpoint', []) %} - job_name: {{ target.name }} dns_sd_configs: - names: @@ -173,4 +174,5 @@ scrape_configs: {%- endfor %} type: {{ target.type }} port: {{ target.port }} -{%- endfor %} + {%- endfor %} +{%- endif %}