Better grain handling for telegraf

Replace '-' to '_' in prometheus config dir
Add possibility to add recording rules

Change-Id: Ifc85a95ae62cb0f1b810a3b4f718e8d92f9c6209

README.rst

@@ -17,7 +17,7 @@ Configure prometheus server
     server:
       enabled: true
       dir:
-        config: /srv/volumes/prometheus-config
+        config: /srv/volumes/prometheus
         config_in_container: /opt/prometheus/config
       bind:
         port: 9090
@@ -30,6 +30,10 @@ Configure prometheus server
           cert_name: kubelet-client.crt
           key_name: kubelet-client.key
         etcd: ${etcd:server:members}
+      recording:
+        - name: 'instance:fd_utilization'
+          query: >-
+            process_open_fds / process_max_fds
       alert:
         PrometheusTargetDown:
           if: 'up != 1'
@@ -63,7 +67,7 @@ Configure alertmanager
     alertmanager:
       enabled: true
       dir:
-        config: /srv/volumes/prometheus-config
+        config: /srv/volumes/prometheus
       bind:
         address: 0.0.0.0
         port: 9093

prometheus/files/alerts.yml

@@ -1,5 +1,11 @@
 {%- from "prometheus/map.jinja" import server with context %}
 
+{%- if server.recording is defined %}
+{%- for recording_rule in server.recording %}
+{{ recording_rule.name }} = {{ recording_rule.query }}
+{%- endfor %}
+{%- endif %}
+
 {%- if server.alert is defined %}
 {%- for alertname, alert in server.alert.iteritems() %}
 ALERT {{ alertname }}

prometheus/files/prometheus.yml

@@ -12,8 +12,9 @@ scrape_configs:
 {%- set telegraf_nodes = [] %}
 {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
   {%- if 'telegraf' in node_grains.get('services') %}
-    {%- set node_ip = node_grains.get('prometheus_client').get('address') %}
-    {%- set node_port = node_grains.get('prometheus_client').get('port') %}
+    {%- set prometheus_client = node_grains.get('telegraf').get('prometheus_client') %}
+    {%- set node_ip = prometheus_client.get('address') %}
+    {%- set node_port = prometheus_client.get('port') %}
     {%- set telegraf_address = "'%s:%d'" | format(node_ip, node_port) %}
     {%- do telegraf_nodes.append(telegraf_address) %}
   {%- endif %}

prometheus/map.jinja

@@ -1,7 +1,7 @@
 {% set server = salt['grains.filter_by']({
     'default': {
         'dir': {
-            'config': '/srv/volumes/prometheus-config',
+            'config': '/srv/volumes/prometheus',
             'config_in_container': '/opt/prometheus/config'
         },
     },
@@ -10,7 +10,7 @@
 {% set alertmanager = salt['grains.filter_by']({
     'default': {
         'dir': {
-            'config': '/srv/volumes/prometheus-config',
+            'config': '/srv/volumes/prometheus',
         },
     },
 }, merge=salt['pillar.get']('prometheus:alertmanager')) %}