Mine updates

README.rst

@@ -42,6 +42,18 @@ Server installed with PostgreSQL database
           user: grafana
           password: passwd
 
+Collector setup
+---------------
+
+Used to aggregate dashboards
+
+.. code-block:: yaml
+
+    grafana:
+      collector:
+        enabled: true
+
+
 Client setups
 -------------
 
@@ -98,12 +110,65 @@ Client enforced dashboards defined in salt-mine
     grafana:
       client:
         enabled: true
-        collect_mine: true
+        remote_data:
+          engine: salt_mine
         server:
           host: grafana.host
           port: 3000
           token: token
 
+Usage
+=====
+
+There's a difference between JSON dashboard representation and models we us. Lists are replaced by dictionaries to support mergings and interpolations.
+
+Client enforced dashboards defined in salt-mine
+
+.. code-block:: yaml
+
+    system_metrics:
+      title: graph
+      editable: true
+      hideControls: false
+      rows:
+      - title: Usage
+        height: 250px
+        panels:
+        - title: Panel Title
+          span: 6
+          editable: false
+          type: graph
+          targets: 
+          - refId: A
+            target: "support_prd.cfg01_iot_tcpcloud_eu.cpu.0.idle"
+          datasource: graphite01
+          renderer: flot
+        showTitle: true
+
+
+.. code-block:: yaml
+
+    system_metrics:
+      title: graph
+      editable: true
+      hideControls: false
+      rows:
+      - title: Usage
+        height: 250px
+        panels:
+        - title: Panel Title
+          span: 6
+          editable: false
+          type: graph
+          targets: 
+          - refId: A
+            target: "support_prd.cfg01_iot_tcpcloud_eu.cpu.0.idle"
+          datasource: graphite01
+          renderer: flot
+        showTitle: true
+
+
+
 
 Read more
 =========

grafana/client.sls

@@ -35,4 +35,47 @@ grafana_client_dashboard_{{ dashboard_name }}:
 
 {%- endfor %}
 
+{%- set raw_dict = {} %}
+{%- set final_dict = {} %}
+
+{%- if client.remote_data.engine == 'salt_mine' %}
+{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
+{%- if node_grains.grafana is defined %}
+{%- set raw_dict = salt['grains.filter_by']({'default': raw_dict}, merge=node_grains.grafana.get('dashboard', {})) %}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+
+{%- if client.dashboard is defined %}
+{%- set raw_dict = salt['grains.filter_by']({'default': raw_dict}, merge=client.dashboard) %}
+{%- endif %}
+
+{%- for dashboard_name, dashboard in raw_dict.iteritems() %}
+{%- set rows = [] %}
+{%- for row_name, row in dashboard.get('row', {}).iteritems() %}
+{%- set panels = [] %}
+{%- for panel_name, panel in row.get('panel', {}).iteritems() %}
+{%- set targets = [] %}
+{%- for target_name, target in panel.get('target', {}).iteritems() %}
+{%- do targets.extend([target]) %}
+{%- endfor %}
+{%- do panel.update({'targets': targets}) %}
+{%- do panels.extend([panel]) %}
+{%- endfor %}
+{%- do row.update({'panels': panels}) %}
+{%- do rows.extend([row]) %}
+{%- endfor %}
+{%- do dashboard.update({'rows': rows}) %}
+{%- do final_dict.update({dashboard_name: dashboard}) %}
+{%- endfor %}
+
+{%- for dashboard_name, dashboard in final_dict.iteritems() %}
+
+grafana_client_dashboard_{{ dashboard_name }}:
+  grafana_dashboard.present:
+  - name: {{ dashboard_name }}
+  - dashboard: {{ dashboard }}
+
+{%- endfor %}
+
 {%- endif %}

grafana/collector.sls

@@ -0,0 +1,50 @@
+{%- from "grafana/map.jinja" import collector with context %}
+{%- if collector.enabled %}
+
+grafana_grains_dir:
+  file.directory:
+  - name: /etc/salt/grains.d
+  - mode: 700
+  - makedirs: true
+  - user: root
+
+{%- set service_grains = {} %}
+
+{# Loading the other service support metadata for localhost #}
+
+{%- for service_name, service in pillar.iteritems() %}
+
+{%- macro load_grains_file(grains_fragment_file) %}{% include grains_fragment_file ignore missing %}{% endmacro %}
+
+{%- set grains_fragment_file = service_name+'/meta/grafana.yml' %}
+{%- set grains_yaml = load_grains_file(grains_fragment_file)|load_yaml %}
+{%- set service_grains = salt['grains.filter_by']({'default': service_grains}, merge=grains_yaml) %}
+
+{%- endfor %}
+
+grafana_grain:
+  file.managed:
+  - name: /etc/salt/grains.d/grafana
+  - source: salt://grafana/files/grafana.grain
+  - template: jinja
+  - user: root
+  - mode: 600
+  - defaults:
+    service_grains:
+      grafana: {{ service_grains|yaml }}
+  - require:
+    - file: grafana_grains_dir
+
+grafana_grains_file:
+  cmd.wait:
+  - name: cat /etc/salt/grains.d/* > /etc/salt/grains
+  - watch:
+    - file: grafana_grain
+
+grafana_grains_publish:
+  module.run:
+  - name: mine.update
+  - watch:
+    - cmd: grafana_grains_file
+
+{%- endif %}

grafana/files/grafana.grain

@@ -0,0 +1 @@
+{{ service_grains|yaml(False) }}

grafana/init.sls

@@ -7,4 +7,7 @@ include:
 {%- if pillar.grafana.client is defined %}
 - grafana.client
 {%- endif %}
+{%- if pillar.grafana.collector is defined %}
+- grafana.collector
+{%- endif %}
 {%- endif %}

grafana/map.jinja

@@ -26,6 +26,8 @@ Debian:
   server:
     host: 127.0.0.1
     port: 3000
+  remote_data:
+    engine: none
   datasource: {}
   dashboard: {}
 {%- endload %}

grafana/meta/grafana.yml

@@ -0,0 +1,45 @@
+{%- if pillar.get('grafana').collector is defined %}
+dashboard:
+  test-single-{{ grains.host }}:
+    title: Dashboard single {{ grains.host }}
+    editable: true
+    hideControls: false
+    row:
+      single:
+        title: Single row
+        height: 250px
+        showTitle: true
+        panel:
+          first:
+            title: Single Panel
+            span: 8
+            editable: false
+            type: graph
+            target:
+              A:
+                refId: A
+                target: "support_prd.cfg01_iot_tcpcloud_eu.cpu.0.idle"
+            datasource: graphite01
+            renderer: flot
+  test-merge:
+    title: Dashboard merge
+    editable: true
+    hideControls: false
+    row:
+      merge:
+        showTitle: true
+        title: Merge
+        height: 250px
+        panel:
+          merge:
+            title: Merge Panel
+            span: 8
+            editable: false
+            type: graph
+            target:
+              {{ grains.host }}:
+                refId: A
+                target: "support_prd.cfg01_iot_tcpcloud_eu.cpu.0.idle"
+            datasource: graphite01
+            renderer: flot
+{%- endif %}