formula-prometheus/prometheus/files/server/prometheus.yml

global:
{%- if server.get('config', {}).global is defined %}
{{ server.config.global | yaml(False) | indent(2, true) }}
{%- endif %}

{%- if server.get('config', {}).get('alertmanager') %}
alerting:
  {%- if server.get('config', {}).get('alert_relabel_configs') %}
  alert_relabel_configs:
    {%- for label in server.config.get('alert_relabel_configs', []) %}
    - action: {{ label.get('action', 'replace') }}
      {%- if label.get('source_labels') %}
        {%- if label.source_labels is string %}
          {%- set label_list = label.source_labels.split(',') %}
        {%- elif label.source_labels is list %}
          {%- set label_list = label.source_labels %}
        {%- endif %}
        {%- if label_list is defined %}
      source_labels: {{ label_list }}
        {%- endif %}
      {%- endif %}
      {%- if label.get('regex') %}
      regex: "{{ label.regex }}"
      {%- endif %}
      {%- if label.get('target_label') %}
      target_label: "{{ label.target_label }}"
      {%- endif %}
      {%- if label.get('replacement') %}
      replacement: "{{ label.replacement }}"
      {%- endif %}
    {%- endfor %}
  {%- endif %}
  alertmanagers:
    {%- for name, alertmanager in server.config.alertmanager.iteritems() %}
    {%- if alertmanager.get('enabled', True) %}
    # {{ name }}
    {%- if alertmanager.static_configs is defined %}
    - static_configs:
      - targets: {{ alertmanager.static_configs.get('target', [])|yaml }}
    {%- elif alertmanager.dns_sd_configs is defined %}
    - dns_sd_configs:
      - names: {{ alertmanager.dns_sd_configs.get('domain', [])|yaml }}
        type: {{ alertmanager.dns_sd_configs.type }}
        port: {{ alertmanager.dns_sd_configs.port }}
    {%- endif %}
    {%- endif %}
    {%- endfor %}
{%- endif %}

{%- if server.get('config', {}).get('remote_write') %}
remote_write:
  {%- for name, remote_endpoint in server.config.remote_write.iteritems() %}
  {%- if remote_endpoint.get('enabled', True) %}
  # {{ name }}
  - url: {{ remote_endpoint.url }}
  {%- endif %}
  {%- endfor %}
{%- endif %}

rule_files:
- alerts.yml

{%- set static_target = server.target.static %}
{%- if server.get('use_grains', {}).get('target', True) %}
  {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
    {%- set static_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('static', {}) %}
    {%- for job_name, job in static_grain.iteritems() %}
      {%- if static_target[job_name] is not defined %}
        {%- do static_target.update({job_name: {
          'enabled': job.get('enabled', True),
          'metrics_path': job.get('metrics_path', '/metrics'),
          'honor_labels': job.get('honor_labels', False),
          'scrape_interval': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')),
          'scrape_timeout': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')),
          'scheme': job.get('scheme', 'http')}
        }) %}
        {%- if job.get('tls_config') %}
          {%- do static_target[job_name].update({
            'tls_config': {
              'skip_verify': job.get('tls_config', {}).get('skip_verify', False),
              'cert_name': job.get('tls_config', {}).get('cert_name'),
              'key_name': job.get('tls_config', {}).get('key_name')
            }
          }) %}
        {%- endif %}
      {%- endif %}
      {%- if static_target[job_name]['endpoint'] is not defined %}
        {%- do static_target[job_name].update({'endpoint': []}) %}
      {%- endif %}
      {%- if static_target[job_name]['metric_relabel'] is not defined %}
        {%- do static_target[job_name].update({'metric_relabel': []}) %}
      {%- endif %}
      {%- if static_target[job_name]['relabel_configs'] is not defined %}
        {%- do static_target[job_name].update({'relabel_configs': []}) %}
      {%- endif %}
      {%- for target in job.get('endpoint', []) %}
        {%- do static_target[job_name]['endpoint'].append(target) %}
      {%- endfor %}
      {%- for label in job.get('metric_relabel', []) %}
        {%- do static_target[job_name]['metric_relabel'].append(label) %}
      {%- endfor %}
      {%- for label in job.get('relabel_configs', []) %}
        {%- do static_target[job_name]['relabel_configs'].append(label) %}
      {%- endfor %}
    {%- endfor %}
  {%- endfor %}
{%- endif %}

scrape_configs:
{%- for job_name, job in static_target.iteritems() %}
  {%- set nodes = [] %}
  {%- for target in job.get('endpoint', []) %}
    {%- set address = "'%s:%d'" | format(target.address, target.port) %}
    {%- do nodes.append(address) %}
  {%- endfor %}
  {%- if nodes|length > 0 and job.get('enabled', True) %}
  - job_name: {{ job_name }}
    {% if job.get('scheme') %}scheme: {{ job.scheme }}{%- endif %}
    {% if job.get('metrics_path') %}metrics_path: {{ job.metrics_path }}{%- endif %}
    {% if job.honor_labels is defined %}honor_labels: {{ job.honor_labels }}{%- endif %}
    {% if job.scrape_interval is defined %}scrape_interval: {{ job.scrape_interval }}{%- endif %}
    {% if job.scrape_timeout is defined %}scrape_timeout: {{ job.scrape_timeout }}{%- endif %}
    {%- if job.get('params') %}
    params:
      {%- for param_name, param_value in job.get('params', {}).iteritems() %}
      {{ param_name }}: {{ param_value }}
      {%- endfor %}
    {%- endif %}
    {%- if job.get('tls_config') %}
    tls_config:
      {% if job.tls_config.get('skip_verify') is defined %}insecure_skip_verify: {{ job.tls_config.skip_verify | lower }}{%- endif %}
      {% if job.tls_config.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ job.tls_config.cert_name }}{%- endif %}
      {% if job.tls_config.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ job.tls_config.key_name }}{%- endif %}
    {%- endif %}
    static_configs:
    - targets: [{{ nodes | join(',') }}]
    {%- if job.get('metric_relabel') %}
    metric_relabel_configs:
      {%- for label in job.get('metric_relabel', []) %}
      - action: {{ label.get('action', 'replace') }}
        {%- if label.get('source_labels') %}
          {%- if label.source_labels is string %}
            {%- set label_list = label.source_labels.split(',') %}
          {%- elif label.source_labels is list %}
            {%- set label_list = label.source_labels %}
          {%- endif %}
          {%- if label_list is defined %}
        source_labels: {{ label_list }}
          {%- endif %}
        {%- endif %}
        {%- if label.get('regex') %}
        regex: "{{ label.regex }}"
        {%- endif %}
        {%- if label.get('target_label') %}
        target_label: "{{ label.target_label }}"
        {%- endif %}
        {%- if label.get('replacement') %}
        replacement: "{{ label.replacement }}"
        {%- endif %}
      {%- endfor %}
    {%- endif %}
    {%- if job.get('relabel_configs') %}
    relabel_configs:
      {%- if server.config.environment_label is defined %}
      - source_labels: ['instance']
        target_label: environment
        replacement: {{ server.config.environment_label }}
      {%- endif %}
      {%- for label in job.get('relabel_configs', []) %}
        {%- if label.source_labels is string %}
          {%- set label_list = label.source_labels.split(',') %}
        {%- elif label.source_labels is list %}
          {%- set label_list = label.source_labels %}
        {%- endif %}
        {%- if label_list is defined %}
      - source_labels: {{ label_list }}
        regex: "{{ label.regex }}"
          {%- if label.get('target_label') %}
        target_label: "{{ label.target_label }}"
          {%- endif %}
          {%- if label.get('replacement') %}
        replacement: "{{ label.replacement }}"
          {%- endif %}
        {%- endif %}
      {%- endfor %}
    {%- endif %}
  {%- endif %}
{%- endfor %}

{%- set kubernetes_target = {} %}
{%- if server.get('use_grains', {}).get('target', True) %}
  {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
    {%- set kubernetes_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('kubernetes', {}) %}
    {%- if kubernetes_grain %}
      {%- do kubernetes_target.update(kubernetes_grain) %}
    {%- endif %}
  {%- endfor %}
{%- endif %}

{%- if kubernetes_target.get('enabled', False) %}

  - job_name: 'kubernetes-api'
    scheme: https
    tls_config:
      insecure_skip_verify: true
      {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
      {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    kubernetes_sd_configs:
    - api_server: {{ kubernetes_target.api_ip }}
      role: endpoints
      tls_config:
        insecure_skip_verify: true
        {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
        {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: default;kubernetes;https

  - job_name: 'kubernetes-node'
    scheme: https
    tls_config:
      insecure_skip_verify: true
      {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
      {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    kubernetes_sd_configs:
    - api_server: {{ kubernetes_target.api_ip }}
      role: node
      tls_config:
        insecure_skip_verify: true
        {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
        {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)

  - job_name: 'kubernetes-cadvisor'
    scheme: https
    tls_config:
      insecure_skip_verify: true
      {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
      {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    kubernetes_sd_configs:
    - api_server: {{ kubernetes_target.api_ip }}
      role: node
      tls_config:
        insecure_skip_verify: true
        {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
        {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
    - action: labelmap
      regex: __meta_kubernetes_node_label_(.+)
    - source_labels: [__address__]
      action: replace
      target_label: __address__
      regex: (.+)(?::\d+)
      replacement: $1:{{ kubernetes_target.get('cadvisor', {}).get('port', 4194) }}
    - source_labels: [__scheme__]
      action: replace
      target_label: __scheme__
      replacement: {{ kubernetes_target.get('cadvisor', {}).get('scheme', 'http') }}

  {%- if kubernetes_target.get('endpoint', {}).get('enabled') %}
  - job_name: 'kubernetes-service-endpoints'
    scheme: https
    tls_config:
      insecure_skip_verify: true
      {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
      {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    kubernetes_sd_configs:
    - api_server: {{ kubernetes_target.api_ip }}
      role: endpoints
      tls_config:
        insecure_skip_verify: true
        {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
        {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
      action: replace
      target_label: __scheme__
      regex: (https?)
    - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
      action: replace
      target_label: __address__
      regex: (.+)(?::\d+);(\d+)
      replacement: $1:$2
    - action: labelmap
      regex: __meta_kubernetes_service_label_(.+)
    - source_labels: [__meta_kubernetes_service_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_service_name]
      action: replace
      target_label: kubernetes_name
    - source_labels: [__meta_kubernetes_pod_node_name]
      action: replace
      target_label: kubernetes_io_hostname
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name
  {%- endif %}

  {%- if kubernetes_target.get('pod', {}).get('enabled') %}
  - job_name: 'kubernetes-pod'
    scheme: https
    tls_config:
      insecure_skip_verify: true
      {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
      {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    kubernetes_sd_configs:
    - api_server: {{ kubernetes_target.api_ip }}
      role: pod
      tls_config:
        insecure_skip_verify: true
        {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
        {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: (.+):(?:\d+);(\d+)
      replacement: ${1}:${2}
      target_label: __address__
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)
    - source_labels: [__meta_kubernetes_pod_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name
  {%- endif %}
{%- endif %}

{%- set dns_endpoints = {} %}
{%- if server.get('use_grains', {}).get('target', True) %}
  {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
    {%- set dns_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('dns', {}) %}
    {%- if dns_grain.get('enabled', False) %}
      {%- for endpoint in dns_grain.get('endpoint', []) %}
        {%- do dns_endpoints.update({endpoint.name: endpoint}) %}
      {%- endfor %}
    {%- endif %}
  {%- endfor %}
{%- endif %}

{%- if server.get('target', {}).get('dns', {}).get('enabled', False) %}
  {%- for endpoint in server.get('target', {}).get('dns', {}).get('endpoint', []) %}
    {%- do dns_endpoints.update({endpoint.name: endpoint}) %}
  {%- endfor %}
{%- endif %}

{%- for name, endpoint in dns_endpoints | dictsort %}
  - job_name: {{ endpoint.name }}
    dns_sd_configs:
    - names:
  {%- for domain in endpoint.domain %}
      - {{ domain }}
  {%- endfor %}
      type: {{ endpoint.type }}
      port: {{ endpoint.port }}
    relabel_configs:
    {%- if server.config.environment_label is defined %}
    - source_labels: ['instance']
      target_label: environment
      replacement: {{ server.config.environment_label }}
    {%- endif %}
{%- endfor %}