global: {%- if server.get('config', {}).global is defined %} {{ server.config.global | yaml(False) | indent(2, true) }} {%- endif %} {%- if server.get('config', {}).get('alertmanager') %} alerting: {%- if server.get('config', {}).get('alert_relabel_configs') %} alert_relabel_configs: {%- for label in server.config.get('alert_relabel_configs', []) %} - action: {{ label.get('action', 'replace') }} {%- if label.get('source_labels') %} {%- if label.source_labels is string %} {%- set label_list = label.source_labels.split(',') %} {%- elif label.source_labels is list %} {%- set label_list = label.source_labels %} {%- endif %} {%- if label_list is defined %} source_labels: {{ label_list }} {%- endif %} {%- endif %} {%- if label.get('regex') %} regex: "{{ label.regex }}" {%- endif %} {%- if label.get('target_label') %} target_label: "{{ label.target_label }}" {%- endif %} {%- if label.get('replacement') %} replacement: "{{ label.replacement }}" {%- endif %} {%- endfor %} {%- endif %} alertmanagers: {%- for name, alertmanager in server.config.alertmanager.iteritems() %} {%- if alertmanager.get('enabled', True) %} # {{ name }} {%- if alertmanager.static_configs is defined %} - static_configs: - targets: {{ alertmanager.static_configs.get('target', [])|yaml }} {%- elif alertmanager.dns_sd_configs is defined %} - dns_sd_configs: - names: {{ alertmanager.dns_sd_configs.get('domain', [])|yaml }} type: {{ alertmanager.dns_sd_configs.type }} port: {{ alertmanager.dns_sd_configs.port }} {%- elif alertmanager.kubernetes_sd_configs is defined %} - kubernetes_sd_configs: - role: {{ alertmanager.kubernetes_sd_configs.role }} namespaces: names: ['{{ alertmanager.kubernetes_sd_configs.namespace }}'] relabel_configs: - source_labels: [__meta_kubernetes_pod_name] action: keep regex: {{ alertmanager.kubernetes_sd_configs.pod_name_regex }} {%- endif %} {%- endif %} {%- endfor %} {%- endif %} {%- if server.get('config', {}).get('remote_write') %} remote_write: {%- for name, remote_endpoint in server.config.remote_write.iteritems() %} {%- if remote_endpoint.get('enabled', True) %} # {{ name }} - url: {{ remote_endpoint.url }} {%- endif %} {%- endfor %} {%- endif %} rule_files: - alerts.yml {%- set static_target = server.target.static %} {%- if server.get('use_grains', {}).get('target', True) %} {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %} {%- set static_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('static', {}) %} {%- for job_name, job in static_grain.iteritems() %} {%- if static_target[job_name] is not defined %} {%- do static_target.update({job_name: { 'enabled': job.get('enabled', True), 'metrics_path': job.get('metrics_path', '/metrics'), 'honor_labels': job.get('honor_labels', False), 'scrape_interval': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')), 'scrape_timeout': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')), 'scheme': job.get('scheme', 'http')} }) %} {%- if job.get('tls_config') %} {%- do static_target[job_name].update({ 'tls_config': { 'skip_verify': job.get('tls_config', {}).get('skip_verify', False), 'cert_name': job.get('tls_config', {}).get('cert_name'), 'key_name': job.get('tls_config', {}).get('key_name') } }) %} {%- endif %} {%- endif %} {%- if static_target[job_name]['endpoint'] is not defined %} {%- do static_target[job_name].update({'endpoint': []}) %} {%- endif %} {%- if static_target[job_name]['metric_relabel'] is not defined %} {%- do static_target[job_name].update({'metric_relabel': []}) %} {%- endif %} {%- if static_target[job_name]['relabel_configs'] is not defined %} {%- do static_target[job_name].update({'relabel_configs': []}) %} {%- endif %} {%- for target in job.get('endpoint', []) %} {%- do static_target[job_name]['endpoint'].append(target) %} {%- endfor %} {%- for label in job.get('metric_relabel', []) %} {%- do static_target[job_name]['metric_relabel'].append(label) %} {%- endfor %} {%- for label in job.get('relabel_configs', []) %} {%- do static_target[job_name]['relabel_configs'].append(label) %} {%- endfor %} {%- endfor %} {%- endfor %} {%- endif %} scrape_configs: {%- for job_name, job in static_target.iteritems() %} {%- set nodes = [] %} {%- for target in job.get('endpoint', []) %} {%- set address = "'%s:%d'" | format(target.address, target.port) %} {%- do nodes.append(address) %} {%- endfor %} {%- if nodes|length > 0 and job.get('enabled', True) %} - job_name: {{ job_name }} {% if job.get('scheme') %}scheme: {{ job.scheme }}{%- endif %} {% if job.get('metrics_path') %}metrics_path: {{ job.metrics_path }}{%- endif %} {% if job.honor_labels is defined %}honor_labels: {{ job.honor_labels }}{%- endif %} {% if job.scrape_interval is defined %}scrape_interval: {{ job.scrape_interval }}{%- endif %} {% if job.scrape_timeout is defined %}scrape_timeout: {{ job.scrape_timeout }}{%- endif %} {%- if job.get('params') %} params: {%- for param_name, param_value in job.get('params', {}).iteritems() %} {{ param_name }}: {{ param_value }} {%- endfor %} {%- endif %} {%- if job.get('tls_config') %} tls_config: {% if job.tls_config.get('skip_verify') is defined %}insecure_skip_verify: {{ job.tls_config.skip_verify | lower }}{%- endif %} {% if job.tls_config.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ job.tls_config.cert_name }}{%- endif %} {% if job.tls_config.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ job.tls_config.key_name }}{%- endif %} {%- endif %} static_configs: - targets: [{{ nodes | join(',') }}] {%- if job.get('metric_relabel') %} metric_relabel_configs: {%- for label in job.get('metric_relabel', []) %} - action: {{ label.get('action', 'replace') }} {%- if label.get('source_labels') %} {%- if label.source_labels is string %} {%- set label_list = label.source_labels.split(',') %} {%- elif label.source_labels is list %} {%- set label_list = label.source_labels %} {%- endif %} {%- if label_list is defined %} source_labels: {{ label_list }} {%- endif %} {%- endif %} {%- if label.get('regex') %} regex: "{{ label.regex }}" {%- endif %} {%- if label.get('target_label') %} target_label: "{{ label.target_label }}" {%- endif %} {%- if label.get('replacement') %} replacement: "{{ label.replacement }}" {%- endif %} {%- endfor %} {%- endif %} {%- if job.get('relabel_configs') %} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} {%- for label in job.get('relabel_configs', []) %} {%- if label.source_labels is string %} {%- set label_list = label.source_labels.split(',') %} {%- elif label.source_labels is list %} {%- set label_list = label.source_labels %} {%- endif %} {%- if label_list is defined %} - source_labels: {{ label_list }} regex: "{{ label.regex }}" {%- if label.get('target_label') %} target_label: "{{ label.target_label }}" {%- endif %} {%- if label.get('replacement') %} replacement: "{{ label.replacement }}" {%- endif %} {%- endif %} {%- endfor %} {%- endif %} {%- endif %} {%- endfor %} {%- set kubernetes_target = {} %} {%- if server.get('use_grains', {}).get('target', True) %} {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %} {%- set kubernetes_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('kubernetes', {}) %} {%- if kubernetes_grain %} {%- do kubernetes_target.update(kubernetes_grain) %} {%- endif %} {%- endfor %} {%- endif %} {%- if kubernetes_target.get('enabled', False) %} - job_name: 'kubernetes-api' scheme: https tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} kubernetes_sd_configs: - api_server: {{ kubernetes_target.api_ip }} role: endpoints tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] action: keep regex: default;kubernetes;https - job_name: 'kubernetes-node' scheme: https tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} kubernetes_sd_configs: - api_server: {{ kubernetes_target.api_ip }} role: node tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} - action: labelmap regex: __meta_kubernetes_node_label_(.+) {%- if kubernetes_target.get('endpoint', {}).get('enabled') %} - job_name: 'kubernetes-service-endpoints' scheme: https tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} kubernetes_sd_configs: - api_server: {{ kubernetes_target.api_ip }} role: endpoints tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ regex: (.+)(?::\d+);(\d+) replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_service_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: kubernetes_io_hostname - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: kubernetes_pod_name {%- endif %} {%- if kubernetes_target.get('pod', {}).get('enabled') %} - job_name: 'kubernetes-pod' scheme: https tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} kubernetes_sd_configs: - api_server: {{ kubernetes_target.api_ip }} role: pod tls_config: insecure_skip_verify: true {% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %} {% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: (.+):(?:\d+);(\d+) replacement: ${1}:${2} target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - source_labels: [__meta_kubernetes_pod_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: kubernetes_pod_name {%- endif %} {%- endif %} {%- set dns_endpoints = {} %} {%- if server.get('use_grains', {}).get('target', True) %} {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %} {%- set dns_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('dns', {}) %} {%- if dns_grain.get('enabled', False) %} {%- for endpoint in dns_grain.get('endpoint', []) %} {%- do dns_endpoints.update({endpoint.name: endpoint}) %} {%- endfor %} {%- endif %} {%- endfor %} {%- endif %} {%- if server.get('target', {}).get('dns', {}).get('enabled', False) %} {%- for endpoint in server.get('target', {}).get('dns', {}).get('endpoint', []) %} {%- do dns_endpoints.update({endpoint.name: endpoint}) %} {%- endfor %} {%- endif %} {%- for name, endpoint in dns_endpoints | dictsort %} - job_name: {{ endpoint.name }} dns_sd_configs: - names: {%- for domain in endpoint.domain %} - {{ domain }} {%- endfor %} type: {{ endpoint.type }} port: {{ endpoint.port }} relabel_configs: {%- if server.config.environment_label is defined %} - source_labels: ['instance'] target_label: environment replacement: {{ server.config.environment_label }} {%- endif %} {%- endfor %}