f945e6091b
Instead of cadvisor we are adviced to use telegraf kubernetes input plugin. Change-Id: Ic9892e2f6a2adc305e550b943a669a606010aa1e Closes-Bug: PROD-24334
384 lines
16 KiB
YAML
384 lines
16 KiB
YAML
global:
|
|
{%- if server.get('config', {}).global is defined %}
|
|
{{ server.config.global | yaml(False) | indent(2, true) }}
|
|
{%- endif %}
|
|
|
|
{%- if server.get('config', {}).get('alertmanager') %}
|
|
alerting:
|
|
{%- if server.get('config', {}).get('alert_relabel_configs') %}
|
|
alert_relabel_configs:
|
|
{%- for label in server.config.get('alert_relabel_configs', []) %}
|
|
- action: {{ label.get('action', 'replace') }}
|
|
{%- if label.get('source_labels') %}
|
|
{%- if label.source_labels is string %}
|
|
{%- set label_list = label.source_labels.split(',') %}
|
|
{%- elif label.source_labels is list %}
|
|
{%- set label_list = label.source_labels %}
|
|
{%- endif %}
|
|
{%- if label_list is defined %}
|
|
source_labels: {{ label_list }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- if label.get('regex') %}
|
|
regex: "{{ label.regex }}"
|
|
{%- endif %}
|
|
{%- if label.get('target_label') %}
|
|
target_label: "{{ label.target_label }}"
|
|
{%- endif %}
|
|
{%- if label.get('replacement') %}
|
|
replacement: "{{ label.replacement }}"
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
alertmanagers:
|
|
{%- for name, alertmanager in server.config.alertmanager.iteritems() %}
|
|
{%- if alertmanager.get('enabled', True) %}
|
|
# {{ name }}
|
|
{%- if alertmanager.static_configs is defined %}
|
|
- static_configs:
|
|
- targets: {{ alertmanager.static_configs.get('target', [])|yaml }}
|
|
{%- elif alertmanager.dns_sd_configs is defined %}
|
|
- dns_sd_configs:
|
|
- names: {{ alertmanager.dns_sd_configs.get('domain', [])|yaml }}
|
|
type: {{ alertmanager.dns_sd_configs.type }}
|
|
port: {{ alertmanager.dns_sd_configs.port }}
|
|
{%- elif alertmanager.kubernetes_sd_configs is defined %}
|
|
- kubernetes_sd_configs:
|
|
- role: {{ alertmanager.kubernetes_sd_configs.role }}
|
|
namespaces:
|
|
names: ['{{ alertmanager.kubernetes_sd_configs.namespace }}']
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
action: keep
|
|
regex: {{ alertmanager.kubernetes_sd_configs.pod_name_regex }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
{%- if server.get('config', {}).get('remote_write') %}
|
|
remote_write:
|
|
{%- for name, remote_endpoint in server.config.remote_write.iteritems() %}
|
|
{%- if remote_endpoint.get('enabled', True) %}
|
|
# {{ name }}
|
|
- url: {{ remote_endpoint.url }}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
rule_files:
|
|
- alerts.yml
|
|
|
|
{%- set static_target = server.target.static %}
|
|
{%- if server.get('use_grains', {}).get('target', True) %}
|
|
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
|
|
{%- set static_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('static', {}) %}
|
|
{%- for job_name, job in static_grain.iteritems() %}
|
|
{%- if static_target[job_name] is not defined %}
|
|
{%- do static_target.update({job_name: {
|
|
'enabled': job.get('enabled', True),
|
|
'metrics_path': job.get('metrics_path', '/metrics'),
|
|
'honor_labels': job.get('honor_labels', False),
|
|
'scrape_interval': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')),
|
|
'scrape_timeout': job.get('scrape_interval', server.get('config', {}).get('global', {}).get('scrape_interval', '15s')),
|
|
'scheme': job.get('scheme', 'http')}
|
|
}) %}
|
|
{%- if job.get('tls_config') %}
|
|
{%- do static_target[job_name].update({
|
|
'tls_config': {
|
|
'skip_verify': job.get('tls_config', {}).get('skip_verify', False),
|
|
'cert_name': job.get('tls_config', {}).get('cert_name'),
|
|
'key_name': job.get('tls_config', {}).get('key_name')
|
|
}
|
|
}) %}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- if static_target[job_name]['endpoint'] is not defined %}
|
|
{%- do static_target[job_name].update({'endpoint': []}) %}
|
|
{%- endif %}
|
|
{%- if static_target[job_name]['metric_relabel'] is not defined %}
|
|
{%- do static_target[job_name].update({'metric_relabel': []}) %}
|
|
{%- endif %}
|
|
{%- if static_target[job_name]['relabel_configs'] is not defined %}
|
|
{%- do static_target[job_name].update({'relabel_configs': []}) %}
|
|
{%- endif %}
|
|
{%- for target in job.get('endpoint', []) %}
|
|
{%- do static_target[job_name]['endpoint'].append(target) %}
|
|
{%- endfor %}
|
|
{%- for label in job.get('metric_relabel', []) %}
|
|
{%- do static_target[job_name]['metric_relabel'].append(label) %}
|
|
{%- endfor %}
|
|
{%- for label in job.get('relabel_configs', []) %}
|
|
{%- do static_target[job_name]['relabel_configs'].append(label) %}
|
|
{%- endfor %}
|
|
{%- endfor %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
scrape_configs:
|
|
{%- for job_name, job in static_target.iteritems() %}
|
|
{%- set nodes = [] %}
|
|
{%- for target in job.get('endpoint', []) %}
|
|
{%- set address = "'%s:%d'" | format(target.address, target.port) %}
|
|
{%- do nodes.append(address) %}
|
|
{%- endfor %}
|
|
{%- if nodes|length > 0 and job.get('enabled', True) %}
|
|
- job_name: {{ job_name }}
|
|
{% if job.get('scheme') %}scheme: {{ job.scheme }}{%- endif %}
|
|
{% if job.get('metrics_path') %}metrics_path: {{ job.metrics_path }}{%- endif %}
|
|
{% if job.honor_labels is defined %}honor_labels: {{ job.honor_labels }}{%- endif %}
|
|
{% if job.scrape_interval is defined %}scrape_interval: {{ job.scrape_interval }}{%- endif %}
|
|
{% if job.scrape_timeout is defined %}scrape_timeout: {{ job.scrape_timeout }}{%- endif %}
|
|
{%- if job.get('params') %}
|
|
params:
|
|
{%- for param_name, param_value in job.get('params', {}).iteritems() %}
|
|
{{ param_name }}: {{ param_value }}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{%- if job.get('tls_config') %}
|
|
tls_config:
|
|
{% if job.tls_config.get('skip_verify') is defined %}insecure_skip_verify: {{ job.tls_config.skip_verify | lower }}{%- endif %}
|
|
{% if job.tls_config.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ job.tls_config.cert_name }}{%- endif %}
|
|
{% if job.tls_config.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ job.tls_config.key_name }}{%- endif %}
|
|
{%- endif %}
|
|
static_configs:
|
|
- targets: [{{ nodes | join(',') }}]
|
|
{%- if job.get('metric_relabel') %}
|
|
metric_relabel_configs:
|
|
{%- for label in job.get('metric_relabel', []) %}
|
|
- action: {{ label.get('action', 'replace') }}
|
|
{%- if label.get('source_labels') %}
|
|
{%- if label.source_labels is string %}
|
|
{%- set label_list = label.source_labels.split(',') %}
|
|
{%- elif label.source_labels is list %}
|
|
{%- set label_list = label.source_labels %}
|
|
{%- endif %}
|
|
{%- if label_list is defined %}
|
|
source_labels: {{ label_list }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- if label.get('regex') %}
|
|
regex: "{{ label.regex }}"
|
|
{%- endif %}
|
|
{%- if label.get('target_label') %}
|
|
target_label: "{{ label.target_label }}"
|
|
{%- endif %}
|
|
{%- if label.get('replacement') %}
|
|
replacement: "{{ label.replacement }}"
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{%- if job.get('relabel_configs') %}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
{%- for label in job.get('relabel_configs', []) %}
|
|
{%- if label.source_labels is string %}
|
|
{%- set label_list = label.source_labels.split(',') %}
|
|
{%- elif label.source_labels is list %}
|
|
{%- set label_list = label.source_labels %}
|
|
{%- endif %}
|
|
{%- if label_list is defined %}
|
|
- source_labels: {{ label_list }}
|
|
regex: "{{ label.regex }}"
|
|
{%- if label.get('target_label') %}
|
|
target_label: "{{ label.target_label }}"
|
|
{%- endif %}
|
|
{%- if label.get('replacement') %}
|
|
replacement: "{{ label.replacement }}"
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
|
|
{%- set kubernetes_target = {} %}
|
|
{%- if server.get('use_grains', {}).get('target', True) %}
|
|
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
|
|
{%- set kubernetes_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('kubernetes', {}) %}
|
|
{%- if kubernetes_grain %}
|
|
{%- do kubernetes_target.update(kubernetes_grain) %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
{%- if kubernetes_target.get('enabled', False) %}
|
|
|
|
- job_name: 'kubernetes-api'
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
kubernetes_sd_configs:
|
|
- api_server: {{ kubernetes_target.api_ip }}
|
|
role: endpoints
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
|
action: keep
|
|
regex: default;kubernetes;https
|
|
|
|
- job_name: 'kubernetes-node'
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
kubernetes_sd_configs:
|
|
- api_server: {{ kubernetes_target.api_ip }}
|
|
role: node
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
|
|
{%- if kubernetes_target.get('endpoint', {}).get('enabled') %}
|
|
- job_name: 'kubernetes-service-endpoints'
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
kubernetes_sd_configs:
|
|
- api_server: {{ kubernetes_target.api_ip }}
|
|
role: endpoints
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: true
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
|
action: replace
|
|
target_label: __scheme__
|
|
regex: (https?)
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
|
action: replace
|
|
target_label: __metrics_path__
|
|
regex: (.+)
|
|
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
|
action: replace
|
|
target_label: __address__
|
|
regex: (.+)(?::\d+);(\d+)
|
|
replacement: $1:$2
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_service_label_(.+)
|
|
- source_labels: [__meta_kubernetes_service_namespace]
|
|
action: replace
|
|
target_label: kubernetes_namespace
|
|
- source_labels: [__meta_kubernetes_service_name]
|
|
action: replace
|
|
target_label: kubernetes_name
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
|
action: replace
|
|
target_label: kubernetes_io_hostname
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
action: replace
|
|
target_label: kubernetes_pod_name
|
|
{%- endif %}
|
|
|
|
{%- if kubernetes_target.get('pod', {}).get('enabled') %}
|
|
- job_name: 'kubernetes-pod'
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
kubernetes_sd_configs:
|
|
- api_server: {{ kubernetes_target.api_ip }}
|
|
role: pod
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
{% if kubernetes_target.get('cert_name') %}cert_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.cert_name }}{%- endif %}
|
|
{% if kubernetes_target.get('key_name') %}key_file: {{ server.dir.config_in_container }}/{{ kubernetes_target.key_name }}{%- endif %}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: true
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
|
action: replace
|
|
target_label: __metrics_path__
|
|
regex: (.+)
|
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
|
action: replace
|
|
regex: (.+):(?:\d+);(\d+)
|
|
replacement: ${1}:${2}
|
|
target_label: __address__
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_pod_label_(.+)
|
|
- source_labels: [__meta_kubernetes_pod_namespace]
|
|
action: replace
|
|
target_label: kubernetes_namespace
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
action: replace
|
|
target_label: kubernetes_pod_name
|
|
{%- endif %}
|
|
{%- endif %}
|
|
|
|
{%- set dns_endpoints = {} %}
|
|
{%- if server.get('use_grains', {}).get('target', True) %}
|
|
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
|
|
{%- set dns_grain = node_grains.get('prometheus', {}).get('server', {}).get('target', {}).get('dns', {}) %}
|
|
{%- if dns_grain.get('enabled', False) %}
|
|
{%- for endpoint in dns_grain.get('endpoint', []) %}
|
|
{%- do dns_endpoints.update({endpoint.name: endpoint}) %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
{%- if server.get('target', {}).get('dns', {}).get('enabled', False) %}
|
|
{%- for endpoint in server.get('target', {}).get('dns', {}).get('endpoint', []) %}
|
|
{%- do dns_endpoints.update({endpoint.name: endpoint}) %}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
|
|
{%- for name, endpoint in dns_endpoints | dictsort %}
|
|
- job_name: {{ endpoint.name }}
|
|
dns_sd_configs:
|
|
- names:
|
|
{%- for domain in endpoint.domain %}
|
|
- {{ domain }}
|
|
{%- endfor %}
|
|
type: {{ endpoint.type }}
|
|
port: {{ endpoint.port }}
|
|
relabel_configs:
|
|
{%- if server.config.environment_label is defined %}
|
|
- source_labels: ['instance']
|
|
target_label: environment
|
|
replacement: {{ server.config.environment_label }}
|
|
{%- endif %}
|
|
{%- endfor %}
|