Linux-Help/formula-prometheus
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge changes from topics 'prometheus/alert', 'rename-prometheus-alerts'

Browse source 
* changes:
  Add Prometheus alerts
  Rename Prometheus alerts for consistency
This commit is contained in:
Ales Komarek 2017-07-26 12:21:50 +00:00 committed by Gerrit Code Review
parent 90b074f356 5b6b583c42
commit dd19c7f616
2 changed files with 45 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
prometheus/map.jinja
View file

@ -17,3 +17,17 @@
},
}, merge=salt['pillar.get']('prometheus:exporters')) %}
{%- do salt['defaults.merge'](exporters, grains.get('prometheus', {}).get('exporters', {})) %}
{%- set monitoring = salt['grains.filter_by']({
'default': {
'remote_storage_adapter': {
'sent_vs_received_ratio': 10.0,
'ignored_vs_sent_ratio': 5.0,
},
},
}, grain='os_family', merge=salt['pillar.get']('prometheus:monitoring')) %}
{% set remote_storage_adapter = salt['grains.filter_by']({
'default': {
},
}, merge=salt['pillar.get']('prometheus:remote_storage_adapter')) %}

33
prometheus/meta/prometheus.yml
View file

@ -1,7 +1,10 @@
{% raw %}
{%- if pillar.prometheus is defined %}
{%- from "prometheus/map.jinja" import server, remote_storage_adapter, monitoring with context %}
server:
alert:
PrometheusUP:
{%- if server.get('enabled', False) %}
{% raw %}
PrometheusTargetDown:
if: 'up != 1'
labels:
severity: critical
@ -10,3 +13,29 @@ server:
summary: 'Prometheus endpoint {{ $labels.instance }} is down'
description: 'Prometheus endpoint {{ $labels.instance }} is down for job {{ $labels.job }}'
{% endraw %}
{%- endif %}
{%- if remote_storage_adapter.get('enabled', False) %}
RemoteStorageAdapterSendingTooSlow:
{%- set threshold = monitoring.remote_storage_adapter.sent_vs_received_ratio|float %}
if: >-
100.0 - (100.0 * sent_samples_total{job="remote_storage_adapter"} / on (job, instance) received_samples_total) > {{ threshold }}
{% raw %}
labels:
severity: warning
service: remote_storage_adapter
annotations:
summary: 'Remote storage adapter too slow on {{ $labels.instance }}'
description: 'Remote storage adapter can not ingest samples fast enough on {{ $labels.instance }} (current value={{ $value }}%, threshold={%- endraw %}{{ threshold }}%).'
RemoteStorageAdapterIgnoredTooHigh:
{%- set threshold = monitoring.remote_storage_adapter.ignored_vs_sent_ratio|float %}
if: >-
100.0 * prometheus_influxdb_ignored_samples_total{job="remote_storage_adapter"} / on (job, instance) sent_samples_total > {{ threshold }}
{% raw %}
labels:
severity: warning
service: remote_storage_adapter
annotations:
summary: 'Remote storage adapter receiving too many invalid metrics on {{ $labels.instance }}'
description: 'Remote storage adapter is receiving too many invalid metrics on {{ $labels.instance }} (current value={{ $value }}%, threshold={%- endraw %}{{ threshold }}%).'
{%- endif %}
{%- endif %}