2017-07-25 10:11:43 -04:00
{%- if pillar.prometheus is defined %}
2017-09-25 06:52:58 -04:00
{%- from "prometheus/map.jinja" import server, alertmanager, remote_storage_adapter, monitoring with context %}
2017-04-11 06:17:08 -04:00
server :
alert :
2017-07-25 10:11:43 -04:00
{%- if server.get('enabled', False) %}
{% raw %}
2017-07-24 09:38:28 -04:00
PrometheusTargetDown :
2018-05-03 09:28:45 -04:00
if : up != 1
2017-09-12 09:14:21 -04:00
for : 2m
2017-04-11 06:17:08 -04:00
labels :
severity : critical
service : prometheus
annotations :
2018-05-03 09:28:45 -04:00
summary : "Prometheus target is down"
2018-06-08 09:40:14 -04:00
description : "The Prometheus target for the {{ $labels.job }} job on the {{ $labels.host or $labels.instance }} node is down for 2 minutes."
2018-05-03 09:28:45 -04:00
PrometheusTargetSamplesOrderWarning :
if : increase(prometheus_target_scrapes_sample_out_of_order_total[1m]) > 0
labels :
severity : warning
service : prometheus
annotations :
summary : "Prometheus samples are out of order"
2018-06-08 09:40:14 -04:00
description : "{{ $value }} Prometheus samples on the {{ $labels.instance }} instance are out of order (as measured over the last minute)."
2018-05-03 09:28:45 -04:00
PrometheusTargetSamplesBoundsWarning :
if : increase(prometheus_target_scrapes_sample_out_of_bounds_total[1m]) > 0
labels :
severity : warning
service : prometheus
annotations :
summary : "Prometheus samples timestamps are out of bounds"
2018-06-08 09:40:14 -04:00
description : "{{ $value }} Prometheus samples on the {{ $labels.instance }} instance have timestamps out of bounds (as measured over the last minute)."
2018-05-03 09:28:45 -04:00
PrometheusTargetSamplesDuplicateWarning :
if : increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]) > 0
labels :
severity : warning
service : prometheus
annotations :
summary : "Prometheus samples have duplicate timestamps"
2018-06-08 09:40:14 -04:00
description : "{{ $value }} Prometheus samples on the {{ $labels.instance }} instance have duplicate timestamps (as measured over the last minute)."
2017-12-20 05:28:41 -05:00
{% endraw %}
{%- if server.version == 1.7 %}
{% raw %}
2018-05-03 09:28:45 -04:00
PrometheusDataIngestionWarning :
if : prometheus_local_storage_rushed_mode != 0
2017-09-25 06:52:58 -04:00
for : 10m
labels :
severity : warning
service : prometheus
annotations :
2018-05-03 09:28:45 -04:00
summary : "Prometheus is in the rushed mode"
2018-06-08 09:40:14 -04:00
description : "The Prometheus service writes on the {{ $labels.instance }} instance do not keep up with data ingestion speed for 10 minutes."
2017-04-11 06:17:08 -04:00
{% endraw %}
2017-07-25 10:11:43 -04:00
{%- endif %}
2017-12-20 05:28:41 -05:00
{%- endif %}
2017-09-25 06:52:58 -04:00
{%- if server.get('config', {}).get('remote_write') %}
2018-05-03 09:28:45 -04:00
PrometheusRemoteStorageQueueFullWarning :
2017-09-25 06:52:58 -04:00
{%- set threshold = monitoring.prometheus.remote_storage_queue_full_percent|float %}
if : >-
prometheus_remote_storage_queue_length / prometheus_remote_storage_queue_capacity * 100 > {{ threshold }}
{%- raw %}
for : 2m
labels :
severity : warning
service : prometheus
annotations :
2018-06-08 09:40:14 -04:00
summary : "Prometheus remote storage queue is {%- endraw %} {{ threshold }}{%- raw %}% full"
description : "The Prometheus remote storage queue on the {{ $labels.instance }} instance is {{ $value }}% full for 2 minutes."
2018-05-03 09:28:45 -04:00
{%- endraw %}
2017-09-25 06:52:58 -04:00
{%- endif %}
2017-07-25 10:11:43 -04:00
{%- if remote_storage_adapter.get('enabled', False) %}
2018-05-03 09:28:45 -04:00
RemoteStorageAdapterMetricsSendingWarning :
2017-07-25 10:11:43 -04:00
{%- set threshold = monitoring.remote_storage_adapter.sent_vs_received_ratio|float %}
if : >-
2018-05-03 09:28:45 -04:00
increase(sent_samples_total{job="remote_storage_adapter"}[1m]) / on (job, instance) increase(received_samples_total[1m]) < {{ threshold }}
2017-07-25 10:11:43 -04:00
{% raw %}
labels :
severity : warning
service : remote_storage_adapter
annotations :
2018-06-08 09:40:14 -04:00
summary : "Remote storage adapter metrics sent/received ratio reached limit of {%- endraw %} {{ threshold }}{%- raw %}"
2018-06-13 07:03:05 -04:00
description : "The remote storage adapter metrics on sent to received ratio on the {{ $labels.instance }} instance is {{ $value }}."
2018-05-03 09:28:45 -04:00
{% endraw %}
RemoteStorageAdapterMetricsIgnoredWarning :
2017-07-25 10:11:43 -04:00
{%- set threshold = monitoring.remote_storage_adapter.ignored_vs_sent_ratio|float %}
if : >-
2018-05-03 09:28:45 -04:00
increase(prometheus_influxdb_ignored_samples_total{job="remote_storage_adapter"}[1m]) / on (job, instance) increase(sent_samples_total[1m]) >= {{ threshold }}
2017-07-25 10:11:43 -04:00
{% raw %}
labels :
severity : warning
service : remote_storage_adapter
annotations :
2018-05-03 09:28:45 -04:00
summary : "{%- endraw %}{{ threshold }}{%- raw %}% of remote storage adapter metrics are invalid"
description : "{{ $value }}% of remote storage adapter metrics on the {{ $labels.instance }} instance are invalid."
{%- endraw %}
{%- endif %}
{%- if alertmanager.get('enabled', False) %}
{%- raw %}
AlertmanagerNotificationFailureWarning :
if : >-
increase(alertmanager_notifications_failed_total[2m]) > 0
for : 2m
labels :
severity : warning
service : alertmanager
annotations :
summary : "Alertmanager notifications fail"
2018-06-08 09:40:14 -04:00
description : "An average of {{ $value }} Alertmanager {{ $labels.integration }} notifications on the {{ $labels.instance }} instance fail for 2 minutes."
2018-05-03 09:28:45 -04:00
AlertmanagerAlertsInvalidWarning :
if : >-
increase(alertmanager_alerts_invalid_total[2m]) > 0
for : 2m
labels :
severity : warning
service : alertmanager
annotations :
summary : "Alertmanager alerts are invalid"
2018-06-08 09:40:14 -04:00
description : "An average of {{ $value }} Alertmanager {{ $labels.integration }} alerts on the {{ $labels.instance }} instance are invalid for 2 minutes."
2018-05-03 09:28:45 -04:00
{%- endraw %}
2017-07-25 10:11:43 -04:00
{%- endif %}
{%- endif %}