Add support for prometheus 2.0

New version changes:
* different alerts format
* rewritten storage (some config flags removed)

Closes-Bug: PROD-16609
Change-Id: I805fa322e4744e98177d6c3e29589ebc6fb917a2
This commit is contained in:
Bartosz Kupidura 2017-12-20 11:28:41 +01:00
parent 94baab3660
commit 8bdf3ed090
14 changed files with 3229 additions and 15 deletions

View file

@ -15,6 +15,7 @@ Configure prometheus server
prometheus:
server:
version: 2.0
enabled: true
dir:
config: /srv/volumes/prometheus
@ -62,11 +63,7 @@ Configure prometheus server
process_open_fds / process_max_fds
storage:
local:
engine: "persisted"
retention: "360h"
memory_chunks: 1048576
max_chunks_to_persist: 524288
num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config:

View file

@ -8,7 +8,6 @@ parameters:
prometheus_region_label: "region1"
prometheus_scrape_interval: "15s"
prometheus_storage_retention: "360h"
prometheus_storage_heap_size: 3221225472
prometheus:
server:
enabled: true
@ -21,10 +20,7 @@ parameters:
address: 0.0.0.0
storage:
local:
engine: "persisted"
retention: ${_param:prometheus_storage_retention}
target_heap_size: ${_param:prometheus_storage_heap_size}
num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config:

View file

@ -0,0 +1,938 @@
{% raw %}
{
"annotations": {
"list": []
},
"description": "Prometheus Stats Dashboard",
"editable": true,
"gnetId": 358,
"graphTooltip": 0,
"hideControls": false,
"id": null,
"links": [],
"refresh": false,
"rows": [
{
"collapse": false,
"height": 178,
"panels": [
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 1,
"editable": true,
"error": false,
"format": "s",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"id": 5,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"span": 3,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"tableColumn": "",
"targets": [
{
"expr": "(time() - process_start_time_seconds{job=\"prometheus\", instance=~\"$Prometheus:[1-9][0-9]*\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
"step": 40
}
],
"thresholds": "",
"title": "Uptime",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "current"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"editable": true,
"error": false,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"id": 6,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"span": 3,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": true
},
"tableColumn": "",
"targets": [
{
"expr": "sum(prometheus_tsdb_head_series{instance=~\"$Prometheus:[1-9][0-9]*\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
"step": 40
}
],
"thresholds": "1,5",
"title": "Total Series",
"type": "singlestat",
"valueFontSize": "70%",
"valueMaps": [],
"valueName": "current"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"editable": true,
"error": false,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"id": 7,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"span": 3,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": true
},
"tableColumn": "",
"targets": [
{
"expr": "sum(prometheus_tsdb_head_chunks{instance=~\"$Prometheus:[1-9][0-9]*\"})",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
"step": 40
}
],
"thresholds": "500,4000",
"title": "Memory Chunks",
"type": "singlestat",
"valueFontSize": "70%",
"valueMaps": [
{
"op": "=",
"text": "Empty",
"value": "0"
}
],
"valueName": "current"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"editable": true,
"error": false,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"id": 16,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"span": 3,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": true
},
"tableColumn": "",
"targets": [
{
"expr": "prometheus_config_last_reload_successful{instance=~\"$Prometheus:[1-9][0-9]*\"}",
"format": "time_series",
"intervalFactor": 2,
"refId": "A",
"step": 40
}
],
"thresholds": "0:1",
"title": "Successful Config Reload",
"type": "singlestat",
"valueFontSize": "70%",
"valueMaps": [
{
"op": "=",
"text": "Failure",
"value": "0"
},
{
"op": "=",
"text": "Success",
"value": "1"
}
],
"valueName": "current"
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": 227,
"panels": [
{
"aliasColors": {
"prometheus": "#C15C17",
"{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(prometheus_tsdb_head_samples_appended_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}",
"metric": "",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Appended Samples per Second",
"tooltip": {
"msResolution": false,
"ordering": "alphabetical",
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": "Samples / Second",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"fill": 1,
"id": 18,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(prometheus_tsdb_head_series_created_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "created",
"refId": "A",
"step": 4
},
{
"expr": "sum(increase(prometheus_tsdb_head_series_removed_total{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "removed",
"refId": "B",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Series Created / Removed",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {
"prometheus": "#F9BA8F",
"{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(prometheus_target_interval_length_seconds_count{instance=~\"$Prometheus:[1-9][0-9]*\"}[5m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Target Scrapes (last 5m)",
"tooltip": {
"msResolution": false,
"ordering": "alphabetical",
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 14,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\", instance=~\"$Prometheus:[1-9][0-9]*\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{quantile}} ({{interval}})",
"metric": "",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Scrape Duration",
"tooltip": {
"msResolution": false,
"ordering": "alphabetical",
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"decimals": null,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 12,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_evaluator_duration_seconds{quantile!=\"0.01\", quantile!=\"0.05\", instance=~\"$Prometheus:[1-9][0-9]*\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{quantile}}",
"refId": "A",
"step": 2
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Rule Eval Duration",
"tooltip": {
"msResolution": false,
"ordering": "alphabetical",
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 17,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(http_request_duration_microseconds_count{handler=~\"query|query_range\", instance=~\"$Prometheus:[1-9][0-9]*\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{handler}}",
"refId": "A",
"step": 2
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Queries to prometheus",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "ms",
"label": "duration",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
}
],
"schemaVersion": 14,
"style": "dark",
"tags": [
"prometheus"
],
"templating": {
"list": [
{
"allValue": null,
"current": {},
"hide": 0,
"includeAll": false,
"label": "Prometheus host",
"multi": false,
"name": "Prometheus",
"options": [],
"query": "query_result(up{job=\"prometheus\"} == 1)",
"refresh": 1,
"regex": ".*instance=\"([^:]+):[1-9][0-9]*\".*",
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"now": true,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"1m",
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Prometheus Stats",
"version": 1
}
{% endraw %}

View file

@ -0,0 +1,65 @@
{%- from "prometheus/map.jinja" import server with context %}
{%- set alerts = {} %}
{%- set recordings = {} %}
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
{%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
{%- for recordingname, recording in server_grain.get('recording', {}).iteritems() %}
{%- do recordings.update({recordingname: recording}) %}
{%- endfor %}
{%- endfor %}
groups:
{%- set recordings = salt['grains.filter_by']({'default': recordings}, merge=server.get('recording', {})) %}
- name: recording.rules
rules:
{%- for recordingname, recording in recordings.iteritems() %}
{%- if recording.get('enabled', True) %}
- record: {{ recordingname }}
expr: >-
{{ recording.query | indent(6, False) }}
{%- endif %}
{%- endfor %}
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
{%- set server_grain = node_grains.get('prometheus', {}).get('server', {}) %}
{%- for alertname, alert in server_grain.get('alert', {}).iteritems() %}
{%- do alerts.update({alertname: alert}) %}
{%- endfor %}
{%- endfor %}
{%- set alerts = salt['grains.filter_by']({'default': alerts}, merge=server.get('alert', {})) %}
- name: alert.rules
rules:
{%- for alertname, alert in alerts.iteritems() %}
{%- if alert.get('enabled', True) %}
- alert: {{ alertname }}
expr: >-
{{ alert.if | indent(6, False) }}
{%- if alert.for is defined %}
for: {{ alert.for }}{%- endif %}
{#- Handle labels #}
{%- if alert.labels is not defined %}
{%- do alert.update({'labels': {}}) %}
{%- endif %}
{%- for label_name, label_value in server.get('config', {}).get('alert', {}).get('labels_add', {}).iteritems() %}
{%- if label_name not in alert.labels %}
{%- do alert.labels.update({label_name: label_value}) %}
{%- endif %}
{%- endfor %}
{%- if alert.labels %}
labels:
{%- for name, value in alert.labels.iteritems() %}
{{ name }}: "{{ value }}"
{%- endfor %}
{%- endif %}
{#- Handle annotations #}
{%- if alert.annotations is defined %}
annotations:
{%- for name, value in alert.annotations.iteritems() %}
{{ name }}: "{{ value }}"
{%- endfor %}
{%- endif %}
{%- endif %}
{%- endfor %}

View file

@ -1,5 +1,6 @@
{% set server = salt['grains.filter_by']({
'default': {
'version': 2.0,
'target': {
'static': {
},

View file

@ -1,8 +1,9 @@
{% from "prometheus/map.jinja" import server with context %}
dashboard:
prometheus-performances_prometheus:
datasource: prometheus
format: json
template: prometheus/files/grafana_dashboards/prometheus_performances_prometheus.json
template: prometheus/files/grafana_dashboards/server/{{server.version}}/prometheus_performances_prometheus.json
remote_storage_adapter_prometheus:
datasource: prometheus
format: json
@ -10,7 +11,7 @@ dashboard:
prometheus_stats_prometheus:
datasource: prometheus
format: json
template: prometheus/files/grafana_dashboards/prometheus_stats_prometheus.json
template: prometheus/files/grafana_dashboards/server/{{server.version}}/prometheus_stats_prometheus.json
alertmanager_prometheus:
datasource: prometheus
format: json

View file

@ -13,6 +13,9 @@ server:
annotations:
summary: 'Prometheus endpoint {{ $labels.instance }} down'
description: 'The Prometheus target {{ $labels.instance }} is down for the job {{ $labels.job }}.'
{% endraw %}
{%- if server.version == 1.7 %}
{% raw %}
PrometheusRushMode:
if: 'prometheus_local_storage_rushed_mode != 0'
for: 10m
@ -24,6 +27,7 @@ server:
description: 'The Prometheus {{ $labels.instance }} is in rush mode for 10m.'
{% endraw %}
{%- endif %}
{%- endif %}
{%- if alertmanager.get('enabled', False) %}
AlertmanagerNotificationFailed:
{%- set threshold = monitoring.alertmanager.notification_failed_rate|float %}

View file

@ -14,14 +14,14 @@
{{server.dir.config}}/prometheus.yml:
file.managed:
- source: salt://prometheus/files/prometheus.yml
- source: salt://prometheus/files/server/prometheus.yml
- template: jinja
- require:
- file: {{server.dir.config}}
{{server.dir.config}}/alerts.yml:
file.managed:
- source: salt://prometheus/files/alerts.yml
- source: salt://prometheus/files/server/{{server.version}}/alerts.yml
- template: jinja
- require:
- file: {{server.dir.config}}

View file

@ -55,10 +55,7 @@ prometheus:
summary: 'Prometheus target down'
storage:
local:
engine: "persisted"
retention: "360h"
target_heap_size: 3221225472
num_fingerprint_mutexes: 4096
alertmanager:
notification_queue_capacity: 10000
config: