Prometheus + grafana + alertmanager 监控体系 $ wget https://github.com/prometheus/prometheus/releases/download/v3.0.1/prometheus-3.0.1.linux-amd64.tar.gz
$ tar -xf prometheus-3.0.1.linux-amd64.tar.gz -C /usr/local/
$ cd ! $ && ln -sv prometheus-2.28.1.linux-amd64 prometheus
$ cd prometheus
$ mkdir rules targets
$ vim prometheus.yml
global:
scrape_interval: 10s
evaluation_interval: 10s
query_log_file: ' query.log' # 告警对接配置' localhost:9093' # 规则引擎具体规则配置* .yaml
scrape_configs:
- job_name: ' prometheus' * .yaml
- job_name: ' alertmanager-server' * .yaml
- job_name: ' nodes' * .yaml
# 创建目录备用# 服务管理文件配置# 开机启动enable prometheus.service
$ systemctl start prometheus.service
docker run -itd -p 9800:3000 --name grafana-server-prod grafana/grafana $ wget https://github.com/prometheus/alertmanager/releases/download/v0.22.2/alertmanager-0.22.2.linux-amd64.tar.gz
$ tar -xf alertmanager-0.22.2.linux-amd64.tar.gz -C /usr/local/
$ cd /usr/local/ && ln -sv alertmanager-0.22.2.linux-amd64 alertmanager
$ cd /usr/local/alertmanager && mkdir config bin data logs
$ mv alertmanager amtool bin/
$ mv alertmanager.yml conf/
$ mv alertmanager.yml config/
$ mv alertmanager amtool bin/
$ vim config/alertmanager.yml
global:
resolve_timeout: 5m
route:
group_by: [' alertname' ' severity' ' telegram' ' telegram' ' http://localhost:9080/alert' " /urs/local/alertmanager/data/" $MAINPID
ExecStop=/bin/kill -KILL $MAINPID
Type=simple
Restart=always
TimeoutStopSec=20s
[Install]
WantedBy=multi-user.target
$ systemctl daemon-reload
$ systemctl enable alertmanager
$ systemctl start alertmanager
$ vim /usr/local/prometheus/rules/hosts.rules.yaml
groups:
- name: Hosts
rules:
- alert: HighCPUUsage
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=" idle" * 100) > 80
for: 15s
labels:
severity: critical
annotations:
summary: " Instance {{ $labels .instance }} CPU usage is high" " CPU usage is above 80% (current value: {{ $value }})" * 100 < 10
for: 15s
labels:
severity: warning
annotations:
summary: " Instance {{ $labels .instance }} memory usage is high" " Memory usage is below 10% (current value: {{ $value }})" * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~ " .+" $labels .instance }})
description: " Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~ " .+" $labels .instance }})
description: " The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" (( node_filesystem_avail_bytes * 100 ) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0 ) * on(instance) group_left (nodename) node_uname_info{nodename=~ ".+ "} for: 1 m
labels:
severity: warning
annotations:
summary: Host out of disk space (instance {{ $labels .instance }})
description: "Disk is almost full (< 10 % left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostHighCpuLoad
expr: (sum by (instance) (avg by (mode, instance) (rate(node_cpu_seconds_total{mode!="idle"}[2 m])) ) > 0.8) * on(instance) group_left (nodename) node_uname_info{nodename=~ " .+" $labels .instance }})
description: " CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" " iowait" * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~ " .+" $labels .instance }})
description: " CPU iowait > 10%. A high iowait means that you are disk or network bound.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" > 0.5) * on(instance) group_left (nodename) node_uname_info{nodename=~ " .+" $labels .instance }})
description: " Time spent in IO is too high on {{ $labels .instance }}. Check storage for issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
根据需求添加更多资源对象
nodes-server.yaml
prometheus-servers.yaml
$ vim /usr/local/prometheus/targets/nodes-server.yaml
- targets:
- ' 172.31.3.43:9100' ' 172.31.12.85:9100' ' 172.31.30.199:9100' ' 172.31.22.190:9100' ' 172.31.26.126:9100' ' 172.31.30.25:9100' ' 172.31.5.56:9100' ' 172.31.4.194:9100'