Etcd Exporter

alert: EtcdMembersCountInvaild
expr: count by (appid) (up{appid!~"uptime",job=~"etcd.*exporter"}) < 3
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群数量基数小于3(当前值:{{$value}})

alert: EtcdNoLeader
expr: (etcd_server_has_leader{appid!~"uptime",job=~"etcd.*exporter"}) == 0
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群未发现leader角色

alert: EtcdHighNumberOfLeaderChanges
expr: ceil(increase(etcd_server_leader_changes_seen_total{appid!~"uptime",job=~"etcd.*exporter"}[10m])) >= 3
for: 0m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 10分钟内,etcd集群leader切换大于3次(当前值:{{$value}})

alert: EtcdMemberCommunicationSlow
expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{appid!~"uptime"}[5m])) >= 0.3
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群99%的成员间同步延迟高于0.3秒(当前值:{{$value}})

alert: EtcdHighNumberOfFailedProposals
expr: ceil(increase(etcd_server_proposals_failed_total{appid!~"uptime"}[1h])) >= 10
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: etcd集群过去1小时内,proposal请求大于10(当前值:{{$value}})

alert: EtcdHighFsyncDurations
expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{appid!~"uptime"}[2m])) >= 1
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群99%的wal fsync延迟高于1秒(当前值:{{$value}})

alert: EtcdHighFsyncDurations
expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{appid!~"uptime"}[2m])) >= 1
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群99%的commit延迟高于1秒(当前值:{{$value}})

alert: EtcdBackendQuotaLowSpace
expr: (etcd_mvcc_db_total_size_in_bytes{appid!~"uptime"}/etcd_server_quota_backend_bytes{appid!~"uptime"}) * 100 > 60
for: 6m
labels:
  api_opsaid_cn_fault_priority: P5
  api_opsaid_cn_pm2_uuid: 99feafb5-bed6-4daf-927a-69a2ab80c485
annotations:
  summary: 2分钟检测一次,持续6分钟,etcd集群空间使用率已达配额60%(当前值:{{$value}})