SaaS Prometheus 告警規則儀表板

(up{job="blackbox-exporter"} == 0)
* on (namespace, pod) group_left(node)
kube_pod_info

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) 
< 604800
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
< 86400
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 604800
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 86400
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 604800
and probe_success == 1

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 86400
and probe_success == 1

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info
) >= 2

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info{node=~".*prometheus.*"}
) >= 10

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info{node!~".*prometheus.*"}
) >= 10

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance=~".*prometheus.*"} != 1

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance!~".*prometheus.*"} != 1

(chrony_up{job="chrony-exporter"} 
* on (namespace, pod) group_left (node)
kube_pod_info
) == 0

abs(chrony_tracking_system_time_seconds{job="ss-tw2-chrony-exporter"}) >= 2

abs(chrony_tracking_system_time_seconds{job="ss-tw2-chrony-exporter"}) >= 10

node_systemd_unit_state{job="ss-tw2-node-exporter", name="chronyd.service", state="active"} != 1

chrony_up{job="ss-tw2-chrony-exporter"} == 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace!="monitoring",node!~".*prometheus.*"} > 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace="monitoring"} > 0

up{job="node-exporter", namespace="monitoring"} == 0

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}) < 600
)

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}) < 600
)

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node=~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node!~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node=~".*prometheus.*"} 
  unless ignoring (key, value) 
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node!~".*prometheus.*"}
  unless ignoring (key, value)
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node=~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node!~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace="monitoring"}
  )
)

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace!="monitoring"}
  )
)

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} 
/ 
node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} * 100)
> 90

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}
/ 
node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"} * 100) 
> 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance!~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance=~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace!="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  node_filefd_allocated{job="node-exporter",instance!~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance!~".*prometheus.*"} > 90
)

(
  node_filefd_allocated{job="node-exporter",instance=~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance=~".*prometheus.*"} > 90
)

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance=~".*prometheus.*"
  }[2m]
) > 2

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance!~".*prometheus.*"
  }[2m]
) > 2

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace!="monitoring",node!~".*prometheus"}

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace="monitoring"}

(kube_pod_init_container_status_terminated_reason{namespace="ss-tw2",reason!="Completed"} == 1)
* on(namespace, pod) group_left(node) kube_pod_info

kube_deployment_status_observed_generation{job="kube-state-metrics"}
  !=
kube_deployment_metadata_generation{job="kube-state-metrics"}

(
  kube_deployment_spec_replicas{job="kube-state-metrics"}
    >
  kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
  changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
  ==
  0
)

kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics"}
!= 0

kube_statefulset_status_observed_generation{job="kube-state-metrics"}
  !=
kube_statefulset_metadata_generation{job="kube-state-metrics"}

(
  kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
    !=
  kube_statefulset_replicas{job="kube-state-metrics"}
) and (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
    ==
  0
)

(
  max by(namespace, statefulset, job, cluster) (
    kube_statefulset_status_current_revision{job="kube-state-metrics"}
      unless
    kube_statefulset_status_update_revision{job="kube-state-metrics"}
  )
    * on(namespace, statefulset, job, cluster)
  (
    kube_statefulset_replicas{job="kube-state-metrics"}
      !=
    kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
  )
)  and on(namespace, statefulset, job, cluster) (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
    ==
  0
)

kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  -
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0

kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0

(
  (
    kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
      !=
    0
  ) or (
    kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_available{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  )
) and (
  changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m])
    ==
  0
)

time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 180

kube_job_status_failed{job="kube-state-metrics"} > 0

kube_cronjob_spec_suspend{job="kube-state-metrics"}  > 0

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring"} 
/ node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring"} * 100) 
> 80

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 80

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 80

sum(increase(container_cpu_cfs_throttled_periods_total{
  container!="", 
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  / on (cluster, namespace, pod, container, instance) group_left
sum(increase(container_cpu_cfs_periods_total{
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  > ( 25 / 100 )

(
  node_filefd_allocated{job="node-exporter"} * 100 
  / 
  node_filefd_maximum{job="node-exporter"} > 80
)

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 0.7

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 1

(redis_up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 0)
and on(instance, job) 
(up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 1)

count by (cluster) (up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 1) < 6

(redis_connected_slaves{job=~".*_redis_cluster_nodes", cluster=~".*"} == 0)
and on(instance, job, cluster)
(redis_instance_info{role="master"} == 1)

up{job="nacos"} == 0

up{job="kafka-exporter"} == 0

(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
  !=
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  >
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
  and
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  ==
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}

count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1

(sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
  /
sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
> 0.01

sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
 / 
( 
sum by(cluster) (rate(apiserver_request_total{job="apiserver"}[10m]))  
+ 
sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
) 
> 0.20

sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0

(1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85

node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10

absent(up{job="apiserver"} == 1)

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume!~".*(grafana|prom).*"} > 0

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume=~".*(grafana|prom).*"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace!="monitoring"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace="monitoring"} > 0

up{job="ss-tw2-node-exporter"} == 0

(
  changes(node_boot_time_seconds{job="ss-tw2-node-exporter"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="ss-tw2-node-exporter"}) < 600
)

(
  100 *
  (1 - (node_memory_MemAvailable_bytes{job="ss-tw2-node-exporter"} / node_memory_MemTotal_bytes{job="ss-tw2-node-exporter"}))
) > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="ss-tw2-node-exporter", mode!~"idle|iowait"}[5m])
  ) 
) * 100 > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        job="ss-tw2-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        job="ss-tw2-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

up{job="ss-tw2-swarm-scheduler-exporter"} == 0

max by (container, job, instance) (
  swarm_container_state{container=~"(prom|monitor)_.*",job="ss-tw2-swarm-scheduler-exporter",orchestrator="compose",state=~"running|healthy"}
) == 0

max by (container, job, instance) (
  swarm_container_state{container!~"(prom|monitor)_.*",job="ss-tw2-swarm-scheduler-exporter",orchestrator="compose",state=~"running|healthy"}
) == 0

(
  node_filefd_allocated{job="ss-tw2-node-exporter"} * 100 
  / 
  node_filefd_maximum{job="ss-tw2-node-exporter"} > 90
)

changes(
  node_network_up{
    job="ss-tw2-node-exporter",
    device=~"^(ens|eth|enp|eno|em).*"
  }[2m]
) > 2

(
  max by (instance, container) (
    max_over_time(swarm_container_state{
      job="ss-tw2-swarm-scheduler-exporter",
      container=~"monitor.*",
      state=~"running|healthy"
    }[1h])
  )
)
unless
(
  max by (instance, container) (
    swarm_container_state{
      job="ss-tw2-swarm-scheduler-exporter",
      container=~"monitor.*",
      state=~"running|healthy"
    }
  )
)

(
  max by (instance, container) (
    max_over_time(swarm_container_state{
      job="ss-tw2-swarm-scheduler-exporter",
      container!~"monitor.*",
      state=~"running|healthy"
    }[1h])
  )
)
unless
(
  max by (instance, container) (
    swarm_container_state{
      job="ss-tw2-swarm-scheduler-exporter",
      container!~"monitor.*",
      state=~"running|healthy"
    }
  )
)

increase(container_start_time_seconds{
  job="ss-tw2-cadvisor",
  container_label_com_docker_swarm_service_name="",
  container_label_com_docker_compose_project=~".+",
  name!~"monitor_.*"
}[10m]) > 2

increase(container_start_time_seconds{
  job="ss-tw2-cadvisor",
  container_label_com_docker_swarm_service_name="",
  container_label_com_docker_compose_project=~".+",
  name=~"monitor_.*"
}[10m]) > 2

(
  100 *
  (1 - (node_memory_MemAvailable_bytes{job="ss-tw2-node-exporter"} / node_memory_MemTotal_bytes{job="ss-tw2-node-exporter"}))
) > 80

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="ss-tw2-node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 80

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        job="ss-tw2-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        job="ss-tw2-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 80

(
  node_filefd_allocated{job="ss-tw2-node-exporter"} * 100 
  / 
  node_filefd_maximum{job="ss-tw2-node-exporter"} > 80
)

(up{job="blackbox-exporter"} == 0)
* on (namespace, pod) group_left(node)
kube_pod_info

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) 
< 604800
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
< 86400
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 604800
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 86400
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 604800
and probe_success == 1

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 86400
and probe_success == 1

abs(chrony_tracking_system_time_seconds{job=~"chrony-exporter.*"}
* on (namespace, pod) group_left (node)
kube_pod_info
) >= 2

abs(chrony_tracking_system_time_seconds{job=~"chrony-exporter.*"}
* on (namespace, pod) group_left (node)
kube_pod_info{node=~".*prometheus.*"}
) >= 10

abs(chrony_tracking_system_time_seconds{job=~"chrony-exporter.*"}
* on (namespace, pod) group_left (node)
kube_pod_info{node!~".*prometheus.*"}
) >= 10

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance=~".*prometheus.*"} != 1

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance!~".*prometheus.*"} != 1

(chrony_up{job=~"chrony-exporter.*"} 
* on (namespace, pod) group_left (node)
kube_pod_info
) == 0

abs(chrony_tracking_system_time_seconds{job="sspre-coreVM-chrony-exporter"}) >= 2

abs(chrony_tracking_system_time_seconds{job="sspre-coreVM-chrony-exporter"}) >= 10

node_systemd_unit_state{job="sspre-coreVM-node-exporter", name="chronyd.service", state="active"} != 1

chrony_up{job="sspre-coreVM-chrony-exporter"} == 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace!="monitoring",node!~".*prometheus.*"} > 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace="monitoring"} > 0

up{job="node-exporter", namespace="monitoring"} == 0

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}) < 600
)

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}) < 600
)

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node=~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node!~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node=~".*prometheus.*"} 
  unless ignoring (key, value) 
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node!~".*prometheus.*"}
  unless ignoring (key, value)
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node=~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node!~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace="monitoring"}
  )
)

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace!="monitoring"}
  )
)

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} / node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} * 100) > 90

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"} / node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"} * 100) > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance!~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance=~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace!="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  node_filefd_allocated{job="node-exporter",instance!~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance!~".*prometheus.*"} > 90
)

(
  node_filefd_allocated{job="node-exporter",instance=~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance=~".*prometheus.*"} > 90
)

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance=~".*prometheus.*"
  }[2m]
) > 2

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance!~".*prometheus.*"
  }[2m]
) > 2

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace!="monitoring",node!~".*prometheus"}

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace="monitoring"}

(kube_pod_init_container_status_terminated_reason{namespace="sspre",reason!="Completed"} == 1)
* on(namespace, pod) group_left(node) kube_pod_info

kube_deployment_status_observed_generation{job="kube-state-metrics"}
  !=
kube_deployment_metadata_generation{job="kube-state-metrics"}

(
  kube_deployment_spec_replicas{job="kube-state-metrics"}
    >
  kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
  changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
  ==
  0
)

kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics"}
!= 0

kube_statefulset_status_observed_generation{job="kube-state-metrics"}
  !=
kube_statefulset_metadata_generation{job="kube-state-metrics"}

(
  kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
    !=
  kube_statefulset_replicas{job="kube-state-metrics"}
) and (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
    ==
  0
)

(
  max by(namespace, statefulset, job, cluster) (
    kube_statefulset_status_current_revision{job="kube-state-metrics"}
      unless
    kube_statefulset_status_update_revision{job="kube-state-metrics"}
  )
    * on(namespace, statefulset, job, cluster)
  (
    kube_statefulset_replicas{job="kube-state-metrics"}
      !=
    kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
  )
)  and on(namespace, statefulset, job, cluster) (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
    ==
  0
)

kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  -
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0

kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0

(
  (
    kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
      !=
    0
  ) or (
    kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_available{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  )
) and (
  changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m])
    ==
  0
)

time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 180

kube_job_status_failed{job="kube-state-metrics"} > 0

kube_cronjob_spec_suspend{job="kube-state-metrics"} > 0

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring"} / node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring"} * 100) > 80

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 80

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 80

sum(increase(container_cpu_cfs_throttled_periods_total{
  container!="", 
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  / on (cluster, namespace, pod, container, instance) group_left
sum(increase(container_cpu_cfs_periods_total{
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  > ( 25 / 100 )

(
  node_filefd_allocated{job="node-exporter"} * 100 
  / 
  node_filefd_maximum{job="node-exporter"} > 80
)

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 0.7

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 1

(redis_up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 0)
and on(instance, job) 
(up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 1)

count by (cluster) (up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 1) < 6

(redis_connected_slaves{job=~".*_redis_cluster_nodes", cluster=~".*"} == 0)
and on(instance, job, cluster)
(redis_instance_info{role="master"} == 1)

up{job="minio"} == 0

up{job="nacos"} == 0

up{job="kafka-exporter"} == 0

kafka_connect_connector_tasks_state{job="kafka-connect-exporter"} != 1

kafka_connect_up{job="kafka-connect-exporter"} == 0

(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
  !=
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  >
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
  and
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  ==
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}

count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1

(sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
  /
sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
> 0.01

sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
 / 
( 
sum by(cluster) (rate(apiserver_request_total{job="apiserver"}[10m]))  
+ 
sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
) 
> 0.20

sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0

(1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85

node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10

absent(up{job="apiserver"} == 1)

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume!~".*(grafana|prom).*"} > 0

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume=~".*(grafana|prom).*"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace!="monitoring"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace="monitoring"} > 0

up{job="sspre-coreVM-node-exporter"} == 0

(
  changes(node_boot_time_seconds{job="sspre-coreVM-node-exporter"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="sspre-coreVM-node-exporter"}) < 600
)

(
  100 *
  (1 - (node_memory_MemAvailable_bytes{job="sspre-coreVM-node-exporter"} / node_memory_MemTotal_bytes{job="sspre-coreVM-node-exporter"}))
) > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="sspre-coreVM-node-exporter", mode!~"idle|iowait"}[5m])
  ) 
) * 100 > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        job="sspre-coreVM-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        job="sspre-coreVM-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

up{job="sspre-coreVM-swarm-scheduler-exporter"} == 0

max by (container, job, instance) (
  swarm_container_state{container=~"(prom|monitor)_.*",job="sspre-coreVM-swarm-scheduler-exporter",orchestrator="compose",state=~"running|healthy"}
) == 0

max by (container, job, instance) (
  swarm_container_state{container!~"(prom|monitor)_.*",job="sspre-coreVM-swarm-scheduler-exporter",orchestrator="compose",state=~"running|healthy"}
) == 0

(
  node_filefd_allocated{job="sspre-coreVM-node-exporter"} * 100 
  / 
  node_filefd_maximum{job="sspre-coreVM-node-exporter"} > 90
)

changes(
  node_network_up{
    job="sspre-coreVM-node-exporter",
    device=~"^(ens|eth|enp|eno|em).*"
  }[2m]
) > 2

(
  max by (instance, container) (
    max_over_time(swarm_container_state{
      job="sspre-coreVM-swarm-scheduler-exporter",
      container=~"monitor.*",
      state=~"running|healthy"
    }[1h])
  )
)
unless
(
  max by (instance, container) (
    swarm_container_state{
      job="sspre-coreVM-swarm-scheduler-exporter",
      container=~"monitor.*",
      state=~"running|healthy"
    }
  )
)

(
  max by (instance, container) (
    max_over_time(swarm_container_state{
      job="sspre-coreVM-swarm-scheduler-exporter",
      container!~"monitor.*",
      state=~"running|healthy"
    }[1h])
  )
)
unless
(
  max by (instance, container) (
    swarm_container_state{
      job="sspre-coreVM-swarm-scheduler-exporter",
      container!~"monitor.*",
      state=~"running|healthy"
    }
  )
)

increase(container_start_time_seconds{
  job="sspre-coreVM-cadvisor",
  container_label_com_docker_swarm_service_name="",
  container_label_com_docker_compose_project=~".+",
  name!~"monitor_.*"
}[10m]) > 2

increase(container_start_time_seconds{
  job="sspre-coreVM-cadvisor",
  container_label_com_docker_swarm_service_name="",
  container_label_com_docker_compose_project=~".+",
  name=~"monitor_.*"
}[10m]) > 2

(
  100 *
  (1 - (node_memory_MemAvailable_bytes{job="sspre-coreVM-node-exporter"} / node_memory_MemTotal_bytes{job="sspre-coreVM-node-exporter"}))
) > 80

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="sspre-coreVM-node-exporter", mode!~"idle|iowait"}[5m])
  ) 
) * 100 > 80

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        job="sspre-coreVM-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        job="sspre-coreVM-node-exporter",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 80

(
  node_filefd_allocated{job="sspre-coreVM-node-exporter"} * 100 
  / 
  node_filefd_maximum{job="sspre-coreVM-node-exporter"} > 80
)

(up{job="blackbox-exporter"} == 0)
* on (namespace, pod) group_left(node)
kube_pod_info

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) 
< 604800
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

histogram_quantile(0.01, sum without (namespace, service, endpoint) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m])))
< 86400
and
on(job, cluster, instance) apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 604800
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

(probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} - time()) < 86400
and probe_ssl_earliest_cert_expiry{job="blackbox-cert-check"} > 0

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 604800
and probe_success == 1

probe_ssl_earliest_cert_expiry{job="blackbox-cert-kubelet"} < 86400
and probe_success == 1

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info
) >= 2

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info{node=~".*prometheus.*"}
) >= 10

abs(chrony_tracking_system_time_seconds{job="chrony-exporter"}
* on (namespace, pod) group_left (node)
kube_pod_info{node!~".*prometheus.*"}
) >= 10

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance=~".*prometheus.*"} != 1

node_systemd_unit_state{job="node-exporter",name="chronyd.service",state="active", instance!~".*prometheus.*"} != 1

(chrony_up{job="chrony-exporter"} 
* on (namespace, pod) group_left (node)
kube_pod_info
) == 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace!="monitoring",node!~".*prometheus.*"} > 0

sum by (namespace, pod, cluster, phase) (
  max by (namespace, pod, cluster, phase) (kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown|Failed"})
  * on (namespace, pod, cluster) group_left (owner_kind)
    topk by (namespace, pod, cluster) (1,
      max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}))
)
* on (namespace, pod) group_left (node, instance) kube_pod_info{namespace="monitoring"} > 0

up{job="node-exporter", namespace="monitoring"} == 0

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}) < 600
)

(
  changes(node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}[10m]) > 0
)
and
(
  (time() - node_boot_time_seconds{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"}) < 600
)

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node=~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true",node!~".*prometheus.*"} == 0
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node=~".*prometheus.*"} 
  unless ignoring (key, value) 
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

(
  kube_node_spec_taint{effect="NoSchedule", job="kube-state-metrics", key="node.kubernetes.io/unreachable", node!~".*prometheus.*"}
  unless ignoring (key, value)
  kube_node_spec_taint{job="kube-state-metrics", key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}
) == 1

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node=~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node=~".*prometheus.*"} == 0

sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready",node!~".*prometheus.*"}[15m])) by (cluster, node) > 2
and on (cluster, node)
kube_node_spec_unschedulable{job="kube-state-metrics",node!~".*prometheus.*"} == 0

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace="monitoring"}
  )
)

(
  max by(namespace, deployment) (
    max_over_time(kube_deployment_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, deployment) (
    kube_deployment_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, statefulset) (
    max_over_time(kube_statefulset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, statefulset) (
    kube_statefulset_created{namespace!="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace="monitoring"}
  )
)

(
  max by(namespace, daemonset) (
    max_over_time(kube_daemonset_created{namespace!="monitoring"}[1h])
  )
)
unless
(
  max by(namespace, daemonset) (
    kube_daemonset_created{namespace!="monitoring"}
  )
)

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} 
/ 
node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance!~".*prometheus.*"} * 100)
> 90

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"}
/ 
node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring", instance=~".*prometheus.*"} * 100) 
> 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance!~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{instance=~".*prometheus", job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance!~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        instance=~".*prometheus.*",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 90

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  max_over_time(
    kube_pod_container_status_waiting_reason{
      job="kube-state-metrics",
      namespace!="monitoring",
      reason="CrashLoopBackOff"
    }[5m]
  ) >= 1
)
* on(namespace, pod) group_left(node)
kube_pod_info

(
  node_filefd_allocated{job="node-exporter",instance!~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance!~".*prometheus.*"} > 90
)

(
  node_filefd_allocated{job="node-exporter",instance=~".*prometheus.*"} * 100
  /
  node_filefd_maximum{job="node-exporter",instance=~".*prometheus.*"} > 90
)

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance=~".*prometheus.*"
  }[2m]
) > 2

changes(
  node_network_up{
    job="node-exporter",
    device=~"^(ens|eth|enp|eno|em).*",
    instance!~".*prometheus.*"
  }[2m]
) > 2

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace!="monitoring",node!~".*prometheus"}

(
  increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 2
) * on(namespace, pod) group_left(node) kube_pod_info{namespace="monitoring"}

(kube_pod_init_container_status_terminated_reason{namespace="sspre-s",reason!="Completed"} == 1)
* on(namespace, pod) group_left(node) kube_pod_info

kube_deployment_status_observed_generation{job="kube-state-metrics"}
  !=
kube_deployment_metadata_generation{job="kube-state-metrics"}

(
  kube_deployment_spec_replicas{job="kube-state-metrics"}
    >
  kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
  changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
  ==
  0
)

kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics"}
!= 0

kube_statefulset_status_observed_generation{job="kube-state-metrics"}
  !=
kube_statefulset_metadata_generation{job="kube-state-metrics"}

(
  kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
    !=
  kube_statefulset_replicas{job="kube-state-metrics"}
) and (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
    ==
  0
)

(
  max by(namespace, statefulset, job, cluster) (
    kube_statefulset_status_current_revision{job="kube-state-metrics"}
      unless
    kube_statefulset_status_update_revision{job="kube-state-metrics"}
  )
    * on(namespace, statefulset, job, cluster)
  (
    kube_statefulset_replicas{job="kube-state-metrics"}
      !=
    kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
  )
)  and on(namespace, statefulset, job, cluster) (
  changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
    ==
  0
)

kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  -
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0

kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0

(
  (
    kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
      !=
    0
  ) or (
    kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  ) or (
    kube_daemonset_status_number_available{job="kube-state-metrics"}
      !=
    kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
  )
) and (
  changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m])
    ==
  0
)

time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 180

kube_job_status_failed{job="kube-state-metrics"} > 0

kube_cronjob_spec_suspend{job="kube-state-metrics"}  > 0

100 - (node_memory_MemAvailable_bytes{job="node-exporter", namespace="monitoring"} 
/ node_memory_MemTotal_bytes{job="node-exporter", namespace="monitoring"} * 100) 
> 80

sum without(mode) (
  avg without (cpu) (
    rate(node_cpu_seconds_total{job="node-exporter", mode!~"idle|iowait"}[5m])
  )
) * 100 > 80

(
  100 *
  (
    1 - (
      node_filesystem_avail_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
      /
      node_filesystem_size_bytes{
        namespace="monitoring",
        mountpoint=~"^(/|/data|/nfs-data|/es-data)$",
        fstype!~"tmpfs|overlay|squashfs|nsfs|tracefs|proc|sysfs|cgroup2?"
      }
    )
  )
) > 80

sum(increase(container_cpu_cfs_throttled_periods_total{
  container!="", 
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  / on (cluster, namespace, pod, container, instance) group_left
sum(increase(container_cpu_cfs_periods_total{
  namespace!="monitoring",
  job="kubelet", 
  metrics_path="/metrics/cadvisor"
}[5m])) without (id, metrics_path, name, image, endpoint, job)
  > ( 25 / 100 )

(
  node_filefd_allocated{job="node-exporter"} * 100 
  / 
  node_filefd_maximum{job="node-exporter"} > 80
)

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 0.7

label_replace(
  prometheus_rule_group_last_duration_seconds
  /
  prometheus_rule_group_interval_seconds,
  "group", "$1", "rule_group", ".*;(.*)"
) > 1

(redis_up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 0)
and on(instance, job) 
(up{job=~".*_redis_cluster_nodes", cluster=~".*"} == 1)

up{job="nacos"} == 0

up{job="kafka-exporter"} == 0

(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
  !=
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  >
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
  and
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
  and
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  ==
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}

kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
  <
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}

count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1

(sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
  /
sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
> 0.01

sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
 / 
( 
sum by(cluster) (rate(apiserver_request_total{job="apiserver"}[10m]))  
+ 
sum by(cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m])) 
) 
> 0.20

sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0

(1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85

node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10

absent(up{job="apiserver"} == 1)

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume!~".*(grafana|prom).*"} > 0

kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics",persistentvolume=~".*(grafana|prom).*"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace!="monitoring"} > 0

kube_persistentvolumeclaim_status_phase{phase=~"Pending|Lost",job="kube-state-metrics",namespace="monitoring"} > 0