8000 Improve ClickHouse Keeper Grafana Dashboard by discostur · Pull Request #1872 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
7 changes: 6 additions & 1 deletion deploy/devspace/yq_transform_clickhouse-operator-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,9 @@ yq eval -e --inplace "(select(.kind == \"Deployment\" and .metadata.name == \"cl
yq eval -e --inplace '(select(.kind == "Deployment" and .metadata.name == "clickhouse-operator") | .spec.template.spec.containers[] | select(.name=="metrics-exporter") | .imagePullPolicy) = "IfNotPresent"' "${CUR_DIR}/clickhouse-operator-install.yaml"
yq eval -e --inplace '(select(.kind == "Deployment" and .metadata.name == "clickhouse-operator") | .spec.template.spec.containers[] | select(.name=="metrics-exporter") | .securityContext.capabilities.add) = ["SYS_PTRACE"]' "${CUR_DIR}/clickhouse-operator-install.yaml"

sed -i "s/namespace: kube-system/namespace: ${OPERATOR_NAMESPACE}/" "${CUR_DIR}/clickhouse-operator-install.yaml"
# Use sed with compatibility for both macOS and Linux
if [[ "$(uname)" == "Darwin" ]]; then
sed -i '' "s/namespace: kube-system/namespace: ${OPERATOR_NAMESPACE}/" "${CUR_DIR}/clickhouse-operator-install.yaml"
else
sed -i "s/namespace: kube-system/namespace: ${OPERATOR_NAMESPACE}/" "${CUR_DIR}/clickhouse-operator-install.yaml"
fi
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ metadata:
labels:
app: grafana
spec:
# Service account for the Grafana Operator to manage datasources and dashboards
serviceAccount:
metadata:
annotations: {}
labels: {}

deployment:
metadata:
annotations:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,12 @@ echo "Setup Grafana operator into ${GRAFANA_NAMESPACE} namespace"
kubectl create namespace "${GRAFANA_NAMESPACE}" || true

# Setup grafana-operator into dedicated namespace
sed -i "s/namespace: system/namespace: ${GRAFANA_NAMESPACE}/g" "${GRAFANA_OPERATOR_DIR}/deploy/kustomize/overlays/namespace_scoped/kustomization.yaml"
# Use sed with compatibility for both macOS and Linux
if [[ "$(uname)" == "Darwin" ]]; then
sed -i '' "s/namespace: system/namespace: ${GRAFANA_NAMESPACE}/g" "${GRAFANA_OPERATOR_DIR}/deploy/kustomize/overlays/namespace_scoped/kustomization.yaml"
else
sed -i "s/namespace: system/namespace: ${GRAFANA_NAMESPACE}/g" "${GRAFANA_OPERATOR_DIR}/deploy/kustomize/overlays/namespace_scoped/kustomization.yaml"
fi
kubectl kustomize "${GRAFANA_OPERATOR_DIR}/deploy/kustomize/overlays/namespace_scoped" --load-restrictor LoadRestrictionsNone | kubectl apply --server-side -f -

kubectl wait deployment/grafana-operator-controller-manager -n "${GRAFANA_NAMESPACE}" --for=condition=available --timeout=300s
Expand Down
22 changes: 15 additions & 7 deletions deploy/minio/install-minio-operator.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,21 @@ echo "Setup minio.io operator ${MINIO_OPERATOR_VERSION} into ${MINIO_NAMESPACE}
## TODO: need to refactor after next minio-operator release

MINIO_KUSTOMIZE_DIR="${MINIO_OPERATOR_DIR}/resources"
sed -i -e "s/replicas: 2/replicas: 1/" $MINIO_KUSTOMIZE_DIR/base/deployment.yaml
sed -i -e "s/name: minio-operator/name: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
sed -i -e "s/: restricted/: baseline/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
sed -i -e "s/namespace: default/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
sed -i -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
sed -i -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/kustomization.yaml
sed -i -e "s/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/" $MINIO_KUSTOMIZE_DIR/base/*.yaml

# Use sed with compatibility for both macOS and Linux
if [[ "$(uname)" == "Darwin" ]]; then
SED_INPLACE="sed -i ''"
else
SED_INPLACE="sed -i"
fi

$SED_INPLACE -e "s/replicas: 2/replicas: 1/" $MINIO_KUSTOMIZE_DIR/base/deployment.yaml
$SED_INPLACE -e "s/name: minio-operator/name: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
$SED_INPLACE -e "s/: restricted/: baseline/" $MINIO_KUSTOMIZE_DIR/base/namespace.yaml
$SED_INPLACE -e "s/namespace: default/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
$SED_INPLACE -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/base/*.yaml
$SED_INPLACE -e "s/namespace: minio-operator/namespace: ${MINIO_NAMESPACE}/" $MINIO_KUSTOMIZE_DIR/kustomization.yaml
$SED_INPLACE -e "s/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/" $MINIO_KUSTOMIZE_DIR/base/*.yaml

# Setup minio-operator into dedicated namespace via kustomize
kubectl --namespace="${MINIO_NAMESPACE}" apply -k "${MINIO_KUSTOMIZE_DIR}"
Expand Down
5 changes: 2 additions & 3 deletions deploy/prometheus/prometheus-alert-rules-chkeeper.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ spec:
echo "ClickHouseKeeper Write $((($writeEnd - $writeBegin) / 5)) b/s"
```


- alert: ClickHouseKeeperHighEphemeralNodes
expr: ClickHouseAsyncMetrics_KeeperEphemeralsCount{app=~'clickhouse-keeper.*'} > 100
for: 10m
Expand Down Expand Up @@ -196,7 +195,7 @@ spec:
description: |-
ClickHouse Keeper is using {{ with printf "(ClickHouseAsyncMetrics_KeeperOpenFileDescriptorCount{pod_name='%s',namespace='%s'} / ClickHouseAsyncMetrics_KeeperMaxFileDescriptorCount{pod_name='%s',namespace='%s'}) * 100" .Labels.pod_name .Labels.namespace .Labels.pod_name .Labels.namespace | query }}{{ . | first | value | printf "%.1f" }}{{ end }}% of available file descriptors.

Current open FDs: {{ with printf "ClickHouseAsyncMetrics_KeeperOpenFileDescriptorCount{pod_name='%s',namespace='%s'}" .Labels.pod_name .Labels.namespace | query }}{{ . | first | value | printf "%.0f" }}{{ end }}
Max FDs: {{ with printf "ClickHouseAsyncMetrics_KeeperMaxFileDescriptorCount{pod_name='%s',namespace='%s'}" .Labels.pod_name .Labels.namespace | query }}{{ . | first | value | printf "%.0f" }}{{ end }}
Current open FDs: {{ with printf "ClickHouseAsyncMetrics_KeeperOpenFileDescriptorCount{pod_name='%s',namespace='%s'}" .Labels.pod_name .Labels.namespace | query }}{{ . | first | value | printf "%d" }}{{ end }}
Max FDs: {{ with printf "ClickHouseAsyncMetrics_KeeperMaxFileDescriptorCount{pod_name='%s',namespace='%s'}" .Labels.pod_name .Labels.namespace | query }}{{ . | first | value | printf "%d" }}{{ end }}

If this continues to increase, the keeper may run out of file descriptors and become unresponsive.
18 changes: 18 additions & 0 deletions grafana-dashboard/ClickHouseKeeper_dashboard.json
Original file line number Diff line number Diff line change
Expand Up @@ -983,6 +983,24 @@
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "prometheus",
"value": "prometheus"
},
"hide": 2,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".+",
"current": {},
Expand Down
0