8000 Add detached parts monitoring by Slach · Pull Request #680 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 69 additions & 19 deletions Vagrantfile
8000
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,26 @@ def total_cpus
Etc.nprocessors
end

def get_provider
provider='virtualbox'
for arg in ARGV
if ['hyperv','docker'].include? arg
provider=arg
end
end
return provider
end


Vagrant.configure(2) do |config|
config.vm.box = "generic/ubuntu2004"
config.vm.box_check_update = false
config.vm.synced_folder ".", "/vagrant"

if get_provider == "hyperv"
config.vm.synced_folder ".", "/vagrant", type: "smb", smb_username: ENV['USERNAME'], smb_password: ENV['PASSWORD'], mount_options: ["domain="+ENV['USERDOMAIN'], "user="+ENV['USERNAME'], "vers=3.0"," mfsymlinks"]
else
config.vm.synced_folder ".", "/vagrant"
end

if Vagrant.has_plugin?("vagrant-vbguest")
config.vbguest.auto_update = false
Expand All @@ -20,6 +35,7 @@ Vagrant.configure(2) do |config|
config.timezone.value = "UTC"
end


config.vm.provider "virtualbox" do |vb|
vb.gui = false
vb.cpus = total_cpus
Expand All @@ -32,6 +48,19 @@ Vagrant.configure(2) do |config|
vb.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
end

config.vm.provider "hyperv" do |hv|
# hv.gui = false
# hv.default_nic_type = "virtio"
hv.cpus = total_cpus
hv.maxmemory = "6144"
hv.memory = "2048"
hv.enable_virtualization_extensions = true
hv.linked_clone = true
hv.vm_integration_services = {
time_synchronization: true,
}
end

config.vm.define :clickhouse_operator do |clickhouse_operator|
clickhouse_operator.vm.network "private_network", ip: "172.16.2.99", nic_type: "virtio"
# port forwarding works only when pair with kubectl port-forward
Expand All @@ -52,24 +81,40 @@ Vagrant.configure(2) do |config|
config.vm.provision "shell", inline: <<-SHELL
set -xeuo pipefail
export DEBIAN_FRONTEND=noninteractive
# make linux fast again
if [[ "0" == $(grep "mitigations" /etc/default/grub | wc -l) ]]; then
echo 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off"' >> /etc/default/grub
echo 'GRUB_CMDLINE_LINUX_DEFAULT="quiet splash noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off"' >> /etc/default/grub
grub-mkconfig
fi
systemctl enable systemd-timesyncd
systemctl start systemd-timesyncd

apt-get update
apt-get install --no-install-recommends -y apt-transport-https ca-certificates software-properties-common curl
apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git ntp
apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git

# yq
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys CC86BB64
add-apt-repository ppa:rmescandon/yq
apt-get install --no-install-recommends -y yq

# clickhouse
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E0C56BD4
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4
add-apt-repository "deb http://repo.clickhouse.tech/deb/stable/ main/"
apt-get install --no-install-recommends -y clickhouse-client

# golang
export GOLANG_VERSION=1.16
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6BC817356A3D45E
add-apt-repository ppa:longsleep/golang-backports
apt-get install --no-install-recommends -y golang-${GOLANG_VERSION}-go
ln -nvsf /usr/lib/go-${GOLANG_VERSION}/bin/go /bin/go
ln -nvsf /usr/lib/go-${GOLANG_VERSION}/bin/gofmt /bin/gofmt

# docker
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8D81803C0EBFCD88
add-apt-repository "deb https://download.docker.com/linux/ubuntu focal test"
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) test"
apt-get install --no-install-recommends -y docker-ce pigz

# docker compose
Expand All @@ -93,7 +138,7 @@ Vagrant.configure(2) do |config|
curl -sL https://github.com/liggitt/audit2rbac/releases/download/v${AUDIT2RBAC_VERSION}/audit2rbac-linux-amd64.tar.gz | tar -zxvf - -C /usr/local/bin

# minikube
MINIKUBE_VERSION=1.18.1
MINIKUBE_VERSION=1.19.0
wget -c --progress=bar:force:noscroll -O /usr/local/bin/minikube https://github.com/kubernetes/minikube/releases/download/v${MINIKUBE_VERSION}/minikube-linux-amd64
chmod +x /usr/local/bin/minikube
# required for k8s 1.18+
Expand All @@ -103,10 +148,10 @@ Vagrant.configure(2) do |config|
# export VALIDATE_YAML=false # only for 1.14
# K8S_VERSION=${K8S_VERSION:-1.15.12}
# K8S_VERSION=${K8S_VERSION:-1.16.15}
# K8S_VERSION=${K8S_VERSION:-1.17.14}
# K8S_VERSION=${K8S_VERSION:-1.18.12}
# K8S_VERSION=${K8S_VERSION:-1.19.7}
K8S_VERSION=${K8S_VERSION:-1.20.5}
# K8S_VERSION=${K8S_VERSION:-1.17.17}
# K8S_VERSION=${K8S_VERSION:-1.18.18}
# K8S_VERSION=${K8S_VERSION:-1.19.10}
K8S_VERSION=${K8S_VERSION:-1.20.6}
export VALIDATE_YAML=true

killall kubectl || true
Expand Down Expand Up @@ -141,9 +186,11 @@ EOF
sudo -H -u vagrant minikube addons enable ingress-dns
sudo -H -u vagrant minikube addons enable metrics-server

# minikube delete
# rm -rf /tmp/juju*
# minikube config set vm-driver none
# minikube config set kubernetes-version ${K8S_VERSION}
# minikube start --vm=true
# minikube start
# minikube addons enable ingress
# minikube addons enable ingress-dns
# minikube addons enable metrics-server
Expand All @@ -160,13 +207,14 @@ EOF
source $HOME/.bashrc
export KREW_ROOT=/home/vagrant/.krew
kubectl krew install tap
kubectl krew install debug
kubectl krew install sniff
kubectl krew install flame
kubectl krew install minio
# look to https://kubernetes.io/docs/tasks/debug-application-cluster/debug-running-pod/#ephemeral-container
# kubectl krew install debug

cd /vagrant/

git_branch=$(git rev-parse --abbrev-ref HEAD)
git_branch=$(git rev-parse --abbrev-ref HEAD /vagrant/)
export OPERATOR_RELEASE=$(cat release)
export BRANCH=${BRANCH:-$git_branch}
export OPERATOR_NAMESPACE=${OPERATOR_NAMESPACE:-kube-system}
Expand Down Expand Up @@ -217,14 +265,16 @@ EOF

pip3 install -r /vagrant/tests/requirements.txt

python3 /vagrant/tests/test_metrics_alerts.py
python3 /vagrant/tests/test.py -->
pyt 10000 hon3 /vagrant/tests/test_examples.py
python3 /vagrant/tests/test_metrics_exporter.py

# audit2rbac
kubectl logs kube-apiserver-minikube -n kube-system | grep audit.k8s.io/v1 > /tmp/audit2rbac.log
audit2rbac -f /tmp/audit2rbac.log --serviceaccount kube-system:clickhouse-operator > /tmp/audit2rbac.yaml
# cp -fv /tmp/audit2rbac.yaml /vagrant/deploy/dev/clickhouse-operator-install-yaml-template-02-section-rbac-restricted.yaml

python3 /vagrant/tests/test_examples.py
python3 /vagrant/tests/test_metrics_exporter.py
python3 /vagrant/tests/test_metrics_alerts.py
SHELL
end

end
14 changes: 14 additions & 0 deletions deploy/prometheus/prometheus-alert-rules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,20 @@ spec:
Read about how to run KILL MUTATION
https://clickhouse.tech/docs/en/sql-reference/statements/kill/#kill-mutation

- alert: ClickHouseDetachedParts
expr: chi_clickhouse_metric_DetachedParts > 0
labels:
severity: high
annotations:
identifier: "{{ $labels.hostname }}"
summary: "Detached parts in system.detached_parts"
description: |-
`chi_clickhouse_metric_DetachedParts{hostname="{{ $labels.hostname }}",disk="{{ $labels.disk }}",reason="{{ $labels.reason }}"}` = {{ with printf "chi_clickhouse_metric_DetachedParts{hostname='%s',exported_namespace='%s',database='%s',table='%s',disk='%s',reason='%s'}" .Labels.hostname .Labels.exported_namespace .Labels.database .Labels.table .Labels.disk .Labels.reason | query }}{{ . | first | value | printf "%.0f" }}{{ end }}
`system.detached_parts` show have detached parts.
Please check detached parts in log ```kubectl logs -n {{ $labels.exported_namespace }} pod/$(kubectl get pods -n {{ $labels.exported_namespace }} | grep $( echo {{ $labels.hostname }} | cut -d '.' -f 1) | cut -d " " -f 1) -c clickhouse-pod --since=1h | grep -i detach```
Read about how ATTACH / DROP detached parts
https://clickhouse.tech/docs/en/operations/system-tables/detached_parts/

# based on https://blog.serverdensity.com/how-to-monitor-zookeeper/
# and https://github.com/apache/zookeeper/blob/master/zookeeper-contrib/zookeeper-contrib-monitoring/nagios/services.cfg
- name: ZookeeperRules
Expand Down
30 changes: 30 additions & 0 deletions pkg/apis/metrics/clickhouse_fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,21 @@ const (
toString(total_space) AS total_space
FROM system.disks
`

queryDetachedPartsSQL = `
SELECT
count() AS detached_parts,
database,
table,
disk,
BD0 Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The rea 10000 son will be displayed to describe this comment to others. Learn more.

I do not think disk is important here.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it will important if you will have to detached part cause only one disk will have bitflip \ bios error etc.
I agree, usually disk label will have a default value

if(coalesce(reason,'unknown')='','detached_by_user',coalesce(reason,'unknown')) AS detach_reason
FROM system.detached_parts
GROUP BY
database,
table,
disk,
reason
`
)

type ClickHouseFetcher struct {
Expand Down Expand Up @@ -212,6 +227,21 @@ func (f *ClickHouseFetcher) getClickHouseQuerySystemDisks() ([][]string, error)
)
}

// getClickHouseQueryDetachedParts requests detached parts reasons from ClickHouse
func (f *ClickHouseFetcher) getClickHouseQueryDetachedParts() ([][]string, error) {
return f.clickHouseQueryScanRows(
queryDetachedPartsSQL,
func(rows *sqlmodule.Rows, data *[][]string) error {
var detachedParts, database, table, disk, reason string
if err := rows.Scan(&detachedParts, &database, &table, &disk, &reason); err == nil {
*data = append(*data, []string{detachedParts, database, table, disk, reason})
}
return nil
},
)
}


// clickHouseQueryScanRows scan all rows by external scan function
func (f *ClickHouseFetcher) clickHouseQueryScanRows(
sql string,
Expand Down
12 changes: 12 additions & 0 deletions pkg/apis/metrics/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,18 @@ func (e *Exporter) collectFromHost(chi *WatchedCHI, hostname string, c chan<- pr
return
}

log.V(2).Infof("Querying detached parts for %s\n", hostname)
if detachedParts, err := fetcher.getClickHouseQueryDetachedParts(); err == nil {
log.V(2).Infof("Extracted %d detached parts info for %s\n", len(detachedParts), hostname)
writer.WriteDetachedParts(detachedParts)
writer.WriteOKFetch("system.detached_parts")
} else {
// In case of an error fetching data from clickhouse store CHI name in e.cleanup
log.V(2).Infof("Error querying detached parts for %s: %s\n", hostname, err)
writer.WriteErrorFetch("system.detached_parts")
//e.enqueueToRemoveFromWatched(chi)
return
}
}

// getWatchedCHI serves HTTP request to get list of watched CHIs
Expand Down
18 changes: 13 additions & 5 deletions pkg/apis/metrics/prometheus_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,16 +137,24 @@ func (w *PrometheusWriter) WriteSystemDisks(data [][]string) {
}
}

func (w *PrometheusWriter) WriteErrorFetch(fetch_type string) {
func (w *PrometheusWriter) WriteDetachedParts(data [][]string) {
for _, metric := range data {
writeSingleMetricToPrometheus(w.out, "metric_DetachedParts", "Count of currently detached parts from system.detached_parts", metric[0], prometheus.GaugeValue,
[]string{"chi", "namespace", "hostname", "database", "table", "disk", "reason"},
w.chi.Name, w.chi.Namespace, w.hostname, metric[1], metric[2], metric[3], metric[4])
}
}

func (w *PrometheusWriter) WriteErrorFetch(fetchType string) {
writeSingleMetricToPrometheus(w.out, "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", "1", prometheus.GaugeValue,
[]string{"chi", "namespace", "hostname", "fetch_type"},
w.chi.Name, w.chi.Namespace, w.hostname, fetch_type)
w.chi.Name, w.chi.Namespace, w.hostname, fetchType)
}

func (w *PrometheusWriter) WriteOKFetch(fetch_type string) {
func (w *PrometheusWriter) WriteOKFetch(fetchType string) {
writeSingleMetricToPrometheus(w.out, "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", "0", prometheus.GaugeValue,
[]string{"chi", "namespace", "hostname", "fetch_type"},
w.chi.Name, w.chi.Namespace, w.hostname, fetch_type)
w.chi.Name, w.chi.Namespace, w.hostname, fetchType)
}

func writeSingleMetricToPrometheus(out chan<- prometheus.Metric, name string, desc string, value string, metricType prometheus.ValueType, labels []string, labelValues ...string) {
Expand Down Expand Up @@ -192,5 +200,5 @@ func convertMetricName(in string) string {
out = append(out, unicode.ToLower(runes[i]))
}*/

return strings.Replace(string(in), ".", "_", -1)
return strings.Replace(in, ".", "_", -1)
}
Loading
0