8000 rollback zookeeper to 3.6.3 and setup standaloneEnabled=false and add server.X by default to zoo.cfg, rollback maxResponseCacheSize, maxGetChildrenResponseCacheSize to default, add test_zookeeper.py which allow test ZK scale-out/scale-down all tests 0.15.0 PASSED by Slach · Pull Request #740 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,8 @@ Vagrant.configure(2) do |config|
# minikube
# MINIKUBE_VERSION=1.18.1
# MINIKUBE_VERSION=1.19.0
MINIKUBE_VERSION=1.20.0
# MINIKUBE_VERSION=1.20.0
MINIKUBE_VERSION=1.22.0
wget -c --progress=bar:force:noscroll -O /usr/local/bin/minikube https://github.com/kubernetes/minikube/releases/download/v${MINIKUBE_VERSION}/minikube-linux-amd64
chmod +x /usr/local/bin/minikube
# required for k8s 1.18+
Expand All @@ -149,13 +150,13 @@ Vagrant.configure(2) do |config|
# export VALIDATE_YAML=false # only for 1.14
# K8S_VERSION=${K8S_VERSION:-1.15.12}
# K8S_VERSION=${K8S_VERSION:-1.16.15}
K8S_VERSION=${K8S_VERSION:-1.17.17}
# K8S_VERSION=${K8S_VERSION:-1.17.17}
# K8S_VERSION=${K8S_VERSION:-1.18.19}
# K8S_VERSION=${K8S_VERSION:-1.19.11}
# K8S_VERSION=${K8S_VERSION:-1.19.13}
# performance issue 1.20.x, 1.21.x, only 1.22 will good to start
# https://github.com/kubernetes/kubeadm/issues/2395
# K8S_VERSION=${K8S_VERSION:-1.20.7}
# K8S_VERSION=${K8S_VERSION:-1.21.1}
# K8S_VERSION=${K8S_VERSION:-1.20.9}
K8S_VERSION=${K8S_VERSION:-1.21.3}
export VALIDATE_YAML=true

killall kubectl || true
Expand Down Expand Up @@ -273,8 +274,9 @@ EOF

pip3 install -r /vagrant/tests/requirements.txt

python3 /vagrant/tests/test_backup_alerts.py
python3 /vagrant/tests/test_metrics_alerts.py
python3 /vagrant/tests/test_zookeeper.py
python3 /vagrant/tests/test_backup_alerts.py
python3 /vagrant/tests/test_examples.py
python3 /vagrant/tests/test_metrics_exporter.py
python3 /vagrant/tests/test.py
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,24 +77,17 @@ spec:
prometheus.io/port: '7000'
prometheus.io/scrape: 'true'
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- zookeeper
topologyKey: "kubernetes.io/hostname"
containers:
- name: kubernetes-zookeeper
imagePullPolicy: IfNotPresent
image: "docker.io/zookeeper:3.6.1"
image: "docker.io/zookeeper:3.6.3"
resources:
requests:
memory: "128M"
cpu: "1"
limits:
memory: "1Gi"
cpu: "0.5"
cpu: "2"
ports:
- containerPort: 2181
name: client
Expand Down Expand Up @@ -136,8 +129,8 @@ spec:
echo 'preAllocSize=131072'
echo 'snapCount=3000000'
echo 'leaderServes=yes'
echo 'standaloneEnabled=true'
echo '4lw.commands.whitelist=stat, ruok, conf, isro, mntr'
echo 'standaloneEnabled=false'
echo '4lw.commands.whitelist=*'
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
} > /conf/zoo.cfg &&
Expand All @@ -162,11 +155,9 @@ spec:
mkdir -p ${ZOO_DATA_LOG_DIR} &&
export MY_ID=$((ORD+1)) &&
echo $MY_ID > $ZOO_DATA_DIR/myid &&
if [[ $SERVERS -gt 1 ]]; then
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done
fi &&
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done &&
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" &&
zkServer.sh start-foreground
readinessProbe:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,14 @@ spec:
containers:
- name: kubernetes-zookeeper
imagePullPolicy: IfNotPresent
image: "docker.io/zookeeper:3.6.1"
image: "docker.io/zookeeper:3.6.3"
resources:
requests:
memory: "4Gi"
memory: "512M"
Comment thread
Slach marked this conversation as resolved.
Resolved
cpu: "1"
limits:
memory: "4Gi"
cpu: "2"
ports:
- containerPort: 2181
name: client
Expand Down Expand Up @@ -138,8 +141,8 @@ spec:
echo 'preAllocSize=131072'
echo 'snapCount=3000000'
echo 'leaderServes=yes'
echo 'standaloneEnabled=true'
echo '4lw.commands.whitelist=stat, ruok, conf, isro, mntr'
echo 'standaloneEnabled=false'
echo '4lw.commands.whitelist=*'
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
} > /conf/zoo.cfg &&
Expand All @@ -164,11 +167,9 @@ spec:
mkdir -p ${ZOO_DATA_LOG_DIR} &&
export MY_ID=$((ORD+1)) &&
echo $MY_ID > $ZOO_DATA_DIR/myid &&
if [[ $SERVERS -gt 1 ]]; then
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done
fi &&
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done &&
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" &&
zkServer.sh start-foreground
readinessProbe:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
# Setup Service to provide access to Zookeeper for clients
apiVersion: v1
kind: Service
metadata:
# DNS would be like zookeeper.zoons
name: zookeeper
labels:
app: zookeeper
spec:
ports:
- port: 2181
name: client
- port: 7000
name: prometheus
selector:
app: zookeeper
what: node
---
# Setup Headless Service for StatefulSet
apiVersion: v1
kind: Service
metadata:
# DNS would be like zookeeper-0.zookeepers.etc
name: zookeepers
labels:
app: zookeeper
spec:
ports:
- port: 2888
name: server
- port: 3888
name: leader-election
clusterIP: None
selector:
app: zookeeper
what: node
---
# Setup max number of unavailable pods in StatefulSet
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: zookeeper-pod-disruption-budget
spec:
selector:
matchLabels:
app: zookeeper
maxUnavailable: 1
---
# Setup Zookeeper StatefulSet
# Possible params:
# 1. replicas
# 2. memory
# 3. cpu
# 4. storage
# 5. storageClassName
# 6. user to run app
apiVersion: apps/v1
kind: StatefulSet
metadata:
# nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2
name: zookeeper
spec:
selector:
matchLabels:
app: zookeeper
se 8 rviceNam 10BC8 e: zookeepers
replicas: 3
updateStrategy:
type: RollingUpdate
podManagementPolicy: Parallel
template:
metadata:
labels:
app: zookeeper
what: node
annotations:
prometheus.io/port: '7000'
prometheus.io/scrape: 'true'
spec:
containers:
- name: kubernetes-zookeeper
imagePullPolicy: IfNotPresent
image: "docker.io/zookeeper:3.6.3"
resources:
requests:
memory: "128M"
cpu: "1"
limits:
memory: "1Gi"
cpu: "2"
ports:
- containerPort: 2181
name: client
- containerPort: 2888
name: server
- containerPort: 3888
name: leader-election
- containerPort: 7000
name: prometheus
# See those links for proper startup settings:
# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
# https://github.com/ClickHouse/ClickHouse/issues/11781
command:
- bash
- -x
- -c
- |
SERVERS=3 &&
HOST=`hostname -s` &&
DOMAIN=`hostname -d` &&
CLIENT_PORT=2181 &&
SERVER_PORT=2888 &&
ELECTION_PORT=3888 &&
PROMETHEUS_PORT=7000 &&
ZOO_DATA_DIR=/var/lib/zookeeper/data &&
ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog &&
{
echo "clientPort=${CLIENT_PORT}"
echo 'tickTime=2000'
echo 'initLimit=300'
echo 'syncLimit=10'
echo 'maxClientCnxns=2000'
echo 'maxSessionTimeout=60000000'
echo "dataDir=${ZOO_DATA_DIR}"
echo "dataLogDir=${ZOO_DATA_LOG_DIR}"
echo 'autopurge.snapRetainCount=10'
echo 'autopurge.purgeInterval=1'
echo 'preAllocSize=131072'
echo 'snapCount=3000000'
echo 'leaderServes=yes'
echo 'standaloneEnabled=false'
echo '4lw.commands.whitelist=*'
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
} > /conf/zoo.cfg &&
{
echo "zookeeper.root.logger=CONSOLE"
echo "zookeeper.console.threshold=INFO"
echo "log4j.rootLogger=\${zookeeper.root.logger}"
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender"
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}"
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout"
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n"
} > /conf/log4j.properties &&
echo 'JVMFLAGS="-Xms128M -Xmx1G -XX:+UseG1GC -XX:+CMSParallelRemarkEnabled"' > /conf/java.env &&
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
NAME=${BASH_REMATCH[1]}
ORD=${BASH_REMATCH[2]}
else
echo "Failed to parse name and ordinal of Pod"
exit 1
fi &&
mkdir -p ${ZOO_DATA_DIR} &&
mkdir -p ${ZOO_DATA_LOG_DIR} &&
export MY_ID=$((ORD+1)) &&
echo $MY_ID > $ZOO_DATA_DIR/myid &&
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done &&
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" &&
zkServer.sh start-foreground
readinessProbe:
exec:
command:
- bash
- -c
- "OK=$(echo ruok | nc 127.0.0.1 2181); if [[ \"$OK\" == \"imok\" ]]; then exit 0; else exit 1; fi"
initialDelaySeconds: 10
timeoutSeconds: 5
livenessProbe:
exec:
command:
- bash
- -c
- "OK=$(echo ruok | nc 127.0.0.1 2181); if [[ \"$OK\" == \"imok\" ]]; then exit 0; else exit 1; fi"
initialDelaySeconds: 10
timeoutSeconds: 5
volumeMounts:
- name: datadir-volume
mountPath: /var/lib/zookeeper
# Run as a non-privileged user
securityContext:
runAsUser: 1000
fsGroup: 1000
volumeClaimTemplates:
- metadata:
10BC8 name: datadir-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,14 @@ spec:
containers:
- name: kubernetes-zookeeper
imagePullPolicy: IfNotPresent
image: "docker.io/zookeeper:3.6.1"
image: "docker.io/zookeeper:3.6.3"
resources:
requests:
memory: "4Gi"
memory: "512M"
cpu: "1"
limits:
memory: "4Gi"
cpu: "2"
ports:
- containerPort: 2181
name: client
Expand Down Expand Up @@ -136,8 +139,8 @@ spec:
echo 'preAllocSize=131072'
echo 'snapCount=3000000'
echo 'leaderServes=yes'
echo 'standaloneEnabled=true'
echo '4lw.commands.whitelist=stat, ruok, conf, isro, mntr'
echo 'standaloneEnabled=false'
echo '4lw.commands.whitelist=*'
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
} > /conf/zoo.cfg &&
Expand All @@ -162,11 +165,9 @@ spec:
mkdir -p ${ZOO_DATA_LOG_DIR} &&
export MY_ID=$((ORD+1)) &&
echo $MY_ID > $ZOO_DATA_DIR/myid &&
if [[ $SERVERS -gt 1 ]]; then
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done
fi &&
for (( i=1; i<=$SERVERS; i++ )); do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
done &&
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" &&
zkServer.sh start-foreground
readinessProbe:
Expand Down
Loading
0