serhii-chechun · Jan 24, 2019 · Jan 14, 2019 · Jan 14, 2019 · Jan 14, 2019 · Jan 15, 2019
diff --git a/cmd/clickhouse-operator/app/clickhouse_operator.go b/cmd/clickhouse-operator/app/clickhouse_operator.go
@@ -94,7 +94,7 @@ func Run() {
 		chiClient, kubeClient,
 		chiInformerFactory.Clickhouse().V1().ClickHouseInstallations(),
 		kubeInformerFactory.Apps().V1().StatefulSets(),
-		kubeInformerFactory.Core().V1().Pods(),
+		kubeInformerFactory.Core().V1().ConfigMaps(),
 		kubeInformerFactory.Core().V1().Services())
 	kubeInformerFactory.Start(ctx.Done())
 	chiInformerFactory.Start(ctx.Done())

diff --git a/manifests/clickhouse/01-service-client.yaml b/manifests/clickhouse/01-service-client.yaml
@@ -0,0 +1,15 @@
+# Setup Service to provide access to Zookeeper for clients
+apiVersion: v1
+kind: Service
+metadata:
+  name: clickhouse-service-client
+  labels:
+    app: clickhouse
+spec:
+  ports:
+  - port: 9000
+    name: client
+  - port: 8123
+    name: http
+  selector:
+    dns: chi1
diff --git a/manifests/clickhouse/02-headless-service.yaml b/manifests/clickhouse/02-headless-service.yaml
@@ -0,0 +1,12 @@
+# Setup Headless Service for StatefulSet
+apiVersion: v1
+kind: Service
+metadata:
+  name: chpod-hs
+  labels:
+    app: chpod
+spec:
+  clusterIP: None
+  selector:
+    dns: chi1
+
diff --git a/manifests/clickhouse/03-config.d-configmap.yaml b/manifests/clickhouse/03-config.d-configmap.yaml
@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: etc-clickhouse-server-config-d-common
+data:
+  zookeeper.xml: |-
+    <yandex>
+        <zookeeper>
+            <node>
+                <host>zookeeper-service-client</host>
+                <port>2181</port>
+            </node>
+        </zookeeper>
+    </yandex>
+
+  remote_servers.xml: |-
+    <yandex>
+        <remote_servers>
+
+            <benchmark_cluster>
+                <shard>
+                    <replica>
+                        <!-- chpod1 -->
+                        <host>chpod1.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                </shard>
+                <shard>
+                    <replica>
+                        <!-- chpod2 -->
+                        <host>chpod2.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                </shard>
+            </benchmark_cluster>
+
+            <repl_cluster>
+                <shard>
+                    <internal_replication>true</internal_replication>
+                    <replica>
+                        <!-- chpod1 -->
+                        <host>chpod1.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                    <replica>
+                        <!-- chpod3 -->
+                        <host>chpod3.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                </shard>
+                <shard>
+                    <internal_replication>true</internal_replication>
+                    <replica>
+                        <!-- chpod2 -->
+                        <host>chpod2.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                    <replica>
+                        <!-- chpod4 -->
+                        <host>chpod4.chpod-hs.default.svc.cluster.local</host>
+                        <port>9000</port>
+                    </replica>
+                </shard>
+            </repl_cluster>
+
+        </remote_servers>
+    </yandex>
+
diff --git a/manifests/clickhouse/04-storageclass-clickhouse.yaml b/manifests/clickhouse/04-storageclass-clickhouse.yaml
@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: storage-clickhouse
+provisioner: kubernetes.io/no-provisioner
+#volumeBindingMode: WaitForFirstConsumer
+volumeBindingMode: Immediate
+
diff --git a/manifests/clickhouse/ch-statefulset-1.yaml b/manifests/clickhouse/ch-statefulset-1.yaml
@@ -0,0 +1,143 @@
+# Setup Headless Service for StatefulSet
+apiVersion: v1
+kind: Service
+metadata:
+  name: ch-node-hs-1
+  labels:
+    app: clickhouse
+    node: ch-node-1
+spec:
+  ports:
+  - port: 8123
+    name: http
+  - port: 9000
+    name: native
+  - port: 9009
+    name: replication
+  clusterIP: None
+  selector:
+    app: clickhouse
+    node: ch-node-1
+    pod: ch-pod-1
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: etc-clickhouse-server-config-d-macros-ch-node-1
+  labels:
+    app: clickhouse
+    node: ch-node-1
+data:
+  macros.xml: |-
+    <yandex>
+        <macros>
+            <shard>1</shard>
+            <replica>1</replica>
+        </macros>
+    </yandex>
+---
+# Setup ClickHouse StatefulSet
+# Possible params:
+# 1. replicas
+# 2. memory
+# 3. cpu
+# 4. storage
+# 5. storageClassName
+# 6. user to run app
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ch-stateful-set-1
+  labels:
+    app: clickhouse
+    node: ch-node-1
+spec:
+  selector:
+    matchLabels:
+      app: clickhouse
+      node: ch-node-1
+      pod: ch-pod-1
+  serviceName: ch-node-hs-1
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        app: clickhouse
+        node: ch-node-1
+        pod: ch-pod-1
+    spec:
+      affinity:
+        # Locate only one pod on each host
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: "app"
+                    operator: In
+                    values:
+                    - clickhouse
+              topologyKey: "kubernetes.io/hostname"
+      containers:
+      - name: clickhouse
+        imagePullPolicy: Always
+        image: yandex/clickhouse-server
+        livenessProbe:
+          tcpSocket:
+            port: 8123
+#           port: 9000
+          # Number of seconds after the container has started before liveness or readiness probes are initiated
+          initialDelaySeconds: 10
+          # How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1
+          periodSeconds: 10
+          # Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1.
+          timeoutSeconds: 10
+          # Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1
+          successThreshold: 1
+          # When a Pod starts and the probe fails, Kubernetes will try failureThreshold times before giving up
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            # Try 5 times to access both native and http API entry points.
+            # In case of failure sleep for i*10 sec and retry
+            # This command can take long to to run, so adjust 'timeoutSeconds' accordingly
+            command:
+            - /bin/bash
+            - -c
+            - for i in 1 2 3 4 5; do echo "Attempt $i" && echo -n "version=" && clickhouse-client -q 'select version()' && HTTP=$(wget localhost:8123 -q --tries=1 --timeout=5 -O /dev/stdout) && echo "HTTP=$HTTP" && [ $HTTP == 'Ok.' ] && exit 0; echo "Attempt $i FAILED with above output"; sleep $(($i * 10)); done; exit 1
+          initialDelaySeconds: 20
+          periodSeconds: 60
+          # Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1.
+          timeoutSeconds: 200
+          # When a Pod starts and the probe fails, Kubernetes will try failureThreshold times before giving up
+          failureThreshold: 3
+        ports:
+        - containerPort: 8123
+          name: http
+        - containerPort: 9000
+          name: native
+        - containerPort: 9009
+          name: replication
+        resources:
+          requests:
+            memory: "1Gi"
+            cpu: "0.5"
+        volumeMounts:
+        - name: etc-clickhouse-server-config-d-common-volume
+          mountPath: /etc/clickhouse-server/config.d/remote_servers.xml
+          subPath: remote_servers.xml
+        - name: etc-clickhouse-server-config-d-common-volume
+          mountPath: /etc/clickhouse-server/config.d/zookeeper.xml
+          subPath: zookeeper.xml
+        - name: etc-clickhouse-server-config-d-macros-volume
+          mountPath: /etc/clickhouse-server/config.d/macros.xml
+          subPath: macros.xml
+      volumes:
+      - name: etc-clickhouse-server-config-d-common-volume
+        configMap:
+          name: etc-clickhouse-server-config-d-common
+      - name: etc-clickhouse-server-config-d-macros-volume
+        configMap:
+          name: etc-clickhouse-server-config-d-macros-ch-node-1
diff --git a/manifests/clickhouse/chpod1.yaml b/manifests/clickhouse/chpod1.yaml
@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: etc-clickhouse-server-config-d-macros-chpod1
+data:
+  macros.xml: |-
+    <yandex>
+        <macros>
+            <shard>1</shard>
+            <replica>1</replica>
+        </macros>
+    </yandex>
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: chpod1
+  labels:
+    app: chpod1
+    dns: chi1
+spec:
+  hostname: chpod1
+  subdomain: chpod-hs
+  containers:
+  - name: chpod1
+    image: yandex/clickhouse-server
+    livenessProbe:
+      tcpSocket:
+        port: 8123
+#        port: 9000
+      # Number of seconds after the container has started before liveness or readiness probes are initiated
+      initialDelaySeconds: 10
+      # How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1
+      periodSeconds: 10
+      # Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1.
+      timeoutSeconds: 10
+      # Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1
+      successThreshold: 1
+      # When a Pod starts and the probe fails, Kubernetes will try failureThreshold times before giving up
+      failureThreshold: 3
+    readinessProbe:
+      exec:
+        # Try 5 times to access both native and http API entry points.
+        # In case of failure sleep for i*10 sec and retry
+        # This command can take long to to run, so adjust 'timeoutSeconds' accordingly
+        command:
+        - /bin/bash
+        - -c
+        - for i in 1 2 3 4 5; do echo "Attempt $i" && echo -n "version=" && clickhouse-client -q 'select version()' && HTTP=$(wget localhost:8123 -q --tries=1 --timeout=5 -O /dev/stdout) && echo "HTTP=$HTTP" && [ $HTTP == 'Ok.' ] && exit 0; echo "Attempt $i FAILED with above output"; sleep $(($i * 10)); done; exit 1
+      initialDelaySeconds: 20
+      periodSeconds: 60
+      # Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1.
+      timeoutSeconds: 200
+      # When a Pod starts and the probe fails, Kubernetes will try failureThreshold times before giving up
+      failureThreshold: 3
+    volumeMounts:
+    - name: etc-clickhouse-server-config-d-common-volume
+      mountPath: /etc/clickhouse-server/config.d/remote_servers.xml
+      subPath: remote_servers.xml
+    - name: etc-clickhouse-server-config-d-common-volume
+      mountPath: /etc/clickhouse-server/config.d/zookeeper.xml
+      subPath: zookeeper.xml
+    - name: etc-clickhouse-server-config-d-macros-volume
+      mountPath: /etc/clickhouse-server/config.d/macros.xml
+      subPath: macros.xml
+  volumes:
+  - name: etc-clickhouse-server-config-d-common-volume
+    configMap:
+      name: etc-clickhouse-server-config-d-common
+  - name: etc-clickhouse-server-config-d-macros-volume
+    configMap:
+      name: etc-clickhouse-server-config-d-macros-chpod2
+