Slach · Jan 29, 2024 · Dec 26, 2023 · Jan 24, 2024 · Jan 29, 2024
diff --git a/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml b/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml
@@ -0,0 +1,44 @@
+---
+# Fake Service to drop-in replacement in tests
+apiVersion: v1
+kind: Service
+metadata:
+  # DNS would be like zookeeper.namespace.svc
+  name: zookeeper
+  labels:
+    app: zookeeper
+spec:
+  ports:
+    - port: 2181
+      name: client
+    - port: 7000
+      name: prometheus
+  selector:
+    app: clickhouse-keeper
+    what: node
+---
+apiVersion: "clickhouse-keeper.altinity.com/v1"
+kind: "ClickHouseKeeperInstallation"
+metadata:
+  name: clickhouse-keeper
+spec:
+  configuration:
+    clusters:
+      - name: "simple-1"
+        layout:
+          replicasCount: 1
+    settings:
+      logger/level: "trace"
+      logger/console: "true"
+      listen_host: "0.0.0.0"
+      keeper_server/storage_path: /var/lib/clickhouse-keeper
+      keeper_server/tcp_port: "2181"
+      keeper_server/four_letter_word_white_list: "*"
+      keeper_server/coordination_settings/raft_logs_level: "information"
+      keeper_server/raft_configuration/server/port: "9444"
+      prometheus/endpoint: "/metrics"
+      prometheus/port: "7000"
+      prometheus/metrics: "true"
+      prometheus/events: "true"
+      prometheus/asynchronous_metrics: "true"
+      prometheus/status_info: "false"
diff --git a/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml b/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml
@@ -0,0 +1,44 @@
+---
+# Fake Service to drop-in replacement in tests
+apiVersion: v1
+kind: Service
+metadata:
+  # DNS would be like zookeeper.namespace.svc
+  name: zookeeper
+  labels:
+    app: zookeeper
+spec:
+  ports:
+    - port: 2181
+      name: client
+    - port: 7000
+      name: prometheus
+  selector:
+    app: clickhouse-keeper
+    what: node
+---
+apiVersion: "clickhouse-keeper.altinity.com/v1"
+kind: "ClickHouseKeeperInstallation"
+metadata:
+  name: clickhouse-keeper
+spec:
+  configuration:
+    clusters:
+      - name: "simple-3"
+        layout:
+          replicasCount: 3
+    settings:
+      logger/level: "trace"
+      logger/console: "true"
+      listen_host: "0.0.0.0"
+      keeper_server/storage_path: /var/lib/clickhouse-keeper
+      keeper_server/tcp_port: "2181"
+      keeper_server/four_letter_word_white_list: "*"
+      keeper_server/coordination_settings/raft_logs_level: "information"
+      keeper_server/raft_configuration/server/port: "9444"
+      prometheus/endpoint: "/metrics"
+      prometheus/port: "7000"
+      prometheus/metrics: "true"
+      prometheus/events: "true"
+      prometheus/asynchronous_metrics: "true"
+      prometheus/status_info: "false"
diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py
@@ -118,7 +118,7 @@ def delete_all_keeper(ns=None):
                     launch(f"delete {resource_type} -n {current().context.test_namespace} {name}", ok_to_fail=True)
 
 
-def create_and_check(manifest, check, ns=None, shell=None, timeout=1800):
+def create_and_check(manifest, check, kind="chi", ns=None, shell=None, timeout=1800):
     chi_name = yaml_manifest.get_chi_name(util.get_full_path(f"{manifest}"))
 
     # state_field = ".status.taskID"
@@ -138,8 +138,10 @@ def create_and_check(manifest, check, ns=None, shell=None, timeout=1800):
     else:
         # Wait for reconcile to start before performing other checks. In some cases it does not start, so we can pass
         # wait_field_changed("chi", chi_name, state_field, prev_state, ns)
-        wait_chi_status(chi_name, "InProgress", ns=ns, retries=3, throw_error=False, shell=shell)
-        wait_chi_status(chi_name, "Completed", ns=ns, shell=shell)
+        wait_field(kind=kind, name=chi_name, field=".status.status", value="InProgress"
+                   , ns=ns, retries=3, throw_error=False, shell=shell)
+        wait_field(kind=kind, name=chi_name, field=".status.status", value="Completed"
+                   , ns=ns, shell=shell)
 
     if "object_counts" in check:
         wait_objects(chi_name, check["object_counts"], ns=ns, shell=shell)

diff --git a/tests/e2e/manifests/chi/test-048-clickhouse-keeper.yaml b/tests/e2e/manifests/chi/test-048-clickhouse-keeper.yaml
@@ -0,0 +1,17 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: test-048-clickhouse-keeper
+spec:
+  useTemplates:
+    - name: clickhouse-version
+  configuration:
+    zookeeper:
+      nodes:
+        - host: clickhouse-keeper
+          port: 2181
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 2
+          replicasCount: 1
diff --git a/tests/e2e/manifests/chi/test-cluster-for-clickhouse-keeper_with_CHKI-1.yaml b/tests/e2e/manifests/chi/test-cluster-for-clickhouse-keeper_with_CHKI-1.yaml
@@ -0,0 +1,18 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: test-cluster-for-zk
+spec:
+  useTemplates:
+    - name: clickhouse-latest-version
+    - name: persistent-volume
+  configuration:
+    zookeeper:
+      nodes:
+        - host: clickhouse-keeper
+          port: 2181
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 1
+          replicasCount: 1
diff --git a/tests/e2e/manifests/chi/test-cluster-for-clickhouse-keeper_with_CHKI-2.yaml b/tests/e2e/manifests/chi/test-cluster-for-clickhouse-keeper_with_CHKI-2.yaml
@@ -0,0 +1,18 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: test-cluster-for-zk
+spec:
+  useTemplates:
+    - name: clickhouse-latest-version
+    - name: persistent-volume
+  configuration:
+    zookeeper:
+      nodes:
+        - host: clickhouse-keeper
+          port: 2181
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 1
+          replicasCount: 2
diff --git a/tests/e2e/test_keeper.py b/tests/e2e/test_keeper.py
@@ -8,19 +8,20 @@
 from testflows.core import *
 from e2e.steps import *
 
+from requirements.requirements import *
 
 def wait_keeper_ready(keeper_type="zookeeper", pod_count=3, retries_number=10):
     svc_name = "zookeeper-client" if keeper_type == "zookeeper-operator" else "zookeeper"
     expected_containers = "1/1"
-    expected_pod_prefix = "clickhouse-keeper" if keeper_type == "clickhouse-keeper" else "zookeeper"
+    expected_pod_prefix = "clickhouse-keeper" if "clickhouse-keeper" in keeper_type else "zookeeper"
     for i in range(retries_number):
         ready_pods = kubectl.launch(
-            f"get pods | grep {expected_pod_prefix} | grep Running | grep '{expected_containers}' | wc -l"
+            f"get pods | grep {expected_pod_prefix} | grep Running | grep '{expected_containers}' | wc -l", ok_to_fail=True
         )
         ready_endpoints = "0"
         if ready_pods == str(pod_count):
             ready_endpoints = kubectl.launch(
-                f"get endpoints {svc_name} -o json | jq '.subsets[].addresses[].ip' | wc -l"
+                f"get endpoints {svc_name} -o json | jq '.subsets[].addresses[].ip' | wc -l", ok_to_fail=True
             )
             if ready_endpoints == str(pod_count):
                 break
@@ -104,6 +105,7 @@ def check_zk_root_znode(chi, keeper_type, pod_count, retry_count=15):
             "zookeeper": "2",
             "zookeeper-operator": "3",
             "clickhouse-keeper": "2",
+            "clickhouse-keeper_with_CHKI": "2",
         }
         if expected_out[keeper_type] != out.strip(" \t\r\n") and i + 1 < retry_count:
             with Then(f"{keeper_type} system.zookeeper not ready, wait {(i + 1) * 3} sec"):
@@ -348,6 +350,18 @@ def test_clickhouse_keeper_rescale(self):
     )
 
 
+@TestScenario
+@Name("test_clickhouse_keeper_rescale_CHKI using ClickHouseKeeperInstallation. Check KEEPER scale-up / scale-down cases")
+@Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_Kind_ClickHouseKeeperInstallation("1.0"))
+def test_clickhouse_keeper_rescale_CHKI(self):
+    test_keeper_rescale_outline(
+        keeper_type="clickhouse-keeper_with_CHKI",
+        pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0",
+        keeper_manifest_1_node="clickhouse-keeper-1-node-for-test-only.yaml",
+        keeper_manifest_3_node="clickhouse-keeper-3-node-for-test-only.yaml",
+    )
+
+
 @TestScenario
 @Name("test_zookeeper_operator_rescale. Check Zookeeper OPERATOR scale-up / scale-down cases")
 def test_zookeeper_operator_rescale(self):
@@ -434,7 +448,7 @@ def test_keeper_probes_outline(
             )
 
     with Then("Check liveness and readiness probes fail"):
-        zk_pod_prefix = "clickhouse-keeper" if keeper_type == "clickhouse-keeper" else "zookeeper"
+        zk_pod_prefix = "clickhouse-keeper" if "clickhouse-keeper" in keeper_type else "zookeeper"
         for zk_pod in range(3):
             out = kubectl.launch(f"describe pod {zk_pod_prefix}-{zk_pod}")
             assert "probe failed" not in out, "all probes shall be successful"
@@ -503,6 +517,20 @@ def test_clickhouse_keeper_probes_workload(self):
     )
 
 
+@TestScenario
+@Name(
+    "test_clickhouse_keeper_probes_workload_with_CHKI. Liveness + Readiness probes shall works fine "
+    "under workload in multi-datacenter installation"
+)
+@Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_Kind_ClickHouseKeeperInstallation("1.0"))
+def test_clickhouse_keeper_probes_workload_with_CHKI(self):
+    test_keeper_probes_outline(
+        keeper_type="clickhouse-keeper_with_CHKI",
+        keeper_manifest_1_node="clickhouse-keeper-1-node-for-test-only.yaml",
+        keeper_manifest_3_node="clickhouse-keeper-3-node-for-test-only.yaml",
+    )
+
+
 @TestModule
 @Name("e2e.test_keeper")
 def test(self):
@@ -517,13 +545,15 @@ def test(self):
     all_tests = [
         test_zookeeper_operator_rescale,
         test_clickhouse_keeper_rescale,
+        test_clickhouse_keeper_rescale_CHKI,
         test_zookeeper_pvc_scaleout_rescale,
         test_zookeeper_rescale,
 
         test_zookeeper_probes_workload,
         test_zookeeper_pvc_probes_workload,
         test_zookeeper_operator_probes_workload,
         test_clickhouse_keeper_probes_workload,
+        test_clickhouse_keeper_probes_workload_with_CHKI,
     ]
 
     util.clean_namespace(delete_chi=True, delete_keeper=True)

diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py
@@ -4312,6 +4312,65 @@ def test_047(self):
             delete_test_namespace()
 
 
+@TestScenario
+@Name("test_048. Clickhouse-keeper")
+@Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_Kind_ClickHouseKeeperInstallation("1.0"))
+def test_048(self):
+    """Check clickhouse-operator support ClickHouseKeeperInstallation."""
+
+    create_shell_namespace_clickhouse_template()
+    util.require_keeper(keeper_type="clickhouse-keeper_with_CHKI",
+                        keeper_manifest="clickhouse-keeper-3-node-for-test-only.yaml")
+    manifest = f"manifests/chi/test-048-clickhouse-keeper.yaml"
+    chi = yaml_manifest.get_chi_name(util.get_full_path(manifest))
+    cluster = "default"
+    with Given("CHI with 2 shards"):
+        kubectl.create_and_check(
+            manifest=manifest,
+            check={
+                "pod_count": 2,
+                "do_not_delete": 1,
+                },
+            )
+    numbers = 100
+    with When("I create distributed table"):
+        create_table = """
+            CREATE TABLE test_local_048 ON CLUSTER 'default' (a UInt32)
+            Engine = ReplicatedMergeTree('/clickhouse/{installation}/tables/{shard}/{database}/{table}', '{replica}')
+            PARTITION BY tuple()
+            ORDER BY a
+            """.replace(
+            "\r", ""
+        ).replace(
+            "\n", ""
+        )
+        clickhouse.query(chi, create_table)
+        clickhouse.query(
+            chi,
+            "CREATE TABLE test_distr_048 ON CLUSTER 'default' AS test_local_048 "
+            "Engine = Distributed('default', default, test_local_048, a%2)",
+        )
+
+    with And("I insert data in the distributed table"):
+        clickhouse.query(chi, f"INSERT INTO test_distr_048 select * from numbers({numbers})")
+
+    with Then("I check clickhouse-keeper properly works"):
+        out = clickhouse.query(chi, "SELECT count(*) from test_local_048", host=f"chi-{chi}-{cluster}-0-0-0")
+        assert out == f"{numbers // 2}", error()
+        out = clickhouse.query(chi, "SELECT count(*) from test_local_048", host=f"chi-{chi}-{cluster}-1-0-0")
+        assert out == f"{numbers // 2}", error()
+        out = clickhouse.query(chi, "SELECT count(*) from test_distr_048", host=f"chi-{chi}-{cluster}-0-0-0")
+        assert out == f"{numbers}", error()
+        out = clickhouse.query(chi, "SELECT count(*) from test_distr_048", host=f"chi-{chi}-{cluster}-1-0-0")
+        assert out == f"{numbers}", error()
+
+    with Finally("I clean up"):
+        with By("deleting chi"):
+            kubectl.delete_chi(chi)
+        with And("deleting test namespace"):
+            delete_test_namespace()
+
+
 @TestModule
 @Name("e2e.test_operator")
 @Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_APIVersion("1.0"),

diff --git a/tests/e2e/util.py b/tests/e2e/util.py
@@ -85,6 +85,11 @@ def require_keeper(keeper_manifest="", keeper_type="zookeeper", force_install=Fa
                 "clickhouse-keeper-1-node-256M-for-test-only.yaml" if keeper_manifest == "" else keeper_manifest
             )
             keeper_manifest = f"../../deploy/clickhouse-keeper/clickhouse-keeper-manually/{keeper_manifest}"
+        if keeper_type == "clickhouse-keeper_with_CHKI":
+            keeper_manifest = (
+                "clickhouse-keeper-1-node-for-test-only.yaml" if keeper_manifest == "" else keeper_manifest
+            )
+            keeper_manifest = f"../../deploy/clickhouse-keeper/clickhouse-keeper-with-CHK-resource/{keeper_manifest}"
         if keeper_type == "zookeeper-operator":
             keeper_manifest = "zookeeper-operator-1-node.yaml" if keeper_manifest == "" else keeper_manifest
             keeper_manifest = f"../../deploy/zookeeper/zookeeper-with-zookeeper-operator/{keeper_manifest}"
@@ -99,12 +104,14 @@ def require_keeper(keeper_manifest="", keeper_type="zookeeper", force_install=Fa
         expected_docs = {
             "zookeeper": 5 if "scaleout-pvc" in keeper_manifest else 4,
             "clickhouse-keeper": 7,
+            "clickhouse-keeper_with_CHKI": 2,
             "zookeeper-operator": 3 if "probes" in keeper_manifest else 1,
         }
         expected_pod_prefix = {
             "zookeeper": "zookeeper",
             "zookeeper-operator": "zookeeper",
             "clickhouse-keeper": "clickhouse-keeper",
+            "clickhouse-keeper_with_CHKI": "clickhouse-keeper"
         }
         assert (
             docs_count == expected_docs[keeper_type]
@@ -154,6 +161,8 @@ def install_clickhouse_and_keeper(
             keeper_manifest = "zookeeper-1-node-1GB-for-tests-only.yaml"
         if keeper_type == "clickhouse-keeper":
             keeper_manifest = "clickhouse-keeper-1-node-256M-for-test-only.yaml"
+        if keeper_type == "clickhouse-keeper_with_CHKI":
+            keeper_manifest = "clickhouse-keeper-1-node-for-test-only.yaml"
         if keeper_type == "zookeeper-operator":
             keeper_manifest = "zookeeper-operator-1-node.yaml"
 
@@ -172,7 +181,7 @@ def install_clickhouse_and_keeper(
 
         chi_manifest_data = yaml_manifest.get_manifest_data(get_full_path(chi_file))
         layout = chi_manifest_data["spec"]["configuration"]["clusters"][0]["layout"]
-        expected_nodes = 1 * layout["shardsCount"] * layout["replicasCount"]
+        expected_nodes = 1 * layout["replicasCount"] if ("clickhouse-keeper" in keeper_type) else 1 * layout["shardsCount"] * layout["replicasCount"]
         check = {
             "apply_templates": [
                 chi_template_file,