sunsingerus · Nov 13, 2023 · Aug 11, 2023
diff --git a/cmd/operator/app/clickhouse_operator.go b/cmd/operator/app/clickhouse_operator.go
@@ -71,7 +71,6 @@ func init() {
 	flag.BoolVar(&versionRequest, "version", false, "Display clickhouse-operator version and exit")
 	flag.BoolVar(&debugRequest, "debug", false, "Debug run")
 	flag.StringVar(&chopConfigFile, "config", "", "Path to clickhouse-operator config file.")
-	flag.StringVar(&kubeConfigFile, "kubeconfig", "", "Path to custom kubernetes config file. Makes sense if runs outside of the cluster only.")
 	flag.StringVar(&masterURL, "master", "", "The address of custom Kubernetes API server. Makes sense if runs outside of the cluster and not being specified in kube config file only.")
 	flag.StringVar(&metricsEP, "metrics-endpoint", defaultMetricsEndpoint, "The Prometheus exporter endpoint.")
 	flag.Parse()

diff --git a/cmd/operator/app/keeper_operator.go b/cmd/operator/app/keeper_operator.go
@@ -0,0 +1,56 @@
+package app
+
+import (
+	"os"
+
+	"github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.com/v1alpha1"
+	"github.com/altinity/clickhouse-operator/pkg/controller/chk"
+	appsv1 "k8s.io/api/apps/v1"
+	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+var (
+	scheme = apimachineryruntime.NewScheme()
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(v1alpha1.AddToScheme(scheme))
+}
+
+func KeeperRun() {
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	var log = ctrl.Log.WithName("keeper-runner")
+	log.Info("KeeperRun() called")
+
+	manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme: scheme,
+	})
+	if err != nil {
+		log.Error(err, "could not create manager")
+		os.Exit(1)
+	}
+
+	err = ctrl.
+		NewControllerManagedBy(manager).   // Create the Controller
+		For(&v1alpha1.ClickHouseKeeper{}). // ReplicaSet is the Application API
+		Owns(&appsv1.StatefulSet{}).       // ReplicaSet owns Pods created by it
+		Complete(&chk.ChkReconciler{
+			Client: manager.GetClient(),
+			Scheme: manager.GetScheme(),
+		})
+	if err != nil {
+		log.Error(err, "could not create controller")
+		os.Exit(1)
+	}
+
+	if err := manager.Start(ctrl.SetupSignalHandler()); err != nil {
+		log.Error(err, "could not start manager")
+		os.Exit(1)
+	}
+}
diff --git a/cmd/operator/main.go b/cmd/operator/main.go
@@ -15,10 +15,25 @@
 package main
 
 import (
+	"sync"
+
 	"github.com/altinity/clickhouse-operator/cmd/operator/app"
 )
 
 func main() {
-	// Application entry point
-	app.Run()
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		// Application entry point for chi, chit and chopconf
+		app.Run()
+	}()
+	go func() {
+		defer wg.Done()
+		// Application entry point for chk
+		app.KeeperRun()
+	}()
+
+	wg.Wait()
 }
diff --git a/deploy/builder/cat-clickhouse-operator-install-yaml.sh b/deploy/builder/cat-clickhouse-operator-install-yaml.sh
@@ -130,6 +130,14 @@ if [[ "${MANIFEST_PRINT_CRD}" == "yes" ]]; then
         OPERATOR_IMAGE="${OPERATOR_IMAGE}"      \
         OPERATOR_VERSION="${OPERATOR_VERSION}"  \
         envsubst
+
+    # Render CHK
+    SECTION_FILE_NAME="clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml"
+    ensure_file "${TEMPLATES_DIR}" "${SECTION_FILE_NAME}" "${REPO_PATH_TEMPLATES_PATH}"
+    render_separator
+    cat "${TEMPLATES_DIR}/${SECTION_FILE_NAME}" | \
+        OPERATOR_VERSION="${OPERATOR_VERSION}"  \
+        envsubst
 fi
 
 # Render RBAC section for ClusterRole

diff --git a/...lates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml b/...lates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml
@@ -0,0 +1,142 @@
+# Template Parameters:
+#
+# OPERATOR_VERSION=${OPERATOR_VERSION}
+#
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clickhousekeepers.clickhouse.com
+  labels:
+    clickhouse.com/chop: ${OPERATOR_VERSION}
+spec:
+  group: clickhouse.com
+  scope: Namespaced
+  names:
+    kind: ClickHouseKeeper
+    singular: clickhousekeeper
+    plural: clickhousekeepers
+    shortNames:
+      - chk
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - name: status
+          type: string
+          description: CHK status
+          jsonPath: .status.status
+        - name: replicas
+          type: integer
+          description: Replica count
+          priority: 1 # show in wide view
+          jsonPath: .status.replicas
+        - name: age
+          type: date
+          description: Age of the resource
+          # Displayed in all priorities
+          jsonPath: .metadata.creationTimestamp
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster"
+          properties:
+            apiVersion:
+              type: string
+              description: 'APIVersion defines the versioned schema of this representation
+                of an object. Servers should convert recognized schemas to the latest
+                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            kind:
+              type: string
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            metadata:
+              type: object
+            status:
+              type: object
+              description: "Current ClickHouseKeeper manifest status, contains many fields like overall status, desired replicas and ready replica list with their endpoints"
+              properties:
+                status:
+                  type: string
+                  description: "Status"
+                replicas:
+                  type: integer
+                  format: int32
+                  description: Replicas is the number of number of desired replicas in the cluster
+                readyReplicas:
+                  type: array
+                  description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster
+                  items:
+                    type: object
+                    properties:
+                      host:
+                        type: string
+                        description: "dns name or ip address for Keeper node"
+                      port:
+                        type: integer
+                        minimum: 0
+                        maximum: 65535
+                        description: "TCP port which used to connect to Keeper node"
+                      secure:
+                        type: string
+                        description: "if a secure connection to Keeper is required"
+            spec:
+              type: object
+              description: KeeperSpec defines the desired state of a Keeper cluster
+              properties:
+                type:
+                  type: string
+                  description: This is used by chi reconciliation.
+                  enum:
+                    - "Container"
+                    - "Embedded"
+                replicas:
+                  type: integer
+                  format: int32
+                  description: "Replicas is the expected size of the keeper cluster.
+                    The valid range of size is from 1 to 7."
+                  minimum: 1
+                  maximum: 7
+                settings:
+                  type: object
+                  description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance"
+                  x-kubernetes-preserve-unknown-fields: true
+                podTemplate:
+                  type: object
+                  description: |
+                    podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone
+                  properties:
+                    metadata:
+                      type: object
+                      description: |
+                        allows passing standard object's metadata from template to Pod
+                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                      x-kubernetes-preserve-unknown-fields: true
+                    spec:
+                      # TODO specify PodSpec
+                      type: object
+                      description: "allows defining whole Pod.spec inside StatefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details"
+                      x-kubernetes-preserve-unknown-fields: true
+                volumeClaimTemplates:
+                  type: array
+                  description: "allows defining template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mounting clickhouse `data`, clickhouse `logs` or something else"
+                  items:
+                    type: object
+                    properties:
+                      metadata:
+                        type: object
+                        description: |
+                          allows to pass standard object's metadata from template to PVC
+                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+                        x-kubernetes-preserve-unknown-fields: true
+                      spec:
+                        type: object
+                        description: |
+                          allows define all aspects of `PVC` resource
+                          More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+                        x-kubernetes-preserve-unknown-fields: true
diff --git a/...tes-install-bundle/clickhouse-operator-install-yaml-template-02-section-rbac-02-role.yaml b/...tes-install-bundle/clickhouse-operator-install-yaml-template-02-section-rbac-02-role.yaml
@@ -148,6 +148,33 @@ rules:
     - patch
     - create
     - delete
+- apiGroups:
+    - clickhouse.com
+  resources:
+    - clickhousekeepers
+  verbs:
+    - get
+    - patch
+    - update
+    - delete
+    - list
+    - watch
+- apiGroups:
+    - clickhouse.com
+  resources:
+    - clickhousekeepers/finalizers
+  verbs:
+    - update
+- apiGroups:
+    - clickhouse.com
+  resources:
+    - clickhousekeepers/status
+  verbs:
+    - get
+    - update
+    - patch
+    - create
+    - delete
 - apiGroups:
     - ""
   resources:

diff --git a/deploy/clickhouse-keeper/chk-1-node-simplest.yaml b/deploy/clickhouse-keeper/chk-1-node-simplest.yaml
@@ -0,0 +1,6 @@
+apiVersion: "clickhouse.com/v1alpha1"
+kind: "ClickHouseKeeper"
+metadata:
+  name: chk-1-node-simplest
+spec:
+  replicas: 1
diff --git a/deploy/clickhouse-keeper/chk-1-node.yaml b/deploy/clickhouse-keeper/chk-1-node.yaml
@@ -0,0 +1,59 @@
+apiVersion: "clickhouse.com/v1alpha1"
+kind: "ClickHouseKeeper"
+metadata:
+  name: chk-1-node
+spec:
+  replicas: 1
+  settings:
+    logger/level: "trace"
+    logger/console: "true"
+    listen_host: "0.0.0.0"
+    keeper_server/storage_path: /var/lib/clickhouse-keeper
+    keeper_server/tcp_port: "2181"
+    keeper_server/four_letter_word_white_list: "*"
+    keeper_server/coordination_settings/raft_logs_level: "information"
+    keeper_server/raft_configuration/server/port: "9444"
+    prometheus/endpoint: "/metrics"
+    prometheus/port: "7000"
+    prometheus/metrics: "true"
+    prometheus/events: "true"
+    prometheus/asynchronous_metrics: "true"
+    prometheus/status_info: "false"
+  podTemplate:
+    metadata:
+      labels:
+        app: clickhouse-keeper
+        what: node
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 50
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: "app"
+                      operator: In
+                      values:
+                        - clickhouse-keeper
+                topologyKey: "kubernetes.io/hostname"
+      containers:
+        - name: clickhouse-keeper
+          imagePullPolicy: IfNotPresent
+          image: "clickhouse/clickhouse-keeper:head-alpine"
+          resources:
+            requests:
+              memory: "256M"
+              cpu: "1"
+            limits:
+              memory: "4Gi"
+              cpu: "2"
+  volumeClaimTemplates:
+    - metadata:
+        name: both-paths
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 25Gi