8000 Reload user passwords from a Secret without restarting ClickHouse by alex-zaitsev · Pull Request #2100 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/metrics_exporter/app/metrics_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func Run() {
log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt)

// Initialize k8s API clients
kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
kubeClient, _, chopClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)

// Create operator instance
chop.New(kubeClient, chopClient, chopConfigFile)
Expand Down
36 changes: 36 additions & 0 deletions cmd/operator/app/informer_scope_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,48 @@ import (
core "k8s.io/api/core/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
fakeKube "k8s.io/client-go/kubernetes/fake"
fakeMetadata "k8s.io/client-go/metadata/fake"
"k8s.io/client-go/tools/cache"
ctrlCache "sigs.k8s.io/controller-runtime/pkg/cache"

"github.com/altinity/clickhouse-operator/pkg/controller/chi"
)

// The password watcher must use PartialObjectMetadata, never full Secret values.
// This tests the factory used by initClickHouse and the cache it actually builds.
func TestPasswordSecretInformerCachesOnlyMetadata(t *testing.T) {
scheme := fakeMetadata.NewTestScheme()
require.NoError(t, meta.AddMetaToScheme(scheme))
client := fakeMetadata.NewSimpleMetadataClient(scheme,
&meta.PartialObjectMetadata{
TypeMeta: meta.TypeMeta{APIVersion: "v1", Kind: "Secret"},
ObjectMeta: meta.ObjectMeta{
Namespace: "ns", Name: "passwords", ResourceVersion: "1",
},
},
&meta.PartialObjectMetadata{
TypeMeta: meta.TypeMeta{APIVersion: "v1", Kind: "Secret"},
ObjectMeta: meta.ObjectMeta{
Namespace: "another-ns", Name: "not-watched", ResourceVersion: "1",
},
},
)
factory, informer := newPasswordSecretMetadataInformerFactory(client, "ns")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer factory.Shutdown()
defer cancel()

factory.Start(ctx.Done())
require.True(t, cache.WaitForCacheSync(ctx.Done(), informer.HasSynced))

cached := informer.GetStore().List()
require.Len(t, cached, 1)
secret, ok := cached[0].(*meta.PartialObjectMetadata)
require.True(t, ok, "Secret data must not be cached")
require.Equal(t, "ns", secret.Namespace)
require.Equal(t, "passwords", secret.Name)
}

// TestKubeInformerFactoryFiltersToCHOPGeneratedObjects pins the CHI wiring, not the helper that
// builds the selector.
//
Expand Down
2 changes: 1 addition & 1 deletion cmd/operator/app/thread_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func newBackupCacheOptions(log logr.Logger, cacheNamespaces []string) cache.Opti
// chop.GetClientset terminates the process when it cannot build a client, so it runs here, once,
// outside the retry; initClickHouse has made the identical call by this point.
func newBackupInitializer() func(context.Context) error {
kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
kubeClient, _, chopClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
cr := chiKube.NewCR(chopClient, kubeClient)
return func(ctx context.Context) error {
if !waitForBackupCRDs(ctx, kubeClient.Discovery(), backupCRDsPollInterval) {
Expand Down
22 changes: 21 additions & 1 deletion cmd/operator/app/thread_chi.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ import (
"github.com/altinity/clickhouse-operator/pkg/chop"
chopinformers "github.com/altinity/clickhouse-operator/pkg/client/informers/externalversions"
"github.com/altinity/clickhouse-operator/pkg/controller/chi"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/metadata"
"k8s.io/client-go/metadata/metadatainformer"
"k8s.io/client-go/tools/cache"
)

// Prometheus exporter defaults
Expand All @@ -43,6 +47,14 @@ func init() {

var chiController *chi.Controller

// newPasswordSecretMetadataInformerFactory watches names and resource versions,
// not Secret data. No periodic resync is needed for rotation events.
func newPasswordSecretMetadataInformerFactory(client metadata.Interface, namespace string) (metadatainformer.SharedInformerFactory, cache.SharedIndexInformer) {
factory := metadatainformer.NewFilteredSharedInformerFactory(client, 0, namespace, nil)
informer := factory.ForResource(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}).Informer()
return factory, informer
}

// initClickHouse is an entry point of the application
func initClickHouse(ctx context.Context) {
log.S().P()
Expand All @@ -54,7 +66,7 @@ func initClickHouse(ctx context.Context) {
}

// Initialize k8s API clients
kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
kubeClient, extClient, chopClient, dynamicClient, metadataClient := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)

// Create operator instance. The chopconf load inside chop.New gates on
// clickhouse.security.kubernetes.allowInsecure BEFORE the first network call,
Expand Down Expand Up @@ -112,13 +124,21 @@ func initClickHouse(ctx context.Context) {
kubeInformerFactory,
)

// Source password Secrets do not carry the operator app label. Watch their
// metadata without caching Secret data; values are fetched only when rendering.
secretMetadataFactory, secretMetadataInformer := newPasswordSecretMetadataInformerFactory(
metadataClient, chop.Config().GetInformerNamespace(),
)
chiController.WatchPasswordSecrets(chopInformerFactory, secretMetadataInformer)

// Start CHK watcher (if enabled by config)
chiController.StartCHKWatcher(ctx)

// Start Informers
kubeInformerFactory.Start(ctx.Done())
chopInformerFactory.Start(ctx.Done())
chopConfigInformerFactory.Start(ctx.Done())
secretMetadataFactory.Start(ctx.Done())
}

// runClickHouse is an entry point of the application
Expand Down
2 changes: 1 addition & 1 deletion cmd/operator/app/thread_keeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func newKeeperCacheOptions(log logr.Logger, cacheNamespaces []string) cache.Opti
func newKeeperInitializer() func(context.Context) error {
// Build the apiextensions client for CRD deletion checks during CHK cleanup.
// Uses the same kubeConfigFile/masterURL package vars as the CHI thread.
_, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
_, extClient, _, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)

return func(ctx context.Context) error {
return initKeeper(ctx, extClient)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1038,8 +1038,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1038,8 +1038,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1038,8 +1038,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
12 changes: 8 additions & 4 deletions deploy/operator/clickhouse-operator-install-ansible.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1045,8 +1045,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down Expand Up @@ -2986,8 +2988,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
12 changes: 8 additions & 4 deletions deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1033,8 +1033,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down Expand Up @@ -2962,8 +2964,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
12 changes: 8 additions & 4 deletions deploy/operator/clickhouse-operator-install-bundle.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1038,8 +1038,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down Expand Up @@ -2979,8 +2981,10 @@ spec:
Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers

any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets
secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml
it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle
by default the secret value is passed in `pod.spec.containers.env` and generated with from_env=XXX in /etc/clickhouse-server/users.d/chop-generated-users.xml
a password change is then picked up only after the Pod is recreated
set `valueFrom.hotReload: true` on user password, password_sha256_hex, or password_double_sha1_hex to publish that user as chop-generated-hot-reload-users.xml in a CHI-owned Secret, mounted into users.d next to chop-generated-users.xml, and reload it without a Pod restart
hotReload on any other field is rejected

look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples

Expand Down
Loading
Loading
0