8000 Backup: sidecar or per-shard Pod transport, multi-period schedules, bash or Python orchestrator as operator config by Slach · Pull Request #2148 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 44 additions & 24 deletions cmd/operator/app/thread_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (

"github.com/go-logr/logr"

batch "k8s.io/api/batch/v1"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
apiMachineryRuntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/discovery"
Expand All @@ -35,14 +34,15 @@ import (
"sigs.k8s.io/controller-runtime/pkg/log/zap"
ctrlManager "sigs.k8s.io/controller-runtime/pkg/manager"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/source"

log "github.com/altinity/clickhouse-operator/pkg/announcer"
api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
"github.com/altinity/clickhouse-operator/pkg/apis/deployment"
"github.com/altinity/clickhouse-operator/pkg/chop"
controller "github.com/altinity/clickhouse-operator/pkg/controller/chbackup"
chiKube "github.com/altinity/clickhouse-operator/pkg/controller/chi/kube"
"github.com/altinity/clickhouse-operator/pkg/interfaces"
"github.com/altinity/clickhouse-operator/pkg/model/chbackup"
"github.com/altinity/clickhouse-operator/pkg/util"
)

Expand Down Expand Up @@ -117,32 +117,50 @@ func missingBackupKinds(discovery discovery.DiscoveryInterface) ([]string, error
return missing, nil
}

// newBackupCacheOptions builds the backup manager cache config. Jobs and CronJobs are narrowed to
// the ones the backup controllers build; without that, the cache would hold every Job and CronJob
// in the watched namespaces. The custom resources and installations stay unfiltered: they are
// user-authored and carry no operator label. Installations are trimmed instead, to the metadata
// newBackupCacheOptions builds the backup manager cache config. The custom resources and installations stay
// unfiltered: they are user-authored and carry no operator label. Installations are trimmed instead, to the metadata
// and status the watches need.
//
// log is a parameter rather than the package-level logger because a logr zero value discards Info
// silently, so reading that var would let this line vanish if the sole call site ever moved.
func newBackupCacheOptions(log logr.Logger, cacheNamespaces []string) cache.Options {
selector := chbackup.JobSelector()
byObject := map[client.Object]cache.ByObject{
&batch.Job{}: {Label: selector},
&batch.CronJob{}: {Label: selector},
&api.ClickHouseInstallation{}: {Transform: controller.TrimInstallation},
}

// A wrong selector fails silently - the cache just stays empty - so the effective one goes in
// the boot log.
log.Info("init backup - cache label selector", "selector", selector.String(), "types", len(byObject))
log.Info("init backup - cache", "types", len(byObject))

return cache.Options{
DefaultNamespaces: newCacheNamespaces(cacheNamespaces),
ByObject: byObject,
}
}

// backupLeaderElectionID names the Lease the backup controllers are elected by.
const backupLeaderElectionID = "clickhouse-operator-backup"

// newBackupManagerOptions builds the options of the backup manager. Its controllers are leader-elected, in the
// namespace of the operator: they run backups and restores, and prune backups by schedule, which two operators side
// by side must never do twice. An operator that runs outside a pod, with no namespace of its own,
// runs them unelected.
func newBackupManagerOptions(log logr.Logger) ctrlRuntime.Options {
options := ctrlRuntime.Options{
Scheme: backupScheme,
Cache: newBackupCacheOptions(log, chop.Config().GetCacheNamespaces()),
// Disable controller-runtime's built-in metrics listener: operator metrics, backup
// metrics included, are served by pkg/metrics/operator on the operator's own endpoint.
Metrics: metricsserver.Options{BindAddress: "0"},
}
if namespace, ok := chop.GetRuntimeParam(deployment.OPERATOR_POD_NAMESPACE); ok && (namespace != "") {
options.LeaderElection = true
options.LeaderElectionID = backupLeaderElectionID
options.LeaderElectionNamespace = namespace
// A stopping operator hands the Lease over at once, so a restart does not wait for it to expire
options.LeaderElectionReleaseOnCancel = true
}
log.Info("init backup - leader election", "enabled", options.LeaderElection, "namespace", options.LeaderElectionNamespace)
return options
}

// newBackupInitializer resolves the clients that must be built exactly once, then returns the
// step that is safe to retry: it waits for the backup CRDs to be served and builds the manager.
//
Expand Down Expand Up @@ -186,13 +204,7 @@ func initBackup(ctx context.Context, cr interfaces.IKubeCR) error {
return err
}

backupManager, err = ctrlRuntime.NewManager(restConfig, ctrlRuntime.Options{
Scheme: backupScheme,
Cache: newBackupCacheOptions(backupLogger, chop.Config().GetCacheNamespaces()),
// Disable controller-runtime's built-in metrics listener: operator metrics, backup
// metrics included, are served by pkg/metrics/operator on the operator's own endpoint.
Metrics: metricsserver.Options{BindAddress: "0"},
})
backupManager, err = ctrlRuntime.NewManager(restConfig, newBackupManagerOptions(backupLogger))
if err != nil {
// Not terminal: building the manager depends on the API server.
backupLogger.Error(err, "init backup - unable to ctrlRuntime.NewManager")
Expand All @@ -206,6 +218,11 @@ func initBackup(ctx context.Context, cr interfaces.IKubeCR) error {
return fmt.Errorf("%w: %w", errInitTerminal, err)
}

// The runs of the backup controllers live as long as the manager leads
if err = backupManager.Add(controller.Runs()); err != nil {
return fmt.Errorf("%w: %w", errInitTerminal, err)
}

// Logged by the manager itself once it runs its controllers, rather than after Start returns,
// which happens only at shutdown.
if err = backupManager.Add(ctrlManager.RunnableFunc(func(context.Context) error {
Expand All @@ -224,10 +241,14 @@ func initBackup(ctx context.Context, cr interfaces.IKubeCR) error {
func setupBackupControllers(manager ctrlRuntime.Manager, cr interfaces.IKubeCR) error {
c, apiReader, scheme := manager.GetClient(), manager.GetAPIReader(), manager.GetScheme()

// A run tells the resource it works for when its status changes, which the resource then records
runs := func(kind string) source.Source {
return source.Channel(controller.Runs().Notifications(kind), &handler.EnqueueRequestForObject{})
}
if err := ctrlRuntime.
NewControllerManagedBy(manager).
For(&api.ClickHouseBackup{}).
Owns(&batch.Job{}).
WatchesRawSource(runs(api.ClickHouseBackupCRDResourceKind)).
WithEventFilter(controller.WatchedNamespacePredicate()).
Complete(controller.NewBackupController(c, apiReader, scheme, cr)); err != nil {
return err
Expand All @@ -237,14 +258,13 @@ func setupBackupControllers(manager ctrlRuntime.Manager, cr interfaces.IKubeCR)
if err := ctrlRuntime.
NewControllerManagedBy(manager).
For(&api.ClickHouseBackupSchedule{}).
Owns(&batch.CronJob{}).
WatchesRawSource(runs(api.ClickHouseBackupScheduleCRDResourceKind)).
Watches(
&api.ClickHouseInstallation{},
handler.EnqueueRequestsFromMapFunc(schedule.SchedulesForCHI),
builder.WithPredicates(controller.CHITopologyPredicate()),
).
Watches(&api.ClickHouseRestore{}, handler.EnqueueRequestsFromMapFunc(schedule.SchedulesForRestore)).
Watches(&batch.Job{}, handler.EnqueueRequestsFromMapFunc(schedule.SchedulesForJob)).
WithEventFilter(controller.WatchedNamespacePredicate()).
Complete(schedule); err != nil {
return err
Expand All @@ -253,7 +273,7 @@ func setupBackupControllers(manager ctrlRuntime.Manager, cr interfaces.IKubeCR)
if err := ctrlRuntime.
NewControllerManagedBy(manager).
For(&api.ClickHouseRestore{}).
Owns(&batch.Job{}).
WatchesRawSource(runs(api.ClickHouseRestoreCRDResourceKind)).
WithEventFilter(controller.WatchedNamespacePredicate()).
Complete(controller.NewRestoreController(c, apiReader, scheme, cr)); err != nil {
return err
Expand Down
27 changes: 7 additions & 20 deletions cmd/operator/app/thread_backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
ctrlCache "sigs.k8s.io/controller-runtime/pkg/cache"

api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
"github.com/altinity/clickhouse-operator/pkg/model/chbackup"
)

// stagedDiscovery serves the backup group version without the restore kind for the first
Expand Down Expand Up @@ -104,19 +103,13 @@ func TestMissingBackupKindsWhenGroupVersionIsNotServed(t *testing.T) {
require.Equal(t, backupKinds, missing)
}

// TestBackupCacheOptionsNarrowsJobsAndCronJobs pins the backup cache wiring. Jobs and CronJobs are
// narrowed to the ones the backup controllers build, by a key of their own - the CHI key would
// hide every backup Job. The custom resources and installations must stay readable: they carry no
// operator label, and a cache-wide selector would turn every read of them into a silent NotFound.
// Installations are trimmed instead, so the cache holds no second full copy of each of them.
func TestBackupCacheOptionsNarrowsJobsAndCronJobs(t *testing.T) {
var logged []string
log := funcr.New(func(prefix, args string) { logged = append(logged, args) }, funcr.Options{})
// TestBackupCacheOptions pins the backup cache wiring. The custom resources and installations must stay readable: they
// carry no operator label, and a cache-wide selector would turn every read of them into a silent NotFound.
// Installations are trimmed instead, so the cache holds no second full copy of each of them. The backup controllers
// run no Job, CronJob or Pod, and cache none.
func TestBackupCacheOptions(t *testing.T) {
opts := newBackupCacheOptions(funcr.New(func(prefix, args string) {}, funcr.Options{}), []string{"ns1", "ns2"})

opts := newBackupCacheOptions(log, []string{"ns1", "ns2"})

require.Contains(t, strings.Join(logged, "\n"), chbackup.LabelBackupCRKind,
"the effective selector must reach the boot log")
require.Nil(t, opts.DefaultLabelSelector,
"a cache-wide selector would hide the user-authored custom resources from the reconcilers")
require.Len(t, opts.DefaultNamespaces, 2, "ByObject must not replace the namespace scoping")
Expand All @@ -125,13 +118,7 @@ func TestBackupCacheOptionsNarrowsJobsAndCronJobs(t *testing.T) {
for obj, byObject := range opts.ByObject {
narrowed[fmt.Sprintf("%T", obj)] = byObject
}
require.Len(t, narrowed, 3, "narrowed set changed - update this test deliberately")
for _, typeName := range []string{"*v1.Job", "*v1.CronJob"} {
byObject, ok := narrowed[typeName]
require.True(t, ok, "%s must be narrowed - an unfiltered type caches every instance in scope", typeName)
require.Equal(t, chbackup.JobSelector().String(), byObject.Label.String())
require.Nil(t, byObject.Namespaces, "%s must inherit DefaultNamespaces", typeName)
}
require.Len(t, narrowed, 1, "narrowed set changed - update this test deliberately")

installations, ok := narrowed["*v1.ClickHouseInstallation"]
require.True(t, ok, "installations must be trimmed - the CHI controller caches them in full already")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1224,19 +1224,19 @@ spec:
description: |
EXPERIMENTAL. Operator-managed backup for this cluster, overriding `chi.spec.backup`.
Fields left empty are inherited from `chi.spec.backup`; set `enabled: "no"` to opt this cluster out.
When enabled, the operator injects a clickhouse-backup sidecar into every Pod of the cluster,
so turning it on or off rolls the cluster's Pods. Customize the sidecar through a container named
clickhouse-backup in the Pod template `podTemplate` names. Backup takes effect only where the cluster
has a `configSecretRef`, or `podTemplate` is set: without either the sidecar would have no backup storage,
so backup stays off and the operator warns.
When enabled in mode sidecar, the operator injects a clickhouse-backup sidecar into every Pod of the
cluster, so turning it on or off rolls the cluster's Pods. Customize the container through a container
named clickhouse-backup in the Pod template `podTemplate` names. Backup takes effect only where the
cluster has a `configSecretRef`: without it clickhouse-backup would have no storage and no API
credentials, so backup stays off and the operator warns.
See docs/backup.md for details.
properties:
enabled:
<<: *TypeStringBool
description: |
Enable operator-managed backup: inject the clickhouse-backup sidecar and allow backup/restore resources to target
the cluster. Takes effect only with `configSecretRef` or `podTemplate` set, which name where the sidecar takes
its storage from: without either, backup stays off
the cluster. Takes effect only with `configSecretRef` set, which names where clickhouse-backup takes its
storage and API credentials from: without it, backup stays off
podTemplate:
type: string
description: |
Expand All @@ -1250,14 +1250,35 @@ spec:
of the sidecar, a YAML mapping of its sections, e.g. general.remote_storage, s3.bucket, s3.path,
s3.access_key, s3.secret_key. The operator generates the configuration file of the sidecar from it,
which follows a change of the Secret without a restart. A cluster that leaves it out takes the
installation's; an explicit `name: ""` on a cluster leaves the cluster without one. Without one, the
storage is configured by the container of the Pod template `podTemplate` names
installation's; an explicit `name: ""` on a cluster leaves the cluster without one. Required: without
one, backup stays off. The configuration must set api.username and api.password of the REST API
properties:
name:
description: |
Name of the referent. More info:
https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
preferredReplica:
type: integer
minimum: 0
description: |
Index, from 0, of the replica of each shard a backup works on, as the backupReplica of the Altinity usual backup workflow.
With mode pod the backup Pod of the shard works next to it: it mounts the volumes of that replica's Pod and runs
on the same node, by a nodeSelector on the node's hostname; a shard with fewer replicas, or whose preferred replica
is not ready, falls back to its next ready replica. With mode sidecar a backup of one replica per shard
(FirstPerShard) runs on the sidecar of that replica, or on that of the first replica when the shard has fewer
replicas or the preferred one does not answer when the Job starts. Empty means 0, the first replica
naming:
type: string
maxLength: 128
pattern: '^([A-Za-z0-9._-]|\{[A-Za-z]+\})+$'
description: |
Template the backups of the cluster are named by, one name for the backup of each shard, e.g.
"shard{shard}-{YYYYMMDDHHIISS}" as the Altinity usual backup workflow names them. Placeholders: {shard}, the number
of the shard from 1, required; {cluster}, {chi}, {namespace}; {prefix}, the backupNamePrefix of the schedule;
the time by the tokens YYYY, MM, DD, HH, II (minutes) and SS, all required, in braces, e.g. {YYYYMMDDHHIISS}.
A backup made at once has a T before the hour. Empty, the default, names a backup "<prefix>-<YYYYMMDDHHMMSS>"
alike on every shard. See docs/backup.md
recovery:
type: object
description: |
Expand Down Expand Up @@ -1884,10 +1905,10 @@ spec:
description: |
EXPERIMENTAL. Operator-managed backup defaults, applied to every cluster. Each cluster can override
them via `chi.spec.configuration.clusters[].backup`, e.g. `enabled: "no"` to opt a cluster out.
When enabled, the operator injects a clickhouse-backup sidecar into every Pod of the cluster,
so turning it on or off rolls the cluster's Pods. Backup takes effect only where a cluster has a
`configSecretRef`, or `podTemplate` is set: without either the sidecar would have no backup storage,
so backup stays off and the operator warns.
When enabled in mode sidecar, the operator injects a clickhouse-backup sidecar into every Pod of the
cluster, so turning it on or off rolls the cluster's Pods. Backup takes effect only where a cluster has
a `configSecretRef`: without it clickhouse-backup would have no storage and no API credentials, so
backup stays off and the operator warns.
See docs/backup.md for details.
useTemplates:
type: array
Expand Down
Loading
Loading
0