8000 Switch to StatefulSet by andreaTP · Pull Request #12757 · keycloak/keycloak · GitHub
[go: up one dir, main page]
Skip to content

Switch to StatefulSet #12757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abstractj merged 1 commit into from
Jul 13, 2022
Merged

Switch to StatefulSet #12757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.keycloak.operator.controllers;

import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
Expand Down Expand Up @@ -61,8 +61,8 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
public List<EventSource> prepareEventSources(EventSourceContext<Keycloak> context) {
String namespace = context.getConfigurationService().getClientConfiguration().getNamespace();

SharedIndexInformer<Deployment> deploymentInformer =
client.apps().deployments().inNamespace(namespace)
SharedIndexInformer<StatefulSet> deploymentInformer =
client.apps().statefulSets().inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.runnableInformer(0);

Expand Down
112 changes: 46 additions & 66 deletions operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeployment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,12 @@
import io.fabric8.kubernetes.api.model.EnvVarSourceBuilder;
import io.fabric8.kubernetes.api.model.ExecActionBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.quarkus.logging.Log;
import org.keycloak.operator.Config;
Expand All @@ -55,13 +54,13 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu

private final Config config;
private final Keycloak keycloakCR;
private final Deployment existingDeployment;
private final Deployment baseDeployment;
private final StatefulSet existingDeployment;
private final StatefulSet baseDeployment;
private final String adminSecretName;

private Set<String> serverConfigSecretsNames;

public KeycloakDeployment(KubernetesClient client, Config config, Keycloak keycloakCR, Deployment existingDeployment, String adminSecretName) {
public KeycloakDeployment(KubernetesClient client, Config config, Keycloak keycloakCR, StatefulSet existingDeployment, String adminSecretName) {
super(client, keycloakCR);
this.config = config;
this.keycloakCR = keycloakCR;
Expand All @@ -81,15 +80,15 @@ public KeycloakDeployment(KubernetesClient client, Config config, Keycloak keycl

@Override
public Optional<HasMetadata> getReconciledResource() {
Deployment baseDeployment = new DeploymentBuilder(this.baseDeployment).build(); // clone not to change the base template
Deployment reconciledDeployment;
StatefulSet baseDeployment = new StatefulSetBuilder(this.baseDeployment).build(); // clone not to change the base template
StatefulSet reconciledDeployment;
if (existingDeployment == null) {
Log.info("No existing Deployment found, using the default");
reconciledDeployment = baseDeployment;
}
else {
Log.info("Existing Deployment found, updating specs");
reconciledDeployment = new DeploymentBuilder(existingDeployment).build();
reconciledDeployment = new StatefulSetBuilder(existingDeployment).build();

// don't overwrite metadata, just specs
reconciledDeployment.setSpec(baseDeployment.getSpec());
Expand All @@ -106,10 +105,10 @@ public Optional<HasMetadata> getReconciledResource() {
return Optional.of(reconciledDeployment);
}

private Deployment fetchExistingDeployment() {
private StatefulSet fetchExistingDeployment() {
return client
.apps()
.deployments()
.statefulSets()
.inNamespace(getNamespace())
.withName(getName())
.get();
Expand Down Expand Up @@ -319,7 +318,7 @@ private void mergePodTemplate(PodTemplateSpec baseTemplate) {
}
}

private void configureHostname(Deployment deployment) {
private void configureHostname(StatefulSet deployment) {
var kcContainer = deployment.getSpec().getTemplate().getSpec().getContainers().get(0);
var hostname = this.keycloakCR.getSpec().getHostname();
var envVars = kcContainer.getEnv();
Expand All @@ -346,7 +345,7 @@ private void configureHostname(Deployment deployment) {
}
}

private void configureTLS(Deployment deployment) {
private void configureTLS(StatefulSet deployment) {
var kcContainer = deployment.getSpec().getTemplate().getSpec().getContainers().get(0);
var tlsSecret = this.keycloakCR.getSpec().getTlsSecret();
var envVars = kcContainer.getEnv();
Expand Down Expand Up @@ -462,8 +461,8 @@ public String readConfigurationValue(String key) {
}
}

private Deployment createBaseDeployment() {
Deployment baseDeployment = new DeploymentBuilder()
private StatefulSet createBaseDeployment() {
StatefulSet baseDeployment = new StatefulSetBuilder()
.withNewMetadata()
.endMetadata()
.withNewSpec()
Expand All @@ -475,40 +474,33 @@ private Deployment createBaseDeployment() {
.addToLabels("app", "")
.endMetadata()
.withNewSpec()
.withRestartPolicy("Always")
.withTerminationGracePeriodSeconds(30L)
.withDnsPolicy("ClusterFirst")
.addNewContainer()
.withName("keycloak")
.withArgs("start")
.addNewPort()
.withContainerPort(8443)
.withProtocol("TCP")
.endPort()
.addNewPort()
.withContainerPort(8080)
.withProtocol("TCP")
.endPort()
.withNewReadinessProbe()
.withInitialDelaySeconds(20)
.withPeriodSeconds(2)
.withFailureThreshold(250)
.endReadinessProbe()
.withNewLivenessProbe()
.withInitialDelaySeconds(20)
.withPeriodSeconds(2)
.withFailureThreshold(150)
.endLivenessProbe()
.withRestartPolicy("Always")
.withTerminationGracePeriodSeconds(30L)
.withDnsPolicy("ClusterFirst")
.addNewContainer()
.withName("keycloak")
.withArgs("start")
.addNewPort()
.withContainerPort(8443)
.withProtocol("TCP")
.endPort()
.addNewPort()
.withContainerPort(8080)
.withProtocol("TCP")
.endPort()
.withNewReadinessProbe()
.withInitialDelaySeconds(20)
.withPeriodSeconds(2)
.withFailureThreshold(250)
.endReadinessProbe()
.withNewLivenessProbe()
.withInitialDelaySeconds(20)
.withPeriodSeconds(2)
.withFailureThreshold(150)
.endLivenessProbe()
.endContainer()
.endSpec()
.endTemplate()
.withNewStrategy()
.withNewRollingUpdate()
// Same defaults as for a StatefulSet: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#maximum-unavailable-pods
.withNewMaxSurge(1) // maximum number of Pods that can be created over the desired number of Pods
.withNewMaxUnavailable(1) // maximum number of Pods that can be unavailable during the update process
.endRollingUpdate()
.endStrategy()
.endSpec()
.build();

Expand Down Expand Up @@ -601,15 +593,7 @@ private List<EnvVar> getEnvVars() {
public void updateStatus(KeycloakStatusBuilder status) {
validatePodTemplate(status);
if (existingDeployment == null) {
status.addNotReadyMessage("No existing Deployment found, waiting for creating a new one");
return;
}

var replicaFailure = existingDeployment.getStatus().getConditions().stream()
.filter(d -> d.getType().equals("ReplicaFailure")).findFirst();
if (replicaFailure.isPresent()) {
status.addNotReadyMessage("Deployment failures");
status.addErrorMessage("Deployment failure: " + replicaFailure.get());
status.addNotReadyMessage("No existing StatefulSet found, waiting for creating a new one");
return;
}

Expand All @@ -619,16 +603,12 @@ public void updateStatus(KeycloakStatusBuilder status) {
status.addNotReadyMessage("Waiting for more replicas");
}

var progressing = existingDeployment.getStatus().getConditions().stream()
.filter(c -> c.getType().equals("Progressing")).findFirst();
progressing.ifPresent(p -> {
String reason = p.getReason();
// https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#progressing-deployment
if (p.getStatus().equals("True") &&
(reason.equals("NewReplicaSetCreated") || reason.equals("FoundNewReplicaSet") || reason.equals("ReplicaSetUpdated"))) {
status.addRollingUpdateMessage("Rolling out deployment update");
}
});
if (existingDeployment.getStatus() != null
&& existingDeployment.getStatus().getCurrentRevision() != null
&& existingDeployment.getStatus().getUpdateRevision() != null
&& !existingDeployment.getStatus().getCurrentRevision().equals(existingDeployment.getStatus().getUpdateRevision())) {
status.addRollingUpdateMessage("Rolling out deployment update");
}
}

public Set<String> getConfigSecretsNames() {
Expand All @@ -645,7 +625,7 @@ public String getName() {
}

public void rollingRestart() {
client.apps().deployments()
client.apps().statefulSets()
.inNamespace(getNamespace())
.withName(getName())
.rolling().restart();
Expand Down
34 changes: 29 additions & 5 deletions operator/src/main/java/org/keycloak/operator/controllers/KeycloakRealmImportJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@
package org.keycloak.operator.controllers;

import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.SecretVolumeSourceBuilder;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
Expand All @@ -37,11 +38,14 @@
import java.util.List;
import java.util.Optional;

import static org.keycloak.operator.Constants.DEFAULT_DIST_CONFIG;
import static org.keycloak.operator.controllers.KeycloakDeployment.getEnvVarName;

public class KeycloakRealmImportJob extends OperatorManagedResource {

private final Keycloak keycloak;
private final KeycloakRealmImport realmCR;
private final Deployment existingDeployment;
private final StatefulSet existingDeployment;
private final Job existingJob;
private final String secretName;
private final String volumeName;
Expand Down Expand Up @@ -80,10 +84,10 @@ private Job fetchExistingJob() {
.get();
}

private Deployment fetchExistingDeployment() {
private StatefulSet fetchExistingDeployment() {
return client
.apps()
.deployments()
.statefulSets()
.inNamespace(getNamespace())
.withName(getKeycloakName())
.get();
Expand Down Expand Up @@ -129,6 +133,26 @@ private Job createImportJob() {
buildKeycloakJobContainer(keycloakPodTemplate.getSpec().getContainers().get(0));
keycloakPodTemplate.getSpec().getVolumes().add(buildSecretVolume());

var labels = keycloakPodTemplate.getMetadata().getLabels();

// The Job should not be selected with app=keycloak
labels.put("app", "keycloak-realm-import");

var envvars = keycloakPodTemplate
.getSpec()
.getContainers()
.get(0)
.getEnv();

var cacheEnvVarName = getEnvVarName("cache");
var healthEnvVarName = getEnvVarName("health-enabled");
envvars.removeIf(e -> e.getName().equals(cacheEnvVarName) || e.getName().equals(healthEnvVarName));

// The Job should not connect to the cache
envvars.add(new EnvVarBuilder().withName(cacheEnvVarName).withValue("local").build());
// The Job doesn't need health to be enabled
envvars.add(new EnvVarBuilder().withName(healthEnvVarName).withValue("false").build());

return buildJob(keycloakPodTemplate);
}

Expand Down Expand Up @@ -205,7 +229,7 @@ protected String getName() {
private String getRealmName() { return realmCR.getSpec().getRealm().getRealm(); }

private void rollingRestart() {
client.apps().deployments()
client.apps().statefulSets()
.inNamespace(getNamespace())
.withName(getKeycloakName())
.rolling().restart();
Expand Down
2 changes: 1 addition & 1 deletion operator/src/main/kubernetes/kubernetes.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ rules:
# https://github.com/fabric8io/kubernetes-client/issues/3996
- extensions
resources:
- deployments
- statefulsets
verbs:
- get
- list
Expand Down
14 changes: 12 additions & 2 deletions operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
Expand Down Expand Up @@ -186,6 +185,17 @@ protected static void deployDBSecret() {
protected static void deleteDB() {
// Delete the Postgres StatefulSet
k8sclient.apps().statefulSets().inNamespace(namespace).withName("postgresql-db").delete();
Awaitility.await()
.ignoreExceptions()
.untilAsserted(() -> {
Log.infof("Waiting for postgres to be deleted");
assertThat(k8sclient
.apps()
.statefulSets()
.inNamespace(namespace)
.withName("postgresql-db")
.get()).isNull();
});
}

// TODO improve this (preferably move to JOSDK)
Expand Down Expand Up @@ -220,7 +230,7 @@ public void cleanup() {
.untilAsserted(() -> {
var kcDeployments = k8sclient
.apps()
.deployments()
.statefulSets()
5342 .inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.list()
Expand Down
Loading
0