8000 Evaluate YugabyteDB as a PostgreSQL database in tests by ahus1 · Pull Request #54731 · keycloak/keycloak · GitHub
[go: up one dir, main page]

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/actions/run-store-tests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ inputs:
db:
description: The database to use for tests
required: true
server-start-timeout:
description: Seconds to wait for the Keycloak server to start, which includes creating the schema
default: "360"

runs:
using: composite
Expand All @@ -16,12 +19,12 @@ runs:
- name: Run new base tests
shell: bash
run: |
KC_TEST_DATABASE=${{ inputs.db }} KC_TEST_DATABASE_REUSE=true TESTCONTAINERS_REUSE_ENABLE=true ./mvnw package -f tests/pom.xml -Dtest=DatabaseTestSuite -Dkc.test.server.start.timeout=360
KC_TEST_DATABASE=${{ inputs.db }} KC_TEST_DATABASE_REUSE=true TESTCONTAINERS_REUSE_ENABLE=true ./mvnw package -f tests/pom.xml -Dtest=DatabaseTestSuite -Dkc.test.server.start.timeout=${{ inputs.server-start-timeout }}

- name: Run stateless tests
shell: bash
run: |
KC_TEST_DATABASE=${{ inputs.db }} KC_TEST_DATABASE_REUSE=true TESTCONTAINERS_REUSE_ENABLE=true ./mvnw package -f tests/pom.xml -Dtest=StatelessTestSuite -Dkc.test.server.start.timeout=360
KC_TEST_DATABASE=${{ inputs.db }} KC_TEST_DATABASE_REUSE=true TESTCONTAINERS_REUSE_ENABLE=true ./mvnw package -f tests/pom.xml -Dtest=StatelessTestSuite -Dkc.test.server.start.timeout=${{ inputs.server-start-timeout }}

- uses: ./.github/actions/upload-flaky-tests
name: Upload flaky tests
Expand Down
37 changes: 37 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }}
ci-azure: ${{ steps.azure-tests.outputs.run-azure-tests }}
ci-additional-dbs: ${{ steps.additional-dbs-tests.outputs.run-additional-dbs-tests }}
ci-yugabyte: ${{ steps.yugabyte-tests.outputs.run-yugabyte-tests }}
ci-admin-v2: ${{ steps.conditional.outputs.admin-v2 }}
ci-jackson3: ${{ steps.conditional.outputs.ci-jackson3 }}
permissions:
Expand Down Expand Up @@ -98,6 +99,24 @@ jobs:
env:
PRIVATE_DBS_QUAY_USERNAME: ${{ secrets.PRIVATE_DBS_QUAY_USERNAME }}
PRIVATE_DBS_QUAY_TOKEN: ${{ secrets.PRIVATE_DBS_QUAY_TOKEN }}
- name: YugabyteDB conditional check
id: yugabyte-tests
# Runs for manually or nightly triggered runs, and for pushes/PRs when any commit message in the PR/push contains "[yugabyte ci]"
run: |
RUN_YUGABYTE_TESTS=false
if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then
COMMIT_MESSAGE="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/commits" --jq '.[].commit.message')"
else
COMMIT_MESSAGE="${PUSH_COMMIT_MESSAGES}"
fi
if [[ $GITHUB_EVENT_NAME == "workflow_dispatch" || $GITHUB_EVENT_NAME == "schedule" || "${COMMIT_MESSAGE}" == *"[yugabyte ci]"* ]]; then
RUN_YUGABYTE_TESTS=true
fi
echo "run-yugabyte-tests=$RUN_YUGABYTE_TESTS" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PUSH_COMMIT_MESSAGES: ${{ join(github.event.commits.*.message, '\n') }}
testsuite-deprecation-check:
name: Testsuite Deprecation Check
runs-on: ubuntu-latest
Expand Down Expand Up @@ -877,6 +896,23 @@ jobs:
with:
db: ${{ matrix.db }}

store-integration-tests-yugabyte:
name: Store IT (YugabyteDB)
needs: [build, conditional]
runs-on: ubuntu-latest
timeout-minutes: 85
if: needs.conditional.outputs.ci-yugabyte == 'true'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- id: run-store-tests
name: Run Store Integration Tests - yugabyte
uses: ./.github/actions/run-store-tests
with:
db: yugabyte
# creating the schema takes several minutes
server-start-timeout: 900

store-model-tests:
name: Store Model Tests
runs-on: ubuntu-latest
Expand Down Expand Up @@ -1362,6 +1398,7 @@ jobs:
- jdk-integration-tests
- store-integration-tests
- store-integration-tests-additional
- store-integration-tests-yugabyte
- volatile-sessions-tests
- http-client-v2-tests
- store-model-tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.keycloak.common.util.Time;
import org.keycloak.connections.jpa.JpaConnectionProvider;
import org.keycloak.connections.jpa.support.EntityManagerProxy;
import org.keycloak.connections.jpa.util.JpaUtils;
import org.keycloak.models.AbstractKeycloakTransaction;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.ModelException;
Expand Down Expand Up @@ -83,6 +84,7 @@ public RootAuthenticationSessionModel createRootAuthenticationSession(RealmModel
// could remove it between the INSERT and the subsequent find. Retry if this happens.
RootAuthenticationSessionEntity entity;
long now = Time.currentTimeSeconds();
JpaUtils.lockForUpsert(em, RootAuthenticationSessionEntity.class, id);
for (;;) {
EntityManagerProxy.allowAsyncCommit(em, em.createNamedQuery("insertRootAuthSessionIfAbsent"))
.setParameter("id", id)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright 2026 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.keycloak.connections.jpa.updater.liquibase;

import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import liquibase.database.Database;
import liquibase.sql.Sql;
import liquibase.sql.UnparsedSql;
import liquibase.sqlgenerator.SqlGeneratorChain;
import liquibase.sqlgenerator.core.CreateIndexGeneratorPostgres;
import liquibase.statement.core.CreateIndexStatement;
import liquibase.structure.DatabaseObject;

/**
* Creates indexes on YugabyteDB using {@code CREATE INDEX NONCONCURRENTLY}.
* <p>
* By default, YugabyteDB builds an index online, which waits for all transactions that were open when the build started.
* Keycloak holds its database lock in a transaction for the whole migration, so an online build would wait for the migration to finish, and never complete.
* Inside a transaction block, YugabyteDB falls back to a non-concurrent build, but {@link YugabyteDatabase} runs Liquibase in auto-commit mode.
* That is necessary because YugabyteDB bumps the table or catalog version on every DDL statement, and a session with an older cached version fails on its next statement in the same transaction with "schema version mismatch".
* In auto-commit mode, each statement starts with a fresh version, which avoids this error for changesets that mix DDL with queries or inserts.
*/
public class YugabyteCreateIndexGenerator extends CreateIndexGeneratorPostgres {

private static final Pattern CREATE_INDEX = Pattern.compile("^CREATE (UNIQUE )?INDEX ", Pattern.CASE_INSENSITIVE);

@Override
public int getPriority() {
return super.getPriority() + 1;
}

@Override
public boolean supports(CreateIndexStatement statement, Database database) {
return database instanceof YugabyteDatabase && super.supports(statement, database);
}

@Override
public Sql[] generateSql(CreateIndexStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) {
return Arrays.stream(super.generateSql(statement, database, sqlGeneratorChain))
.map(sql -> {
Matcher matcher = CREATE_INDEX.matcher(sql.toSql());
if (!matcher.find()) {
return sql;
}
String rewritten = "CREATE " + (matcher.group(1) != null ? "UNIQUE " : "") + "INDEX NONCONCURRENTLY " + sql.toSql().substring(matcher.end());
return (Sql) new UnparsedSql(rewritten, sql.getEndDelimiter(), sql.getAffectedDatabaseObjects().toArray(new DatabaseObject[0]));
})
.toArray(Sql[]::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2026 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.keycloak.connections.jpa.updater.liquibase;

import liquibase.database.core.PostgresDatabase;

/**
* PostgreSQL-compatible Liquibase database for YugabyteDB.
* <p>
* YugabyteDB doesn't make DDL transactional by default: every DDL statement commits on its own, even inside a transaction block.
* A statement following a DDL statement in the same transaction block can then fail with
* {@code schema version mismatch for table ...} as it still uses the previous schema version.
* Declaring that DDL isn't supported in transactions makes Liquibase run with auto-commit enabled,
* so each statement is its own transaction and sees the current schema version.
* See <a href="https://github.com/yugabyte/yugabyte-db/issues/23882">yugabyte/yugabyte-db#23882</a>.
*/
public class YugabyteDatabase extends PostgresDatabase {

@Override
public boolean supportsDDLInTransaction() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@
package org.keycloak.connections.jpa.updater.liquibase.conn;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

import org.keycloak.Config;
import org.keycloak.config.DatabaseOptions;
import org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider;
import org.keycloak.connections.jpa.updater.liquibase.YugabyteDatabase;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;

Expand All @@ -33,6 +35,7 @@
import liquibase.database.AbstractJdbcDatabase;
import liquibase.database.Database;
import liquibase.database.DatabaseConnection;
import liquibase.database.core.PostgresDatabase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.exception.LiquibaseException;
import liquibase.resource.ClassLoaderResourceAccessor;
Expand Down Expand Up @@ -171,6 +174,9 @@ private Database getLiquibaseDatabase(Connection connection) {
} catch (Exception e) {
throw new RuntimeException("Failed to create instance of " + liquibaseDatabaseClazz.getName());
}
if (liquibaseDatabase.getClass() == PostgresDatabase.class && isYugabyte(connection)) {
liquibaseDatabase = new YugabyteDatabase();
}
DatabaseConnection liquibaseConnection = new JdbcConnection(connection);
try {
logger.debugf("DB Product Name: %s", liquibaseConnection.getDatabaseProductName());
Expand All @@ -182,4 +188,14 @@ private Database getLiquibaseDatabase(Connection connection) {
return liquibaseDatabase;
}

// YugabyteDB reports itself as PostgreSQL, the version looks like "15.12-YB-2025.2.8.1-b0"
private boolean isYugabyte(Connection connection) {
try {
return connection.getMetaData().getDatabaseProductVersion().contains("-YB-");
} catch (SQLException e) {
logger.debug("Failed to detect DB product version", e);
return false;
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,14 @@ protected boolean isApplicable() throws CustomChangeException {
// We're inside a liquibase managed transaction at this point. Some RDBMS don't like updates to tables
// that were queried in the same transaction. So we need to create a savepoint and rollback to it so that
// this select is effectively removed from a transaction and doesn't interfere with an update that will come later.
Savepoint savepoint = connection.setSavepoint();
// There is no transaction and no savepoint support when Liquibase runs in auto-commit mode (for example on YugabyteDB).
Savepoint savepoint = connection.getAutoCommit() ? null : connection.setSavepoint();
try (Statement st = connection.createStatement(); ResultSet resultSet = st.executeQuery("SELECT ID FROM " + getTableName(correctedTableName))) {
return (resultSet.next());
} finally {
connection.rollback(savepoint);
if (savepoint != null) {
connection.rollback(savepoint);
}
}
} else {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.keycloak.common.util.Retry;
import org.keycloak.connections.jpa.JpaConnectionProvider;
import org.keycloak.connections.jpa.JpaConnectionProviderFactory;
import org.keycloak.connections.jpa.updater.liquibase.YugabyteDatabase;
import org.keycloak.connections.jpa.updater.liquibase.conn.LiquibaseConnectionProvider;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.dblock.DBLockProvider;
Expand Down Expand Up @@ -69,11 +70,16 @@ private void lazyInit() {
try {
Liquibase liquibase = liquibaseProvider.getLiquibase(dbConnection, defaultSchema);

if (liquibase.getDatabase() instanceof YugabyteDatabase) {
// The lock is held by the transaction of the SELECT FOR UPDATE, therefore it can't run in the auto-commit mode Liquibase enables for Yugabyte
dbConnection.setAutoCommit(false);
}

this.lockService = new CustomLockService();
lockService.setChangeLogLockWaitTime(factory.getLockWaitTimeoutMillis());
lockService.setDatabase(liquibase.getDatabase());
initialized = true;
} catch (LiquibaseException exception) {
} catch (LiquibaseException | SQLException exception) {
safeRollbackConnection();
safeCloseConnection();
throw new IllegalStateException(exception);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -41,6 +43,7 @@
import org.hibernate.dialect.Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.hibernate.dialect.OracleDialect;
import org.hibernate.dialect.PostgreSQLDialect;
import org.hibernate.engine.jdbc.env.spi.IdentifierHelper;
import org.hibernate.engine.spi.SessionFactoryImplementor;
import org.hibernate.internal.SessionFactoryImpl;
Expand Down Expand Up @@ -81,6 +84,47 @@ public static boolean isConflictUpdateRacy(EntityManager em) {
.getJdbcServices().getDialect() instanceof OracleDialect;
}

private static final String YUGABYTE_PROPERTY = "keycloak.yugabyte";

/**
* Concurrent {@code insert ... on conflict} statements for the same new key deadlock on YugabyteDB when the table has a
* secondary index, and the query layer can't retry the aborted READ COMMITTED transaction. This
* serializes them with a transaction-scoped advisory lock on the key, before the statement is executed.
* Does nothing on other databases. See https://github.com/yugabyte/yugabyte-db/issues/37250 for the upstream issue.
*
* @param entityClass the entity the upsert writes to
* @param key the values of the unique key the upsert conflicts on
*/
public static void lockForUpsert(EntityManager em, Class<?> entityClass, String... key) {
if (!isYugabyte(em)) {
return;
}
// the lock is only a mutex on this string, not on the table; sharing it across schemas only adds harmless contention
em.createNativeQuery("select 1 from (select pg_advisory_xact_lock(hashtext(?1))) l")
.setParameter(1, entityClass.getSimpleName() + ":" + String.join(":", key))
.getSingleResult();
}

/**
* Detects database products that are used via the dialect of another database and can only be identified by their
* version string. Needs to be called once while the server starts, before any request is handled, so the result can be
* read without synchronization afterwards.
*/
public static void detectDatabaseProduct(EntityManagerFactory emf, Connection connection) throws SQLException {
SessionFactoryImplementor factory = emf.unwrap(SessionFactoryImplementor.class);
boolean yugabyte = false;
if (factory.getJdbcServices().getDialect() instanceof PostgreSQLDialect) {
// YugabyteDB reports a version like 15.12-YB-2025.2.8.1-b0
String version = connection.getMetaData().getDatabaseProductVersion();
yugabyte = version != null && version.contains("-YB-");
}
emf.getProperties().put(YUGABYTE_PROPERTY, yugabyte);
}

private static boolean isYugabyte(EntityManager em) {
return Boolean.TRUE.equals(em.getEntityManagerFactory().getProperties().get(YUGABYTE_PROPERTY));
}

public static String getTableNameForNativeQuery(String tableName, EntityManager em) {
final Dialect dialect = em.getEntityManagerFactory().unwrap(SessionFactoryImpl.class).getJdbcServices().getDialect();
IdentifierHelper identifierHelper = em.getEntityManagerFactory().unwrap(SessionFactoryImpl.class).getJdbcServices().getJdbcEnvironment().getIdentifierHelper();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ public UserLoginFailureModel addUserLoginFailure(RealmModel realm, String userId
notInDatabaseCache.remove(key);
// INSERT ON CONFLICT DO NOTHING does not lock the conflicting row, so a concurrent DELETE
// could remove it between the INSERT and the subsequent find. Retry if this happens.
JpaUtils.lockForUpsert(em, LoginFailureEntity.class, realm.getId(), userId);
// One retry should normally be sufficient for that race; allow five attempts as an extra safety
// margin while ensuring an unexpected database behavior cannot retain resources until the transaction timeout.
for (int attempt = 0; attempt < MAX_ADD_ATTEMPTS; attempt++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.keycloak.connections.jpa.util.JpaUtils;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.RevokedTokenProvider;
import org.keycloak.models.jpa.entities.RevokedTokenEntity;

public class JpaRevokedTokenProvider implements RevokedTokenProvider {

Expand All @@ -44,6 +45,7 @@ public boolean put(String id, long lifespanSeconds) {
var currentTime = Time.currentTimeSeconds();
var expire = currentTime + lifespanSeconds;
int rows;
JpaUtils.lockForUpsert(em, RevokedTokenEntity.class, id);
if (JpaUtils.isUpsertRowCountUnreliable(em)) {
String table = JpaUtils.getTableNameForNativeQuery("REVOKED_TOKEN", em);
em.createNativeQuery("DELETE FROM " + table + " WHERE ID = ?1 AND EXPIRE <= ?2")
Expand Down
Loading
Loading
0