8000 Batch composite-role expansion to avoid an N+1 of getChildRoles on the cache-miss path by hakdogan · Pull Request #50370 · keycloak/keycloak · GitHub
[go: up one dir, main page]

Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -50,6 +51,7 @@
import org.keycloak.models.cache.infinispan.entities.CachedClient;
import org.keycloak.models.cache.infinispan.entities.CachedClientRole;
import org.keycloak.models.cache.infinispan.entities.CachedClientScope;
import org.keycloak.models.cache.infinispan.entities.CachedCompositeRoles;
import org.keycloak.models.cache.infinispan.entities.CachedGroup;
import org.keycloak.models.cache.infinispan.entities.CachedRealm;
import org.keycloak.models.cache.infinispan.entities.CachedRealmRole;
Expand Down Expand Up @@ -1507,6 +1509,50 @@ public void addClientScopeToAllClients(RealmModel realm, ClientScopeModel client
invalidationEvents.add(ClientScopeRemovedEvent.create(clientScope.getId(), realm.getId()));
}

public Stream<RoleModel> getCompositeRolesStream(RealmModel realm, Set<String> parentRoleIds) {
String id = realm.getId() + "::" + String.join(",", parentRoleIds.stream().sorted().collect(Collectors.toList()));
CachedCompositeRoles cached = cache.get(id, CachedCompositeRoles.class);
if (cached != null && !cached.getRealm().equals(realm.getId())) {
cached = null;
}
if (cached != null) {
for (String compositeId : cached.getCompositeIds()) {
if (invalidations.contains(compositeId)) {
// If this composite role has been invalidated, it might be faster to fetch it and others from the DB as a batch
cached = null;
break;
}
if (managedRoles.containsKey(compositeId)) {
// If this composite role is already managed, continue
continue;
}
if (getCachedRole(realm, compositeId) == null) {
// If this composite role is not cached, it might be faster to fetch it and others from the DB as a batch
cached = null;
break;
}
}
}
if (cached != null) {
for (String parentId : cached.getParentIds()) {
CachedRole cachedParent = cache.get(parentId, CachedRole.class);
if (cachedParent == null || cachedParent.getRevision() > cached.getRevision()) {
// The parent has been modified or evicted, cache is invalid
cached = null;
break;
}
}
}
if (cached != null) {
return cached.getCompositeIds().stream().map(s -> getRoleById(realm, s)).filter(Objects::nonNull);
}
List<RoleModel> compositeRoles = getRoleDelegate().getCompositeRolesStream(realm, parentRoleIds).toList();
long loaded = cache.getCurrentRevision(id);
cached = new CachedCompositeRoles(loaded, id, new HashSet<>(parentRoleIds), compositeRoles.stream().map(RoleModel::getId).toList(), realm);
cache.addRevisioned(cached, startupRevision);
return compositeRoles.stream();
}

// Don't cache ClientInitialAccessModel for now
@Override
public ClientInitialAccessModel createClientInitialAccessModel(RealmModel realm, int expiration, int count) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright 2016 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.keycloak.models.cache.infinispan.entities;

import java.util.List;
import java.util.Set;

import org.keycloak.models.RealmModel;

/**
* @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
* @version $Revision: 1 $
*/
public class CachedCompositeRoles extends AbstractRevisioned implements InRealm {

private final Set<String> parentIds;
final protected String realm;
private final List<String> compositeIds;

public CachedCompositeRoles(long revision, String id, Set<String> parentIds, List<String> compositeIds, RealmModel realm) {
super(revision, id);
this.parentIds = parentIds;
this.realm = realm.getId();
this.compositeIds = compositeIds;
}

@Override
public String getRealm() {
return realm;
}

public Set<String> getParentIds() {
return parentIds;
}

public List<String> getCompositeIds() {
return compositeIds;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.keycloak.marshalling.Marshalling;
import org.keycloak.models.cache.infinispan.entities.CachedClient;
import org.keycloak.models.cache.infinispan.entities.CachedClientScope;
import org.keycloak.models.cache.infinispan.entities.CachedCompositeRoles;
import org.keycloak.models.cache.infinispan.entities.CachedGroup;
import org.keycloak.models.cache.infinispan.entities.CachedRole;
import org.keycloak.models.cache.infinispan.entities.Revisioned;
Expand Down Expand Up @@ -44,6 +45,7 @@ void setRole(String role) {
public boolean test(Map.Entry<String, Revisioned> entry) {
Object value = entry.getValue();
return (value instanceof CachedRole cachedRole && cachedRole.getCachedComposites().ids().contains(role)) ||
(value instanceof CachedCompositeRoles cachedCompositeRoles && (cachedCompositeRoles.getCompositeIds().contains(role) || cachedCompositeRoles.getParentIds().contains(role))) ||
(value instanceof CachedGroup cachedGroup && cachedGroup.getCachedRoleMappings().contains(role)) ||
(value instanceof RoleQuery roleQuery && roleQuery.getRoles().contains(role)) ||
(value instanceof CachedClient cachedClient && cachedClient.getScope().contains(role)) ||
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,22 @@ public RoleModel getRoleById(RealmModel realm, String id) {
return adapter;
}

@Override
public Stream<RoleModel> getCompositeRolesStream(RealmModel realm, Set<String> parentRoleIds) {
if (parentRoleIds == null || parentRoleIds.isEmpty()) {
return Stream.empty();
}
// One query fetches the child roles for every parent in the frontier, so a composite-role
// tree expands with a single getChildRolesFromParentIds query per breadth-first level. The
// query hydrates the child RoleEntity rows, so the getRoleById calls below are served from
// the persistence context without extra round-trips.
TypedQu 10000 ery<RoleEntity> query = em.createNamedQuery("getChildRolesFromParentIds", RoleEntity.class)
.setParameter("parentRoleIds", parentRoleIds);
return closing(query.getResultStream())
.map(roleEntity -> session.roles().getRoleById(realm, roleEntity.getId()))
.filter(Objects::nonNull);
}

@Override
public GroupModel getGroupById(RealmModel realm, String id) {
GroupEntity groupEntity = em.find(GroupEntity.class, id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
@NamedQuery(name="getRoleIdsFromIdList", query="select role.id from RoleEntity role where role.realmId = :realm and role.id in :ids order by role.name ASC"),
@NamedQuery(name="getRoleIdsByNameContainingFromIdList", query="select role.id from RoleEntity role where role.realmId = :realm and lower(role.name) like lower(concat('%',:search,'%')) and role.id in :ids order by role.name ASC"),
@NamedQuery(name="getChildRoles", query="select r from RoleEntity r join CompositeRoleEntity c on r.id = c.childRole.id where c.parentRole.id = :parentRoleId"),
@NamedQuery(name="getChildRolesFromParentIds", query="select distinct r from RoleEntity r join CompositeRoleEntity c on r.id = c.childRole.id where c.parentRole.id in :parentRoleIds"),
})

public class RoleEntity {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.keycloak.storage;

import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.keycloak.common.util.reflections.Types;
Expand Down Expand Up @@ -161,6 +163,32 @@ public Stream<RoleModel> getRolesStream(RealmModel realm, Stream<String> ids, St
return localStorage().getRolesStream(realm, ids, search, first, max);
}

@Override
public Stream<RoleModel> getCompositeRolesStream(RealmModel realm, Set<String> parentRoleIds) {
if (parentRoleIds == null || parentRoleIds.isEmpty()) {
return Stream.empty();
}
// Split local and external IDs. Local IDs are handled by a single batched query; external
// (federated) IDs are resolved individually via getRoleById and then expanded per-role.
Set<String> localIds = parentRoleIds.stream()
.filter(StorageId::isLocalStorage)
.collect(Collectors.toSet());
Set<String> externalIds = parentRoleIds.stream()
.filter(id -> !StorageId.isLocalStorage(id))
.collect(Collectors.toSet());

Stream<RoleModel> localComposites = localIds.isEmpty()
? Stream.empty()
: localStorage().getCompositeRolesStream(realm, localIds);

Stream<RoleModel> externalComposites = externalIds.stream()
.map(id -> getRoleById(realm, id))
.filter(Objects::nonNull)
.flatMap(RoleModel::getCompositesStream);

return Stream.concat(localComposites, externalComposites).distinct();
}

/**
* Obtaining roles from an external role storage is time-bounded. In case the external role storage
* isn't available at least roles from a local storage are returned. For this purpose
Expand Down
25 changes: 25 additions & 0 deletions server-spi/src/main/java/org/keycloak/models/RoleProvider.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.keycloak.models;

import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;

import org.keycloak.provider.Provider;
Expand Down Expand Up @@ -80,6 +82,29 @@ default Stream<RoleModel> getRealmRolesStream(RealmModel realm) {
*/
Stream<RoleModel> getRolesStream(RealmModel realm, Stream<String> ids, String search, Integer first, Integer max);

/**
* Returns the direct composite child roles of all the given parent roles, de-duplicated.
* <p>
* This is the bulk counterpart of {@link RoleModel#getCompositesStream()}: it lets callers expand
* a composite-role tree one breadth-first level at a time with a single lookup per level instead
* of one per role. The default implementation resolves each parent individually; storage-backed
* providers should override it with a batched lookup.
*
* @param realm Realm. Cannot be {@code null}.
* @param parentRoleIds Ids of the roles whose direct composite children should be returned.
* @return Stream of the child roles. Never returns {@code null}.
*/
default Stream<RoleModel> getCompositeRolesStream(RealmModel realm, Set<String> parentRoleIds) {
if (parentRoleIds == null || parentRoleIds.isEmpty()) {
return Stream.empty();
}
return parentRoleIds.stream()
.map(id -> getRoleById(realm, id))
.filter(Objects::nonNull)
.flatMap(RoleModel::getCompositesStream)
.distinct();
}

/**
* Removes given realm role from the given realm.
* @param role Role to be removed.
Expand Down
Loading
Loading
0