8000 fix: cutting down on the memory footprint for import by shawkins · Pull Request #41196 · keycloak/keycloak · GitHub
[go: up one dir, main page]
Skip to content

fix: cutting down on the memory footprint for import #41196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shawkins merged 1 commit into from
Aug 4, 2025
Merged

fix: cutting down on the memory footprint for import #41196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 131 additions & 117 deletions common/src/main/java/org/keycloak/common/util/StringPropertyReplacer.java
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,17 @@
*/
package org.keycloak.common.util;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;

import org.jboss.logging.Logger;
import org.jboss.logging.Logger.Level;

/**
* A utility class for replacing properties in strings.
*
Expand All @@ -32,6 +40,8 @@
public final class StringPropertyReplacer
{

private static final Logger logger = Logger.getLogger(StringPropertyReplacer.class);

/** File separator value */
private static final String FILE_SEPARATOR = File.separator;

Expand All @@ -44,14 +54,11 @@ public final class StringPropertyReplacer
/** Path separator alias */
private static final String PATH_SEPARATOR_ALIAS = ":";

// States used in property parsing
private static final int NORMAL = 0;
private static final int SEEN_DOLLAR = 1;
private static final int IN_BRACKET = 2;

private static final PropertyResolver NULL_RESOLVER = property -> null;
private static PropertyResolver DEFAULT_PROPERTY_RESOLVER;

private static final int MAX_KEY_LENGTH = 1<<22;

public static void setDefaultPropertyResolver(PropertyResolver systemVariables) {
DEFAULT_PROPERTY_RESOLVER = systemVariables;
}
Expand Down Expand Up @@ -99,138 +106,145 @@ public static String replaceProperties(final String string) {
* @return the input string with all property references replaced if any.
* If there are no valid references the input string will be returned.
*/
public static String replaceProperties(final String string, PropertyResolver resolver)
{
if(string == null) {
public static String replaceProperties(final String string, PropertyResolver resolver) {
if (string == null) {
return null;
}
final char[] chars = string.toCharArray();
StringBuilder buffer = new StringBuilder();
boolean properties = false;
int state = NORMAL;
int start = 0;
int openBracketsCount = 0;
for (int i = 0; i < chars.length; ++i)
{
char c = chars[i];
int index = string.indexOf("${");
if (index == -1) {
return string;
}
try {
return string.substring(0, index).concat(StreamUtil
.readString(replaceProperties(new ByteArrayInputStream(string.substring(index).getBytes(StandardCharsets.UTF_8)),
resolver), StandardCharsets.UTF_8));
} catch (IOException e) {
throw new RuntimeException(e); // not expected
}
}

// Dollar sign outside brackets
if (c ==  8000 9;$' && state != IN_BRACKET)
state = SEEN_DOLLAR;
public static InputStream replaceProperties(final InputStream source, PropertyResolver resolver) {
return replaceProperties(source, false, resolver);
}

// Open bracket immediately after dollar
else if (c == '{' && state == SEEN_DOLLAR)
{
buffer.append(string.substring(start, i - 1));
state = IN_BRACKET;
start = i - 1;
openBracketsCount = 1;
private static InputStream replaceProperties(final InputStream source, boolean readUntilCurlyBrace, PropertyResolver resolver) {
return new InputStream() {
private ByteArrayInputStream buffer;
private boolean closed;

@Override
public int read() throws IOException {
if (closed) {
throw new IOException("Stream closed");
}
// read off of the buffer first if possible
if (buffer != null) {
int c = buffer.read();
if (c != -1) {
return c;
}
buffer = null;
}
// if no buffer, scan for } or ${
int c = source.read();
if (c == '}' && readUntilCurlyBrace) {
return -2;
}
if (c != '$') {
return c;
}
int next = source.read();
if (next != '{') {
buffer = new ByteArrayInputStream(new byte[] {(byte)c, (byte)next});
return read();
}
// determine the key
int keyChar = -1;
InputStream keyStream = replaceProperties(source, true, resolver);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
while ((keyChar = keyStream.read()) > -1) {
bytes.write((byte)keyChar);
if (bytes.size() == MAX_KEY_LENGTH) {
logger.log(Level.WARN, "Detected an unclosed ${, replacement will not be performed");
keyChar = -1;
break;
}
}
String keyString = bytes.toString(StandardCharsets.UTF_8.name());
String replacement = null;
if (keyChar == -1) {
// eof before } - prepend ${ and output directly
replacement = "${" + keyString;
} else {
replacement = replaceProperty(resolver, keyString);
if (replacement == null) {
replacement = "${" + keyString + "}";
} else {
try {
replacement = replaceProperties(replacement, resolver);
} catch (StackOverflowError ex) {
throw new IllegalStateException("Infinite recursion happening when replacing properties on '" + replacement + "'");
}
}
}
buffer = new ByteArrayInputStream(replacement.getBytes(StandardCharsets.UTF_8));
return read();
}

// Seeing open bracket after we already saw some open bracket without corresponding closed bracket. This causes "nested" expressions. For example ${foo:${bar}}
else if (c == '{' && state == IN_BRACKET)
openBracketsCount++;
@Override
public void close() throws IOException {
closed = true;
source.close();
}
};
}

// No open bracket after dollar
else if (state == SEEN_DOLLAR)
state = NORMAL;
private static String replaceProperty(PropertyResolver resolver, String key) {
String value = null;

// Seeing closed bracket, but we already saw more than one open bracket before. Hence "nested" expression is still not fully closed.
// For example expression ${foo:${bar}} is closed after the second closed bracket, not after the first closed bracket.
else if (c == '}' && state == IN_BRACKET && openBracketsCount > 1)
openBracketsCount--;
// check for alias
if (FILE_SEPARATOR_ALIAS.equals(key))
{
value = FILE_SEPARATOR;
}
else if (PATH_SEPARATOR_ALIAS.equals(key))
{
value = PATH_SEPARATOR;
}
else
{
// check from the properties
value = resolveValue(resolver, key);

// Closed bracket after open bracket
else if (c == '}' && state == IN_BRACKET)
if (value == null) 8000
{
// No content
if (start + 2 == i)
// Check for a default value ${key:default}
int colon = key.indexOf(':');
if (colon > 0)
{
buffer.append("${}"); // REVIEW: Correct?
}
else // Collect the system property
{
String value = null;

String key = string.substring(start + 2, i);
String realKey = key.substring(0, colon);
value = resolveValue(resolver, realKey);

// check for alias
if (FILE_SEPARATOR_ALIAS.equals(key))
if (value == null)
{
value = FILE_SEPARATOR;
}
else if (PATH_SEPARATOR_ALIAS.equals(key))
{
value = PATH_SEPARATOR;
}
else
{
// check from the properties
value = resolveValue(resolver, key);
// Check for a composite key, "key1,key2"
value = resolveCompositeKey(realKey, resolver);

if (value == null)
{
// Check for a default value ${key:default}
int colon = key.indexOf(':');
if (colon > 0)
{
String realKey = key.substring(0, colon);
value = resolveValue(resolver, realKey);

if (value == null)
{
// Check for a composite key, "key1,key2"
value = resolveCompositeKey(realKey, resolver);

// Not a composite key either, use the specified default
if (value == null)
value = key.substring(colon+1);
}
}
else
{
// No default, check for a composite key, "key1,key2"
value = resolveCompositeKey(key, resolver);
}
// Not a composite key either, use the specified default
if (value == null) {
value = key.substring(colon+1);
}
}

if (value != null)
{
properties = true;
buffer.append(value);
}
else
{
buffer.append("${");
buffer.append(key);
buffer.append('}');
}

}
start = i + 1;
state = NORMAL;
}
}

// No properties
if (!properties)
return string;

// Collect the trailing characters
if (start != chars.length)
buffer.append(string.substring(start, chars.length));

if (buffer.indexOf("${") != -1) {
try {
return replaceProperties(buffer.toString(), resolver);
} catch (StackOverflowError ex) {
throw new IllegalStateException("Infinite recursion happening when replacing properties on '" + buffer + "'");
else
{
// No default, check for a composite key, "key1,key2"
value = resolveCompositeKey(key, resolver);
}
}
}

// Done
return buffer.toString();
return value;
}

private static String resolveCompositeKey(String key, PropertyResolver resolver)
Expand Down
3 changes: 3 additions & 0 deletions common/src/test/java/org/keycloak/common/util/StringPropertyReplacerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public class StringPropertyReplacerTest {
public void testSystemProperties() throws NoSuchAlgorithmException {
System.setProperty("prop1", "val1");
Assert.assertEquals("foo-val1", replaceProperties("foo-${prop1}"));
// non-matching scenarios
Assert.assertEquals("foo-${prop1", replaceProperties("foo-${prop1"));
Assert.assertEquals("foo-$prop1${", replaceProperties("foo-$prop1${"));

Assert.assertEquals("foo-def", replaceProperties("foo-${prop2:def}"));
System.setProperty("prop2", "val2");
Expand Down
7 changes: 2 additions & 5 deletions ...age-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,11 @@

import static org.keycloak.common.util.StringPropertyReplacer.replaceProperties;

import java.io.ByteArrayInputStream;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Optional;
import org.keycloak.common.util.StringPropertyReplacer;

Expand All @@ -39,9 +38,7 @@ public String resolve(String property) {

protected InputStream parseFile(File importFile) throws IOException {
if (ExportImportConfig.isReplacePlaceholders()) {
String raw = Files.readString(importFile.toPath());
String parsed = replaceProperties(raw, ENV_VAR_PROPERTY_RESOLVER);
return new ByteArrayInputStream(parsed.getBytes());
return replaceProperties(new BufferedInputStream(new FileInputStream(importFile)), ENV_VAR_PROPERTY_RESOLVER);
}

return new FileInputStream(importFile);
Expand Down
14 changes: 0 additions & 14 deletions model/storage-services/src/main/java/org/keycloak/exportimport/util/ImportUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,8 @@

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;

Expand Down Expand Up @@ -147,20 +145,9 @@ public static Map<String, RealmRepresentation> getRealmsFromStream(ObjectMapper
// Case with more realms in stream
parser.nextToken();

List<RealmRepresentation> realmReps = new ArrayList<RealmRepresentation>();
while (parser.getCurrentToken() == JsonToken.START_OBJECT) {
RealmRepresentation realmRep = parser.readValueAs(RealmRepresentation.class);
parser.nextToken();

// Ensure that master realm is imported first
if (Config.getAdminRealm().equals(realmRep.getRealm())) {
realmReps.add(0, realmRep);
} else {
realmReps.add(realmRep);
}
}

for (RealmRepresentation realmRep : realmReps) {
result.put(realmRep.getRealm(), realmRep);
}
} else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
Expand All @@ -175,7 +162,6 @@ public static Map<String, RealmRepresentation> getRealmsFromStream(ObjectMapper
return result;
}


// Assuming that it's invoked inside transaction
public static void importUsersFromStream(KeycloakSession session, String realmName, ObjectMapper mapper, InputStream is, boolean federated, Consumer<KeycloakSession> onUserCreated) throws IOException {
RealmProvider model = session.realms();
Expand Down
0