8000 pre-compute password denylist Bloom filter to speed up server startup by kfaseela · Pull Request #48515 · keycloak/keycloak · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,28 @@ The current implementation uses a BloomFilter for fast and memory efficient cont
* By default a false positive probability of `0.01%` is used.
* To change the false positive probability by CLI configuration, use `+--spi-password-policy--password-blacklist--false-positive-probability=0.00001+`.

.Pre-computing the Bloom filter

For large denylist files, {project_name} builds the Bloom filter from the plaintext file on every startup or reload, which can take several seconds.
To reduce load time to milliseconds, pre-compute the Bloom filter once using the `build-password-denylist` CLI command:

[source,bash]
----
bin/kc.sh tools build-password-denylist /path/to/100k_passwords
----

This generates a `100k_passwords.bloom` file next to the input file.
Place it in the password-blacklists folder and configure the realm password policy to use the `.bloom` filename (for example, `100k_passwords.bloom`) instead of the plaintext file.
{project_name} detects the file type by extension: files ending in `.bloom` are loaded as pre-computed Bloom filter binaries; all other files are read as plaintext.
Re-run the command and update the policy value each time the denylist is updated.

You can also control the false positive probability for the pre-computed filter:

[source,bash]
----
bin/kc.sh tools build-password-denylist /path/to/100k_passwords --fpp 0.00001
----

[[maximum-authentication-age]]
===== Maximum Authentication Age

Expand Down
8000
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
* Copyright 2026 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.keycloak.quarkus.runtime.cli.command;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

import org.keycloak.policy.BlacklistPasswordPolicyProviderFactory;

import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Parameters;

// Builds a pre-computed Bloom filter (.bloom) alongside a plaintext password denylist.
// The server loads the .bloom file instead of rebuilding from plaintext, reducing reload latency.
@Command(name = BuildPasswordDenylist.NAME,
header = BuildPasswordDenylist.HEADER,
sortOptions = false,
description = "%n" + BuildPasswordDenylist.HEADER
+ "%n%nKeycloak's password-blacklist policy rejects passwords found in a plaintext denylist file."
+ " For large lists, loading from plaintext on every startup or reload can take seconds."
+ " Run this command once after creating or updating DENYLIST_FILE to generate a pre-computed"
+ " .bloom file. To use it, configure the password policy with the .bloom filename instead of"
+ " the plaintext file. The server detects the file type by extension and loads it accordingly,"
+ " reducing load time to milliseconds.",
footerHeading = "%nExamples:%n",
footer = {
" ${PARENT-COMMAND-FULL-NAME:-$PARENTCOMMAND} ${COMMAND-NAME} /path/to/denylist.txt%n",
" ${PARENT-COMMAND-FULL-NAME:-$PARENTCOMMAND} ${COMMAND-NAME} /path/to/denylist.txt --fpp 0.00001 -o /path/to/out.bloom%n"
})
public class BuildPasswordDenylist extends AbstractCommand {

public static final String NAME = "build-password-denylist";
public static final String HEADER = "Pre-compute a Bloom filter for a password denylist.";

@Parameters(index = "0",
paramLabel = "DENYLIST_FILE",
description = "Path to the plaintext password denylist file (one password per line, UTF-8).")
private Path inputFile;

@Option(names = "--fpp",
paramLabel = "PROBABILITY",
description = "Desired false-positive probability for the Bloom filter, defaults to 0.0001.",
defaultValue = "0.0001")
private double fpp;

@Option(names = {"-o", "--output"},
paramLabel = "OUTPUT_FILE",
description = "Path for the generated .bloom file. Must end with '.bloom'. Defaults to <DENYLIST_FILE>.bloom in the same directory.")
private Path outputFile;

@Override
public String getName() {
return NAME;
}

@Override
public boolean isHelpAll() {
return false;
}

@Override
protected void runCommand() {
if (!Files.isRegularFile(inputFile)) {
executionError(spec.commandLine(), "File not found or not a regular file: " + inputFile);
}
if (fpp <= 0.0 || fpp >= 1.0) {
executionError(spec.commandLine(), "--fpp must be between 0 and 1 (exclusive), got: " + fpp);
}

if (outputFile == null) {
outputFile = inputFile.resolveSibling(inputFile.getFileName() + ".bloom");
} else if (!outputFile.getFileName().toString().endsWith(".bloom")) {
executionError(spec.commandLine(), "--output must end with '.bloom', got: " + outputFile);
} else {
Path outputParent = outputFile.toAbsolutePath().getParent();
if (outputParent != null && !Files.isDirectory(outputParent)) {
executionError(spec.commandLine(), "Output directory does not exist: " + outputParent);
}
}
picocli.println("Building Bloom filter from: " + inputFile);
picocli.println(" False-positive probability: " + fpp);

try {
long startMs = System.currentTimeMillis();
BlacklistPasswordPolicyProviderFactory.buildBloomFile(inputFile, outputFile, fpp);
long elapsedMs = System.currentTimeMillis() - startMs;
long outputSizeBytes = Files.size(outputFile);
String sizeStr;
if (outputSizeBytes < 1024) {
sizeStr = outputSizeBytes + " B";
} else if (outputSizeBytes < 1024 * 1024) {
sizeStr = (outputSizeBytes / 1024) + " KB";
} else {
sizeStr = (outputSizeBytes / (1024 * 1024)) + " MB";
}
picocli.println("Done in " + elapsedMs + " ms. Output: " + outputFile + " (" + sizeStr + ")");
picocli.println("Next step: place " + outputFile.getFileName() + " in your password-blacklists folder and"
+ " configure the password blacklist policy value to '" + outputFile.getFileName() + "'.");
} catch (IOException e) {
executionError(spec.commandLine(), "Failed to build Bloom filter: " + e.getMessage(), e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

@Command(name = Tools.NAME,
description = "Utilities for use and interaction with the server.",
subcommands = {Completion.class, WindowsService.class})
subcommands = {Completion.class, WindowsService.class, BuildPasswordDenylist.class})
public class Tools {

public static final String NAME = "tools";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ Commands:
windows-service Manage Keycloak as a Windows service.
install Install Keycloak as a Windows service.
uninstall Uninstall Keycloak Windows service.
build-password-denylist
Pre-compute a Bloom filter for a password denylist.
bootstrap-admin Commands for bootstrapping admin access
user Add an admin user with a password
service Add an admin service account
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ Commands:
windows-service Manage Keycloak as a Windows service.
install Install Keycloak as a Windows service.
uninstall Uninstall Keycloak Windows service.
build-password-denylist
Pre-compute a Bloom filter for a password denylist.
bootstrap-admin Commands for bootstrapping admin access
user Add an admin user with a password
service Add an admin service account
Expand Down
BD0
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ Commands:
windows-service Manage Keycloak as a Windows service.
install Install Keycloak as a Windows service.
uninstall Uninstall Keycloak Windows service.
build-password-denylist
Pre-compute a Bloom filter for a password denylist.
bootstrap-admin Commands for bootstrapping admin access
user Add an admin user with a password
service Add an admin service account
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,19 @@

package org.keycloak.policy;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DecimalFormat;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
Expand Down Expand Up @@ -247,6 +251,32 @@ public List<ProviderConfigProperty> getConfigMetadata() {
return builder.build();
}

/**
* Builds a pre-computed Bloom filter (.bloom) file from a plaintext password denylist file.
* Each line is treated as one password (lowercased before insertion).
*
* @param inputFile path to the plaintext password list (one password per line, UTF-8)
* @param outputFile path for the generated .bloom file
* @param fpp desired false-positive probability (e.g. 0.0001)
* @throws IOException if the input file cannot be read or the output file cannot be written
*/
public static void buildBloomFile(Path inputFile, Path outputFile, double fpp) throws IOException {
long count;
try (var lines = Files.lines(inputFile, StandardCharsets.UTF_8)) {
count = lines.count();
}
BloomFilter<String> filter = BloomFilter.create(
Funnels.stringFunnel(StandardCharsets.UTF_8), Math.max(count, 1), fpp);
try (var lines = Files.lines(inputFile, StandardCharsets.UTF_8)) {
lines.map(s -> s.toLowerCase(Locale.ROOT)).forEach(filter::put);
}
try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(outputFile))) {
filter.writeTo(out);
}
LOG.infof("Built pre-computed denylist: input=%s passwords=%d fpp=%f output=%s",
inputFile, count, fpp, outputFile);
}

/**
* A {@link PasswordBlacklist} describes a list of too easy to guess
* or potentially leaked passwords that users should not be able to use.
Expand Down Expand Up @@ -371,12 +401,56 @@ private void reloadIfNeeded() {
}

/**
* Loads the referenced blacklist into a {@link BloomFilter}.
* Loads the denylist into a {@link BloomFilter}.
* If the configured file ends with {@code .bloom}, it is loaded as a pre-computed Bloom filter binary.
* Otherwise, it is read as a plaintext password list.
*
* @return the {@link BloomFilter} backing a password blacklist
* @return the {@link BloomFilter} backing a password denylist
*/
private BloomFilter<String> load() {
if (name.endsWith(".bloom")) {
return loadFromBloom();
}
return loadFromPlaintext();
}

/**
* Fast path: deserialise a pre-computed Bloom filter binary (.bloom).
* Emits a warning when the stored false-positive probability differs from the configured value.
*
* @return the deserialised {@link BloomFilter}
* @throws IOException if the binary file cannot be read
*/
private BloomFilter<String> loadFromBloom() {
try {
LOG.infof("Loading pre-computed denylist start: name=%s path=%s", name, path);
long loadStartMillis = System.currentTimeMillis();
BloomFilter<String> filter;
try (BufferedInputStream in = new BufferedInputStream(
Files.newInputStream(path), BUFFER_SIZE_IN_BYTES)) {
filter = BloomFilter.readFrom(in, Funnels.stringFunnel(StandardCharsets.UTF_8));
}
long loadTimeMillis = System.currentTimeMillis() - loadStartMillis;
LOG.infof("Loading pre-computed denylist finished: name=%s path=%s expectedFpp=%s loadTime=%dms",
name, path, filter.expectedFpp(), loadTimeMillis);
if (Math.abs(filter.expectedFpp() - falsePositiveProbability) > 1e-9) {
LOG.warnf("Pre-computed denylist '%s' has fpp=%.6f but configured fpp=%.6f. "
+ "Regenerate the .bloom file with 'kc.sh tools build-password-denylist' if this is unintended.",
name, filter.expectedFpp(), falsePositiveProbability);
}
return filter;
} catch (IOException e) {
throw new RuntimeException("Loading pre-computed denylist failed: path=" + path, e);
}
}

/**
* Slow path: build a BloomFilter from the plaintext denylist file.
* Requires two passes: one to count passwords, one to insert them.
*
* @return a newly constructed {@link BloomFilter} populated from the plaintext file
*/
private BloomFilter<String> loadFromPlaintext() {
try {
LOG.infof("Loading blacklist start: name=%s path=%s", name, path);
long loadStartMillis = System.currentTimeMillis();
Expand Down
Loading
Loading
0