8000 Store outbox errors that contain null characters or exceed the column in bytes by Semih702 · Pull Request #54320 · keycloak/keycloak · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.keycloak.events.outbox;

import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Collection;
import java.util.EnumMap;
Expand Down Expand Up @@ -59,9 +60,12 @@ public class OutboxStore {
* Hard cap on the {@code last_error} column width — matches the
* VARCHAR(2048) defined in the changelog. Truncation is applied
* here so callers can pass arbitrarily long exception messages
* without worrying about persistence-layer rejection.
* without worrying about persistence-layer rejection. The cap is in
* UTF-8 bytes, not characters, as some databases measure the column
* width in bytes (Oracle by default, SQL Server with a UTF-8
* collation).
*/
public static final int MAX_LAST_ERROR_LENGTH = 2048;
public static final int MAX_LAST_ERROR_BYTES = 2048;

protected final KeycloakSession session;

Expand Down Expand Up @@ -731,14 +735,46 @@ private int scopedDelete(String namedQuery, String entryKind, String scopeParam,
* Truncates an error message to the column width with an ellipsis
* marker. Returns {@code null} unchanged so an explicit "no error"
* value (used by {@code markDelivered}) survives.
*
* <p>Error messages can contain text from remote parties (e.g. a
* receiver's response body). Null characters, which PostgreSQL
* cannot store, are removed, and the length is limited in UTF-8
* bytes. Otherwise storing the error would fail and roll back the
* whole transaction, including the outcome of other rows.
*/
protected String truncateError(String error) {
if (error == null) {
return null;
}
if (error.length() <= MAX_LAST_ERROR_LENGTH) {
if (error.indexOf('\0') >= 0) {
error = error.replace("\0", "");
}
if (error.length() * 3 <= MAX_LAST_ERROR_BYTES
|| error.getBytes(StandardCharsets.UTF_8).length <= MAX_LAST_ERROR_BYTES) {
return error;
}
return error.substring(0, MAX_LAST_ERROR_LENGTH - 3) + "...";
int limit = MAX_LAST_ERROR_BYTES - 3;
int bytes = 0;
int end = 0;
while (end < error.length()) {
int codePoint = error.codePointAt(end);
bytes += utf8Length(codePoint);
if (bytes > limit) {
break;
}
end += Character.charCount(codePoint);
}
return error.substring(0, end) + "...";
}

private static int utf8Length(int codePoint) {
if (codePoint < 0x80) {
return 1;
} else if (codePoint < 0x800) {
return 2;
} else if (codePoint < 0x10000) {
return 3;
}
return 4;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,10 @@ protected PushDeliveryOutcome deliverEncoded(KeycloakSession session, StreamConf
* </ul>
*
* <p>The body / exception message is truncated at
* {@link #LAST_ERROR_DETAIL_MAX} so the column ({@code VARCHAR(2048)})
* comfortably absorbs the prefix + url + detail. The store's own
* {@code truncateError} provides a final hard cap as defense in
* depth.
* {@link #LAST_ERROR_DETAIL_MAX} characters to keep the stored error
* short. The column ({@code VARCHAR(2048)}) can be measured in bytes,
* so a non-ASCII detail may still be too long; the store's own
* {@code truncateError} applies the final cap in UTF-8 bytes.
*/
protected String formatLastError(PushDeliveryOutcome push) {
if (push.status() != null) {
Expand Down Expand Up @@ -207,7 +207,12 @@ protected static String truncateDetail(String detail) {
if (detail.length() <= LAST_ERROR_DETAIL_MAX) {
return detail;
}
return detail.substring(0, LAST_ERROR_DETAIL_MAX) + "...";
int end = LAST_ERROR_DETAIL_MAX;
// do not split a surrogate pair
if (Character.isHighSurrogate(detail.charAt(end - 1))) {
end--;
}
return detail.substring(0, end) + "...";
}

protected static String nullToEmpty(String s) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package org.keycloak.ssf.transmitter.outbox;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;

class SsfPushDeliveryHandlerTruncateDetailTest {

@Test
void keepsShortDetail() {
assertNull(SsfPushDeliveryHandler.truncateDetail(null));
String detail = "a".repeat(SsfPushDeliveryHandler.LAST_ERROR_DETAIL_MAX);
assertSame(detail, SsfPushDeliveryHandler.truncateDetail(detail));
}

@Test
void truncatesLongDetail() {
String detail = "a".repeat(SsfPushDeliveryHandler.LAST_ERROR_DETAIL_MAX + 1);
assertEquals("a".repeat(SsfPushDeliveryHandler.LAST_ERROR_DETAIL_MAX) + "...",
SsfPushDeliveryHandler.truncateDetail(detail));
}

@Test
void doesNotSplitSurrogatePair() {
// the emoji's high surrogate is the last char that fits
String detail = "a".repeat(SsfPushDeliveryHandler.LAST_ERROR_DETAIL_MAX - 1) + "😀" + "b";
String truncated = SsfPushDeliveryHandler.truncateDetail(detail);
assertEquals("a".repeat(SsfPushDeliveryHandler.LAST_ERROR_DETAIL_MAX - 1) + "...", truncated);
assertFalse(Character.isHighSurrogate(truncated.charAt(truncated.length() - 4)));
}
}
8000 75A7
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.keycloak.tests.events.outbox;

import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
Expand All @@ -32,9 +33,11 @@
import org.keycloak.models.jpa.entities.OutboxEntryEntity;
import org.keycloak.models.jpa.entities.OutboxEntryStatus;
import org.keycloak.testframework.annotations.KeycloakIntegrationTest;
import org.keycloak.testframework.conditions.DisabledForDatabases;
import org.keycloak.testframework.remote.runonserver.InjectRunOnServer;
import org.keycloak.testframework.remote.runonserver.RunOnServerClient;
import org.keycloak.testframework.server.DefaultKeycloakServerConfig;
import org.keycloak.tests.suites.DatabaseTest;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
Expand Down Expand Up @@ -290,13 +293,45 @@ public void recordFailure_bumpsAttemptsAndSchedulesRetryAndTruncatesLongError()
Assertions.assertEquals(3, after.getAttempts(), "attempts must be incremented");
Assertions.assertEquals(nextAttempt, after.getNextAttemptAt());
Assertions.assertNotNull(after.getLastError());
Assertions.assertTrue(after.getLastError().length() <= OutboxStore.MAX_LAST_ERROR_LENGTH,
Assertions.assertTrue(after.getLastError().getBytes(StandardCharsets.UTF_8).length <= OutboxStore.MAX_LAST_ERROR_BYTES,
"lastError must be truncated to the column width");
Assertions.assertTrue(after.getLastError().endsWith("..."),
"truncated lastError must carry an ellipsis marker");
});
}

@Test
@DatabaseTest
@DisabledForDatabases("oracle") // ORA-18716 reading outbox timestamps until #54139
public void recordFailure_storesErrorWithNullCharacterAndMultiByteCharacters() {
Comment thread
Semih702 marked this conversation as resolved.
Resolved
final String realmId = testRealmId;
runOnServer.run(session -> {
Instant now = Instant.now();
OutboxEntryEntity row = persistRaw(session, TEST_KIND, realmId, "owner-f", null,
"corr-fail-chars", OutboxEntryStatus.PENDING, 0,
now.minusSeconds(1), now);
em(session).flush();
em(session).clear();

// PostgreSQL cannot store U+0000 in text; with byte length semantics (Oracle) 2048 characters of
// a two-byte character do not fit LAST_ERROR VARCHAR(2048)
String error = "HTTP 500 https://receiver.example/push: invalid\u0000request " + "é".repeat(2000);
new OutboxStore(session)
.recordFailure(findById(session, row.getId()), now.plus(Duration.ofMinutes(5)), error);
em(session).flush();
em(session).clear();

OutboxEntryEntity after = findById(session, row.getId());
Assertions.assertEquals(1, after.getAttempts(), "attempts must be incremented");
Assertions.assertTrue(after.getLastError().startsWith("HTTP 500 https://receiver.example/push: invalidrequest "),
"lastError must keep the message without the null character");
Assertions.assertTrue(after.getLastError().getBytes(StandardCharsets.UTF_8).length <= OutboxStore.MAX_LAST_ERROR_BYTES,
"lastError must fit the column width in bytes");
Assertions.assertTrue(after.getLastError().endsWith("..."),
"truncated lastError must carry an ellipsis marker");
});
}

@Test
public void markDeadLetter_setsStatusAndIncrementsAttemptsAndStoresError() {
final String realmId = testRealmId;
Expand Down
Loading
0