mhajas · Aug 1, 2023 · Jul 17, 2023
@@ -13,6 +13,9 @@ include::topics/templates/document-attributes.adoc[]
 :release_header_latest_link: {releasenotes_link_latest}
 include::topics/templates/release-header.adoc[]
 
+== {project_name_full} 23.0.0
+include::topics/23_0_0.adoc[leveloffset=2]
+
 == {project_name_full} 22.0.0
 include::topics/22_0_0.adoc[leveloffset=2]
 

@@ -0,0 +1,7 @@
+= Localization files for themes default to UTF-8 encoding
+
+Message properties files for themes are now read in UTF-8 encoding, with an automatic fallback to ISO-8859-1 encoding.
+
+See the migration guide for more details.
+
+
@@ -302,14 +302,11 @@ locales=en,no
 ----
 locale_no=Norsk
 ----
-+
-By default message properties files should be encoded using ISO-8859-1. It's also possible to specify the encoding using a special header. For example to use UTF-8 encoding:
-+
-[source]
-----
-# encoding: UTF-8
-usernameOrEmail=....
-----
+
+By default, message properties files should be encoded using UTF-8.
+Keycloak falls back to ISO-8859-1 handling if it can't read the contents as UTF-8.
+Unicode characters can be escaped as described in Java's documentation for https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/PropertyResourceBundle.html[PropertyResourceBundle].
+Previous versions of Keycloak supported specifying the encoding in the first line with a comment like `# encoding: UTF-8`, which is no longer supported.
 
 [role="_additional-resources"]
 .Additional resources

@@ -14,9 +14,19 @@ which can be turned on to prevent adding the `iss` parameter to the authenticati
 
 = Wildcard characters handling
 
-JPA allows wildcards `%` and `_` when searching, while other providers like LDAP allow only `*`. 
-As `*` is a natural wildcard character in LDAP, it works in all places, while with JPA it only 
-worked at the beginning and the end of the search string. Starting with this release the only 
-wildcard character is `*` which work consistently across all providers in all places in the search 
-string. All special characters in a specific provider like `%` and `_` for JPA are escaped. For exact 
+JPA allows wildcards `%` and `_` when searching, while other providers like LDAP allow only `*`.
+As `*` is a natural wildcard character in LDAP, it works in all places, while with JPA it only
+worked at the beginning and the end of the search string. Starting with this release the only
+wildcard character is `*` which work consistently across all providers in all places in the search
+string. All special characters in a specific provider like `%` and `_` for JPA are escaped. For exact
 search, with added quotes e.g. `"w*ord"`, the behavior remains the same as in previous releases.
+
+= Language files for themes default to UTF-8 encoding
+
+This release now follows the standard mechanisms of Java and later, which assumes resource bundle files to be encoded in UTF-8.
+
+Previous versions of Keycloak supported specifying the encoding in the first line with a comment like `# encoding: UTF-8`, which is no longer supported and is ignored.
+
+Message properties files for themes are now read in UTF-8 encoding, with an automatic fallback to ISO-8859-1 encoding.
+If you are using a different encoding, convert the files to UTF-8.
+
@@ -22,10 +22,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
 import java.net.URL;
-import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
@@ -72,9 +69,8 @@ public void init(String name, Type type, ClassLoader classLoader) throws IOExcep
 
         URL p = classLoader.getResource(themeRoot + "theme.properties");
         if (p != null) {
-            Charset encoding = PropertiesUtil.detectEncoding(p.openStream());
-            try (Reader reader = new InputStreamReader(p.openStream(), encoding)) {
-                properties.load(reader);
+            try (InputStream stream = p.openStream()) {
+                PropertiesUtil.readCharsetAware(properties, stream);
             }
             this.parentName = properties.getProperty("parent");
             this.importName = properties.getProperty("import");
@@ -143,11 +139,10 @@ public Properties getMessages(String baseBundlename, Locale locale) throws IOExc
         }
         Properties m = new Properties();
 
-        URL url = classLoader.getResource(this.messageRoot + baseBundlename + "_" + locale.toString() + ".properties");
+        URL url = classLoader.getResource(this.messageRoot + baseBundlename + "_" + locale + ".properties");
         if (url != null) {
-            Charset encoding = PropertiesUtil.detectEncoding(url.openStream());
-            try (Reader reader = new InputStreamReader(url.openStream(), encoding)) {
-                m.load(reader);
+            try (InputStream stream = url.openStream()) {
+                PropertiesUtil.readCharsetAware(m, stream);
             }
         }
         return m;

@@ -2,10 +2,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
 import java.net.URL;
-import java.nio.charset.Charset;
 import java.util.Locale;
 import java.util.Properties;
 
@@ -71,10 +68,8 @@ public Properties getMessages(String baseBundlename, Locale locale) throws IOExc
 
     protected void loadMessages(Properties messages, URL resource) throws IOException {
         if (resource != null) {
-            Charset encoding = PropertiesUtil.detectEncoding(resource.openStream());
-            // detectEncoding closes the stream
-            try (Reader reader = new InputStreamReader(resource.openStream(), encoding)) {
-                messages.load(reader);
+            try (InputStream stream = resource.openStream()) {
+                PropertiesUtil.readCharsetAware(messages, stream);
             }
         }
     }

@@ -21,17 +21,15 @@
 import org.keycloak.services.util.LocaleUtil;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
 import java.net.URL;
-import java.nio.charset.Charset;
+import java.nio.file.Files;
 import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.regex.Pattern;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
@@ -53,10 +51,9 @@ public FolderTheme(File themeDir, String name, Type type) throws IOException {
         this.properties = new Properties();
 
         File propertiesFile = new File(themeDir, "theme.properties");
-        if (propertiesFile .isFile()) {
-            Charset encoding = PropertiesUtil.detectEncoding(new FileInputStream(propertiesFile));
-            try (Reader reader = new InputStreamReader(new FileInputStream(propertiesFile), encoding)) {
-                properties.load(reader);
+        if (propertiesFile.isFile()) {
+            try (InputStream stream = Files.newInputStream(propertiesFile.toPath())) {
+                PropertiesUtil.readCharsetAware(properties, stream);
             }
             parentName = properties.getProperty("parent");
             importName = properties.getProperty("import");
@@ -110,6 +107,8 @@ public Properties getMessages(Locale locale) throws IOException {
         return getMessages("messages", locale);
     }
 
+    private static final Pattern LEGAL_LOCALE = Pattern.compile("[a-zA-Z0-9-_]*");
+
     @Override
     public Properties getMessages(String baseBundlename, Locale locale) throws IOException {
         if (locale == null){
@@ -118,11 +117,16 @@ public Properties getMessages(String baseBundlename, Locale locale) throws IOExc
 
         Properties m = new Properties();
 
-        File file = new File(themeDir, "messages" + File.separator + baseBundlename + "_" + locale.toString() + ".properties");
+        String filename = baseBundlename + "_" + locale;
+
+        if (!LEGAL_LOCALE.matcher(filename).matches()) {
+            throw new RuntimeException("Found illegal characters in locale or bundle name: " + filename);
+        }
+
+        File file = new File(themeDir, "messages" + File.separator + filename + ".properties");
         if (file.isFile()) {
-            Charset encoding = PropertiesUtil.detectEncoding(new FileInputStream(file));
-            try (Reader reader = new InputStreamReader(new FileInputStream(file), encoding)) {
-                m.load(reader);
+            try (InputStream stream = Files.newInputStream(file.toPath())) {
+                PropertiesUtil.readCharsetAware(m, stream);
             }
         }
         return m;

@@ -17,56 +17,36 @@
 
 package org.keycloak.theme;
 
-import org.jboss.logging.Logger;
-
-import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.io.Reader;
+import java.util.Enumeration;
+import java.util.Properties;
+import java.util.PropertyResourceBundle;
 
 /**
  * @author <a href="mailto:wadahiro@gmail.com">Hiroyuki Wada</a>
  */
 public class PropertiesUtil {
 
-    private static final Logger logger = Logger.getLogger(PropertiesUtil.class);
-
-    public static final Pattern DETECT_ENCODING_PATTERN = Pattern.compile("^#\\s*encoding:\\s*([\\w.:-]+)",
-            Pattern.CASE_INSENSITIVE);
-
-    public static final Charset DEFAULT_ENCODING = Charset.forName("ISO-8859-1");
-
     /**
-     * <p>
-     * Detect file encoding from the first line of the property file. If the first line in the file doesn't contain the
-     * comment with the encoding, it uses ISO-8859-1 as default encoding for backwards compatibility.
-     * </p>
-     * <p>
-     * The specified stream is closed before this method returns.
-     * </p>
-     * 
-     * @param in The input stream
-     * @return Encoding
-     * @throws IOException
+     * Read a properties file either UTF-8 or if that doesn't work in ISO-8895-1 format.
+     * This utilizes the functionality present in JDK 9 to automatically detect the encoding of the resource.
+     * A user can specify the standard Java system property <code>java.util.PropertyResourceBundle.encoding</code>
+     * to change this.
+     * <p />
+     * Unfortunately the standard {@link Properties#load(Reader)} doesn't support this automatic decoding,
+     * as it is only been implemented for resource files.
+     *
+     * @see PropertyResourceBundle
      */
-    public static Charset detectEncoding(InputStream in) throws IOException {
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(in, DEFAULT_ENCODING))) {
-            String firstLine = br.readLine();
-            if (firstLine != null) {
-                Matcher matcher = DETECT_ENCODING_PATTERN.matcher(firstLine);
-                if (matcher.find()) {
-                    String encoding = matcher.group(1);
-                    if (Charset.isSupported(encoding)) {
-                        return Charset.forName(encoding);
-                    } else {
-                        logger.warnv("Unsupported encoding: {0}", encoding);
-                    }
-                }
-            }
+    public static void readCharsetAware(Properties properties, InputStream stream) throws IOException {
+        PropertyResourceBundle propertyResourceBundle = new PropertyResourceBundle(stream);
+        Enumeration<String> keys = propertyResourceBundle.getKeys();
+        while(keys.hasMoreElements()) {
+            String s = keys.nextElement();
+            properties.put(s, propertyResourceBundle.getString(s));
         }
-        return DEFAULT_ENCODING;
     }
+
 }
@@ -20,36 +20,47 @@
 import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
 /**
  * @author <a href="mailto:wadahiro@gmail.com">Hiroyuki Wada</a>
  */
 public class PropertiesUtilTest {
 
-    @Test
-    public void testDetectEncoding() throws Exception {
-        Charset encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("# encoding: utf-8\nkey=value".getBytes()));
-        assertEquals(Charset.forName("utf-8"), encoding);
+    String valueWithUmlaut = "Umlaut: \u00E4\u00F6\u00FC";
 
-        encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("# encoding: Shift_JIS\nkey=value".getBytes()));
-        assertEquals(Charset.forName("Shift_JIS"), encoding);
-    }
+    String key = "key";
+
+    String propertyLine = key + "=" + valueWithUmlaut;
 
     @Test
-    public void testDefaultEncoding() throws Exception {
-        Charset encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("key=value".getBytes()));
-        assertEquals(Charset.forName("ISO-8859-1"), encoding);
+    public void testEncodingIso() throws Exception {
+        testWithEncoding(StandardCharsets.ISO_8859_1);
+    }
 
-        encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("# encoding: unknown\nkey=value".getBytes()));
-        assertEquals(Charset.forName("ISO-8859-1"), encoding);
+    @Test
+    public void testEncodingUtf8() throws Exception {
+        testWithEncoding(StandardCharsets.UTF_8);
+    }
 
-        encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("\n# encoding: utf-8\nkey=value".getBytes()));
-        assertEquals(Charset.forName("ISO-8859-1"), encoding);
+    @Test
+    public void testIfValueContainsSpecialCharacters() {
+        assertNotEquals(valueWithUmlaut.getBytes(StandardCharsets.UTF_8), valueWithUmlaut.getBytes(StandardCharsets.ISO_8859_1));
+    }
 
-        encoding = PropertiesUtil.detectEncoding(new ByteArrayInputStream("".getBytes()));
-        assertEquals(Charset.forName("ISO-8859-1"), encoding);
+    private void testWithEncoding(Charset charset) throws IOException {
+        Properties p = new Properties();
+        try (InputStream stream = new ByteArrayInputStream(propertyLine.getBytes(charset))) {
+            PropertiesUtil.readCharsetAware(p, stream);
+        }
+        assertEquals(p.get(key), valueWithUmlaut);
     }
+
 }
@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=حفظ
 doCancel=إلغاء
 doLogOutAllSessions=تسجيل الخروج لجميع الجلسات

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Desa
 doCancel=Cancel·la
 doLogOutAllSessions=Desconnecta de totes les sessions

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Uložit
 doCancel=Zrušit
 doLogOutAllSessions=Odhlásit všechny relace

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Gem
 doCancel=Annuller
 doLogOutAllSessions=Log alle sessioner ud

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Speichern
 doCancel=Abbrechen
 doLogOutAllSessions=Alle Sitzungen abmelden

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 role_manage-identity-providers=Διαχείριση παρόχων ταυτότητας
 doRemove=Αφαίρεση
 doAdd=Προσθήκη

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Guardar
 doCancel=Cancelar
 doLogOutAllSessions=Desconectar de todas las sesiones

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=ذخیره
 doCancel=لغو کنید
 doLogOutAllSessions=از تمام جلسات خارج شوید

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Tallenna
 doCancel=Peruuta
 doLogOutAllSessions=Kirjaudu ulos kaikista sessioista

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Sauvegarder
 doCancel=Annuler
 doLogOutAllSessions=Déconnexion de toutes les sessions

@@ -1,5 +1,3 @@
-# encoding: UTF-8
-
 doSave=Mentés
 doCancel=Mégsem
 doLogOutAllSessions=Minden munkamenet kiléptetése