fix: Strip UTF-8 BOM from resources to prevent corruption in XML and properties files

internal/documentation/docs/pages/Configuration.md

@@ -208,6 +208,8 @@ If your project uses a different encoding for `*.properties` files, you need to
 
 UI5 CLI will read the corresponding files of the project in the given encoding. Any non-ASCII characters will be replaced with the respective Unicode escape sequences. This allows you to deploy the resulting files to any environment, independent of how it expects `*.properties` files to be encoded. Please refer to [RFC 7](https://github.com/UI5/cli/blob/main/rfcs/0007-properties-file-encoding.md) for details.
 
+If a `*.properties` file was accidentally saved with a leading UTF-8 Byte Order Mark (BOM), the BOM is removed during this process. Otherwise it would be escaped into a literal `\uFEFF` sequence at the start of the file, corrupting the first key.
+
 ## Custom Configuration
 
 ::: info

packages/builder/lib/lbt/bundle/AutoSplitter.js

@@ -3,7 +3,7 @@ import {pd} from "pretty-data";
 import {toRequireJSName} from "../utils/ModuleName.js";
 import {SectionType} from "./BundleDefinition.js";
 import escapePropertiesFile from "../utils/escapePropertiesFile.js";
-import {makeStringLiteral} from "../utils/stringUtils.js";
+import {makeStringLiteral, stripBOM} from "../utils/stringUtils.js";
 import {getLogger} from "@ui5/logger";
 const log = getLogger("lbt:bundle:AutoSplitter");
 
@@ -312,19 +312,22 @@ class AutoSplitter {
 				return fileContent.length;
 			} else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) {
 				// needs to be activated when it gets activated in JSMergedModuleBuilderExt
-				let fileContent = await resource.buffer();
+				let fileContent = stripBOM((await resource.buffer()).toString());
 				if ( this.optimize ) {
 					// For XML we use the pretty data
 					// Do not minify if XML(View) contains an <*:pre> tag because whitespace of
 					//	HTML <pre> should be preserved (should only happen rarely)
-					if (!xmlHtmlPrePattern.test(fileContent.toString())) {
-						fileContent = pd.xmlmin(fileContent.toString(), false);
+					if (!xmlHtmlPrePattern.test(fileContent)) {
+						fileContent = pd.xmlmin(fileContent, false);
 					}
 				}
 				return fileContent.length;
 			}
 
 			// if there is no precompiled information about the resource, just determine its length
+			// Note: this is a rough size estimate used only for split-point selection, so (unlike the
+			// Builder and the optimizeXMLViews branch above) it does not strip a leading UTF-8 BOM.
+			// A BOM'd resource is therefore counted ~3 bytes larger here, which is immaterial.
 			if ( !resource.info && /\.(js|json|xml)$/.test(module) ) {
 				const fileContent = await resource.buffer();
 				return fileContent.length;

packages/builder/lib/lbt/bundle/Builder.js

@@ -11,7 +11,7 @@ import {
 	MODULE__UI5LOADER, MODULE__UI5LOADER_AUTOCONFIG,
 	MODULE__JQUERY_SAP_GLOBAL, MODULE__SAP_UI_CORE_CORE} from "../UI5ClientConstants.js";
 import escapePropertiesFile from "../utils/escapePropertiesFile.js";
-import {makeStringLiteral, removeHashbang} from "../utils/stringUtils.js";
+import {makeStringLiteral, removeHashbang, stripBOM} from "../utils/stringUtils.js";
 import BundleResolver from "./Resolver.js";
 import BundleSplitter from "./AutoSplitter.js";
 import {SectionType} from "./BundleDefinition.js";
@@ -453,6 +453,11 @@ class BundleBuilder {
 			outW.writeln(`function(){`);
 			// The module should be written to a new line in order for dev-tools to map breakpoints to it
 			outW.ensureNewLine();
+			// Note: a leading UTF-8 BOM is not stripped here. The JS code is written verbatim, where a
+			// U+FEFF is a valid whitespace character to the parser, so it does not corrupt the module.
+			// Stripping it would also shift all byte offsets and invalidate the source map handled below.
+			// (removeHashbang below still drops a hashbang that sits behind such a BOM, which otherwise
+			// would be an invalid HashbangComment and break the bundle.)
 			let moduleContent = (await resource.buffer()).toString();
 			moduleContent = removeHashbang(moduleContent);
 			if (this.options.sourceMap) {
@@ -473,6 +478,8 @@ class BundleBuilder {
 		} else if ( /\.js$/.test(moduleName) /* implicitly: && info != null && info.requiresTopLevelScope */ ) {
 			log.warn(
 				`Module ${moduleName} requires top level scope and can only be embedded as a string (requires 'eval')`);
+			// Note: a leading UTF-8 BOM is not stripped here either. The string literal is eval'd as JS,
+			// where a leading U+FEFF is whitespace and does not corrupt the module (unlike XML/JSON/properties).
 			let moduleContent = (await resource.buffer()).toString();
 			moduleContent = removeHashbang(moduleContent);
 			if (this.options.sourceMap) {
@@ -489,10 +496,10 @@ class BundleBuilder {
 			}
 			outW.write( makeStringLiteral(moduleContent) );
 		} else if ( /\.html$/.test(moduleName) ) {
-			const fileContent = (await resource.buffer()).toString();
+			const fileContent = stripBOM((await resource.buffer()).toString());
 			outW.write( makeStringLiteral( fileContent ) );
 		} else if ( /\.json$/.test(moduleName) ) {
-			let fileContent = (await resource.buffer()).toString();
+			let fileContent = stripBOM((await resource.buffer()).toString());
 			if ( this.optimize ) {
 				try {
 					fileContent = JSON.stringify( JSON.parse( fileContent) );
@@ -503,7 +510,7 @@ class BundleBuilder {
 			}
 			outW.write(makeStringLiteral(fileContent));
 		} else if ( /\.xml$/.test(moduleName) ) {
-			let fileContent = (await resource.buffer()).toString();
+			let fileContent = stripBOM((await resource.buffer()).toString());
 			if ( this.optimize ) {
 				// For XML we use the pretty data
 				// Do not minify if XML(View) contains an <*:pre> tag,

packages/builder/lib/lbt/utils/stringUtils.js

@@ -13,5 +13,30 @@ export function makeStringLiteral(str) {
 }
 
 export function removeHashbang(str) {
-	return str.replace(/^#!(.*)/, "");
+	// A hashbang is only a valid ECMAScript HashbangComment when "#!" are the first characters of the
+	// source. If the file was saved with a UTF-8 BOM, the "#!" is no longer first and would become a
+	// syntax error once embedded in a bundle. Match (and keep) an optional leading BOM so the hashbang
+	// line is removed in that case too; the BOM is valid JS whitespace and is left untouched.
+	return str.replace(/^(\uFEFF)?#!(.*)/, "$1");
+}
+
+/**
+ * Removes a leading UTF-8 Byte Order Mark (BOM, U+FEFF) from the given string.
+ *
+ * When a source file is saved as "UTF-8 with BOM", decoding its buffer to a string keeps the BOM
+ * as a leading U+FEFF character. Inside a bundle this character ends up within the inlined string
+ * literal, which breaks strict XML engines (e.g. the Rust-based DOMParser in Chromium 153+) and
+ * corrupts the first key of *.properties files. A BOM in a UI5 source file is always unintended,
+ * so it is stripped before the content is embedded.
+ *
+ * Note: We intentionally do not decode via TextDecoder (which strips a BOM on its own). It operates
+ * on a Buffer rather than the already-decoded strings handled here, and its "latin1" label maps to
+ * windows-1252 instead of the ISO-8859-1 that *.properties files rely on. This string-based helper
+ * matches the de-facto "strip-bom" contract (remove exactly one leading U+FEFF) without those pitfalls.
+ *
+ * @param {string} str input string that might start with a BOM
+ * @returns {string} the input string without a leading BOM
+ */
+export function stripBOM(str) {
+	return str.charCodeAt(0) === 0xFEFF ? str.slice(1) : str;
 }

packages/builder/lib/processors/nonAsciiEscaper.js

@@ -1,4 +1,5 @@
 import {escapeUnicode} from "escape-unicode";
+import {stripBOM} from "../lbt/utils/stringUtils.js";
 
 /**
  * @public
@@ -57,6 +58,13 @@ const escapeNonAscii = function(string) {
 /**
  * Escapes non ASCII characters with unicode escape sequences.
  *
+ * A leading UTF-8 Byte Order Mark (BOM, U+FEFF) is removed before escaping. Otherwise it would be
+ * escaped to a literal "\\uFEFF" sequence at the start of the content, corrupting the first key of
+ * a *.properties file. The removal is gated on the decoded content starting with U+FEFF, not on the
+ * encoding name: a UTF-8 BOM decodes to a leading U+FEFF and is removed, whereas an ISO-8859-1
+ * (latin1) decode can never produce U+FEFF (its code points are limited to U+0000-U+00FF), so
+ * latin1 files are left untouched.
+ *
  * ```js
  * const encoding = nonAsciiEscaper.getEncodingFromAlias("ISO-8859-1");
  * nonAsciiEscaper({resources, options: {encoding}});
@@ -79,9 +87,16 @@ async function nonAsciiEscaper({resources, options: {encoding}}) {
 
 	async function processResource(resource) {
 		const resourceString = (await resource.getBuffer()).toString(encoding);
-		const escaped = escapeNonAscii(resourceString);
-		// only modify the resource's string if it was changed
-		if (escaped.modified) {
+		// Strip a leading UTF-8 BOM before escaping. Otherwise the U+FEFF character would be escaped
+		// to a literal "\\uFEFF" sequence at the very start of the content, silently corrupting the
+		// first key of a *.properties file. This is the common choke point for both the standalone
+		// escapeNonAsciiCharacters task and the bundler (via escapePropertiesFile), so handling it
+		// here covers every path. Only UTF-8 input is affected: a BOM decoded as latin1 becomes the
+		// three characters "" (not U+FEFF), so ISO-8859-1 byte semantics are preserved.
+		const strippedString = stripBOM(resourceString);
+		const escaped = escapeNonAscii(strippedString);
+		// modify the resource if a BOM was stripped and/or non-ascii characters were escaped
+		if (escaped.modified || strippedString !== resourceString) {
 			resource.setString(escaped.string);
 			return resource;
 		}

packages/builder/test/lib/lbt/bundle/AutoSplitter.js

@@ -341,6 +341,31 @@ test("_calcMinSize: properties resource", async (t) => {
 	t.is(await autoSplitter._calcMinSize("mymodule.properties"), 10, "length of 1234\\u00df");
 });
 
+test("_calcMinSize: properties resource with UTF-8 BOM does not count the BOM", async (t) => {
+	const pool = {
+		findResourceWithInfo: function() {
+			let content = "\uFEFFKEY=val";
+			return {
+				buffer: async () => Buffer.from(content, "utf8"),
+				resource: {
+					setString: (string) => {
+						content = string;
+					},
+					getBuffer: async () => Buffer.from(content, "utf8")
+				},
+				getProject: () => {
+					return {
+						getPropertiesFileSourceEncoding: () => "UTF-8"
+					};
+				}
+			};
+		}
+	};
+	const autoSplitter = new AutoSplitter(pool);
+	t.is(await autoSplitter._calcMinSize("mymodule.properties"), "KEY=val".length,
+		"leading BOM is stripped and not counted in the size");
+});
+
 test("_calcMinSize: xml view resource", async (t) => {
 	const pool = {
 		findResourceWithInfo: function() {
@@ -355,6 +380,22 @@ test("_calcMinSize: xml view resource", async (t) => {
 	t.is(await autoSplitter._calcMinSize("mymodule.view.xml"), 5);
 });
 
+test("_calcMinSize: xml view resource with UTF-8 BOM does not count BOM bytes in size", async (t) => {
+	const xmlContent = "<xml/>";
+	const pool = {
+		findResourceWithInfo: function() {
+			return {
+				buffer: async () => Buffer.from("\uFEFF" + xmlContent, "utf8"),
+				getProject: () => undefined
+			};
+		}
+	};
+	const autoSplitter = new AutoSplitter(pool);
+	autoSplitter.optimizeXMLViews = true;
+	t.is(await autoSplitter._calcMinSize("mymodule.view.xml"), xmlContent.length,
+		"the 3 BOM bytes are stripped and not counted");
+});
+
 test("_calcMinSize: xml view resource without optimizeXMLViews", async (t) => {
 	const pool = {
 		findResourceWithInfo: function() {