From 8dd0e5f80970777fb063078a1738eab1df6bcddf Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Tue, 15 Sep 2026 02:17:09 -0400 Subject: [PATCH 1/2] Read package manifests with utf-8-sig so a BOM does not drop the package Python's utf-8 codec does not consume a byte order mark. It decodes the three bytes to U+FEFF, and json.load then raises "Unexpected UTF-8 BOM (decode using utf-8-sig)". packagedcode opens every manifest with encoding='utf-8', and recognize.py _parse() swallows the exception and continues, so a manifest that carries a BOM disappears from the scan with scan_errors left empty. The input is valid. npm, Composer and the other package managers read these files, and a BOM is what several Windows editors and PowerShell's Out-File write by default. utf-8-sig differs from utf-8 only in consuming a leading BOM if one is there, so this is a safe swap at all 25 manifest read sites in packagedcode. There was no BOM-aware decode anywhere in the tree before this. Signed-off-by: Arpit Jain --- CHANGELOG.rst | 3 ++ src/packagedcode/about.py | 2 +- src/packagedcode/bower.py | 2 +- src/packagedcode/chef.py | 4 +- src/packagedcode/conan.py | 4 +- src/packagedcode/conda.py | 6 +-- src/packagedcode/cran.py | 2 +- src/packagedcode/freebsd.py | 2 +- src/packagedcode/go_mod.py | 4 +- src/packagedcode/godeps.py | 2 +- src/packagedcode/haxe.py | 2 +- src/packagedcode/npm.py | 6 +-- src/packagedcode/phpcomposer.py | 4 +- src/packagedcode/readme.py | 2 +- src/packagedcode/rpm.py | 2 +- src/packagedcode/swift.py | 6 +-- tests/packagedcode/test_utf8_bom.py | 63 +++++++++++++++++++++++++++++ 17 files changed, 91 insertions(+), 25 deletions(-) create mode 100644 tests/packagedcode/test_utf8_bom.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf4db410925..acf3c2af2b9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog Next release -------------- +- Read package manifests with the ``utf-8-sig`` codec so that a UTF-8 BOM does + not make the package disappear from the scan. + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/packagedcode/about.py b/src/packagedcode/about.py index 5a1192fdd28..4468678779a 100644 --- a/src/packagedcode/about.py +++ b/src/packagedcode/about.py @@ -52,7 +52,7 @@ def parse(cls, location, package_only=False): Yield one or more Package manifest objects given a file ``location`` pointing to a package archive, manifest or similar. """ - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_data = saneyaml.load(loc.read()) # About files can contain any purl and also have a namespace diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py index 5bf5c949020..49cfcc018b6 100644 --- a/src/packagedcode/bower.py +++ b/src/packagedcode/bower.py @@ -26,7 +26,7 @@ class BowerJsonHandler(models.DatafileHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_data = json.load(loc) name = package_data.get('name') diff --git a/src/packagedcode/chef.py b/src/packagedcode/chef.py index d5df6a2f6c4..28b9f755e68 100644 --- a/src/packagedcode/chef.py +++ b/src/packagedcode/chef.py @@ -188,7 +188,7 @@ def parse(cls, location, package_only=False): Yield one or more Package manifest objects given a file ``location`` pointing to a package archive, manifest or similar. """ - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_data = json.load(loc) yield build_package( package_data=package_data, @@ -207,7 +207,7 @@ class ChefMetadataRbHandler(BaseChefMetadataHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: file_contents = loc.read() # we use a Pygments formatter for parsing lexed Ruby code diff --git a/src/packagedcode/conan.py b/src/packagedcode/conan.py index 86e6ae93353..adc1a2b24c2 100644 --- a/src/packagedcode/conan.py +++ b/src/packagedcode/conan.py @@ -167,7 +167,7 @@ def _parse(cls, conan_recipe, package_only=False): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding="utf-8") as loc: + with io.open(location, encoding="utf-8-sig") as loc: conan_recipe = loc.read() yield cls._parse(conan_recipe, package_only) @@ -186,7 +186,7 @@ class ConanDataHandler(models.DatafileHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding="utf-8") as loc: + with io.open(location, encoding="utf-8-sig") as loc: conan_data = loc.read() conan_data = saneyaml.load(conan_data) diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index 19e0462f883..5ae0a4eec97 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -322,7 +322,7 @@ class CondaMetaJsonHandler(CondaBaseHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: conda_metadata = json.load(loc) name = conda_metadata.get('name') @@ -614,7 +614,7 @@ def get_meta_yaml_data(location): # FIXME: use Jinja to process these variables = get_variables(location) yaml_lines = [] - with io.open(location, encoding='utf-8') as metayaml: + with io.open(location, encoding='utf-8-sig') as metayaml: for line in metayaml: if not line: continue @@ -656,7 +656,7 @@ def get_variables(location): {% set sha256 = "bc7512f2eef785b037d836f4cc6faded457ac277f75c6e34eccd12da7c85258f" %} """ result = {} - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: for line in loc.readlines(): if not line: continue diff --git a/src/packagedcode/cran.py b/src/packagedcode/cran.py index 827ff38f9d3..14a410b056b 100644 --- a/src/packagedcode/cran.py +++ b/src/packagedcode/cran.py @@ -115,7 +115,7 @@ def get_cran_description(location): Parse a CRAN DESCRIPTION file as YAML and return a mapping of metadata. """ yaml_lines = [] - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: for line in loc.readlines(): if not line: continue diff --git a/src/packagedcode/freebsd.py b/src/packagedcode/freebsd.py index d170e8f8222..dcd5442824a 100644 --- a/src/packagedcode/freebsd.py +++ b/src/packagedcode/freebsd.py @@ -113,7 +113,7 @@ def parse(cls, location, package_only=False): Yield one or more Package manifest objects given a file ``location`` pointing to a package archive, manifest or similar. """ - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: yaml_data = saneyaml.load(loc) yield cls._parse(yaml_data, package_only) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 527931c2dcc..4e449ec5ef2 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -114,7 +114,7 @@ def parse_gomod(location): >>> assert p.group('ns_name') == ('github.com/davecgh/go-spew') >>> assert p.group('version') == ('v1.1.1') """ - with io.open(location, encoding='utf-8', closefd=True) as data: + with io.open(location, encoding='utf-8-sig', closefd=True) as data: lines = data.readlines() gomods = GoModule() @@ -229,7 +229,7 @@ def parse_gosum(location): >>> assert p.group('version') == ('v0.3.1') >>> assert p.group('checksum') == ('WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=') """ - with io.open(location, encoding='utf-8', closefd=True) as data: + with io.open(location, encoding='utf-8-sig', closefd=True) as data: lines = data.readlines() gosums = [] diff --git a/src/packagedcode/godeps.py b/src/packagedcode/godeps.py index b584d23c2a2..51ee4b6eb6e 100644 --- a/src/packagedcode/godeps.py +++ b/src/packagedcode/godeps.py @@ -138,7 +138,7 @@ def load(self, location): Load self from a location string or a file-like object containing a Godeps JSON. """ - with io.open(location, encoding='utf-8') as godep: + with io.open(location, encoding='utf-8-sig') as godep: text = godep.read() return self.loads(text) diff --git a/src/packagedcode/haxe.py b/src/packagedcode/haxe.py index 24ff870db53..b6ded0cd5c2 100644 --- a/src/packagedcode/haxe.py +++ b/src/packagedcode/haxe.py @@ -109,7 +109,7 @@ def parse(cls, location, package_only=False): "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"] } """ - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: json_data = json.load(loc) yield cls._parse(json_data, package_only) diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 7618e830c19..89c40c7c3ed 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -609,7 +609,7 @@ def _parse(cls, json_data, package_only=False): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: json_data = json.load(loc) yield cls._parse(json_data, package_only) @@ -620,7 +620,7 @@ class BaseNpmLockHandler(BaseNpmHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_data = json.load(loc) # we have two formats: v1 and v2 @@ -1004,7 +1004,7 @@ def parse(cls, location, package_only=False): Yield a single PackageData """ - with io.open(location, encoding='utf-8') as yl: + with io.open(location, encoding='utf-8-sig') as yl: yl_dependencies = yl.read().split('\n\n') dependencies_by_purl = {} diff --git a/src/packagedcode/phpcomposer.py b/src/packagedcode/phpcomposer.py index 5a3ee59af1f..d448a03ac3c 100644 --- a/src/packagedcode/phpcomposer.py +++ b/src/packagedcode/phpcomposer.py @@ -66,7 +66,7 @@ def parse(cls, location, package_only=False): Note that this is NOT exactly the packagist.json format (all are closely related of course but have important (even if minor) differences. """ - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_json = json.load(loc) yield build_package_data(package_json, package_only) @@ -175,7 +175,7 @@ class PhpComposerLockHandler(BasePhpComposerHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as loc: + with io.open(location, encoding='utf-8-sig') as loc: package_data = json.load(loc) packages = [ diff --git a/src/packagedcode/readme.py b/src/packagedcode/readme.py index a03886929ba..18e255f487b 100644 --- a/src/packagedcode/readme.py +++ b/src/packagedcode/readme.py @@ -56,7 +56,7 @@ class ReadmeHandler(models.NonAssemblableDatafileHandler): @classmethod def parse(cls, location, package_only=False): - with open(location, encoding='utf-8') as loc: + with open(location, encoding='utf-8-sig') as loc: readme_manifest = loc.read() package_data = build_package(readme_manifest, package_only) diff --git a/src/packagedcode/rpm.py b/src/packagedcode/rpm.py index 4cce7e8d1f3..1e88fe287ee 100644 --- a/src/packagedcode/rpm.py +++ b/src/packagedcode/rpm.py @@ -410,7 +410,7 @@ class RpmMarinerContainerManifestHandler(models.DatafileHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding='utf-8') as data: + with io.open(location, encoding='utf-8-sig') as data: lines = data.readlines() for line in lines: diff --git a/src/packagedcode/swift.py b/src/packagedcode/swift.py index 40a2c7742b9..e2f720fca0d 100644 --- a/src/packagedcode/swift.py +++ b/src/packagedcode/swift.py @@ -203,7 +203,7 @@ def _parse(cls, swift_dependency_relation, package_only=False): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding="utf-8") as loc: + with io.open(location, encoding="utf-8-sig") as loc: swift_dependency_relation = json.load(loc) yield cls._parse(swift_dependency_relation, package_only) @@ -242,7 +242,7 @@ def _parse(cls, swift_manifest, package_only=False): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding="utf-8") as loc: + with io.open(location, encoding="utf-8-sig") as loc: swift_manifest = json.load(loc) yield cls._parse(swift_manifest, package_only) @@ -261,7 +261,7 @@ class SwiftPackageResolvedHandler(BaseSwiftDatafileHandler): @classmethod def parse(cls, location, package_only=False): - with io.open(location, encoding="utf-8") as loc: + with io.open(location, encoding="utf-8-sig") as loc: package_resolved = json.load(loc) resolved_doc_version = package_resolved.get("version") diff --git a/tests/packagedcode/test_utf8_bom.py b/tests/packagedcode/test_utf8_bom.py new file mode 100644 index 00000000000..9f68888daeb --- /dev/null +++ b/tests/packagedcode/test_utf8_bom.py @@ -0,0 +1,63 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest + +from packagedcode import bower +from packagedcode import haxe +from packagedcode import npm +from packagedcode import phpcomposer + + +BOM = '' + +MANIFESTS = [ + (npm.NpmPackageJsonHandler, 'package.json', + '{"name": "demo", "version": "1.0.0", "license": "MIT"}'), + (bower.BowerJsonHandler, 'bower.json', + '{"name": "demo", "version": "1.0.0", "license": "MIT"}'), + (phpcomposer.PhpComposerJsonHandler, 'composer.json', + '{"name": "acme/demo", "version": "1.0.0", "license": "MIT"}'), + (haxe.HaxelibJsonHandler, 'haxelib.json', + '{"name": "demo", "version": "1.0.0", "license": "MIT"}'), +] + + +@pytest.mark.parametrize( + 'handler, filename, content', + MANIFESTS, + ids=[filename for _, filename, _ in MANIFESTS], +) +def test_manifest_with_a_utf8_bom_is_parsed(handler, filename, content, tmp_path): + """A BOM is legal in a package manifest and package managers accept one. + + Python's utf-8 codec does not consume it, it decodes the three bytes to + U+FEFF and json.load then raises, so the whole package used to disappear + from the scan. + """ + location = tmp_path / filename + location.write_text(BOM + content, encoding='utf-8') + + packages = list(handler.parse(location=str(location))) + + assert [(p.name, p.version) for p in packages] == [('demo', '1.0.0')] + + +@pytest.mark.parametrize( + 'handler, filename, content', + MANIFESTS, + ids=[filename for _, filename, _ in MANIFESTS], +) +def test_manifest_without_a_bom_is_unaffected(handler, filename, content, tmp_path): + location = tmp_path / filename + location.write_text(content, encoding='utf-8') + + packages = list(handler.parse(location=str(location))) + + assert [(p.name, p.version) for p in packages] == [('demo', '1.0.0')] From 5fea395e53c682473cfcaf599f72aad3dfef7f56 Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Tue, 15 Sep 2026 02:17:40 -0400 Subject: [PATCH 2/2] Link the changelog entry to the pull request Signed-off-by: Arpit Jain --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index acf3c2af2b9..734ba64f042 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,7 @@ Next release - Read package manifests with the ``utf-8-sig`` codec so that a UTF-8 BOM does not make the package disappear from the scan. + https://github.com/aboutcode-org/scancode-toolkit/pull/5321 - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``.