From 4143c67a48d0cd70c1c07077b5f5377117cfaf55 Mon Sep 17 00:00:00 2001 From: lipengyu Date: Fri, 18 Sep 2026 23:45:28 +0800 Subject: [PATCH 1/4] Fix RFC 2231 charset handling in the email header parser get_parameter() iterated over the outer Value node when looking for the extended-attrtext terminal that contains the RFC 2231 charset. Since the terminal is nested inside an Attribute or QuotedString, the loop never reached it. Descend into the charset container before retyping the terminal as attrtext, and use stripped_value for Parameter.charset so trailing CFWS is not included. --- Lib/email/_header_value_parser.py | 11 ++++--- .../test_email/test__header_value_parser.py | 32 +++++++++++++++++++ ...-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst | 3 ++ 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 7c03fb49b3b5e7..f7ed37022a54c0 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2580,12 +2580,15 @@ def get_parameter(value): return param, value else: if token is not None: - for t in token: + # The charset is parsed as a Value child, but belongs to the + # parameter itself. Retype its extended-attrtext as plain attrtext. + charset = token[0] + for t in charset: if t.token_type == 'extended-attrtext': + t.token_type = 'attrtext' break - t.token_type == 'attrtext' - appendto.append(t) - param.charset = t.value + appendto.append(charset) + param.charset = charset.stripped_value if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 9d9fe418ee4d06..eb29bf6e525b99 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3031,6 +3031,38 @@ def test_parse_message_ids_broken_ang(self): +class Test_get_parameter(TestParserMixin, TestEmailBase): + + def test_rfc2231_charset_is_retyped_as_attrtext(self): + # gh-150474: the charset of an initial extended parameter is moved + # out of the value and into the parameter itself, and while doing so + # its text is retyped as an attrtext token. + param = self._test_get_x(parser.get_parameter, + "title*=us-ascii'en'This%20is%20a%20test", + "title*=us-ascii'en'This%20is%20a%20test", + "title*=us-ascii'en'This%20is%20a%20test", + [], + '') + self.assertEqual(param.token_type, 'parameter') + self.assertEqual(param[3].token_type, 'attribute') + self.assertEqual(param[3][0].token_type, 'attrtext') + self.assertEqual(param.charset, 'us-ascii') + + def test_rfc2231_charset_strips_cfws(self): + # gh-150474: CFWS around the charset is part of the parse tree, but + # must not end up in the charset itself. + param = self._test_get_x(parser.get_parameter, + "title*=us-ascii 'en'This%20is%20a%20test", + "title*=us-ascii 'en'This%20is%20a%20test", + "title*=us-ascii 'en'This%20is%20a%20test", + [], + '') + self.assertEqual(param[3].token_type, 'attribute') + self.assertEqual(param[3][0].token_type, 'attrtext') + self.assertEqual(param[3][1].token_type, 'cfws') + self.assertEqual(param.charset, 'us-ascii') + + @parameterize class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): diff --git a/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst new file mode 100644 index 00000000000000..f5cf73b79b6707 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst @@ -0,0 +1,3 @@ +Fixed an issue in :mod:`email` where an incorrect comparison in the header +value parser :func:`get_parameter` could lead to improper parameter +handling. From 0c10e9f4e31eaa935a834ab148ff960953c38d98 Mon Sep 17 00:00:00 2001 From: lipengyu Date: Sat, 19 Sep 2026 00:01:32 +0800 Subject: [PATCH 2/4] update --- Lib/test/test_email/test__header_value_parser.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index eb29bf6e525b99..21bc3fea015f86 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3029,8 +3029,6 @@ def test_parse_message_ids_broken_ang(self): ) self.assertEqual(message_ids.token_type, 'message-id-list') - - class Test_get_parameter(TestParserMixin, TestEmailBase): def test_rfc2231_charset_is_retyped_as_attrtext(self): @@ -3062,7 +3060,6 @@ def test_rfc2231_charset_strips_cfws(self): self.assertEqual(param[3][1].token_type, 'cfws') self.assertEqual(param.charset, 'us-ascii') - @parameterize class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): From 69c706e232262a386207c5f00fb4b8acfd7ad8dd Mon Sep 17 00:00:00 2001 From: lipengyu Date: Sat, 19 Sep 2026 00:04:01 +0800 Subject: [PATCH 3/4] update news --- .../Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst index f5cf73b79b6707..798bfebf2aebf9 100644 --- a/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst +++ b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst @@ -1,3 +1,2 @@ -Fixed an issue in :mod:`email` where an incorrect comparison in the header -value parser :func:`get_parameter` could lead to improper parameter -handling. +Fix parsing of extended email parameters when CFWS follows the charset, +preventing the charset value from incorrectly including the trailing CFWS. From a80fc9ba22e8f00c8daa62b3b35784351e06c500 Mon Sep 17 00:00:00 2001 From: lipengyu Date: Sat, 19 Sep 2026 21:55:17 +0800 Subject: [PATCH 4/4] update --- Lib/test/test_email/test__header_value_parser.py | 2 ++ .../Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 21bc3fea015f86..d4ccd551299b4b 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3029,6 +3029,7 @@ def test_parse_message_ids_broken_ang(self): ) self.assertEqual(message_ids.token_type, 'message-id-list') + class Test_get_parameter(TestParserMixin, TestEmailBase): def test_rfc2231_charset_is_retyped_as_attrtext(self): @@ -3060,6 +3061,7 @@ def test_rfc2231_charset_strips_cfws(self): self.assertEqual(param[3][1].token_type, 'cfws') self.assertEqual(param.charset, 'us-ascii') + @parameterize class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): diff --git a/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst index 798bfebf2aebf9..449933e43a67b8 100644 --- a/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst +++ b/Misc/NEWS.d/next/Library/2026-09-18-23-41-48.gh-issue-150474.uJ3cL-.rst @@ -1,2 +1,4 @@ -Fix parsing of extended email parameters when CFWS follows the charset, -preventing the charset value from incorrectly including the trailing CFWS. +Fix the parsing of the character set of :rfc:`2231` encoded parameters in the +:mod:`email` header parser. The character set is now correctly located in +the parsed value, so whitespace or a comment following it is no longer +included in the parameter's character set name.