From 1c60884e949310002be310ee17c6220918ae450e Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 2 Sep 2026 14:12:23 -0700 Subject: [PATCH 1/5] gh-156955: Speed up csv.writer by not scanning lineterminator per character join_append_data() tested every character of every field for membership in dialect->lineterminator with PyUnicode_FindChar(), an out-of-line call made twice per character (the function runs a count pass and a copy pass). At 54% of cycles it was the single hottest symbol in csv.writer, which made writing CSV slower than parsing it back. Cache the terminator's highest code point on the dialect, which is immutable, and compare inline. Ordinary text exceeds that maximum, so the membership test is skipped without touching the terminator at all; when it does run it is a short loop over the terminator's characters rather than a cross-module call. Membership semantics are unchanged, including multi-character, empty and non-BMP terminators. Interleaved A/B, median of 25 per-round ratios, pinned to one CPU: mixed 2000x4 1.430 ms -> 0.547 ms 2.62x text 2000x4 1.486 ms -> 0.561 ms 2.64x wide 500x2 (200ch) 3.159 ms -> 1.094 ms 2.88x mixed QUOTE_ALL 1.438 ms -> 0.606 ms 2.37x mixed lineterm='\n' 1.335 ms -> 0.605 ms 2.21x mixed lineterm='END' 1.535 ms -> 0.822 ms 1.87x quoted 2000x4 0.486 ms -> 0.323 ms 1.50x short 5000x4 0.906 ms -> 0.650 ms 1.39x csv.reader (control) 0.887 ms -> 0.887 ms 1.00x The maximum is cached on the dialect rather than recomputed per field so that the change never loses. Degenerate inputs (rows of empty fields, or a 4096-character lineterminator) measure 1.00-1.01x, and a long terminator with short fields improves from 0.11x to 2.94x against a per-field variant. --- ...-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst | 3 ++ Modules/_csv.c | 41 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst new file mode 100644 index 00000000000000..b0c058771451aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst @@ -0,0 +1,3 @@ +Speed up :func:`csv.writer` by up to 2.9x. Writing a field no longer calls +:c:func:`PyUnicode_FindChar` on the dialect's ``lineterminator`` once per +character. diff --git a/Modules/_csv.c b/Modules/_csv.c index c640f2d36a8464..e0e5c3050cefe5 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -116,6 +116,7 @@ typedef struct { Py_UCS4 delimiter; /* field separator */ Py_UCS4 quotechar; /* quote character */ Py_UCS4 escapechar; /* escape character */ + Py_UCS4 lineterm_maxchar; /* highest code point in lineterminator */ PyObject *lineterminator; /* string to write between records */ } DialectObj; @@ -332,6 +333,22 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) return 0; } +static Py_UCS4 +str_maxchar(PyObject *s) +{ + int kind = PyUnicode_KIND(s); + const void *data = PyUnicode_DATA(s); + Py_ssize_t len = PyUnicode_GET_LENGTH(s); + Py_UCS4 maxchar = 0; + for (Py_ssize_t i = 0; i < len; i++) { + Py_UCS4 c = PyUnicode_READ(kind, data, i); + if (c > maxchar) { + maxchar = c; + } + } + return maxchar; +} + static int dialect_check_quoting(int quoting) { @@ -533,6 +550,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); #undef DIASET + self->lineterm_maxchar = str_maxchar(self->lineterminator); /* validate options */ if (dialect_check_quoting(self->quoting)) @@ -1165,6 +1183,21 @@ join_reset(WriterObj *self) #define MEM_INCR 32768 +static inline int +in_lineterminator(Py_UCS4 c, DialectObj *dialect) +{ + PyObject *lt = dialect->lineterminator; + int kind = PyUnicode_KIND(lt); + const void *data = PyUnicode_DATA(lt); + Py_ssize_t len = PyUnicode_GET_LENGTH(lt); + for (Py_ssize_t i = 0; i < len; i++) { + if (PyUnicode_READ(kind, data, i) == c) { + return 1; + } + } + return 0; +} + /* Calculate new record length or append field to record. Return new * record length. */ @@ -1176,6 +1209,10 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, DialectObj *dialect = self->dialect; Py_ssize_t i; Py_ssize_t rec_len; + /* A character above this cannot be in the line terminator, so the + scan below is skipped; the default "\r\n" rejects all ordinary + text that way. */ + Py_UCS4 term_maxchar = dialect->lineterm_maxchar; #define INCLEN \ do {\ @@ -1213,9 +1250,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, c == dialect->quotechar || c == '\n' || c == '\r' || - PyUnicode_FindChar( - dialect->lineterminator, c, 0, - PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { + (c <= term_maxchar && in_lineterminator(c, dialect))) { if (dialect->quoting == QUOTE_NONE) want_escape = 1; else { From 3b2ec50f872a8dff92fa6dcacb8e1eb6752c66a0 Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 9 Sep 2026 13:18:32 -0400 Subject: [PATCH 2/5] Update Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maurycy Pawłowski-Wieroński --- .../Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst index b0c058771451aa..f3ed172027c07f 100644 --- a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst +++ b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst @@ -1,3 +1,2 @@ -Speed up :func:`csv.writer` by up to 2.9x. Writing a field no longer calls -:c:func:`PyUnicode_FindChar` on the dialect's ``lineterminator`` once per -character. +Speed up :func:`csv.writer` by up to 2.9x when fields contain no special +characters. From 40e6756f458a392b05ab3a82a9d5506a5620ac02 Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 9 Sep 2026 10:30:25 -0700 Subject: [PATCH 3/5] gh-156955: Cover line terminator quoting in the csv writer tests Apply the reviewer's suggestion on GH-156956: exercise csv.writer with a field that embeds the line terminator, over terminators spanning the latin-1, BMP and non-BMP string kinds. Add two more tests for the paths the optimization introduced but nothing pinned: * test_write_lineterminator_quoting -- every character of a multi-character terminator forces quoting, not just the last, and characters bracketing the terminator in code point order do not. * test_write_empty_lineterminator -- an empty terminator has no characters, so it separates nothing and quotes nothing, including '\0'. All three pass against Modules/_csv.c as it stood before the optimization, so they pin existing csv.writer semantics rather than new behavior. --- Lib/test/test_csv.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 73e282d1abf717..aa0efcc1e4f390 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -260,17 +260,44 @@ def test_write_escape(self): escapechar='\\', quoting=csv.QUOTE_MINIMAL) def test_write_lineterminator(self): - for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': + for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0', '\x85', + '\u2028', '\U0001f600'): with self.subTest(lineterminator=lineterminator): with StringIO() as sio: writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) writer.writerow(['\r', '\n']) + writer.writerow([f'a{lineterminator[-1]}b', 'c']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' f'1,2{lineterminator}' - f'"\r","\n"{lineterminator}') + f'"\r","\n"{lineterminator}' + f'"a{lineterminator[-1]}b",c{lineterminator}') + + def test_write_lineterminator_quoting(self): + # Every character of the line terminator forces quoting, not just the + # last one, and no other character does. Each terminator is paired + # with characters that bracket it in code point order, to pin that + # boundary. + for lineterminator, plain in ('!@#', ' ?A'), ('\u2028', '\u2027\u2029'): + with self.subTest(lineterminator=lineterminator): + for c in lineterminator: + self._write_test([f'a{c}b', 'c'], f'"a{c}b",c', + lineterminator=lineterminator) + self._write_test([f'a{plain}b', 'c'], f'a{plain}b,c', + lineterminator=lineterminator) + + def test_write_empty_lineterminator(self): + # An empty line terminator separates nothing and, having no characters + # of its own, forces no quoting -- not even of '\0', the lowest code + # point. '\r' and '\n' are quoted whatever the terminator is. + with StringIO() as sio: + writer = csv.writer(sio, lineterminator='') + writer.writerow(['a', 'b']) + writer.writerow(['\0', 'c']) + writer.writerow(['\r', '\n']) + self.assertEqual(sio.getvalue(), 'a,b\0,c"\r","\n"') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') From aeb530f525ef731e26dabf4b5ea28aa3e10a2afd Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 9 Sep 2026 10:43:31 -0700 Subject: [PATCH 4/5] gh-156955: Keep the line terminator test additions in their own tests Restore test_write_lineterminator to its original body and move the reviewer's suggestion into test_write_lineterminator_in_field, so the existing test keeps its charter -- the terminator is emitted between records -- and the new coverage stands on its own. Also strip a trailing space from the NEWS entry, which was failing the trim-trailing-whitespace pre-commit hook and turning the lint job red. --- Lib/test/test_csv.py | 20 ++++++++++++------- ...-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index aa0efcc1e4f390..21ec0c76955e2a 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -260,6 +260,19 @@ def test_write_escape(self): escapechar='\\', quoting=csv.QUOTE_MINIMAL) def test_write_lineterminator(self): + for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': + with self.subTest(lineterminator=lineterminator): + with StringIO() as sio: + writer = csv.writer(sio, lineterminator=lineterminator) + writer.writerow(['a', 'b']) + writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) + self.assertEqual(sio.getvalue(), + f'a,b{lineterminator}' + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') + + def test_write_lineterminator_in_field(self): for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0', '\x85', '\u2028', '\U0001f600'): with self.subTest(lineterminator=lineterminator): @@ -276,10 +289,6 @@ def test_write_lineterminator(self): f'"a{lineterminator[-1]}b",c{lineterminator}') def test_write_lineterminator_quoting(self): - # Every character of the line terminator forces quoting, not just the - # last one, and no other character does. Each terminator is paired - # with characters that bracket it in code point order, to pin that - # boundary. for lineterminator, plain in ('!@#', ' ?A'), ('\u2028', '\u2027\u2029'): with self.subTest(lineterminator=lineterminator): for c in lineterminator: @@ -289,9 +298,6 @@ def test_write_lineterminator_quoting(self): lineterminator=lineterminator) def test_write_empty_lineterminator(self): - # An empty line terminator separates nothing and, having no characters - # of its own, forces no quoting -- not even of '\0', the lowest code - # point. '\r' and '\n' are quoted whatever the terminator is. with StringIO() as sio: writer = csv.writer(sio, lineterminator='') writer.writerow(['a', 'b']) diff --git a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst index f3ed172027c07f..2cd00704e66bc8 100644 --- a/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst +++ b/Misc/NEWS.d/next/Library/2026-09-02-14-30-00.gh-issue-156955.Kv3Qa1.rst @@ -1,2 +1,2 @@ -Speed up :func:`csv.writer` by up to 2.9x when fields contain no special +Speed up :func:`csv.writer` by up to 2.9x when fields contain no special characters. From 50e89bed58230c949b5df4f29f2ab35e0330a01d Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Wed, 9 Sep 2026 10:53:11 -0700 Subject: [PATCH 5/5] Merge the line terminator tests back together test_write_lineterminator_in_field was a strict superset of test_write_lineterminator -- same rows, same assertion plus one line, and the original five terminators were all in its list -- so the older test asserted nothing the newer one did not. Apply the reviewer's suggestion in place, as it was written, and keep only the cases the merged test cannot reach in a second test: every character of a multi-character terminator rather than just the last, non-members bracketing each terminator in code point order, and the empty terminator, which cannot join the first loop because lineterminator[-1] raises IndexError on ''. Verified by mutating Modules/_csv.c: the two tests catch an off-by-one on the cached maximum (c <= -> c <), a maximum that is never populated, and a dropped membership scan. They also still pass against _csv.c as it stood before the optimization. --- Lib/test/test_csv.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 21ec0c76955e2a..d125131e26de9d 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -260,19 +260,6 @@ def test_write_escape(self): escapechar='\\', quoting=csv.QUOTE_MINIMAL) def test_write_lineterminator(self): - for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': - with self.subTest(lineterminator=lineterminator): - with StringIO() as sio: - writer = csv.writer(sio, lineterminator=lineterminator) - writer.writerow(['a', 'b']) - writer.writerow([1, 2]) - writer.writerow(['\r', '\n']) - self.assertEqual(sio.getvalue(), - f'a,b{lineterminator}' - f'1,2{lineterminator}' - f'"\r","\n"{lineterminator}') - - def test_write_lineterminator_in_field(self): for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0', '\x85', '\u2028', '\U0001f600'): with self.subTest(lineterminator=lineterminator): @@ -289,7 +276,9 @@ def test_write_lineterminator_in_field(self): f'"a{lineterminator[-1]}b",c{lineterminator}') def test_write_lineterminator_quoting(self): - for lineterminator, plain in ('!@#', ' ?A'), ('\u2028', '\u2027\u2029'): + for lineterminator, plain in (('!@#', ' ?A'), + ('\u2028', '\u2027\u2029'), + ('', '\0')): with self.subTest(lineterminator=lineterminator): for c in lineterminator: self._write_test([f'a{c}b', 'c'], f'"a{c}b",c', @@ -297,14 +286,6 @@ def test_write_lineterminator_quoting(self): self._write_test([f'a{plain}b', 'c'], f'a{plain}b,c', lineterminator=lineterminator) - def test_write_empty_lineterminator(self): - with StringIO() as sio: - writer = csv.writer(sio, lineterminator='') - writer.writerow(['a', 'b']) - writer.writerow(['\0', 'c']) - writer.writerow(['\r', '\n']) - self.assertEqual(sio.getvalue(), 'a,b\0,c"\r","\n"') - def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') self._write_test(iter(['a', 1, None]), 'a,1,')