gh-159184: Make the marshal order of set elements with equal dumps stable

Set elements with the same dump (NaNs which are distinct objects) were
written in set iteration order, which depends on their addresses.  When
one of them is also referenced elsewhere in the dump, the position of
the reference among them changed from run to run.

Elements with equal dumps are now ordered by when they were first
written in this dump; floats and complex numbers not written yet are
interchangeable, and which one is which is decided when one of them is
referenced again.  This relies on the writer knowing exactly which
objects it has written (gh-129724).

The elements are sorted with qsort() on the bytes of their dumps,
in an array instead of a list of tuples.

Author: gpshead

Lib/test/test_marshal.py

@@ -617,7 +617,6 @@ def test_unorderable_set_items(self):
                     self.assertEqual(sorted(map(repr, new)),
                                      sorted(map(repr, sample)))
 
-
     def test_unorderable_set_items_in_code(self):
         # A constant set containing NaNs which are distinct objects.
         # Run in a subprocess: constants of code objects are immortal
@@ -662,7 +661,6 @@ def shrink(sample):
                                                 "set changed size"):
                         marshal.dumps(sample, version)
 
-
     def test_error_in_buffer(self):
         # The BufferError raised for a non-contiguous buffer is not replaced
         # with a generic error.
@@ -1238,7 +1236,8 @@ def test_nan_in_set(self):
                     nan1 = float('nan')
                     nan2 = float('nan')
                     nan3 = float('nan')
-                    s = kind([nan1, 1.5, nan2, nan3])
+                    s = kind([nan1, 1.5, nan2, nan3,
+                              complex('nan'), complex('nan')])
                     for sample in (s, [s], [nan1, s], [s, nan2],
                                    [nan3, s, nan1], [nan1, nan2, nan3, s],
                                    [s, s], (nan2, [s, nan2], s)):
@@ -1247,6 +1246,59 @@ def test_nan_in_set(self):
                         new = marshal.loads(data)
                         self.check_same_structure(new, sample, version >= 3)
 
+    @support.cpython_only
+    def test_nan_in_set_order(self):
+        # The output does not depend on the order of the elements of a set
+        # which have the same marshalled representation, even if some of
+        # them are also referenced elsewhere.
+        version = self.VERSION
+        def flagged(data):
+            data = bytearray(data)
+            data[0] |= self.FLAG_REF
+            return bytes(data)
+        def ref(index):
+            return b'r' + struct.pack('<i', index)
+        keep = []
+        for make_nan in (lambda: float('nan'), lambda: complex('nan')):
+            data = marshal.dumps(make_nan(), version)
+            fdata = flagged(data)
+            data15 = marshal.dumps(1.5, version)
+            for kind, typecode in (set, b'<'), (frozenset, b'>'):
+                head = typecode + b'\4\0\0\0' + data15
+                for i in range(20):
+                    # Different objects at different addresses every time.
+                    keep.append(bytes(i * 8))
+                    nans = [make_nan() for j in range(3)]
+                    keep.append(nans)
+                    s = kind([1.5, *nans])
+                    a, b, c = nans[i % 3:] + nans[:i % 3]
+                    for sample, expected in [
+                        (s, head + data * 3),
+                        ([s, a],
+                         b'[\2\0\0\0' + head + fdata + data * 2 + ref(0)),
+                        ([a, s],
+                         b'[\2\0\0\0' + fdata + head + ref(0) + data * 2),
+                        ([s, a, b, a],
+                         b'[\4\0\0\0' + head + fdata * 2 + data +
+                         ref(0) + ref(1) + ref(0)),
+                        ([a, b, s, c],
+                         b'[\4\0\0\0' + fdata * 2 + head + ref(0) + ref(1) +
+                         flagged(data) + ref(2)),
+                        ([b, s, c, a, kind([a, b, c])],
+                         b'[\5\0\0\0' + fdata + head + ref(0) + fdata * 2 +
+                         ref(1) + ref(2) + typecode + b'\3\0\0\0' +
+                         ref(0) + ref(1) + ref(2)),
+                        ([s, kind([b, a]), kind([c, make_nan()]), b, a, c],
+                         b'[\6\0\0\0' + head + fdata * 3 +
+                         typecode + b'\2\0\0\0' + ref(0) + ref(1) +
+                         typecode + b'\2\0\0\0' + ref(2) + data +
+                         ref(0) + ref(1) + ref(2)),
+                    ]:
+                        self.assertEqual(marshal.dumps(sample, version),
+                                         expected)
+                        new = marshal.loads(expected)
+                        self.check_same_structure(new, sample, True)
+
     def check_same_structure(self, new, sample, shared):
         # Both object graphs have the same values and, if shared is true,
         # share the same objects.  Elements of sets are matched in any
@@ -1261,7 +1313,7 @@ def known(x):
                 if shared:
                     return seen.get(id(x), -1)
             def visit(x):
-                if isinstance(x, float):
+                if isinstance(x, (float, complex)):
                     result.append((repr(x), number(x)))
                 elif isinstance(x, (set, frozenset)):
                     # Elements are only numbered if they occur elsewhere.

Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst

@@ -0,0 +1,3 @@
+The output of :func:`marshal.dumps` no longer depends on the order of the
+elements of a set which have the same marshalled form, such as NaNs which
+are distinct objects.