Address review: keep optional imports nested, keep shutdown ones eager

Keep ssl and multiprocessing.queues as function-level imports. A
module-level lazy import is resolved by inspect.getmembers() and pydoc,
so ssl breaks them on a build without _ssl, and multiprocessing defeats
the deferral the comment there asks for. Neither costs anything: a
nested import and a module-level lazy import defer identically.

Leave pickle, socket and struct eager in handlers.py. A record emitted
from a __del__ during finalization can no longer import, so a
SocketHandler pickled its first record too late and dropped it.

Resolve logging.handlers in fileConfig() before it evaluates any config
expression against vars(logging), so the documented handlers.X spelling
works again in a fresh process, both in class= and in defaults=.

Add tests for each, and pin the laziness that is left.

Author: brittanyrey

Lib/logging/config.py

@@ -40,7 +40,6 @@
 lazy import struct
 lazy from bisect import bisect_left
 lazy from logging import handlers as logging_handlers
-lazy from multiprocessing.queues import Queue as MPQueue
 lazy from socketserver import StreamRequestHandler, ThreadingTCPServer
 
 
@@ -83,6 +82,9 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non
         except configparser.ParsingError as e:
             raise RuntimeError(f'{fname} is invalid: {e}')
 
+    # the eval()s below resolve "handlers.X" names against vars(logging)
+    _ = logging_handlers
+
     formatters = _create_formatters(cp)
 
     # critical section
@@ -511,6 +513,9 @@ def _is_queue_like_object(obj):
     """Check that *obj* implements the Queue API."""
     if isinstance(obj, (queue.Queue, queue.SimpleQueue)):
         return True
+    # defer importing multiprocessing as much as possible; a lazy import at
+    # module level would still be resolved by getmembers() and pydoc
+    from multiprocessing.queues import Queue as MPQueue
     if isinstance(obj, MPQueue):
         return True
     # Depending on the multiprocessing start context, we cannot create

Lib/logging/handlers.py

@@ -26,19 +26,18 @@
 import io  # must stay eager to support finalization
 import logging
 import os
+import pickle
 import re
+import socket
+import struct
 import threading
 import time
 lazy import base64
 lazy import copy
 lazy import email.utils
 lazy import http.client
-lazy import pickle
 lazy import queue
 lazy import smtplib
-lazy import socket
-lazy import ssl
-lazy import struct
 lazy import urllib.parse
 lazy from email.message import EmailMessage
 
@@ -1132,6 +1131,8 @@ def emit(self, record):
             msg.set_content(self.format(record))
             if self.username:
                 if self.secure is not None:
+                    import ssl  # not lazy: breaks getmembers() without _ssl
+
                     try:
                         keyfile = self.secure[0]
                     except IndexError:

Lib/test/test_logging.py

@@ -1865,6 +1865,45 @@ def test_defaults_do_no_interpolation(self):
         finally:
             os.unlink(fn)
 
+    def test_names_from_handlers_module(self):
+        # gh-156777: "handlers.X" in a class or defaults entry is evaluated
+        # against vars(logging), so logging.handlers must be imported first.
+        # Run it in a subprocess, since importing this module already
+        # imports logging.handlers.
+        ini = textwrap.dedent("""
+            [loggers]
+            keys=root
+
+            [handlers]
+            keys=hand1
+
+            [formatters]
+            keys=form1
+
+            [logger_root]
+            handlers=hand1
+
+            [handler_hand1]
+            class=handlers.MemoryHandler
+            formatter=form1
+            args=(10,)
+
+            [formatter_form1]
+            format=%(levelname)s ++ %(message)s ++ %(port)s
+            defaults={'port': handlers.DEFAULT_TCP_LOGGING_PORT}
+            """).strip()
+        fd, fn = tempfile.mkstemp(prefix='test_logging_', suffix='.ini')
+        self.addCleanup(os.unlink, fn)
+        os.write(fd, ini.encode('ascii'))
+        os.close(fd)
+        code = textwrap.dedent(f"""
+            import logging, logging.config
+            logging.config.fileConfig({fn!r}, encoding="utf-8")
+            h = logging.getLogger().handlers[0]
+            assert isinstance(h, logging.handlers.MemoryHandler), h
+        """)
+        assert_python_ok("-c", code)
+
 
 @support.requires_working_socket()
 @threading_helper.requires_working_threading()
@@ -5437,6 +5476,35 @@ def __del__(self):
         with open(filename, encoding="utf-8") as fp:
             self.assertEqual(fp.read().rstrip(), "ERROR:root:log in __del__")
 
+    def test_socket_handler_at_shutdown(self):
+        # gh-156777: SocketHandler pickles the record before sending it, and
+        # that must keep working when importing no longer can.
+        code = textwrap.dedent("""
+            import logging
+            import logging.handlers
+            import os
+
+            class Handler(logging.handlers.SocketHandler):
+                # report what emit() did instead of doing network I/O
+                def send(self, s):
+                    os.write(1, b"sent %d bytes" % len(s))
+
+                def handleError(self, record):
+                    os.write(1, b"record dropped")
+
+            h = Handler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
+            r = logging.LogRecord('n', logging.INFO, 'p', 1, 'msg', None, None)
+
+            class A:
+                # the module globals are already cleared when __del__ runs
+                def __del__(self, h=h, r=r):
+                    h.emit(r)
+
+            a = A()
+        """)
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertStartsWith(out.decode(), "sent ")
+
     def test_recursion_error(self):
         # Issue 36272
         code = textwrap.dedent("""
@@ -7536,6 +7604,38 @@ def test_without_pywin32(self):
         h.emit(r)
 
 
+class LazyImportTest(unittest.TestCase):
+
+    """Tests for the module level lazy imports of the logging package."""
+
+    def test_lazy_imports_config(self):
+        import_helper.ensure_lazy_imports(
+            "logging.config",
+            {"configparser", "json", "logging.handlers", "multiprocessing",
+             "select", "socket", "socketserver", "struct"},
+            additional_code="logging.config.dictConfig({'version': 1})\n",
+        )
+
+    def test_lazy_imports_handlers(self):
+        import_helper.ensure_lazy_imports(
+            "logging.handlers",
+            {"base64", "copy", "email", "http", "queue", "smtplib", "ssl",
+             "urllib"},
+        )
+
+    def test_getmembers_without_ssl(self):
+        # gh-156777: getmembers() and pydoc resolve lazy imports, so a module
+        # level "lazy import ssl" would break them on a build without _ssl.
+        code = textwrap.dedent("""
+            import sys
+            sys.modules['_ssl'] = None
+            import inspect
+            import logging.handlers
+            inspect.getmembers(logging.handlers)
+        """)
+        assert_python_ok("-c", code)
+
+
 class MiscTestCase(unittest.TestCase):
     def test__all__(self):
         not_exported = {