Repository navigation
[C API] Add all macros to Misc/stable_abi.toml #158886
Description
Activity
I wrote some very ugly scripts to:
- macros.py: extract macros from
Include/*.hheader files - get_version.py: check if a macro is really defined in the limited C API and get the Python version which introduced it (using
git show VERSION:FILENAME) - update.py: generate an ouput in
stable_abi.tomlformat. It checks if a macro is already documented as a function inDoc/c-api/*.rst.
$ python macros.py > macros $ python update.py macros >output 2>stderrThe
get_version.pyrequires many checkout of the Python Git repository, one per version (3.2 to 3.15 + "main"). Just comment th call to "double_check()" if you don't want that (it's not really needed, it's to check if the script has bugs).Check
stderrfor anything suspicious. There are many warnings aboutdatetime.hmacros which are not in the limited C API.
macros.py:
Details
import glob import os.path import re import sys EXCLUDE_HEADERS = { # Header files not included by Python.h and ignored by this script 'dynamic_annotations.h', 'errcode.h', 'opcode.h', 'opcode_ids.h', 'osdefs.h', 'pyexpat.h', 'structmember.h', # Header files not included by Python.h but parsed by this script: # - datetime.h # - frameobject.h # - marshal.h # - py_curses.h # - pydtrace.h } SRC_DIR = os.path.dirname(__file__) DEFINE_REGEX = re.compile(r'\s*#\s*(?P<directive>define|undef)\s+(?P<macro>.*)') PYTHON_PREFIX = re.compile(r'(Py|PY)') ID_REGEX = r'(?:[a-zA-Z_][a-zA-Z0-9_]*)' ARGS_REGEX = fr'(?:(?:{ID_REGEX})(?:,\s*{ID_REGEX})*\s*|)' NAME_REGEX = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\b\s*\(\s*' + ARGS_REGEX + r'\)') CAUSES_BY_DIRECTIVE = { "define": "defined", "undef": "undefined", } def parse_file(filename, names): with open(filename, encoding='utf8') as fp: for lineno, line in enumerate(fp, start=1): # Check for '#define MACRO' match = DEFINE_REGEX.match(line) if not match: continue macro = match['macro'] directive = match['directive'] cause = CAUSES_BY_DIRECTIVE.get(directive, directive) if not PYTHON_PREFIX.match(macro): continue match = NAME_REGEX.match(macro) if not match: continue name = match[1] names.append((name, filename, lineno, cause)) def main(): # Parse header files include_dir = os.path.join(SRC_DIR, 'Include') files = glob.glob(os.path.join(include_dir, '*.h')) files = [filename for filename in files if os.path.basename(filename) not in EXCLUDE_HEADERS] names = [] # list of (name: str, filename: str, lineno: int, cause: str) for filename in files: parse_file(filename, names) names = [name for name, *ignored in names] for name in sorted(set(names)): print(name) if __name__ == "__main__": main()
get_version.py:
Details
import sys import glob import subprocess import re def git_tag(ver): if ver == 15: return ('3.15', 'v3.15.0rc3') if ver in (0, 1, 2): return (f'3.{ver}', f'v3.{ver}') return (f'3.{ver}', f'v3.{ver}.0') def old_filename(filename): if filename == 'Include/refcount.h': return 'Include/object.h' if filename == 'Include/exports.h': return 'Include/pyport.h' return None def command(args): proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) return (proc.returncode, proc.stdout) IF_LIMITED_API_LINE = re.compile(r'#if.*(Py_LIMITED_API|Py_BUILD_CORE).*$', re.MULTILINE) IF_LIMITED_API = re.compile( r' defined\(Py_LIMITED_API\) &&' r'| !defined\(Py_LIMITED_API\) \|\| \(?Py_LIMITED_API\+0 (<|>=)' r'|#ifdef Py_LIMITED_API' r'|#if Py_LIMITED_API == 3' r'|#if defined\(Py_DEBUG\) \|\| defined\(Py_LIMITED_API\)' r'|#if defined\(Py_REF_DEBUG\) && !\(defined\(Py_LIMITED_API\) && Py_LIMITED_API\+0 ' r'|#if \(defined\(Py_LIMITED_API\) && \(?Py_LIMITED_API\+0 ' ) IFNDEF_LIMITED_API = re.compile( r'#ifndef Py_LIMITED_API' r'|#if !defined\(Py_LIMITED_API\)' r'|#if .* && !defined\(Py_LIMITED_API\)' # r'|#if .* && \(!defined\(Py_LIMITED_API\) && ' r'|#if !defined\(Py_LIMITED_API\) && ' r'|#if !defined\(Py_LIMITED_API\) && ' r'|#ifdef Py_BUILD_CORE' r'|#if defined\(Py_BUILD_CORE\)' r'|#if defined\(Py_BUILD_CORE_BUILTIN\) && ' r'|#if defined\(Py_BUILD_CORE_MODULE\) && ' ) FIND_ENDIF = re.compile(r'#\s*(endif)|#\s*(else)|#\s*(if)') # Very ugly parser to exclude non-limited API. # For example, remove '#ifndef Py_LIMITED_API (...) #endif'. # and remove '#ifdef Py_BUILD_CORE (...) #else'. def strip_non_limited_api(filename, code): start = 0 while True: line_match = IF_LIMITED_API_LINE.search(code, start) if line_match is None: break line = line_match.group(0) match = IF_LIMITED_API.search(line) if match is not None: start += match.end() continue if IFNDEF_LIMITED_API.match(line): ifndef_start = line_match.start() match = FIND_ENDIF.search(code, line_match.end()) if match is None: print(f"Error: {filename}: unable to find #endif") sys.exit(1) depth = 1 while True: what = match.group(1) or match.group(2) or match.group(3) if depth == 1 and what in ('endif', 'else'): if what == 'else': end = match.start() else: end = match.end() code = code[:ifndef_start] + code[end:] start = ifndef_start break if what == 'endif': depth -= 1 elif what == 'if': # we found a nested #if, oh no! depth += 1 # elif kind == 'else': keep the same depth start = match.end() match = FIND_ENDIF.search(code, start) if match is None: print(f"Error: {filename}: unable to find #endif") sys.exit(1) continue print(f"Error: {filename}: unable to parse: {line!r}") sys.exit(1) return code def double_check(version, name): for filename in sorted(glob.glob(f'../{version}/Include/*.h')): with open(filename) as fp: code = fp.read() #code = strip_non_limited_api(filename, code) if name in code: print(f"WARNING: {name} found in {version}:{filename}", file=sys.stderr) return def get_version(name): tags = [('3.16', 'main')] tags += [git_tag(version) for version in list(range(15, -1, -1))] regex = re.compile(r'#\s*define\s+' + name) for filename in sorted(glob.glob('Include/*.h')): with open(filename) as fp: code = fp.read() code = strip_non_limited_api(filename, code) if regex.search(code): break else: double_check('main', name) #raise ValueError(f"unable to find {name!r}") return (None, None) found = None for version, tag in tags: cmd = ['git', 'show', f'{tag}:{filename}'] returncode, code = command(cmd) if returncode == 128: filename2 = old_filename(filename) if filename2: cmd = ['git', 'show', f'{tag}:{filename2}'] returncode, code = command(cmd) if returncode == 0: filename = filename2 else: # file doesn't exist in old Python version double_check(version, name) break if returncode: print(f"{cmd} failed with exit code {returncode}") sys.exit(1) code = strip_non_limited_api(filename, code) if regex.search(code): found = version else: double_check(version, name) #print(f"{name} doesn't exist in {version}:{filename}") break if found is None: print("Unable to get version") sys.exit(1) return (found, filename) def main(): name = sys.argv[1] version, filename = get_version(name) if version: print(f"{name} was added to Python {version} in {filename}") else: print(f"ERROR: {name} is NOT part of the limited C API") sys.exit(1) if __name__ == "__main__": main()
update.py:
Details
from get_version import get_version import glob import sys import re with open("Misc/stable_abi.toml") as fp: stable_abi = fp.read() def get_kind(name): pattern = re.compile(r'\.\. c:function:: .*' + name) for filename in glob.glob('Doc/c-api/*.rst'): with open(filename) as fp: doc = fp.read() if pattern.search(doc): return 'function' return 'macro' per_version = {} with open("macros") as fp: for name in fp: name = name.strip() if not name: continue kind = get_kind(name) pattern = f"[{kind}.{name}]" if pattern in stable_abi: print(f"Already in stable_abi.toml: {pattern}", file=sys.stderr) continue version, filename = get_version(name) if version is None: print(f"Not in the stable ABI: {name}", file=sys.stderr) continue #print(kind, name, version, file=sys.stderr) # '3.10' => 10 version = int(version[2:]) if version < 2: # Stable ABI was added to Python 3.2 version = 2 if version in per_version: per_version[version].append((kind, name)) else: per_version[version] = [(kind, name)] for version, names in sorted(per_version.items()): print(f"### Python 3.{version} ###") for kind, name in sorted(names): print(f"[{kind}.{name}]") print(f" added = '3.{version}'") print()
- macros.py: extract macros from
Oh, there are 3 bugs;
- PyAPI_DATA() and PyAPI_FUNC() version is wrong: they exist since Python 3.2
- Py_INVALID_SIZE is not a function, just a macro defining a constant:
#define Py_INVALID_SIZE (Py_ssize_t)-1
The
macros.pyscripts only checks for macros which have arguments:#define Py_MIN(x, y)is listed, whereas#define Py_T_SHORT 0is not listed.More bugs:
PyBUF_CONTIG_ROandPyBUF_STRIDED_ROare constants, not functions. By the way, they were already listed bystable_abi.toml!To avoid error on building the documentation, I declared
PyBool_Check()macro as a function since it's documented as a function byDoc/c-api/bool.rst:.. c:function:: int PyBool_Check(PyObject *o)Same for other macros documented as functions.
Problem: if it's declared as a function by stable_abi.toml,
Lib/test/test_stable_abi_ctypes.pylooks for symbols in Python... and it fails.More bugs: PyDoc_STR, PyDoc_STRVAR, PyDoc_VAR and Py_CHARMASK were already present in Python 3.2 in fact. I already updated the PR.
For some macros like
Py_ULLorPyAPI_DATA, I think it would be better to treat them as exposed by mistake, rather than part of limited API.
There are multiple macros in the limited C API which are missing in
Misc/stable_abi.toml. I propose to add them :-)Linked PRs