diff --git a/changes/4339.bugfix.md b/changes/4339.bugfix.md new file mode 100644 index 0000000000..0697e5d232 --- /dev/null +++ b/changes/4339.bugfix.md @@ -0,0 +1 @@ +The documentation build and the documentation test suite no longer delete a `data/` directory relative to the current working directory. Two executable docs sessions opened with `shutil.rmtree('data', ignore_errors=True)` to make their examples re-runnable; because executed docs blocks run in the process working directory rather than the docs tree, `mkdocs build -f /mkdocs.yml` or `pytest tests/test_docs.py` started from any directory containing a `data/` folder — a project checkout, or `/` — silently emptied it. The sdist ships `docs/` and `tests/` and `testpaths` collects `docs/user-guide`, so this reached anyone running the shipped test suite, not only contributors. The deletions are gone; the on-disk examples in the quick start, arrays, groups, storage and performance guides now create with `overwrite=True` (or `zarr.save_array(..., mode="w")`), which is also what a reader re-running an example needs, and a new docs test rejects any executed block that calls a filesystem deletion. diff --git a/docs/quick-start.md b/docs/quick-start.md index 123f05d5e9..48d92caa4e 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -10,8 +10,6 @@ if you have not installed it yet. To get started, you can create a simple Zarr array: ```python exec="true" session="quickstart" -import shutil -shutil.rmtree('data', ignore_errors=True) import numpy as np from pprint import pprint import io @@ -34,7 +32,8 @@ z = zarr.create_array( store="data/example-1.zarr", shape=(100, 100), chunks=(10, 10), - dtype="f4" + dtype="f4", + overwrite=True, ) # Assign data to the array @@ -58,6 +57,7 @@ z = zarr.create_array( shape=(100, 100), chunks=(10, 10), dtype="f4", + overwrite=True, compressors=zarr.codecs.BloscCodec( cname="zstd", clevel=3, @@ -79,7 +79,7 @@ Zarr allows you to create hierarchical groups, similar to directories: ```python exec="true" session="quickstart" source="above" result="ansi" # Create nested groups and add arrays -root = zarr.group("data/example-3.zarr") +root = zarr.group("data/example-3.zarr", overwrite=True) foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" @@ -104,7 +104,7 @@ Suppose we want to copy existing groups and arrays into a new storage backend: ```python exec="true" session="quickstart" source="above" result="code" # Create nested groups and add arrays -root = zarr.group("data/example-4.zarr", attributes={'name': 'root'}) +root = zarr.group("data/example-4.zarr", attributes={'name': 'root'}, overwrite=True) foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 707fc1a1a5..140bc4bd09 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -5,8 +5,6 @@ Zarr has several functions for creating arrays. For example: ```python exec="true" session="arrays" -import shutil -shutil.rmtree('data', ignore_errors=True) import numpy as np ``` @@ -84,7 +82,7 @@ persistence of data between sessions. To do this, we can change the store argument to point to a filesystem path: ```python exec="true" session="arrays" source="above" -z1 = zarr.create_array(store='data/example-1.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') +z1 = zarr.create_array(store='data/example-1.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', overwrite=True) ``` The array above will store its configuration metadata and all compressed chunk @@ -112,12 +110,13 @@ print(np.all(z1[:] == z2[:])) If you are just looking for a fast and convenient way to save NumPy arrays to disk then load back into memory later, the functions -[`zarr.save`][] and [`zarr.load`][] may be -useful. E.g.: +[`zarr.save`][], [`zarr.save_array`][] and [`zarr.load`][] may be +useful. `zarr.save` refuses to replace an array that already exists at the +path; `zarr.save_array` accepts `mode="w"` to do so. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10) -zarr.save('data/example-2.zarr', a) +zarr.save_array('data/example-2.zarr', a, mode="w") print(zarr.load('data/example-2.zarr')) ``` @@ -130,7 +129,7 @@ A Zarr array can be resized, which means that any of its dimensions can be increased or decreased in length. For example: ```python exec="true" session="arrays" source="above" result="ansi" -z = zarr.create_array(store='data/example-3.zarr', shape=(10000, 10000), dtype='int32', chunks=(1000, 1000)) +z = zarr.create_array(store='data/example-3.zarr', shape=(10000, 10000), dtype='int32', chunks=(1000, 1000), overwrite=True) z[:] = 42 print(f"Original shape: {z.shape}") z.resize((20000, 10000)) @@ -146,7 +145,7 @@ used to append data to any axis. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10000000, dtype='int32').reshape(10000, 1000) -z = zarr.create_array(store='data/example-4.zarr', shape=a.shape, dtype=a.dtype, chunks=(1000, 100)) +z = zarr.create_array(store='data/example-4.zarr', shape=a.shape, dtype=a.dtype, chunks=(1000, 100), overwrite=True) z[:] = a print(f"Original shape: {z.shape}") z.append(a) @@ -207,7 +206,7 @@ argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=3, shuffle='bitshuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) +z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors, overwrite=True) z[:] = data print(z.compressors) ``` @@ -242,7 +241,7 @@ compressor. To create an array without any compression, set `compressors=None`: ```python exec="true" session="arrays" source="above" result="ansi" -z_no_compress = zarr.create_array(store='data/example-uncompressed.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', compressors=None) +z_no_compress = zarr.create_array(store='data/example-uncompressed.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', compressors=None, overwrite=True) print(f"Compressors: {z_no_compress.compressors}") ``` @@ -251,7 +250,7 @@ here is an array using Gzip compression, level 1: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1)) +z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1), overwrite=True) z[:] = data print(f"Compressors: {z.compressors}") ``` @@ -266,7 +265,7 @@ from zarr.codecs.numcodecs import LZMA lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), dict(id=lzma.FILTER_LZMA2, preset=1)] compressors = LZMA(filters=lzma_filters) data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) +z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors, overwrite=True) print(f"Compressors: {z.compressors}") ``` @@ -291,7 +290,7 @@ from zarr.codecs.numcodecs import Delta filters = [Delta(dtype='int32')] compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=1, shuffle='shuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) -z = zarr.create_array(store='data/example-9.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors) +z = zarr.create_array(store='data/example-9.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors, overwrite=True) print(z.info_complete()) ``` @@ -316,7 +315,7 @@ coordinates. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 -z = zarr.create_array(store='data/example-10.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-10.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) print(z.get_coordinate_selection([2, 5])) @@ -334,7 +333,7 @@ e.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-11.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-11.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` @@ -378,7 +377,7 @@ Items can also be extracted by providing a Boolean mask. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 -z = zarr.create_array(store='data/example-12.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-12.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` @@ -399,7 +398,7 @@ Here's a multidimensional example: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-13.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-13.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` @@ -442,7 +441,7 @@ example, this allows selecting a subset of rows and/or columns from a ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-14.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-14.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` @@ -470,7 +469,7 @@ For convenience, the orthogonal indexing functionality is also available via the ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-15.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-15.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z.oindex[[0, 2], :]) # select first and third rows ``` @@ -496,7 +495,7 @@ orthogonal indexing is also available directly on the array: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) -z = zarr.create_array(store='data/example-16.zarr', shape=data.shape, dtype=data.dtype) +z = zarr.create_array(store='data/example-16.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :])) ``` @@ -509,7 +508,7 @@ a subset of chunk aligned rows and/or columns from a 2-dimensional array. E.g.: ```python exec="true" session="arrays" source="above" data = np.arange(100).reshape(10, 10) -z = zarr.create_array(store='data/example-17.zarr', shape=data.shape, dtype=data.dtype, chunks=(3, 3)) +z = zarr.create_array(store='data/example-17.zarr', shape=data.shape, dtype=data.dtype, chunks=(3, 3), overwrite=True) z[:] = data ``` @@ -542,7 +541,7 @@ print(z.blocks[0, 1:3]) Data can also be modified. Let's start by a simple 2D array: ```python exec="true" session="arrays" source="above" -z = zarr.create_array(store='data/example-18.zarr', shape=(6, 6), dtype=int, chunks=(2, 2)) +z = zarr.create_array(store='data/example-18.zarr', shape=(6, 6), dtype=int, chunks=(2, 2), overwrite=True) ``` Set data for a selection of items: @@ -585,7 +584,7 @@ performance guide. Sharded arrays can be created by providing the `shards` parameter to [`zarr.create_array`][]. ```python exec="true" session="arrays" source="above" result="ansi" -a = zarr.create_array('data/example-20.zarr', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8') +a = zarr.create_array('data/example-20.zarr', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8', overwrite=True) a[:] = (np.arange(10000 * 10000) % 256).astype('uint8').reshape(10000, 10000) print(a.info_complete()) ``` diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index 7429a03847..ada5c5ec84 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -148,7 +148,7 @@ from pprint import pprint import io node_spec = {'a/b/c': GroupMetadata()} -nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec)) +nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec, overwrite=True)) # Report nodes (pprint is used for cleaner rendering in the docs) output = io.StringIO() pprint(nodes_created, stream=output, width=60) diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md index 52c1cf0d71..d1ff0e1b75 100644 --- a/docs/user-guide/performance.md +++ b/docs/user-guide/performance.md @@ -164,6 +164,7 @@ def timed_write(write_empty_chunks): chunks=chunks, dtype=dtype, fill_value=0, + overwrite=True, config={'write_empty_chunks': write_empty_chunks} ) # initialize all chunks @@ -324,7 +325,7 @@ E.g., pickle/unpickle a local store array: ```python exec="true" session="performance" source="above" result="ansi" import pickle data = np.arange(100000) -z1 = zarr.create_array(store='data/perf-example-2.zarr', shape=data.shape, chunks=data.shape, dtype=data.dtype) +z1 = zarr.create_array(store='data/perf-example-2.zarr', shape=data.shape, chunks=data.shape, dtype=data.dtype, overwrite=True) z1[:] = data s = pickle.dumps(z1) z2 = pickle.loads(s) diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index a34e2e2874..88ee2f1f32 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -19,7 +19,7 @@ being created automatically: import zarr # Implicitly creates a writable LocalStore -group = zarr.create_group(store='data/foo/bar') +group = zarr.create_group(store='data/foo/bar', overwrite=True) print(group) ``` diff --git a/tests/test_docs.py b/tests/test_docs.py index 0ed429f173..874d90d4e5 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -12,6 +12,7 @@ from __future__ import annotations +import re from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Any @@ -156,6 +157,42 @@ def test_no_unvalidated_blocks() -> None: ) +_DESTRUCTIVE_FS_CALL = re.compile( + r"shutil\.rmtree|os\.(?:remove|unlink|rmdir|removedirs)\b|\.unlink\(|\.rmdir\(|rm -rf" +) + + +def test_no_destructive_filesystem_calls() -> None: + """Executed docs blocks must not delete files or directories. + + Every python block that runs at build (exec="true") or under this harness + (test="true") executes in the *process* working directory: markdown-exec runs it + inside the mkdocs process and pytest-examples inside the pytest process, and neither + changes directory to the docs tree. A relative path in a deletion call therefore + resolves against wherever `mkdocs build` or `pytest` was started. Two sessions used to + open with `shutil.rmtree('data', ignore_errors=True)` to make their examples + re-runnable; started from any directory that happened to contain a `data/` folder -- + a project checkout, or `/` -- the docs build silently emptied it. The sdist ships + `docs/` and `tests/` and `testpaths` collects `docs/user-guide`, so that reached users + running the shipped test suite, not just contributors. + + Make examples re-runnable by creating with `overwrite=True` (or `mode="w"`) instead, + which is also what a reader copy-pasting the example a second time needs.""" + offenders: list[str] = [] + for example in find_examples(str(DOCS_ROOT)): + if not _is_tested(example.prefix_settings()): + continue + rel = Path(example.path).relative_to(DOCS_ROOT) + for offset, line in enumerate(example.source.splitlines()): + if _DESTRUCTIVE_FS_CALL.search(line): + offenders.append(f"{rel}:{example.start_line + offset}: {line.strip()}") + + assert not offenders, ( + "Executed docs blocks must not delete files or directories (they run in the " + "caller's working directory); create with overwrite=True instead:\n" + "\n".join(offenders) + ) + + def test_test_only_blocks_come_last() -> None: """A conservative placement convention: a test="true"-only block must come after every exec="true" block in the same file.