Add adata convert: matrix dtype, layout and density on disk

CHANGELOG.md

@@ -5,6 +5,38 @@ Notable changes to `adata-cli`. Versions are `MAJOR.MINOR.PATCH`; tags carry no
 
 ## Unreleased
 
+### Added
+
+- **`adata convert` changes a matrix's dtype, layout or density on disk**
+  ([#13](https://github.com/cellgeni/adata-cli/issues/13)). Counts stored as
+  float64 halve with `--dtype float32`; `--indices-dtype int32` halves the
+  index arrays of a matrix small enough to address that way; `--layout
+  csr|csc` transposes between the sparse encodings; `--layout dense|sparse`
+  changes the density. Works on `X`, any layer, `raw/X` or any 2-D array by
+  path, and on all of them at once with `--all`.
+
+  A cast that would not round-trip is refused before anything is written --
+  every value is cast and cast back, because whether float64 counts survive
+  float32 depends on the counts, not on the dtypes. So is a densification
+  that would inflate the store beyond four times its size. `--force`
+  overrides either, and both messages say what they measured.
+
+  Transposing streams by default, holding one bucket of nonzeros rather than
+  the matrix, so it works on files too large to load; `--in-memory` is
+  faster when the matrix fits. Buckets are balanced by nonzero count rather
+  than by coordinate range, because a single-cell matrix is skewed -- a few
+  genes carry most of the counts -- and equal-width bounds put most of one
+  in a single bucket.
+
+  `--indices-dtype` is checked against both what `indices` must address and
+  what `indptr` must reach, which differ: a narrow matrix with more than
+  2^31 nonzeros needs int64 offsets over int32 coordinates. Both are
+  preserved from the source when not specified.
+
+- **`concat` now names the command to run** when inputs disagree about a
+  matrix encoding. The check itself is not new, but nothing tested it and it
+  could not suggest a fix, because there was none.
+
 ### Fixed
 
 - **`concat --merge` never finished on a real store.** Aligning a var column

README.md

@@ -56,6 +56,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
 - `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
 - `split` – write one store per distinct value of an annotation column, with a CSV manifest.
 - `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
+- `convert` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced.
 - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
 - `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
 
@@ -78,6 +79,19 @@ adata split  data.h5ad --by sample -o per_sample/
 adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
 ```
 
+### Shrinking a store, and making encodings agree
+
+```bash
+adata convert data.h5ad X --inplace --dtype float32
+adata convert data.h5ad --all -o small.h5ad --dtype float32 --indices-dtype int32
+adata convert data.h5ad X -o csc.h5ad --layout csc
+```
+
+Counts written as float64 halve with no loss — and `convert` proves that
+before it writes, by casting every value and casting it back. `concat`
+refuses inputs whose matrices disagree about CSR versus CSC; `--layout` is
+how you make them agree.
+
 ## Documentation
 
 - [Get started](docs/GET_STARTED.md) — a short tutorial

benchmarks/cases.py

@@ -485,6 +485,61 @@ def _py(body: str) -> str:
             ),
         ],
     ),
+    # -- convert ------------------------------------------------------------
+    Case(
+        name="convert-dtype",
+        question="Rewrite X as float32.",
+        contenders=[
+            Contender(
+                "adata-cli",
+                argv=[
+                    "adata", "convert", "{input0}", "X", "-o", "{output}",
+                    "--dtype", "float32", "--force",
+                ],
+            ),
+            Contender(
+                "anndata (in memory)",
+                script=_py(
+                    "obj = ad.read_h5ad(IN[0])\n"
+                    "obj.X = obj.X.astype('float32')\n"
+                    "obj.write_h5ad(OUT, compression=COMPRESSION)\n"
+                ),
+            ),
+            Contender(
+                "scanpy",
+                unsupported="no dtype rewrite; scanpy defers to anndata",
+            ),
+        ],
+    ),
+    Case(
+        name="convert-layout",
+        question="Transpose X from CSR to CSC.",
+        contenders=[
+            Contender(
+                "adata-cli (streaming)",
+                argv=[
+                    "adata", "convert", "{input0}", "X", "-o", "{output}",
+                    "--layout", "csc",
+                ],
+            ),
+            Contender(
+                "adata-cli (--in-memory)",
+                argv=[
+                    "adata", "convert", "{input0}", "X", "-o", "{output}",
+                    "--layout", "csc", "--in-memory",
+                ],
+            ),
+            Contender(
+                "anndata (in memory)",
+                script=_py(
+                    "obj = ad.read_h5ad(IN[0])\n"
+                    "obj.X = obj.X.tocsc()\n"
+                    "obj.write_h5ad(OUT, compression=COMPRESSION)\n"
+                ),
+            ),
+        ],
+        tags=["headline"],
+    ),
     # -- the claim itself -------------------------------------------------
     Case(
         name="rss-vs-size",

docs/COMMANDS.md

@@ -81,6 +81,52 @@ adata subset data.h5ad --inplace --obs barcodes.txt
 `raw/` is carried over and matched against its **own** var axis, which usually
 holds more genes than the main object.
 
+## `convert`
+
+Change a matrix's dtype, layout or density. Counts held as float64 cost twice
+the disk and twice the read for no information; a tool that wants CSC cannot
+use a CSR store; and `concat` refuses inputs whose encodings disagree.
+
+```bash
+adata convert data.h5ad X -o out.h5ad --dtype float32
+adata convert data.h5ad X -o out.h5ad --layout csc
+adata convert data.h5ad X --inplace --dtype float32 --indices-dtype int32
+adata convert data.h5ad --all -o out.h5ad --dtype float32
+adata convert data.h5ad layers/counts -o out.h5ad --layout dense --force
+```
+
+| Flag | Meaning |
+|---|---|
+| `--output`, `-o` | Output path. Required unless `--inplace` |
+| `--inplace` | Replace the source (written to a temporary path first) |
+| `--all` | Convert `X`, every layer and `raw/X` |
+| `--dtype` | New dtype for the values, e.g. `float32` |
+| `--indices-dtype` | New dtype for sparse indices: `int32` or `int64` |
+| `--layout` | `csr`, `csc`, `dense` or `sparse` |
+| `--force` | Convert despite a lossy cast or a large size increase |
+| `--in-memory` | Transpose in memory rather than streaming |
+| `--chunk`, `-C` | Row chunk size for dense matrices |
+| `--zarr-format` | Zarr version to write; defaults to the source's |
+
+Two things are refused before anything is written. A cast that would not
+round-trip -- checked by casting every value and casting it back, not by
+comparing dtypes -- and a densification that would inflate the store beyond
+four times its size. `--force` overrides either, and the message says how
+many values would change or how large the result would be.
+
+The index dtype is **preserved** unless `--indices-dtype` asks otherwise, so
+narrowing the values does not silently widen the indices and leave the file
+bigger than it started. `indices` and `indptr` keep their own widths, which
+can differ: a narrow matrix with more than 2^31 nonzeros needs int64 offsets
+over int32 coordinates, and both are range-checked before writing.
+
+The output path may not name the input; use `--inplace`, which writes to a
+temporary file and swaps it in only once the conversion has finished.
+
+Transposing streams by default and works on matrices too large to load, at
+the cost of two extra passes over the nonzeros. `--in-memory` is faster when
+the matrix fits.
+
 ## `split`
 
 One store per distinct value of a column.

docs/index.md

@@ -51,6 +51,8 @@ adata subset data.h5ad -o cortex.h5ad --obs-query "cluster == Cortex_2"
 adata split  data.h5ad --by sample -o per_sample/
 adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
 
+adata convert data.h5ad X -o small.h5ad --dtype float32
+
 adata create new.h5ad --n-obs 5000 --n-var 2000
 adata import sparse new.h5ad X counts.mtx --inplace
 ```

src/adata/cli.py

@@ -6,9 +6,11 @@
 from rich.console import Console
 import typer
 
+from adata.core.convert import LAYOUTS
 from adata.commands import (
     MERGE_CHOICES,
     concat_stores,
+    convert_store,
     create_store,
     split_store,
     list_store,
@@ -476,6 +478,134 @@ def concat(
         raise typer.Exit(code=1)
 
 
+# ============================================================================
+# CONVERT command
+# ============================================================================
+@app.command("convert")
+def convert(
+    file: Path = typer.Argument(
+        ...,
+        help="Input .h5ad/.zarr",
+        exists=True,
+        readable=True,
+        dir_okay=True,
+        file_okay=True,
+    ),
+    entries: Optional[List[str]] = typer.Argument(
+        None,
+        help="Matrix paths to convert, e.g. 'X', 'layers/counts', 'raw/X'",
+    ),
+    output: Optional[Path] = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Output .h5ad/.zarr path. Required unless --inplace.",
+        dir_okay=True,
+        file_okay=True,
+    ),
+    inplace: bool = typer.Option(
+        False,
+        "--inplace",
+        help="Modify source file directly.",
+    ),
+    convert_all: bool = typer.Option(
+        False,
+        "--all",
+        help="Convert X, every layer, and raw/X",
+    ),
+    dtype: Optional[str] = typer.Option(
+        None,
+        "--dtype",
+        help="New dtype for the values, e.g. float32",
+    ),
+    indices_dtype: Optional[str] = typer.Option(
+        None,
+        "--indices-dtype",
+        help="New dtype for sparse indices: int32 or int64",
+    ),
+    layout: Optional[str] = typer.Option(
+        None,
+        "--layout",
+        help="Target layout: csr, csc, dense or sparse",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Convert despite a lossy cast or a large size increase",
+    ),
+    in_memory: bool = typer.Option(
+        False,
+        "--in-memory",
+        help="Transpose in memory instead of streaming (faster if it fits)",
+    ),
+    chunk_rows: int = typer.Option(
+        1024,
+        "--chunk",
+        "-C",
+        help="Row chunk size for dense matrices",
+    ),
+    zarr_format: Optional[int] = typer.Option(
+        None,
+        "--zarr-format",
+        help="Zarr spec version to write (defaults to the source store's)",
+    ),
+) -> None:
+    """
+    Change a matrix's dtype, layout or density.
+
+    Counts held as float64 cost twice the disk and twice the read for no
+    information; a tool that wants CSC cannot use a CSR store; and concat
+    refuses inputs whose encodings disagree. All three are this command.
+
+    A cast that would not round-trip is refused before anything is written,
+    as is a densification that would inflate the store; --force overrides
+    both. Transposing streams by default, so it works on matrices too large
+    to load.
+
+    Examples:
+        adata convert data.h5ad X -o out.h5ad --dtype float32
+        adata convert data.h5ad X -o out.h5ad --layout csc
+        adata convert data.h5ad X --inplace --dtype float32 --indices-dtype int32
+        adata convert data.h5ad --all -o out.h5ad --dtype float32
+    """
+    if not inplace and output is None:
+        console.print(
+            "[bold red]Error:[/] Output file is required. "
+            "Use --output/-o or --inplace.",
+        )
+        raise typer.Exit(code=1)
+
+    if layout is not None and layout not in LAYOUTS:
+        console.print(
+            f"[bold red]Error:[/] --layout must be one of: {', '.join(LAYOUTS)}."
+        )
+        raise typer.Exit(code=1)
+
+    if zarr_format is not None and zarr_format not in (2, 3):
+        console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
+        raise typer.Exit(code=1)
+
+    try:
+        convert_store(
+            file=file,
+            entries=list(entries) if entries else None,
+            output=output,
+            console=console,
+            dtype=dtype,
+            indices_dtype=indices_dtype,
+            layout=layout,
+            convert_all=convert_all,
+            inplace=inplace,
+            chunk_rows=chunk_rows,
+            in_memory=in_memory,
+            force=force,
+            zarr_format=zarr_format,
+        )
+    except Exception as e:
+        console.print(f"[bold red]Error:[/] {e}")
+        raise typer.Exit(code=1)
+
+
 # ============================================================================
 # SPLIT command
 # ============================================================================

src/adata/commands/__init__.py

@@ -6,3 +6,4 @@
 from adata.commands.create import create_store
 from adata.commands.split import split_store
 from adata.commands.concat import MERGE_CHOICES, MERGE_STRATEGIES, concat_stores
+from adata.commands.convert import convert_store