From 9eb7237f04d87513a4a7935d52e489411a9248b6 Mon Sep 17 00:00:00 2001 From: Oleks Date: Wed, 8 Jul 2026 16:24:13 +0300 Subject: [PATCH 1/2] =?UTF-8?q?graph:=20tri-state=20to=5Fbuild=5Fset=20?= =?UTF-8?q?=E2=80=94=20empty=20means=20nothing=20to=20build=20(#5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor dry-run text parsing into parse_dry_run_text(text, returncode) so an empty set (warm cache) is distinct from None (unparseable). Only None triggers the cold-cache whole-closure fallback in cli.py; an empty set now correctly reports 'nothing to build'. --- nix_estimator/graph.py | 56 +++++++++++++++++++++++++++++---------- tests/test_graph.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 tests/test_graph.py diff --git a/nix_estimator/graph.py b/nix_estimator/graph.py index e6d6cbb..f269571 100644 --- a/nix_estimator/graph.py +++ b/nix_estimator/graph.py @@ -61,29 +61,31 @@ def derivation_closure( return {_basename(k): v for k, v in raw.items()} -def to_build_set( - attr: str, system: str | None = None, extra_args: list[str] | None = None -) -> set[str] | None: - """Set of ``.drv`` paths Nix says *will be built*, or ``None`` if unparseable. +def parse_dry_run_text(text: str, returncode: int = 0) -> set[str] | None: + """Parse the *human* text of ``nix build --dry-run`` into a to-build set. - ``None`` signals the caller to fall back to "the whole closure" (a cold-cache - over-estimate) rather than silently reporting zero work. + Tri-state result (see :func:`to_build_set`): + + - a ``set`` of drv basenames when the dry-run structure was recognized + (possibly empty — a fully warm cache prints only "will be fetched"); + - ``None`` when no recognizable dry-run structure was found at all, so the + caller cannot tell built-nothing from parse-failure. + + A dry-run is considered "recognized" when the process exited cleanly *or* a + known section header ("will be built"/"will be fetched"/"will be copied") was + seen. ``returncode`` is the ``nix`` exit status (0 = success). """ - cmd = ["nix", "build", "--dry-run", attr] - if system: - cmd += ["--system", system] - if extra_args: - cmd += extra_args - proc = _run(cmd, check=False) - text = proc.stderr + "\n" + proc.stdout built: set[str] = set() + saw_header = False grabbing = False for line in text.splitlines(): s = line.strip() if re.search(r"will be built", s): + saw_header = True grabbing = True continue if re.search(r"will be fetched|will be copied", s): + saw_header = True grabbing = False continue if grabbing: @@ -92,7 +94,29 @@ def to_build_set( built.add(_basename(m.group(1))) elif s and not s.startswith("/nix/store"): grabbing = False - return built or None + if not saw_header and returncode != 0: + return None + return built + + +def to_build_set( + attr: str, system: str | None = None, extra_args: list[str] | None = None +) -> set[str] | None: + """Set of ``.drv`` basenames Nix says *will be built*, tri-state. + + Returns an empty ``set`` when the dry-run clearly succeeded but nothing will + be built (fully warm cache), a populated ``set`` of the drvs to build, and + ``None`` *only* when the dry-run produced nothing recognizable — the sole + case in which the caller should fall back to costing the whole closure. + """ + cmd = ["nix", "build", "--dry-run", attr] + if system: + cmd += ["--system", system] + if extra_args: + cmd += extra_args + proc = _run(cmd, check=False) + text = proc.stderr + "\n" + proc.stdout + return parse_dry_run_text(text, proc.returncode) def build_dag( @@ -101,6 +125,10 @@ def build_dag( """Return ``(preds, nodes)`` where ``preds[d]`` are the in-graph derivations ``d`` depends on. Restricted to ``to_build`` when given (edges to already-cached inputs are dropped — they contribute no build time). + + ``to_build`` is tri-state: ``None`` means "cache state unknown" so the whole + closure is costed; an *empty* set means "nothing to build" (warm cache) and + yields no nodes; a populated set restricts to those drvs. """ nodes = set(closure) if to_build is None else (set(to_build) & set(closure)) preds: dict[str, list[str]] = {} diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 0000000..13ffd98 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,59 @@ +"""Tests for dry-run parsing and DAG restriction in :mod:`nix_estimator.graph`.""" + +from nix_estimator import graph + +# --- captured `nix build --dry-run` text (issue #5) ------------------------ + +WARM = """\ +this path will be fetched (0.02 MiB download, 0.10 MiB unpacked): + /nix/store/aaa-hello-2.12.1 +""" + +PARTIAL = """\ +these 2 derivations will be built: + /nix/store/bbb-foo-1.0.drv + /nix/store/ccc-bar-2.0.drv +these 3 paths will be fetched (1.0 MiB download): + /nix/store/ddd-baz-3.0 +""" + +GARBAGE = """\ +error: some totally unrelated failure with no dry-run structure +traceback blah blah +""" + + +def test_warm_cache_only_fetched_is_empty_set(): + # dry-run succeeded, nothing to build: distinct from a parse failure + result = graph.parse_dry_run_text(WARM, returncode=0) + assert result == set() + + +def test_partial_returns_only_built_drvs(): + result = graph.parse_dry_run_text(PARTIAL, returncode=0) + assert result == {"bbb-foo-1.0.drv", "ccc-bar-2.0.drv"} + + +def test_garbage_without_header_and_failure_is_none(): + assert graph.parse_dry_run_text(GARBAGE, returncode=1) is None + + +def test_clean_exit_without_header_is_empty_not_none(): + # exit 0 but no recognizable header still counts as "nothing to build" + assert graph.parse_dry_run_text("", returncode=0) == set() + + +def test_build_dag_empty_set_yields_no_nodes(): + closure = {"a.drv": {"inputDrvs": {}}, "b.drv": {"inputDrvs": {"a.drv": {}}}} + preds, nodes = graph.build_dag(closure, set()) + assert nodes == set() + assert preds == {} + + +def test_build_dag_none_costs_whole_closure(): + closure = { + "a.drv": {"inputDrvs": {}}, + "b.drv": {"inputDrvs": {"/nix/store/x-a.drv": {}}}, + } + _, nodes = graph.build_dag(closure, None) + assert nodes == {"a.drv", "b.drv"} From ffc44944864a6e3902591ce805c4428f70757b84 Mon Sep 17 00:00:00 2001 From: Oleks Date: Wed, 8 Jul 2026 16:25:29 +0300 Subject: [PATCH 2/2] graph: prefer nix build --dry-run --json, fall back to text scraper (#8) Probe --json support via nix build --help and parse structured output when available (robust across Nix versions and locales). Falls back to the #5 text scraper on older Nix or unexpected payloads, feeding the same tri-state contract. --- nix_estimator/graph.py | 61 +++++++++++++++++++++++++++++++++++++++--- tests/test_graph.py | 36 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/nix_estimator/graph.py b/nix_estimator/graph.py index f269571..b33351d 100644 --- a/nix_estimator/graph.py +++ b/nix_estimator/graph.py @@ -99,6 +99,44 @@ def parse_dry_run_text(text: str, returncode: int = 0) -> set[str] | None: return built +def parse_dry_run_json(text: str) -> set[str] | None: + """Parse ``nix build --dry-run --json`` output into a to-build set. + + The JSON form is a list of build results; each entry names its derivation + under ``drvPath`` (or, on some schemas, ``drv``). Locale- and wording-proof, + unlike the human text. Returns a ``set`` of drv basenames (possibly empty), + or ``None`` when the payload is not the expected structured shape so the + caller can fall back to the text scraper. + """ + try: + data = json.loads(text) + except (ValueError, TypeError): + return None + if not isinstance(data, list): + return None + built: set[str] = set() + for entry in data: + if not isinstance(entry, dict): + return None + drv = entry.get("drvPath") or entry.get("drv") + if drv: + built.add(_basename(drv)) + return built + + +def _supports_dry_run_json() -> bool: + """Probe whether this ``nix`` accepts ``build --dry-run --json`` at all. + + ``--json`` on ``nix build`` is unsupported on older Nix; parsing ``--help`` + is cheap and detects support without evaluating anything. + """ + try: + help_text = _run(["nix", "build", "--help"], check=False).stdout + except (OSError, subprocess.SubprocessError): + return False + return "--json" in help_text + + def to_build_set( attr: str, system: str | None = None, extra_args: list[str] | None = None ) -> set[str] | None: @@ -108,13 +146,28 @@ def to_build_set( be built (fully warm cache), a populated ``set`` of the drvs to build, and ``None`` *only* when the dry-run produced nothing recognizable — the sole case in which the caller should fall back to costing the whole closure. + + Prefers structured ``--dry-run --json`` output when this ``nix`` supports it + (robust across versions and locales), falling back to scraping human text. """ - cmd = ["nix", "build", "--dry-run", attr] + base = ["nix", "build", "--dry-run", attr] if system: - cmd += ["--system", system] + base += ["--system", system] if extra_args: - cmd += extra_args - proc = _run(cmd, check=False) + base += extra_args + + if _supports_dry_run_json(): + try: + proc = _run(base + ["--json"], check=False) + except (OSError, subprocess.SubprocessError): + proc = None + if proc is not None: + parsed = parse_dry_run_json(proc.stdout) + if parsed is not None: + return parsed + # JSON unavailable/unusable at runtime: fall through to text scraper. + + proc = _run(base, check=False) text = proc.stderr + "\n" + proc.stdout return parse_dry_run_text(text, proc.returncode) diff --git a/tests/test_graph.py b/tests/test_graph.py index 13ffd98..4a4c405 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -43,6 +43,42 @@ def test_clean_exit_without_header_is_empty_not_none(): assert graph.parse_dry_run_text("", returncode=0) == set() +# --- structured `nix build --dry-run --json` output (issue #8) ------------- + +JSON_PARTIAL = """\ +[ + {"drvPath": "/nix/store/bbb-foo-1.0.drv", "outputs": {"out": "/nix/store/e-foo"}}, + {"drvPath": "/nix/store/ccc-bar-2.0.drv", "outputs": {"out": "/nix/store/f-bar"}} +] +""" + +JSON_WARM = "[]" + +JSON_ALT_KEY = '[{"drv": "/nix/store/ggg-qux-9.drv"}]' + + +def test_json_partial_returns_built_drvs(): + assert graph.parse_dry_run_json(JSON_PARTIAL) == { + "bbb-foo-1.0.drv", + "ccc-bar-2.0.drv", + } + + +def test_json_warm_cache_is_empty_set(): + assert graph.parse_dry_run_json(JSON_WARM) == set() + + +def test_json_alternate_drv_key(): + assert graph.parse_dry_run_json(JSON_ALT_KEY) == {"ggg-qux-9.drv"} + + +def test_json_non_structured_is_none(): + # human text or an unexpected shape must fall back (None), not crash + assert graph.parse_dry_run_json(WARM) is None + assert graph.parse_dry_run_json('{"not": "a list"}') is None + assert graph.parse_dry_run_json("not json at all") is None + + def test_build_dag_empty_set_yields_no_nodes(): closure = {"a.drv": {"inputDrvs": {}}, "b.drv": {"inputDrvs": {"a.drv": {}}}} preds, nodes = graph.build_dag(closure, set())