Oleks
ae74db435a
--json now emits a stable sizing:{version:1,arch,nodes,cores,ram_gb,est_makespan_min}
block reusing provision.arch_for_system + estimate.peak_ram_gb_per_node. render_ephemeral_builders
now derives requirements.maxDuration + parking + policy from est_makespan_min for class ci.
430 lines
16 KiB
Python
430 lines
16 KiB
Python
"""CLI-level tests for grid parsing / normalization (issue #11)."""
|
||
|
||
import json
|
||
|
||
import pytest
|
||
|
||
from nix_estimator import cli
|
||
|
||
|
||
def _p(name):
|
||
return f"/nix/store/{'0' * 32}-{name}.drv"
|
||
|
||
|
||
@pytest.fixture
|
||
def stub_graph(monkeypatch, tmp_path):
|
||
"""Stub out the nix-invoking graph layer with a deterministic toy DAG."""
|
||
leaves = [_p(f"heavylib{i}") for i in range(4)]
|
||
root, sink = _p("root"), _p("sink")
|
||
closure = {d: {} for d in [root, *leaves, sink]}
|
||
preds = {root: [], sink: leaves, **{leaf: [root] for leaf in leaves}}
|
||
nodes = set(closure)
|
||
|
||
monkeypatch.setattr(cli.graph, "derivation_closure", lambda *a, **k: closure)
|
||
monkeypatch.setattr(cli.graph, "to_build_set", lambda *a, **k: nodes)
|
||
monkeypatch.setattr(cli.graph, "build_dag", lambda *a, **k: (preds, nodes))
|
||
|
||
hist = tmp_path / "history.json"
|
||
hist.write_text(
|
||
json.dumps(
|
||
{"root": 1.0, "sink": 1.0, **{f"heavylib{i}": 20.0 for i in range(4)}}
|
||
)
|
||
)
|
||
return str(hist)
|
||
|
||
|
||
def _run_json(capsys, argv):
|
||
assert cli.main(argv) == 0
|
||
return json.loads(capsys.readouterr().out)
|
||
|
||
|
||
def test_unsorted_cores_match_sorted(stub_graph, capsys):
|
||
sorted_out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--cores", "8,16,32", "--json"]
|
||
)
|
||
unsorted_out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--cores", "32,8,16", "--json"]
|
||
)
|
||
assert unsorted_out["recommendation"] == sorted_out["recommendation"]
|
||
assert unsorted_out["grid"] == sorted_out["grid"]
|
||
|
||
|
||
def test_unsorted_nodes_match_sorted(stub_graph, capsys):
|
||
sorted_out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--nodes", "1,2,4,8", "--json"]
|
||
)
|
||
unsorted_out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--nodes", "8,2,1,4", "--json"]
|
||
)
|
||
assert unsorted_out["recommendation"] == sorted_out["recommendation"]
|
||
|
||
|
||
def test_non_positive_cores_rejected(stub_graph, capsys):
|
||
with pytest.raises(SystemExit) as exc:
|
||
cli.main([".#x", "--history", stub_graph, "--cores", "8,0,16", "--json"])
|
||
assert exc.value.code != 0
|
||
assert "positive" in capsys.readouterr().err
|
||
|
||
|
||
def test_negative_nodes_rejected(stub_graph, capsys):
|
||
with pytest.raises(SystemExit) as exc:
|
||
cli.main([".#x", "--history", stub_graph, "--nodes", "1,-2,4", "--json"])
|
||
assert exc.value.code != 0
|
||
|
||
|
||
def test_non_integer_cores_rejected(stub_graph, capsys):
|
||
with pytest.raises(SystemExit) as exc:
|
||
cli.main([".#x", "--history", stub_graph, "--cores", "8,abc", "--json"])
|
||
assert exc.value.code != 0
|
||
assert "integer" in capsys.readouterr().err
|
||
|
||
|
||
# --- new feature surfaces (issues #12–#16) ---------------------------------
|
||
|
||
|
||
def test_backcompat_plain_estimate(stub_graph, capsys):
|
||
"""Plain `nix-estimate <attr> --json` still parses with no subcommand."""
|
||
out = _run_json(capsys, [".#foo", "--history", stub_graph, "--json"])
|
||
assert out["attr"] == ".#foo"
|
||
assert out["max_jobs"] == 1
|
||
assert out["node_ram_gb"] is None
|
||
|
||
|
||
def test_max_jobs_accepted_and_affects_output(stub_graph, capsys):
|
||
base = _run_json(capsys, [".#x", "--history", stub_graph, "--json"])
|
||
wide = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--max-jobs", "4", "--json"]
|
||
)
|
||
assert base["max_jobs"] == 1 and wide["max_jobs"] == 4
|
||
# Sharing cores across 4 jobs changes at least one grid makespan.
|
||
assert wide["grid"] != base["grid"]
|
||
|
||
|
||
def test_node_ram_gb_accepted(stub_graph, capsys):
|
||
out = _run_json(
|
||
capsys,
|
||
[
|
||
".#x",
|
||
"--history",
|
||
stub_graph,
|
||
"--max-jobs",
|
||
"4",
|
||
"--node-ram-gb",
|
||
"8",
|
||
"--json",
|
||
],
|
||
)
|
||
assert out["node_ram_gb"] == 8.0
|
||
|
||
|
||
def test_timeline_renders_report(stub_graph, capsys):
|
||
assert cli.main([".#x", "--history", stub_graph, "--timeline"]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "timeline — recommended shape" in out
|
||
assert "node0" in out
|
||
|
||
|
||
def test_timeline_json_carries_assignments(stub_graph, capsys):
|
||
out = _run_json(capsys, [".#x", "--history", stub_graph, "--timeline", "--json"])
|
||
assert "timeline" in out
|
||
tl = out["timeline"]
|
||
assert tl["makespan_min"] > 0
|
||
# every scheduled task name appears in some lane
|
||
tasks = [t[0] for lane in tl["assignments"].values() for t in lane]
|
||
assert "sink" in tasks
|
||
|
||
|
||
def test_provision_emits_yaml_to_stdout(stub_graph, capsys):
|
||
assert cli.main([".#x", "--history", stub_graph, "--provision"]) == 0
|
||
cap = capsys.readouterr()
|
||
assert "kind: EphemeralBuilder" in cap.out
|
||
assert "apiVersion: arbitrage.oleks.space/v1alpha1" in cap.out
|
||
assert "class: nix-builder" in cap.out
|
||
assert "requirements:" in cap.out and "cores:" in cap.out
|
||
assert "namespace: builder-arbitrage" in cap.out
|
||
# human report is diverted to stderr, not stdout
|
||
assert "RECOMMENDATION" in cap.err
|
||
assert "RECOMMENDATION" not in cap.out
|
||
|
||
|
||
def test_provision_custom_class_and_name(stub_graph, capsys):
|
||
assert (
|
||
cli.main(
|
||
[
|
||
".#x",
|
||
"--history",
|
||
stub_graph,
|
||
"--provision",
|
||
"runner",
|
||
"--provision-name",
|
||
"my-builder",
|
||
]
|
||
)
|
||
== 0
|
||
)
|
||
out = capsys.readouterr().out
|
||
assert "class: runner" in out
|
||
assert "name: my-builder-1" in out
|
||
|
||
|
||
def test_json_sizing_block(stub_graph, capsys):
|
||
"""#25: --json carries a stable versioned sizing block for arbitrage."""
|
||
out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--system", "aarch64-linux", "--json"]
|
||
)
|
||
sizing = out["sizing"]
|
||
assert sizing["version"] == 1
|
||
assert sizing["arch"] == "arm64"
|
||
# sizing mirrors the recommendation shape
|
||
assert sizing["nodes"] == out["recommendation"]["nodes"]
|
||
assert sizing["cores"] == out["recommendation"]["cores_per_node"]
|
||
assert sizing["est_makespan_min"] == out["recommendation"]["est_makespan_min"]
|
||
# ram_gb is a concrete positive int (modeled or defaulted)
|
||
assert isinstance(sizing["ram_gb"], int)
|
||
assert sizing["ram_gb"] >= 1
|
||
|
||
|
||
def test_json_sizing_arch_defaults_amd64(stub_graph, capsys):
|
||
"""#25: sizing.arch falls back to amd64 when --system is unset/unknown."""
|
||
out = _run_json(capsys, [".#x", "--history", stub_graph, "--json"])
|
||
assert out["sizing"]["arch"] == "amd64"
|
||
|
||
|
||
def test_json_sizing_ram_from_model(stub_graph_heavy, capsys):
|
||
"""#25: sizing.ram_gb reuses the modeled peak (covers 10 GB chromium)."""
|
||
out = _run_json(capsys, [".#x", "--json"])
|
||
assert out["sizing"]["ram_gb"] >= 10
|
||
|
||
|
||
def test_json_sizing_explicit_ram_wins(stub_graph, capsys):
|
||
"""#25: --node-ram-gb overrides the modeled sizing.ram_gb."""
|
||
out = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--node-ram-gb", "48", "--json"]
|
||
)
|
||
assert out["sizing"]["ram_gb"] == 48
|
||
|
||
|
||
def test_provision_ci_window(stub_graph, capsys):
|
||
"""#25: --provision ci emits the on-demand window derived from makespan."""
|
||
assert cli.main([".#x", "--history", stub_graph, "--provision", "ci"]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "class: ci" in out
|
||
assert "maxDuration:" in out
|
||
assert "parking:" in out
|
||
assert "policy: cost" in out
|
||
assert "prefer: dependable" in out
|
||
assert "allowNixbuildFallback: false" in out
|
||
|
||
|
||
def test_provision_nix_builder_has_no_window(stub_graph, capsys):
|
||
"""#25: the default nix-builder class keeps its window-free output."""
|
||
assert cli.main([".#x", "--history", stub_graph, "--provision"]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "maxDuration" not in out
|
||
assert "parking" not in out
|
||
|
||
|
||
def test_mine_subcommand_writes_json(monkeypatch, capsys, tmp_path):
|
||
"""`mine` runs mine_history (stubbed) and writes {name: minutes} JSON."""
|
||
calls = []
|
||
|
||
def fake_mine_history(installables, **kw):
|
||
calls.append(list(installables))
|
||
return {"hello": 2.0, "world": 5.0}
|
||
|
||
monkeypatch.setattr(cli.miner, "mine_history", fake_mine_history)
|
||
|
||
out_file = tmp_path / "hist.json"
|
||
rc = cli.main(["mine", ".#hello", ".#world", "--repeat", "3", "-o", str(out_file)])
|
||
assert rc == 0
|
||
assert len(calls) == 3 # repeated
|
||
data = json.loads(out_file.read_text())
|
||
assert data == {"hello": 2.0, "world": 5.0}
|
||
|
||
|
||
def test_mine_subcommand_to_stdout(monkeypatch, capsys):
|
||
monkeypatch.setattr(cli.miner, "mine_history", lambda inst, **kw: {"pkg": 1.5})
|
||
assert cli.main(["mine", ".#pkg"]) == 0
|
||
assert json.loads(capsys.readouterr().out) == {"pkg": 1.5}
|
||
|
||
|
||
@pytest.fixture
|
||
def stub_graph_heavy(monkeypatch):
|
||
"""Toy DAG with a RAM-heavy chromium build (10 GB) to exercise --provision sizing."""
|
||
root = _p("root")
|
||
heavy = _p("chromium")
|
||
leaf = _p("lib")
|
||
closure = {root: {}, heavy: {}, leaf: {}}
|
||
preds = {root: [], heavy: [root], leaf: [root]}
|
||
nodes = set(closure)
|
||
monkeypatch.setattr(cli.graph, "derivation_closure", lambda *a, **k: closure)
|
||
monkeypatch.setattr(cli.graph, "to_build_set", lambda *a, **k: nodes)
|
||
monkeypatch.setattr(cli.graph, "build_dag", lambda *a, **k: (preds, nodes))
|
||
|
||
|
||
def _ramgb_from_yaml(text):
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
if s.startswith("ramGb:"):
|
||
return int(s.split(":", 1)[1])
|
||
raise AssertionError(f"no ramGb in:\n{text}")
|
||
|
||
|
||
def test_provision_ramgb_from_model(stub_graph_heavy, capsys):
|
||
# No --node-ram-gb: ramGb is sized from the modeled peak, which must cover
|
||
# the 10 GB chromium build (issue #19), not just fall back to 2*cores blindly.
|
||
assert cli.main([".#x", "--provision"]) == 0
|
||
ram = _ramgb_from_yaml(capsys.readouterr().out)
|
||
assert ram >= 10
|
||
|
||
|
||
def test_provision_explicit_node_ram_overrides(stub_graph_heavy, capsys):
|
||
assert cli.main([".#x", "--provision", "--node-ram-gb", "64"]) == 0
|
||
ram = _ramgb_from_yaml(capsys.readouterr().out)
|
||
assert ram == 64
|
||
|
||
|
||
def test_provision_fallback_when_nothing_to_build(monkeypatch, capsys):
|
||
# Empty to_build -> no model figure -> renderer's 2 GiB/core default (16c -> 32).
|
||
root = _p("root")
|
||
closure = {root: {}}
|
||
monkeypatch.setattr(cli.graph, "derivation_closure", lambda *a, **k: closure)
|
||
monkeypatch.setattr(cli.graph, "to_build_set", lambda *a, **k: set())
|
||
monkeypatch.setattr(cli.graph, "build_dag", lambda *a, **k: ({}, set()))
|
||
# nothing-to-build short-circuits the human report; force provision path via a
|
||
# closure that still yields a recommendation is not possible when to_build==0,
|
||
# so assert the CLI exits cleanly and prints the nothing-to-build notice.
|
||
assert cli.main([".#x", "--provision"]) == 0
|
||
|
||
|
||
# --- new backend wiring (issues #20–#23) -----------------------------------
|
||
|
||
|
||
def test_work_line_relabelled_serial(stub_graph, capsys):
|
||
"""#20: work T1 line is annotated (1 node, serial), not (1 core, 1 node)."""
|
||
assert cli.main([".#x", "--history", stub_graph]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "work T1" in out
|
||
assert "(1 node, serial)" in out
|
||
assert "(1 core, 1 node)" not in out
|
||
|
||
|
||
@pytest.fixture
|
||
def stub_graph_fetch(monkeypatch):
|
||
"""Toy DAG with one fixed-output fetch drv (has outputs.out.hash)."""
|
||
root = _p("root")
|
||
fetch = _p("source")
|
||
lib = _p("lib")
|
||
closure = {
|
||
root: {},
|
||
fetch: {
|
||
"outputs": {
|
||
"out": {"path": "/nix/store/zzz-source", "hash": "sha256:deadbeef"}
|
||
}
|
||
},
|
||
lib: {},
|
||
}
|
||
preds = {root: [], fetch: [], lib: [root, fetch]}
|
||
nodes = set(closure)
|
||
monkeypatch.setattr(cli.graph, "derivation_closure", lambda *a, **k: closure)
|
||
monkeypatch.setattr(cli.graph, "to_build_set", lambda *a, **k: nodes)
|
||
monkeypatch.setattr(cli.graph, "build_dag", lambda *a, **k: (preds, nodes))
|
||
|
||
|
||
def test_fetch_line_and_cpu_bound_label(stub_graph_fetch, capsys):
|
||
"""#21: report surfaces fetches and relabels the peak as CPU-bound width."""
|
||
assert cli.main([".#x"]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "fixed-output fetches" in out
|
||
assert "peak CPU-bound width" in out
|
||
|
||
|
||
def test_fetch_json_fields(stub_graph_fetch, capsys):
|
||
"""#21: --json carries fetch_count / fetch_min / cpu_bound_count."""
|
||
out = _run_json(capsys, [".#x", "--json"])
|
||
assert out["fetch_count"] == 1
|
||
assert out["cpu_bound_count"] == 2 # root + lib
|
||
assert out["fetch_min"] >= 0
|
||
|
||
|
||
def test_assume_cache_reduces_set(stub_graph, monkeypatch, capsys):
|
||
"""#22: --assume-cache uses the reduced set from filter_upstream_substitutable."""
|
||
baseline = _run_json(capsys, [".#x", "--history", stub_graph, "--json"])
|
||
|
||
def fake_filter(closure, to_build, cache_url="x", _query=None):
|
||
base = set(closure) if to_build is None else (set(to_build) & set(closure))
|
||
# drop one derivation as "substitutable"
|
||
reduced = set(base)
|
||
reduced.discard(next(iter(reduced)))
|
||
return reduced, True
|
||
|
||
monkeypatch.setattr(cli.graph, "filter_upstream_substitutable", fake_filter)
|
||
# Honor the (reduced) to_build set instead of the fixture's fixed stub, so the
|
||
# count reflects what --assume-cache dropped.
|
||
monkeypatch.setattr(
|
||
cli.graph,
|
||
"build_dag",
|
||
lambda closure, to_build: (
|
||
{d: [] for d in to_build},
|
||
set(to_build) if to_build is not None else set(closure),
|
||
),
|
||
)
|
||
reduced = _run_json(
|
||
capsys, [".#x", "--history", stub_graph, "--assume-cache", "--json"]
|
||
)
|
||
assert reduced["to_build"] == baseline["to_build"] - 1
|
||
|
||
|
||
def test_assume_cache_noop_warns(stub_graph, monkeypatch, capsys):
|
||
"""#22: a no-op cache query (changed=False) prints a stderr warning."""
|
||
monkeypatch.setattr(
|
||
cli.graph,
|
||
"filter_upstream_substitutable",
|
||
lambda closure, to_build, cache_url="x", _query=None: (
|
||
set(closure) if to_build is None else set(to_build) & set(closure),
|
||
False,
|
||
),
|
||
)
|
||
assert cli.main([".#x", "--history", stub_graph, "--assume-cache"]) == 0
|
||
assert "dropped nothing" in capsys.readouterr().err
|
||
|
||
|
||
def test_cold_assume_cache_together(stub_graph, monkeypatch, capsys):
|
||
"""#22: --cold --assume-cache works together (to_build starts as None)."""
|
||
seen = {}
|
||
|
||
def fake_filter(closure, to_build, cache_url="x", _query=None):
|
||
seen["to_build"] = to_build
|
||
return set(closure), True
|
||
|
||
monkeypatch.setattr(cli.graph, "filter_upstream_substitutable", fake_filter)
|
||
assert cli.main([".#x", "--history", stub_graph, "--cold", "--assume-cache"]) == 0
|
||
assert seen["to_build"] is None # cold passes the whole closure
|
||
assert "minus upstream cache" in capsys.readouterr().out
|
||
|
||
|
||
def test_default_min_changes_estimate(stub_graph, capsys):
|
||
"""#23: --default-min moves the headline work figure."""
|
||
base = _run_json(capsys, [".#x", "--json"])
|
||
bumped = _run_json(capsys, [".#x", "--default-min", "10", "--json"])
|
||
assert bumped["work_min"] != base["work_min"]
|
||
|
||
|
||
def test_default_min_shown_in_header(stub_graph, capsys):
|
||
assert cli.main([".#x", "--default-min", "7"]) == 0
|
||
assert "DEFAULT bucket override" in capsys.readouterr().out
|
||
|
||
|
||
def test_calibrate_prints_medians(monkeypatch, capsys, tmp_path):
|
||
"""#23: calibrate classifies the closure and prints per-bucket medians."""
|
||
# DEFAULT-bucket drvs (plain libs) with measured history minutes.
|
||
closure = {_p("liba"): {}, _p("libb"): {}, _p("libc"): {}}
|
||
monkeypatch.setattr(cli.graph, "derivation_closure", lambda *a, **k: closure)
|
||
hist = tmp_path / "h.json"
|
||
hist.write_text(json.dumps({"liba": 2.0, "libb": 4.0, "libc": 6.0}))
|
||
assert cli.main(["calibrate", ".#x", "--history", str(hist)]) == 0
|
||
out = capsys.readouterr().out
|
||
assert "DEFAULT" in out
|
||
assert "4.00 min" in out # median of 2,4,6
|
||
assert "--default-min 4.00" in out
|