# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Which CI jobs are red, and which of those actually gate a merge.
repomatic names the jobs it generates, prefixing each matrix cell with a
glyph that records whether the cell is allowed to fail: `✅` for a required
one, `⁉️` for a probe running under `continue-on-error`. Reading that back
was left to whoever was watching CI, and it is easy to get wrong in four
specific ways this module exists to settle:
- **The glyph, not the position.** Job names differ in shape across
workflows: `tests.yaml` emits `✅ ubuntu-26.04 / py3.11` while the release
engine emits `workflow / ✅ ubuntu-26.04, abc1234 build`. Anything that
splits on `" / "` and reads a fixed field strips the glyph off one of the
two and files a required red as a probe, which reads as green.
- **Jobs, not the run.** A run's own `conclusion` is `success` while a
`continue-on-error` probe inside it crashed, and its `status` still reads
`queued` while a dozen of its jobs have already finished. Neither answers
"is anything broken".
- **A run that failed around its jobs.** A `failure` conclusion with no
failed job is a workflow-level error: an invalid `strategy.matrix`
expression, malformed YAML, a missing secret. There is no job log to read,
and treating it as benign is how a persistently red workflow gets written
off as a known artifact. A required job that GitHub cancelled, like one no
runner picked up, fails its run the same way: that job is named instead,
and it still gates the merge.
- **A run that settled without a result.** A newer push cancels the run of
the commit under it, and a job-level `if:` skips every job of a run: the
bundled `tests.yaml` does that for a version-bump commit. Neither run has a
failed job, and neither says anything about its commit. Reading one as
green hides that the matrix never ran on the newest commits. A cancelled run
on the tip is not one: no newer push stopped it, so a required job that
GitHub cancelled there, like one past its timeout, gates the merge.
A job carrying no stability glyph is required. That covers every non-matrix job
(`1️⃣ Run-once tests`, `📦 Package install`, `🛡️ Lint types`), where the absence
of a marker means the job was never optional rather than that its status is
unknown. Only the two stability glyphs count: a job name may carry any other
emoji and still be required, which is why the test is for `⁉️` specifically
rather than for a decorated name. A name that holds both glyphs is the one
exception: see {meth}`JobStatus.required`.
"""
from __future__ import annotations
import json
import logging
import re
import time
from dataclasses import dataclass, field, replace
from urllib.parse import quote
from click_extra import ColumnSpec
from .gh import GhCommandError, GhOutputError, gh_api_pages, gh_json
from .outputs import AnnotationLevel
from .runs import commit_runs, job_log, read_run, workflow_runs
from .workflow_files import load_workflows, workflow_triggers
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator
from pathlib import Path
TIP_HISTORY_DEPTH = 10
"""Commits {func}`read_ci_status` walks back from the branch tip, tip included.
A push starts most workflows on the tip itself. A workflow whose `paths:` filter
skipped the tip last ran on an earlier commit, which a walk this deep reaches on
any active branch. A workflow with no run in the window falls back to
{func}`latest_run`.
"""
STABLE_GLYPH = "✅"
"""Marks a matrix cell that must pass. See {data}`UNSTABLE_GLYPH`."""
UNSTABLE_GLYPH = "⁉️"
"""Marks a matrix cell running under `continue-on-error`.
A red one never gates a merge, which is exactly why it has to be told apart
from a required cell rather than counted with it. A release still fixes what
it can: see the `repomatic-ship` skill on the genuinely-green goal.
"""
TERMINAL_STATUSES = frozenset({"completed"})
"""Job statuses meaning the job will not change again."""
CI_STATUS_HEADER_DEFS: tuple[ColumnSpec, ...] = (
ColumnSpec("workflow", "Workflow"),
ColumnSpec("commit", "Commit"),
ColumnSpec("run-status", "Run status"),
ColumnSpec("verdict", "Verdict"),
)
"""Column definitions for the `ci-status` table, filled by {meth}`RunStatus.cells`."""
LOG_NAME_RE = re.compile(r"[^\w.-]+")
"""What a log filename drops from a workflow or job name: spaces, glyphs, slashes."""
RUN_FIELDS = ("conclusion", "headSha", "jobs", "status", "workflowName")
"""The `gh run view --json` fields a {class}`RunStatus` is built from."""
FAILED_POLL_LIMIT = 3
"""Failed polls in a row that end a watch.
{func}`watch_ci_status` skips a poll that meets a transient error, like the
`HTTP 504` GitHub answers when it cannot respond in time: a fault that clears
within a few intervals costs a few polls, not the watch. A fault that lasts this
many polls in a row ends the watch with its error, as a persistent one must.
"""
[docs]
@dataclass(frozen=True)
class JobStatus:
"""One job of one workflow run."""
name: str
"""The job's name, glyph included."""
status: str
"""`queued`, `in_progress` or `completed`."""
conclusion: str
"""`success`, `failure`, `cancelled`, `skipped`, or empty while running."""
job_id: int = 0
"""The job's numeric ID, for its log endpoint. `0` when the listing had none."""
@property
def unexpanded(self) -> bool:
"""Whether the job is a matrix GitHub never evaluated.
An expanded cell carries its values in its name. One that still holds
its `name:` expression, `${{` and all, is a stage the run never
reached: no cell of it ran, whatever the run's own conclusion says.
"""
return "${{" in self.name
@property
def required(self) -> bool:
"""Whether a failure here gates a merge.
Looks for the glyph anywhere in the raw name rather than at its
start. The two shapes disagree on where it sits: `tests.yaml` leads
with it (`⁉️ ubuntu-26.04 / py3.15-dev`) while the release engine
prefixes the workflow first (`release / ⁉️ windows-11-arm, abc1234
build`). A leading-position test passes the first and silently files
the second as required; splitting on `" / "` gets it wrong the other
way round. Containment is the one form both satisfy, and the
templates emit exactly one glyph per expanded name.
A name that holds both glyphs is a matrix GitHub never expanded: the
job keeps its `name:` expression, which spells the two of them. A run
stopped before the matrix got its cells leaves one. Some of those
cells are required, so the job is.
"""
return STABLE_GLYPH in self.name or UNSTABLE_GLYPH not in self.name
@property
def failed(self) -> bool:
"""Whether this job reached a failing conclusion."""
return self.conclusion == "failure"
@property
def running(self) -> bool:
"""Whether this job has yet to reach a terminal state."""
return self.status not in TERMINAL_STATUSES
@property
def skipped(self) -> bool:
"""Whether this job never ran: its `if:`, or a job it needs, ruled it out."""
return self.conclusion == "skipped"
@property
def cancelled(self) -> bool:
"""Whether this job was stopped before it finished."""
return self.conclusion == "cancelled"
[docs]
@dataclass(frozen=True)
class RunStatus:
"""The latest run of one workflow on one branch."""
workflow: str
"""Workflow name, as GitHub reports it."""
run_id: int
"""Numeric run ID, for `gh run view`."""
head_sha: str
"""Commit the run was created for."""
status: str
"""The run's own status. Lags its jobs, so it gates nothing once one exists."""
conclusion: str
"""The run's own conclusion, empty while it is still going."""
jobs: tuple[JobStatus, ...] = ()
"""Every job of the run."""
on_tip: bool = False
"""Whether the run was created for the tip of its branch.
{func}`read_ci_status` sets it. Only a newer push moves the tip, so no newer
push stopped a run on it: {attr}`stopped_required` relies on that.
"""
@property
def failed_required(self) -> tuple[JobStatus, ...]:
"""Failing jobs that gate a merge."""
return tuple(job for job in self.jobs if job.failed and job.required)
@property
def failed_probes(self) -> tuple[JobStatus, ...]:
"""Failing jobs allowed to fail."""
return tuple(job for job in self.jobs if job.failed and not job.required)
@property
def running_jobs(self) -> tuple[JobStatus, ...]:
"""Jobs that have not settled yet."""
return tuple(job for job in self.jobs if job.running)
@property
def unexpanded_jobs(self) -> tuple[JobStatus, ...]:
"""Matrix jobs the run never expanded into cells."""
return tuple(job for job in self.jobs if job.unexpanded)
@property
def workflow_level_failure(self) -> bool:
"""Whether the run failed around its jobs rather than inside one.
No job log explains this one: read the run's error annotations and
fix the workflow itself. A run that failed on a required job GitHub
cancelled is not one: see {attr}`stopped_required`.
"""
return (
self.conclusion == "failure"
and not any(job.failed for job in self.jobs)
and not self.cancelled_required
)
@property
def waiting(self) -> bool:
"""Whether GitHub has yet to create a job of the run.
A run waits with no job for a runner, or behind the run ahead of it in
its concurrency group. Its own status is then the only signal.
"""
return not self.jobs and self.status not in TERMINAL_STATUSES
@property
def blocking(self) -> bool:
"""Whether this run holds up a merge."""
return (
bool(self.failed_required)
or bool(self.stopped_required)
or self.workflow_level_failure
)
@property
def cancelled_required(self) -> tuple[JobStatus, ...]:
"""Required jobs stopped before they finished."""
return tuple(job for job in self.jobs if job.cancelled and job.required)
@property
def stopped_required(self) -> tuple[JobStatus, ...]:
"""Required jobs cancelled in a run that no newer run replaces.
Such a job gives its commit no result, and no newer run brings one.
GitHub cancels a job that no runner picked up, and concludes its run
`failure`, with no failed job and no job log. It also cancels a job past
its `timeout-minutes`, but concludes that run `cancelled`, like a run
that a newer push stopped. That one is told apart by {attr}`on_tip`: a
newer push leaves the run it stopped under the tip.
"""
if self.conclusion == "failure" or self.on_tip:
return self.cancelled_required
return ()
@property
def cancelled(self) -> bool:
"""Whether the run was stopped before its required jobs all finished.
A newer push to the same concurrency group does that to the run of the
commit under it. A run stopped before GitHub created any job counts
too.
"""
if self.jobs:
return bool(self.cancelled_required)
return self.conclusion == "cancelled"
@property
def skipped(self) -> bool:
"""Whether the run ran no job at all.
A job-level `if:` ruled every one of them out, which GitHub concludes as
a `skipped` run.
"""
if self.jobs:
return all(job.skipped for job in self.jobs)
return self.conclusion == "skipped"
@property
def inconclusive(self) -> bool:
"""Whether the run settled without a result for its commit.
Neither red nor green: this workflow says nothing about the tree the run
was created for, so nothing here holds up a merge and nothing clears it.
"""
return (
not self.blocking
and not self.running_jobs
and (self.cancelled or self.skipped)
)
@property
def verdict(self) -> str:
"""One-phrase outcome, for the table's last column."""
if self.workflow_level_failure:
return "workflow-level failure"
if self.failed_required:
return f"{len(self.failed_required)} required job(s) failed"
if self.stopped_required:
return f"{len(self.stopped_required)} required job(s) cancelled"
if self.running_jobs:
return f"{len(self.running_jobs)} job(s) still running"
if self.waiting:
return "not started"
if self.cancelled_required:
return f"cancelled ({len(self.cancelled_required)} required job(s))"
if self.cancelled:
return "cancelled (no job ran)"
if self.skipped:
return "skipped (no job ran)"
if self.failed_probes:
return f"green ({len(self.failed_probes)} probe(s) failed)"
return "green"
[docs]
def cells(self) -> tuple[str, str, str, str]:
"""The row of the `ci-status` table, in {data}`CI_STATUS_HEADER_DEFS` order."""
return (self.workflow, self.head_sha[:8], self.status, self.verdict)
[docs]
def findings(self) -> Iterator[tuple[AnnotationLevel, str]]:
"""What the reader must know about this run, one line each.
An error gates a merge: a workflow-level failure, a failed required job,
or a required job that GitHub cancelled in a run no newer run replaces.
A warning does not: a failed probe, or a run with no result for its
commit. A notice asks for nothing: a matrix whose `if:` skipped it
before it evaluated, like a release-only stage on an ordinary push or
the binary stages of a project that compiles none.
:yields: `(level, message)` pairs, errors first.
"""
if self.workflow_level_failure:
message = (
f"{self.workflow}: run {self.run_id} failed with no failed job. "
"This is a workflow-level error (an invalid matrix expression, "
"malformed YAML, a missing secret): read the run's error "
"annotations, there is no job log."
)
yield AnnotationLevel.ERROR, message
for job in self.failed_required:
yield (
AnnotationLevel.ERROR,
f"{self.workflow}: required job failed: {job.name}",
)
for job in self.stopped_required:
yield (
AnnotationLevel.ERROR,
(
f"{self.workflow}: required job cancelled: {job.name}. Its"
" annotations give the reason, like a runner that never"
" picked it up or a timeout. Re-run it with"
f" `gh run rerun {self.run_id} --failed`."
),
)
for job in self.failed_probes:
yield (
AnnotationLevel.WARNING,
f"{self.workflow}: allowed-failure probe failed: {job.name}",
)
for job in self.unexpanded_jobs:
if job.skipped:
yield (
AnnotationLevel.NOTICE,
(
f"{self.workflow}: the matrix behind {job.name!r} never"
" evaluated: its `if:` skipped it, so no cell of it ran."
),
)
continue
yield (
AnnotationLevel.WARNING,
(
f"{self.workflow}: the matrix behind {job.name!r} never"
" evaluated, so no cell of it ran and nothing it builds or"
" tests has a result."
),
)
if self.inconclusive:
reason = (
"was cancelled before its required jobs finished"
if self.cancelled
else "ran no job"
)
message = (
f"{self.workflow}: run {self.run_id} {reason}, so"
f" {self.head_sha[:8]} has no result from it."
)
yield AnnotationLevel.WARNING, message
[docs]
@dataclass
class CIStatus:
"""Every monitored workflow's latest run on a branch."""
branch: str
"""Branch the runs were read from."""
tip_sha: str = ""
"""Commit at the tip of the branch when the runs were read.
Empty when the branch history lists no commit.
"""
runs: list[RunStatus] = field(default_factory=list)
"""One entry per workflow that has a run, newest first."""
skipped_on_tip: list[str] = field(default_factory=list)
"""Workflow filenames that ran no job on the tip.
Only the ones read on an earlier commit: the row of each says nothing about
the tip. A workflow with no other run in the window keeps its tip run, whose
verdict says the same.
"""
@property
def blocking(self) -> list[RunStatus]:
"""Runs holding up a merge."""
return [run for run in self.runs if run.blocking]
@property
def runs_on_tip(self) -> list[RunStatus]:
"""Runs created for the tip commit itself.
Empty right after a push, while GitHub has yet to create the tip's runs:
every run listed then belongs to an earlier commit.
"""
return [run for run in self.runs if run.head_sha == self.tip_sha]
@property
def settled(self) -> bool:
"""Whether every run reached a terminal state."""
return all(not run.running_jobs and not run.waiting for run in self.runs)
def _workflow_triggers(workflow_dir: Path) -> list[tuple[str, set[str]]]:
"""Pair each workflow file in the directory with the triggers it declares.
Both public readers below select on those triggers, and neither should
parse the tree twice or disagree on which files count as workflows.
:param workflow_dir: Directory holding the workflow files.
:return: `(filename, triggers)` pairs, sorted by path. Empty when the
directory is missing. A file that does not parse is skipped.
"""
return [
(path.name, set(workflow_triggers(data)))
for path, data in load_workflows(workflow_dir).items()
]
[docs]
def workflow_files(workflow_dir: Path) -> tuple[str, ...]:
"""Every workflow in the directory that has runs of its own.
{func}`monitored_workflows` narrows this to the ones a push starts, which
is the right default for a status report and the wrong set to *accept*: a
schedule-only or dispatch-only workflow has runs worth reading too. A
reusable workflow is excluded for the reason it is there, since its jobs
only ever appear under a caller's run.
:param workflow_dir: Directory holding the workflow files.
:return: Workflow filenames, sorted.
"""
return tuple(
name
for name, triggers in _workflow_triggers(workflow_dir)
if triggers - {"workflow_call"}
)
[docs]
def monitored_workflows(workflow_dir: Path) -> list[str]:
"""Every workflow a push to the default branch can start.
Derived from the tree rather than listed by hand, so a workflow added
later is watched without anyone remembering to add it here. A reusable
workflow is excluded: it has no runs of its own, only the ones its
callers create.
:param workflow_dir: Directory holding the workflow files.
:return: Workflow filenames, sorted.
"""
return [
name
for name, triggers in _workflow_triggers(workflow_dir)
if "push" in triggers
]
def _run_status(run_id: int, workflow: str) -> RunStatus:
"""Read one run and its jobs by ID.
Every field comes from the run itself rather than from the listing that
found it, so the report never mixes a listing's view of a run with a
fresher view of its jobs.
:param run_id: The run's numeric ID.
:param workflow: Workflow filename, the fallback display name.
:return: The run with its jobs attached.
:raises GhCommandError: When `gh` cannot read the run.
:raises GhOutputError: When `gh` answers with something other than a run. A
run with no status and no jobs would otherwise read as green.
"""
detail = read_run(run_id, RUN_FIELDS)
if detail is None:
msg = f"Could not read run {run_id} of {workflow}."
raise GhOutputError(msg)
return RunStatus(
workflow=detail.get("workflowName") or workflow,
run_id=run_id,
head_sha=detail.get("headSha") or "",
status=detail.get("status") or "",
conclusion=detail.get("conclusion") or "",
jobs=tuple(
JobStatus(
name=job.get("name") or "",
status=job.get("status") or "",
conclusion=job.get("conclusion") or "",
job_id=int(job.get("databaseId") or 0),
)
for job in detail.get("jobs") or []
),
)
[docs]
def latest_run(workflow: str, branch: str) -> RunStatus | None:
"""Read a workflow's most recent run on *branch*, jobs included.
Lists the workflow's runs through {func}`~repomatic.github.runs.workflow_runs`,
which sends a `created` filter to keep a stale snapshot out.
{func}`read_ci_status` asks it only about a workflow with no run on the
newest commits.
:param workflow: Workflow filename, like `tests.yaml`.
:param branch: Branch to read runs from.
:return: The run, or `None` when the workflow has none in the listing
window. An empty listing is not proof the workflow was filtered out:
GitHub can sit on a push event for hours before materializing a run.
"""
# `strict`: an unreachable `gh` must fail the read rather than report the
# workflow as run-less, which `/babysit-ci` would read as a green hole.
runs = workflow_runs(workflow, branch, strict=True)
if not runs:
return None
return _run_status(int(runs[0]["id"]), workflow)
def _commit_history(ref: str) -> list[str]:
"""SHAs of the newest commits reachable from *ref*, tip first.
:param ref: Branch, or any other ref the commits endpoint resolves.
:return: At most {data}`TIP_HISTORY_DEPTH` SHAs. Empty when the answer
lists no commit.
:raises GhCommandError: When the read fails.
:raises GhOutputError: When the answer is not JSON.
"""
endpoint = (
f"repos/{{owner}}/{{repo}}/commits?sha={quote(ref, safe='')}"
f"&per_page={TIP_HISTORY_DEPTH}"
)
commits = gh_json(["api", endpoint])
if not isinstance(commits, list):
return []
return [commit["sha"] for commit in commits if commit.get("sha")]
def _fork_pull_ref(branch: str) -> str | None:
"""The ref of the open pull request sent from the *branch* of a fork.
GitHub keeps the head of every pull request under `refs/pull/<number>/head`,
in the repository the pull request targets.
:param branch: Head branch of the pull request, as its runs name it.
:return: The ref, or `None` when no open pull request comes from *branch*.
:raises RuntimeError: When several open pull requests come from a branch of
that name, sent from different forks.
"""
pulls = gh_api_pages("repos/{owner}/{repo}/pulls?state=open&per_page=100")
numbers = sorted(
int(pull["number"])
for pull in pulls
if isinstance(pull, dict) and (pull.get("head") or {}).get("ref") == branch
)
if len(numbers) > 1:
listed = ", ".join(f"#{number}" for number in numbers)
msg = f"open pull requests {listed} all come from a branch named {branch!r}"
raise RuntimeError(msg)
return f"refs/pull/{numbers[0]}/head" if numbers else None
def _branch_history(branch: str) -> list[str]:
"""SHAs of the newest commits on *branch*, tip first.
The repository answers `404` for the branch of a pull request sent from a
fork, as the branch lives in the fork. Its commits are then read from the ref
of that pull request, while its runs still carry the name of the branch.
:param branch: Branch to read.
:return: At most {data}`TIP_HISTORY_DEPTH` SHAs. Empty when the answer
lists no commit.
:raises GhCommandError: When the read fails.
:raises GhOutputError: When the answer is not JSON.
:raises RuntimeError: When pull requests from several forks share the
branch name.
"""
try:
return _commit_history(branch)
except GhCommandError as error:
if error.status != 404:
raise
pull_ref = _fork_pull_ref(branch)
if pull_ref is None:
raise
return _commit_history(pull_ref)
def _runs_on_commit(sha: str, branch: str) -> tuple[dict[str, int], dict[str, int]]:
"""The newest run of each workflow GitHub created for commit *sha*.
A run GitHub concluded `skipped` ran no job, so it is kept apart from the
runs that did: {func}`read_ci_status` looks for a run with a result on an
earlier commit before it settles for that one.
:param sha: Full commit SHA.
:param branch: Branch the runs must belong to. The same commit also carries
runs for a tag pushed on it, or for a pull request branch.
:return: Two maps of run ID by workflow filename: the runs that ran a job,
then the skipped ones.
"""
ran: dict[str, int] = {}
skipped: dict[str, int] = {}
# Newest first, per the API's ordering: the first entry seen for a
# workflow is its latest run of that kind on this commit.
for entry in commit_runs(sha):
if entry.get("head_branch") != branch:
continue
filename = str(entry.get("path") or "").rpartition("/")[2]
runs = skipped if entry.get("conclusion") == "skipped" else ran
if filename and filename not in runs:
runs[filename] = int(entry["id"])
return ran, skipped
[docs]
def read_ci_status(workflows: Iterable[str], branch: str) -> CIStatus:
"""Read the latest run of each workflow on *branch*.
Anchored on the branch tip: reads the runs of each of the newest commits,
tip first, and stops as soon as every workflow has one. A workflow whose
`paths:` filter skipped the tip is found on the commit that last started
it. Only a workflow with no run in the whole window falls back to
{func}`latest_run`.
A run that ran no job counts as no run at all for that walk, so a workflow
that skipped every job on the tip is also read on an earlier commit. A
cancelled run stops the walk instead: the runs under it are older than the
commits it left without a result, and a green one among them would hide
that. A workflow with nothing but skipped runs in the window is reported on
the newest of them. Each run records whether it sits on the tip, which
decides whether a cancelled job in it blocks.
```{warning}
The branch-wide run listing can answer with a stale snapshot: runs from
commits months old, presented as the latest ones. Reading runs commit by
commit, from a commit list that comes from git data rather than from the
run index, keeps that snapshot out. The fallback listing sends a `created`
filter for the same reason.
```
:param workflows: Workflow filenames to read.
:param branch: Branch to read runs from.
:return: The collected status.
"""
status = CIStatus(branch=branch)
names = list(workflows)
if not names:
return status
history = _branch_history(branch)
if history:
status.tip_sha = history[0]
wanted = set(names)
found: dict[str, int] = {}
skipped: dict[str, int] = {}
idle_on_tip: set[str] = set()
for sha in history:
ran, idle = _runs_on_commit(sha, branch)
for filename, run_id in ran.items():
if filename in wanted and filename not in found:
found[filename] = run_id
for filename, run_id in idle.items():
if filename in wanted and filename not in found:
skipped.setdefault(filename, run_id)
if sha == status.tip_sha:
idle_on_tip = set(skipped)
if len(found) == len(wanted):
break
status.skipped_on_tip = sorted(idle_on_tip & found.keys())
for workflow in names:
chosen = found.get(workflow, skipped.get(workflow))
run = (
_run_status(chosen, workflow)
if chosen is not None
else latest_run(workflow, branch)
)
if run is None:
logging.info(f"No run found for {workflow} on {branch}.")
continue
on_tip = bool(status.tip_sha) and run.head_sha == status.tip_sha
status.runs.append(replace(run, on_tip=on_tip))
return status
def _log_slug(name: str) -> str:
"""A filename-safe spelling of a workflow or job name."""
return LOG_NAME_RE.sub("-", name).strip("-") or "job"
[docs]
def write_failed_logs(
status: CIStatus,
directory: Path,
*,
fetch: Callable[[int], str] | None = None,
) -> list[Path]:
"""Write the log of every failed required job of *status* under *directory*.
One file per job, named after its workflow and job, readable the moment the
job settled: a failed cell is diagnosed while its slowest sibling still
builds, instead of after the whole run drains. The control sequences are
stripped by {func}`~repomatic.github.runs.job_log`. A log that cannot be
read is reported and skipped, so it costs its own file alone.
:param status: The runs to read the failed jobs from.
:param directory: Where the files go, created when missing.
:param fetch: Reads one job's log by ID. Defaults to
{func}`~repomatic.github.runs.job_log`, resolved at call time so a
test can stand in for it.
:return: The files written, in report order.
"""
fetch = fetch or job_log
directory.mkdir(parents=True, exist_ok=True)
written = []
for run in status.runs:
for job in run.failed_required:
if not job.job_id:
logging.warning(
f"{run.workflow}: no job ID for {job.name!r}, so its log cannot"
" be fetched."
)
continue
try:
log = fetch(job.job_id)
except RuntimeError as error:
logging.warning(
f"{run.workflow}: cannot fetch the log of {job.name!r}: {error}"
)
continue
path = directory / f"{_log_slug(run.workflow)}--{_log_slug(job.name)}.log"
path.write_text(log, encoding="UTF-8")
written.append(path)
return written
[docs]
@dataclass
class WatchState:
"""The verdicts a watch has reported, so a re-armed one repeats none.
Keyed by workflow and commit: a run moving from running to green is one
change, and the run of a newer commit is another.
"""
path: Path | None = None
"""The JSON file holding the verdicts across invocations.
`None` keeps them in memory, for the span of one watch.
"""
seen: dict[str, str] = field(default_factory=dict)
"""Verdict by `workflow@commit`."""
[docs]
@classmethod
def load(cls, path: Path | None) -> WatchState:
"""Read the verdicts *path* holds, or start empty.
A missing or unreadable file starts empty too: the first poll then
reports every run, which is the baseline a fresh watch wants.
"""
state = cls(path=path)
if path is not None and path.is_file():
try:
loaded = json.loads(path.read_text(encoding="UTF-8"))
except (OSError, ValueError):
loaded = None
if isinstance(loaded, dict):
state.seen = {str(key): str(value) for key, value in loaded.items()}
return state
[docs]
def record(self, status: CIStatus) -> list[RunStatus]:
"""Note the verdict of each run, and write the file when one is new.
:param status: The runs of one poll.
:return: The runs not seen before, or seen with another verdict.
"""
changed = []
for run in status.runs:
key = f"{run.workflow}@{run.head_sha}"
if self.seen.get(key) != run.verdict:
self.seen[key] = run.verdict
changed.append(run)
if changed and self.path is not None:
self.path.write_text(
json.dumps(self.seen, indent=2, sort_keys=True) + "\n",
encoding="UTF-8",
)
return changed
[docs]
def watch_ci_status(
workflows: Iterable[str],
branch: str,
*,
state: WatchState,
interval: float,
until_change: bool = False,
timeout: float = 0,
sleep: Callable[[float], None] | None = None,
clock: Callable[[], float] | None = None,
) -> Iterator[tuple[CIStatus, list[RunStatus]]]:
"""Poll the runs of *branch* until something is worth acting on.
Each poll yields the status read and the runs whose verdict *state* had
not seen. The watch ends on the first poll where a run blocks a merge,
since that is the moment to act and never one to wait through, or where
every run settled and the tip has a run of its own. Until then GitHub may
still be creating the tip's runs, and a settled report would read as
green for a commit nothing has tested yet.
With *until_change*, it ends after the first poll reporting a change, so a
watcher re-armed on the same state file wakes once per event: a run that
blocks a merge ends it when its verdict is new, and a block the state file
already holds keeps it polling. With *timeout*, it ends once that many
seconds elapsed, whatever the state: the last sleep shrinks to the time
left, so a caller with a hard time limit of its own can set *timeout* one
poll under it.
Every poll pays the REST quota of {func}`read_ci_status`, a handful of
calls, and *interval* is what keeps a long watch inside it.
A poll that fails with a transient error, a GitHub server error or a rate
limit (see {attr}`~repomatic.github.gh.GhCommandError.transient`), yields
nothing: the watch logs a warning and polls again after *interval*.
{data}`FAILED_POLL_LIMIT` failed polls in a row end it with the error of the
last one, and any other error ends it at once. A *timeout* that passes after
a failed poll ends the watch on the last status it yielded, or with the
error when it yielded none.
:param workflows: Workflow filenames to read.
:param branch: Branch to read runs from.
:param state: The verdicts already reported, updated on each poll.
:param interval: Seconds between two polls.
:param until_change: Stop after the first poll that reports a change.
:param timeout: Seconds after which the watch stops, `0` for no limit.
:param sleep: Waits between two polls. Defaults to {func}`time.sleep`,
resolved at call time so a test can stand in for it.
:param clock: Reads a monotonic clock, for *timeout*. Defaults to
{func}`time.monotonic`, resolved the same way.
:yields: The status of each poll, and the runs whose verdict changed.
:raises GhCommandError: When a poll fails with an error that is not
transient, when {data}`FAILED_POLL_LIMIT` polls in a row fail, or when
the *timeout* passes before a poll read the runs.
:raises GhOutputError: When a poll reads an answer that is not a run.
"""
sleep = sleep or time.sleep
clock = clock or time.monotonic
names = list(workflows)
deadline = clock() + timeout if timeout else None
status: CIStatus | None = None
failures: list[GhCommandError] = []
while True:
try:
status = read_ci_status(names, branch)
except GhCommandError as error:
failures.append(error)
if not error.transient or len(failures) >= FAILED_POLL_LIMIT:
raise
logging.warning(
f"Cannot read the CI runs of {branch!r}, polling again"
f" ({len(failures)}/{FAILED_POLL_LIMIT} failed polls in a row):"
f" {str(error).strip()}"
)
else:
failures.clear()
changed = state.record(status)
yield status, changed
if until_change and changed:
return
if status.blocking and not until_change:
return
if status.settled and (status.runs_on_tip or not status.tip_sha):
return
if deadline is None:
sleep(interval)
continue
left = deadline - clock()
if left <= 0:
if status is None:
raise failures[-1]
return
sleep(min(interval, left))