Source code for repomatic.github.ci_status

# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

"""Which CI jobs are red, and which of those actually gate a merge.

repomatic names the jobs it generates, prefixing each matrix cell with a
glyph that records whether the cell is allowed to fail: `✅` for a required
one, `⁉️` for a probe running under `continue-on-error`. Reading that back
was left to whoever was watching CI, and it is easy to get wrong in four
specific ways this module exists to settle:

- **The glyph, not the position.** Job names differ in shape across
  workflows: `tests.yaml` emits `✅ ubuntu-26.04 / py3.11` while the release
  engine emits `workflow / ✅ ubuntu-26.04, abc1234 build`. Anything that
  splits on `" / "` and reads a fixed field strips the glyph off one of the
  two and files a required red as a probe, which reads as green.
- **Jobs, not the run.** A run's own `conclusion` is `success` while a
  `continue-on-error` probe inside it crashed, and its `status` still reads
  `queued` while a dozen of its jobs have already finished. Neither answers
  "is anything broken".
- **A run that failed around its jobs.** A `failure` conclusion with no
  failed job is a workflow-level error: an invalid `strategy.matrix`
  expression, malformed YAML, a missing secret. There is no job log to read,
  and treating it as benign is how a persistently red workflow gets written
  off as a known artifact. A required job that GitHub cancelled, like one no
  runner picked up, fails its run the same way: that job is named instead,
  and it still gates the merge.
- **A run that settled without a result.** A newer push cancels the run of
  the commit under it, and a job-level `if:` skips every job of a run: the
  bundled `tests.yaml` does that for a version-bump commit. Neither run has a
  failed job, and neither says anything about its commit. Reading one as
  green hides that the matrix never ran on the newest commits. A cancelled run
  on the tip is not one: no newer push stopped it, so a required job that
  GitHub cancelled there, like one past its timeout, gates the merge.

A job carrying no stability glyph is required. That covers every non-matrix job
(`1️⃣ Run-once tests`, `📦 Package install`, `🛡️ Lint types`), where the absence
of a marker means the job was never optional rather than that its status is
unknown. Only the two stability glyphs count: a job name may carry any other
emoji and still be required, which is why the test is for `⁉️` specifically
rather than for a decorated name. A name that holds both glyphs is the one
exception: see {meth}`JobStatus.required`.
"""

from __future__ import annotations

import json
import logging
import re
import time
from dataclasses import dataclass, field, replace
from urllib.parse import quote

from click_extra import ColumnSpec

from .gh import GhCommandError, GhOutputError, gh_api_pages, gh_json
from .outputs import AnnotationLevel
from .runs import commit_runs, job_log, read_run, workflow_runs
from .workflow_files import load_workflows, workflow_triggers

TYPE_CHECKING = False
if TYPE_CHECKING:
    from collections.abc import Callable, Iterable, Iterator
    from pathlib import Path

TIP_HISTORY_DEPTH = 10
"""Commits {func}`read_ci_status` walks back from the branch tip, tip included.

A push starts most workflows on the tip itself. A workflow whose `paths:` filter
skipped the tip last ran on an earlier commit, which a walk this deep reaches on
any active branch. A workflow with no run in the window falls back to
{func}`latest_run`.
"""

STABLE_GLYPH = "✅"
"""Marks a matrix cell that must pass. See {data}`UNSTABLE_GLYPH`."""

UNSTABLE_GLYPH = "⁉️"
"""Marks a matrix cell running under `continue-on-error`.

A red one never gates a merge, which is exactly why it has to be told apart
from a required cell rather than counted with it. A release still fixes what
it can: see the `repomatic-ship` skill on the genuinely-green goal.
"""

TERMINAL_STATUSES = frozenset({"completed"})
"""Job statuses meaning the job will not change again."""

CI_STATUS_HEADER_DEFS: tuple[ColumnSpec, ...] = (
    ColumnSpec("workflow", "Workflow"),
    ColumnSpec("commit", "Commit"),
    ColumnSpec("run-status", "Run status"),
    ColumnSpec("verdict", "Verdict"),
)
"""Column definitions for the `ci-status` table, filled by {meth}`RunStatus.cells`."""

LOG_NAME_RE = re.compile(r"[^\w.-]+")
"""What a log filename drops from a workflow or job name: spaces, glyphs, slashes."""

RUN_FIELDS = ("conclusion", "headSha", "jobs", "status", "workflowName")
"""The `gh run view --json` fields a {class}`RunStatus` is built from."""

FAILED_POLL_LIMIT = 3
"""Failed polls in a row that end a watch.

{func}`watch_ci_status` skips a poll that meets a transient error, like the
`HTTP 504` GitHub answers when it cannot respond in time: a fault that clears
within a few intervals costs a few polls, not the watch. A fault that lasts this
many polls in a row ends the watch with its error, as a persistent one must.
"""


[docs] @dataclass(frozen=True) class JobStatus: """One job of one workflow run.""" name: str """The job's name, glyph included.""" status: str """`queued`, `in_progress` or `completed`.""" conclusion: str """`success`, `failure`, `cancelled`, `skipped`, or empty while running.""" job_id: int = 0 """The job's numeric ID, for its log endpoint. `0` when the listing had none.""" @property def unexpanded(self) -> bool: """Whether the job is a matrix GitHub never evaluated. An expanded cell carries its values in its name. One that still holds its `name:` expression, `${{` and all, is a stage the run never reached: no cell of it ran, whatever the run's own conclusion says. """ return "${{" in self.name @property def required(self) -> bool: """Whether a failure here gates a merge. Looks for the glyph anywhere in the raw name rather than at its start. The two shapes disagree on where it sits: `tests.yaml` leads with it (`⁉️ ubuntu-26.04 / py3.15-dev`) while the release engine prefixes the workflow first (`release / ⁉️ windows-11-arm, abc1234 build`). A leading-position test passes the first and silently files the second as required; splitting on `" / "` gets it wrong the other way round. Containment is the one form both satisfy, and the templates emit exactly one glyph per expanded name. A name that holds both glyphs is a matrix GitHub never expanded: the job keeps its `name:` expression, which spells the two of them. A run stopped before the matrix got its cells leaves one. Some of those cells are required, so the job is. """ return STABLE_GLYPH in self.name or UNSTABLE_GLYPH not in self.name @property def failed(self) -> bool: """Whether this job reached a failing conclusion.""" return self.conclusion == "failure" @property def running(self) -> bool: """Whether this job has yet to reach a terminal state.""" return self.status not in TERMINAL_STATUSES @property def skipped(self) -> bool: """Whether this job never ran: its `if:`, or a job it needs, ruled it out.""" return self.conclusion == "skipped" @property def cancelled(self) -> bool: """Whether this job was stopped before it finished.""" return self.conclusion == "cancelled"
[docs] @dataclass(frozen=True) class RunStatus: """The latest run of one workflow on one branch.""" workflow: str """Workflow name, as GitHub reports it.""" run_id: int """Numeric run ID, for `gh run view`.""" head_sha: str """Commit the run was created for.""" status: str """The run's own status. Lags its jobs, so it gates nothing once one exists.""" conclusion: str """The run's own conclusion, empty while it is still going.""" jobs: tuple[JobStatus, ...] = () """Every job of the run.""" on_tip: bool = False """Whether the run was created for the tip of its branch. {func}`read_ci_status` sets it. Only a newer push moves the tip, so no newer push stopped a run on it: {attr}`stopped_required` relies on that. """ @property def failed_required(self) -> tuple[JobStatus, ...]: """Failing jobs that gate a merge.""" return tuple(job for job in self.jobs if job.failed and job.required) @property def failed_probes(self) -> tuple[JobStatus, ...]: """Failing jobs allowed to fail.""" return tuple(job for job in self.jobs if job.failed and not job.required) @property def running_jobs(self) -> tuple[JobStatus, ...]: """Jobs that have not settled yet.""" return tuple(job for job in self.jobs if job.running) @property def unexpanded_jobs(self) -> tuple[JobStatus, ...]: """Matrix jobs the run never expanded into cells.""" return tuple(job for job in self.jobs if job.unexpanded) @property def workflow_level_failure(self) -> bool: """Whether the run failed around its jobs rather than inside one. No job log explains this one: read the run's error annotations and fix the workflow itself. A run that failed on a required job GitHub cancelled is not one: see {attr}`stopped_required`. """ return ( self.conclusion == "failure" and not any(job.failed for job in self.jobs) and not self.cancelled_required ) @property def waiting(self) -> bool: """Whether GitHub has yet to create a job of the run. A run waits with no job for a runner, or behind the run ahead of it in its concurrency group. Its own status is then the only signal. """ return not self.jobs and self.status not in TERMINAL_STATUSES @property def blocking(self) -> bool: """Whether this run holds up a merge.""" return ( bool(self.failed_required) or bool(self.stopped_required) or self.workflow_level_failure ) @property def cancelled_required(self) -> tuple[JobStatus, ...]: """Required jobs stopped before they finished.""" return tuple(job for job in self.jobs if job.cancelled and job.required) @property def stopped_required(self) -> tuple[JobStatus, ...]: """Required jobs cancelled in a run that no newer run replaces. Such a job gives its commit no result, and no newer run brings one. GitHub cancels a job that no runner picked up, and concludes its run `failure`, with no failed job and no job log. It also cancels a job past its `timeout-minutes`, but concludes that run `cancelled`, like a run that a newer push stopped. That one is told apart by {attr}`on_tip`: a newer push leaves the run it stopped under the tip. """ if self.conclusion == "failure" or self.on_tip: return self.cancelled_required return () @property def cancelled(self) -> bool: """Whether the run was stopped before its required jobs all finished. A newer push to the same concurrency group does that to the run of the commit under it. A run stopped before GitHub created any job counts too. """ if self.jobs: return bool(self.cancelled_required) return self.conclusion == "cancelled" @property def skipped(self) -> bool: """Whether the run ran no job at all. A job-level `if:` ruled every one of them out, which GitHub concludes as a `skipped` run. """ if self.jobs: return all(job.skipped for job in self.jobs) return self.conclusion == "skipped" @property def inconclusive(self) -> bool: """Whether the run settled without a result for its commit. Neither red nor green: this workflow says nothing about the tree the run was created for, so nothing here holds up a merge and nothing clears it. """ return ( not self.blocking and not self.running_jobs and (self.cancelled or self.skipped) ) @property def verdict(self) -> str: """One-phrase outcome, for the table's last column.""" if self.workflow_level_failure: return "workflow-level failure" if self.failed_required: return f"{len(self.failed_required)} required job(s) failed" if self.stopped_required: return f"{len(self.stopped_required)} required job(s) cancelled" if self.running_jobs: return f"{len(self.running_jobs)} job(s) still running" if self.waiting: return "not started" if self.cancelled_required: return f"cancelled ({len(self.cancelled_required)} required job(s))" if self.cancelled: return "cancelled (no job ran)" if self.skipped: return "skipped (no job ran)" if self.failed_probes: return f"green ({len(self.failed_probes)} probe(s) failed)" return "green"
[docs] def cells(self) -> tuple[str, str, str, str]: """The row of the `ci-status` table, in {data}`CI_STATUS_HEADER_DEFS` order.""" return (self.workflow, self.head_sha[:8], self.status, self.verdict)
[docs] def findings(self) -> Iterator[tuple[AnnotationLevel, str]]: """What the reader must know about this run, one line each. An error gates a merge: a workflow-level failure, a failed required job, or a required job that GitHub cancelled in a run no newer run replaces. A warning does not: a failed probe, or a run with no result for its commit. A notice asks for nothing: a matrix whose `if:` skipped it before it evaluated, like a release-only stage on an ordinary push or the binary stages of a project that compiles none. :yields: `(level, message)` pairs, errors first. """ if self.workflow_level_failure: message = ( f"{self.workflow}: run {self.run_id} failed with no failed job. " "This is a workflow-level error (an invalid matrix expression, " "malformed YAML, a missing secret): read the run's error " "annotations, there is no job log." ) yield AnnotationLevel.ERROR, message for job in self.failed_required: yield ( AnnotationLevel.ERROR, f"{self.workflow}: required job failed: {job.name}", ) for job in self.stopped_required: yield ( AnnotationLevel.ERROR, ( f"{self.workflow}: required job cancelled: {job.name}. Its" " annotations give the reason, like a runner that never" " picked it up or a timeout. Re-run it with" f" `gh run rerun {self.run_id} --failed`." ), ) for job in self.failed_probes: yield ( AnnotationLevel.WARNING, f"{self.workflow}: allowed-failure probe failed: {job.name}", ) for job in self.unexpanded_jobs: if job.skipped: yield ( AnnotationLevel.NOTICE, ( f"{self.workflow}: the matrix behind {job.name!r} never" " evaluated: its `if:` skipped it, so no cell of it ran." ), ) continue yield ( AnnotationLevel.WARNING, ( f"{self.workflow}: the matrix behind {job.name!r} never" " evaluated, so no cell of it ran and nothing it builds or" " tests has a result." ), ) if self.inconclusive: reason = ( "was cancelled before its required jobs finished" if self.cancelled else "ran no job" ) message = ( f"{self.workflow}: run {self.run_id} {reason}, so" f" {self.head_sha[:8]} has no result from it." ) yield AnnotationLevel.WARNING, message
[docs] @dataclass class CIStatus: """Every monitored workflow's latest run on a branch.""" branch: str """Branch the runs were read from.""" tip_sha: str = "" """Commit at the tip of the branch when the runs were read. Empty when the branch history lists no commit. """ runs: list[RunStatus] = field(default_factory=list) """One entry per workflow that has a run, newest first.""" skipped_on_tip: list[str] = field(default_factory=list) """Workflow filenames that ran no job on the tip. Only the ones read on an earlier commit: the row of each says nothing about the tip. A workflow with no other run in the window keeps its tip run, whose verdict says the same. """ @property def blocking(self) -> list[RunStatus]: """Runs holding up a merge.""" return [run for run in self.runs if run.blocking] @property def runs_on_tip(self) -> list[RunStatus]: """Runs created for the tip commit itself. Empty right after a push, while GitHub has yet to create the tip's runs: every run listed then belongs to an earlier commit. """ return [run for run in self.runs if run.head_sha == self.tip_sha] @property def settled(self) -> bool: """Whether every run reached a terminal state.""" return all(not run.running_jobs and not run.waiting for run in self.runs)
def _workflow_triggers(workflow_dir: Path) -> list[tuple[str, set[str]]]: """Pair each workflow file in the directory with the triggers it declares. Both public readers below select on those triggers, and neither should parse the tree twice or disagree on which files count as workflows. :param workflow_dir: Directory holding the workflow files. :return: `(filename, triggers)` pairs, sorted by path. Empty when the directory is missing. A file that does not parse is skipped. """ return [ (path.name, set(workflow_triggers(data))) for path, data in load_workflows(workflow_dir).items() ]
[docs] def workflow_files(workflow_dir: Path) -> tuple[str, ...]: """Every workflow in the directory that has runs of its own. {func}`monitored_workflows` narrows this to the ones a push starts, which is the right default for a status report and the wrong set to *accept*: a schedule-only or dispatch-only workflow has runs worth reading too. A reusable workflow is excluded for the reason it is there, since its jobs only ever appear under a caller's run. :param workflow_dir: Directory holding the workflow files. :return: Workflow filenames, sorted. """ return tuple( name for name, triggers in _workflow_triggers(workflow_dir) if triggers - {"workflow_call"} )
[docs] def monitored_workflows(workflow_dir: Path) -> list[str]: """Every workflow a push to the default branch can start. Derived from the tree rather than listed by hand, so a workflow added later is watched without anyone remembering to add it here. A reusable workflow is excluded: it has no runs of its own, only the ones its callers create. :param workflow_dir: Directory holding the workflow files. :return: Workflow filenames, sorted. """ return [ name for name, triggers in _workflow_triggers(workflow_dir) if "push" in triggers ]
def _run_status(run_id: int, workflow: str) -> RunStatus: """Read one run and its jobs by ID. Every field comes from the run itself rather than from the listing that found it, so the report never mixes a listing's view of a run with a fresher view of its jobs. :param run_id: The run's numeric ID. :param workflow: Workflow filename, the fallback display name. :return: The run with its jobs attached. :raises GhCommandError: When `gh` cannot read the run. :raises GhOutputError: When `gh` answers with something other than a run. A run with no status and no jobs would otherwise read as green. """ detail = read_run(run_id, RUN_FIELDS) if detail is None: msg = f"Could not read run {run_id} of {workflow}." raise GhOutputError(msg) return RunStatus( workflow=detail.get("workflowName") or workflow, run_id=run_id, head_sha=detail.get("headSha") or "", status=detail.get("status") or "", conclusion=detail.get("conclusion") or "", jobs=tuple( JobStatus( name=job.get("name") or "", status=job.get("status") or "", conclusion=job.get("conclusion") or "", job_id=int(job.get("databaseId") or 0), ) for job in detail.get("jobs") or [] ), )
[docs] def latest_run(workflow: str, branch: str) -> RunStatus | None: """Read a workflow's most recent run on *branch*, jobs included. Lists the workflow's runs through {func}`~repomatic.github.runs.workflow_runs`, which sends a `created` filter to keep a stale snapshot out. {func}`read_ci_status` asks it only about a workflow with no run on the newest commits. :param workflow: Workflow filename, like `tests.yaml`. :param branch: Branch to read runs from. :return: The run, or `None` when the workflow has none in the listing window. An empty listing is not proof the workflow was filtered out: GitHub can sit on a push event for hours before materializing a run. """ # `strict`: an unreachable `gh` must fail the read rather than report the # workflow as run-less, which `/babysit-ci` would read as a green hole. runs = workflow_runs(workflow, branch, strict=True) if not runs: return None return _run_status(int(runs[0]["id"]), workflow)
def _commit_history(ref: str) -> list[str]: """SHAs of the newest commits reachable from *ref*, tip first. :param ref: Branch, or any other ref the commits endpoint resolves. :return: At most {data}`TIP_HISTORY_DEPTH` SHAs. Empty when the answer lists no commit. :raises GhCommandError: When the read fails. :raises GhOutputError: When the answer is not JSON. """ endpoint = ( f"repos/{{owner}}/{{repo}}/commits?sha={quote(ref, safe='')}" f"&per_page={TIP_HISTORY_DEPTH}" ) commits = gh_json(["api", endpoint]) if not isinstance(commits, list): return [] return [commit["sha"] for commit in commits if commit.get("sha")] def _fork_pull_ref(branch: str) -> str | None: """The ref of the open pull request sent from the *branch* of a fork. GitHub keeps the head of every pull request under `refs/pull/<number>/head`, in the repository the pull request targets. :param branch: Head branch of the pull request, as its runs name it. :return: The ref, or `None` when no open pull request comes from *branch*. :raises RuntimeError: When several open pull requests come from a branch of that name, sent from different forks. """ pulls = gh_api_pages("repos/{owner}/{repo}/pulls?state=open&per_page=100") numbers = sorted( int(pull["number"]) for pull in pulls if isinstance(pull, dict) and (pull.get("head") or {}).get("ref") == branch ) if len(numbers) > 1: listed = ", ".join(f"#{number}" for number in numbers) msg = f"open pull requests {listed} all come from a branch named {branch!r}" raise RuntimeError(msg) return f"refs/pull/{numbers[0]}/head" if numbers else None def _branch_history(branch: str) -> list[str]: """SHAs of the newest commits on *branch*, tip first. The repository answers `404` for the branch of a pull request sent from a fork, as the branch lives in the fork. Its commits are then read from the ref of that pull request, while its runs still carry the name of the branch. :param branch: Branch to read. :return: At most {data}`TIP_HISTORY_DEPTH` SHAs. Empty when the answer lists no commit. :raises GhCommandError: When the read fails. :raises GhOutputError: When the answer is not JSON. :raises RuntimeError: When pull requests from several forks share the branch name. """ try: return _commit_history(branch) except GhCommandError as error: if error.status != 404: raise pull_ref = _fork_pull_ref(branch) if pull_ref is None: raise return _commit_history(pull_ref) def _runs_on_commit(sha: str, branch: str) -> tuple[dict[str, int], dict[str, int]]: """The newest run of each workflow GitHub created for commit *sha*. A run GitHub concluded `skipped` ran no job, so it is kept apart from the runs that did: {func}`read_ci_status` looks for a run with a result on an earlier commit before it settles for that one. :param sha: Full commit SHA. :param branch: Branch the runs must belong to. The same commit also carries runs for a tag pushed on it, or for a pull request branch. :return: Two maps of run ID by workflow filename: the runs that ran a job, then the skipped ones. """ ran: dict[str, int] = {} skipped: dict[str, int] = {} # Newest first, per the API's ordering: the first entry seen for a # workflow is its latest run of that kind on this commit. for entry in commit_runs(sha): if entry.get("head_branch") != branch: continue filename = str(entry.get("path") or "").rpartition("/")[2] runs = skipped if entry.get("conclusion") == "skipped" else ran if filename and filename not in runs: runs[filename] = int(entry["id"]) return ran, skipped
[docs] def read_ci_status(workflows: Iterable[str], branch: str) -> CIStatus: """Read the latest run of each workflow on *branch*. Anchored on the branch tip: reads the runs of each of the newest commits, tip first, and stops as soon as every workflow has one. A workflow whose `paths:` filter skipped the tip is found on the commit that last started it. Only a workflow with no run in the whole window falls back to {func}`latest_run`. A run that ran no job counts as no run at all for that walk, so a workflow that skipped every job on the tip is also read on an earlier commit. A cancelled run stops the walk instead: the runs under it are older than the commits it left without a result, and a green one among them would hide that. A workflow with nothing but skipped runs in the window is reported on the newest of them. Each run records whether it sits on the tip, which decides whether a cancelled job in it blocks. ```{warning} The branch-wide run listing can answer with a stale snapshot: runs from commits months old, presented as the latest ones. Reading runs commit by commit, from a commit list that comes from git data rather than from the run index, keeps that snapshot out. The fallback listing sends a `created` filter for the same reason. ``` :param workflows: Workflow filenames to read. :param branch: Branch to read runs from. :return: The collected status. """ status = CIStatus(branch=branch) names = list(workflows) if not names: return status history = _branch_history(branch) if history: status.tip_sha = history[0] wanted = set(names) found: dict[str, int] = {} skipped: dict[str, int] = {} idle_on_tip: set[str] = set() for sha in history: ran, idle = _runs_on_commit(sha, branch) for filename, run_id in ran.items(): if filename in wanted and filename not in found: found[filename] = run_id for filename, run_id in idle.items(): if filename in wanted and filename not in found: skipped.setdefault(filename, run_id) if sha == status.tip_sha: idle_on_tip = set(skipped) if len(found) == len(wanted): break status.skipped_on_tip = sorted(idle_on_tip & found.keys()) for workflow in names: chosen = found.get(workflow, skipped.get(workflow)) run = ( _run_status(chosen, workflow) if chosen is not None else latest_run(workflow, branch) ) if run is None: logging.info(f"No run found for {workflow} on {branch}.") continue on_tip = bool(status.tip_sha) and run.head_sha == status.tip_sha status.runs.append(replace(run, on_tip=on_tip)) return status
def _log_slug(name: str) -> str: """A filename-safe spelling of a workflow or job name.""" return LOG_NAME_RE.sub("-", name).strip("-") or "job"
[docs] def write_failed_logs( status: CIStatus, directory: Path, *, fetch: Callable[[int], str] | None = None, ) -> list[Path]: """Write the log of every failed required job of *status* under *directory*. One file per job, named after its workflow and job, readable the moment the job settled: a failed cell is diagnosed while its slowest sibling still builds, instead of after the whole run drains. The control sequences are stripped by {func}`~repomatic.github.runs.job_log`. A log that cannot be read is reported and skipped, so it costs its own file alone. :param status: The runs to read the failed jobs from. :param directory: Where the files go, created when missing. :param fetch: Reads one job's log by ID. Defaults to {func}`~repomatic.github.runs.job_log`, resolved at call time so a test can stand in for it. :return: The files written, in report order. """ fetch = fetch or job_log directory.mkdir(parents=True, exist_ok=True) written = [] for run in status.runs: for job in run.failed_required: if not job.job_id: logging.warning( f"{run.workflow}: no job ID for {job.name!r}, so its log cannot" " be fetched." ) continue try: log = fetch(job.job_id) except RuntimeError as error: logging.warning( f"{run.workflow}: cannot fetch the log of {job.name!r}: {error}" ) continue path = directory / f"{_log_slug(run.workflow)}--{_log_slug(job.name)}.log" path.write_text(log, encoding="UTF-8") written.append(path) return written
[docs] @dataclass class WatchState: """The verdicts a watch has reported, so a re-armed one repeats none. Keyed by workflow and commit: a run moving from running to green is one change, and the run of a newer commit is another. """ path: Path | None = None """The JSON file holding the verdicts across invocations. `None` keeps them in memory, for the span of one watch. """ seen: dict[str, str] = field(default_factory=dict) """Verdict by `workflow@commit`."""
[docs] @classmethod def load(cls, path: Path | None) -> WatchState: """Read the verdicts *path* holds, or start empty. A missing or unreadable file starts empty too: the first poll then reports every run, which is the baseline a fresh watch wants. """ state = cls(path=path) if path is not None and path.is_file(): try: loaded = json.loads(path.read_text(encoding="UTF-8")) except (OSError, ValueError): loaded = None if isinstance(loaded, dict): state.seen = {str(key): str(value) for key, value in loaded.items()} return state
[docs] def record(self, status: CIStatus) -> list[RunStatus]: """Note the verdict of each run, and write the file when one is new. :param status: The runs of one poll. :return: The runs not seen before, or seen with another verdict. """ changed = [] for run in status.runs: key = f"{run.workflow}@{run.head_sha}" if self.seen.get(key) != run.verdict: self.seen[key] = run.verdict changed.append(run) if changed and self.path is not None: self.path.write_text( json.dumps(self.seen, indent=2, sort_keys=True) + "\n", encoding="UTF-8", ) return changed
[docs] def watch_ci_status( workflows: Iterable[str], branch: str, *, state: WatchState, interval: float, until_change: bool = False, timeout: float = 0, sleep: Callable[[float], None] | None = None, clock: Callable[[], float] | None = None, ) -> Iterator[tuple[CIStatus, list[RunStatus]]]: """Poll the runs of *branch* until something is worth acting on. Each poll yields the status read and the runs whose verdict *state* had not seen. The watch ends on the first poll where a run blocks a merge, since that is the moment to act and never one to wait through, or where every run settled and the tip has a run of its own. Until then GitHub may still be creating the tip's runs, and a settled report would read as green for a commit nothing has tested yet. With *until_change*, it ends after the first poll reporting a change, so a watcher re-armed on the same state file wakes once per event: a run that blocks a merge ends it when its verdict is new, and a block the state file already holds keeps it polling. With *timeout*, it ends once that many seconds elapsed, whatever the state: the last sleep shrinks to the time left, so a caller with a hard time limit of its own can set *timeout* one poll under it. Every poll pays the REST quota of {func}`read_ci_status`, a handful of calls, and *interval* is what keeps a long watch inside it. A poll that fails with a transient error, a GitHub server error or a rate limit (see {attr}`~repomatic.github.gh.GhCommandError.transient`), yields nothing: the watch logs a warning and polls again after *interval*. {data}`FAILED_POLL_LIMIT` failed polls in a row end it with the error of the last one, and any other error ends it at once. A *timeout* that passes after a failed poll ends the watch on the last status it yielded, or with the error when it yielded none. :param workflows: Workflow filenames to read. :param branch: Branch to read runs from. :param state: The verdicts already reported, updated on each poll. :param interval: Seconds between two polls. :param until_change: Stop after the first poll that reports a change. :param timeout: Seconds after which the watch stops, `0` for no limit. :param sleep: Waits between two polls. Defaults to {func}`time.sleep`, resolved at call time so a test can stand in for it. :param clock: Reads a monotonic clock, for *timeout*. Defaults to {func}`time.monotonic`, resolved the same way. :yields: The status of each poll, and the runs whose verdict changed. :raises GhCommandError: When a poll fails with an error that is not transient, when {data}`FAILED_POLL_LIMIT` polls in a row fail, or when the *timeout* passes before a poll read the runs. :raises GhOutputError: When a poll reads an answer that is not a run. """ sleep = sleep or time.sleep clock = clock or time.monotonic names = list(workflows) deadline = clock() + timeout if timeout else None status: CIStatus | None = None failures: list[GhCommandError] = [] while True: try: status = read_ci_status(names, branch) except GhCommandError as error: failures.append(error) if not error.transient or len(failures) >= FAILED_POLL_LIMIT: raise logging.warning( f"Cannot read the CI runs of {branch!r}, polling again" f" ({len(failures)}/{FAILED_POLL_LIMIT} failed polls in a row):" f" {str(error).strip()}" ) else: failures.clear() changed = state.record(status) yield status, changed if until_change and changed: return if status.blocking and not until_change: return if status.settled and (status.runs_on_tip or not status.tip_sha): return if deadline is None: sleep(interval) continue left = deadline - clock() if left <= 0: if status is None: raise failures[-1] return sleep(min(interval, left))