# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Measure how long each runner image actually takes, from finished runs.
Runner selection rests on measurement, and the measurement that matters is the
whole job. A benchmark that times only the tool pass misses checkout and
install, which is where most of the difference between two images sits. The
jobs API reports `startedAt` and `completedAt`, so a duration read from it is
*whole-job by construction*, and that mistake cannot be made here.
```{note} Why the job name, and not a second API call
Attributing a duration to an image needs no extra request, because the matrix
job names already carry it (`⁉️ ubuntu-26.04 / py3.15-dev`). Matching them
against {data}`~repomatic.matrix.axes.KNOWN_RUNNERS` is the same technique
{func}`~repomatic.github.workflow_files.literal_runners` uses on `runs-on:`
values, and it degrades honestly: a job whose name carries no known image is
reported under {data}`UNATTRIBUTED` rather than guessed at.
```
```{caution}
These numbers are a sample of a shared, noisy fleet, not a benchmark. A cold
image, a queue stall or a flaky network inflates a single cell, which is why
the report is a median across several runs rather than a mean of one. Read a
gap of a few percent as noise and act only on the systematic ones.
```
"""
from __future__ import annotations
import logging
import statistics
from dataclasses import dataclass
from click_extra import ColumnSpec
from ..humanize import parse_iso_datetime
from ..matrix.axes import KNOWN_RUNNERS
from ..tabular import render_markdown_table
from .runs import read_run, workflow_runs
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
UNATTRIBUTED = "(no image in job name)"
"""Bucket for a job whose name matches no known runner image.
Non-matrix jobs land here by design: they carry no image in their name because
they never had a choice of one. Keeping them visible rather than dropping them
is what stops the report reading as though it covered the whole workflow.
"""
JOB_TIMINGS_HEADER_DEFS: tuple[ColumnSpec, ...] = (
ColumnSpec("runner", "Runner"),
ColumnSpec("jobs", "Jobs"),
ColumnSpec("median", "Median"),
ColumnSpec("slowest-job", "Slowest job"),
ColumnSpec("slowest", "Slowest"),
)
"""Column definitions for the `job-timings` table."""
[docs]
@dataclass(frozen=True)
class JobTiming:
"""One finished job, and how long it occupied a runner."""
name: str
"""The job's name, glyph and matrix cell included."""
runner: str
"""Runner image matched out of {attr}`name`, or {data}`UNATTRIBUTED`."""
seconds: float
"""Whole-job wall-clock: the `completedAt` minus `startedAt` delta.
Deliberately not the compute time. This is what the run costs in billed
minutes and in wall-clock waiting.
"""
_RUNNERS_LONGEST_FIRST = tuple(sorted(KNOWN_RUNNERS, key=len, reverse=True))
"""Runner images by descending label length, sorted once.
Longest first, so `ubuntu-26.04-arm` is never shadowed by `ubuntu-26.04`; a
module constant because {func}`match_runner` runs once per job across every
sampled run.
"""
[docs]
def match_runner(job_name: str) -> str:
"""Attribute a job to the runner image named in it.
:param job_name: Job name as GitHub reports it.
:return: The matched image, or {data}`UNATTRIBUTED`.
"""
for image in _RUNNERS_LONGEST_FIRST:
if image in job_name:
return image
return UNATTRIBUTED
def _duration(job: dict) -> float | None:
"""Whole-job wall-clock in seconds, or `None` when it cannot be read.
A job that never started (cancelled in the queue) carries no usable pair,
and a still-running one has no end: both are skipped rather than counted
as zero, which would drag a median toward a runner's queue behaviour
instead of its speed. Timestamps go through the package-wide
{func}`~repomatic.humanize.parse_iso_datetime`, which reads the `Z`
suffix GitHub always writes.
"""
started, completed = job.get("startedAt"), job.get("completedAt")
if not started or not completed:
return None
started_at = parse_iso_datetime(started)
completed_at = parse_iso_datetime(completed)
if started_at is None or completed_at is None:
logging.warning(f"Unparsable job timestamps: {started!r}, {completed!r}")
return None
seconds = (completed_at - started_at).total_seconds()
return seconds if seconds > 0 else None
[docs]
def fetch_job_timings(
workflow: str, branch: str = "main", limit: int = 5
) -> list[JobTiming]:
"""Read finished job durations from the most recent successful runs.
Only successful runs are sampled. A failed run's jobs stop early, so their
durations measure where the failure landed rather than what the image
costs, and a cancelled matrix reports whatever fraction ran before the
cancellation swept it. The runs come from
{func}`~repomatic.github.runs.workflow_runs`, whose `created` filter keeps
a stale snapshot of old runs out of the sample.
:param workflow: Workflow filename, like `tests.yaml`.
:param branch: Branch whose runs to sample.
:param limit: How many successful runs to sample. A median over several is
what smooths a queue stall into noise.
:return: One {class}`JobTiming` per finished job across the sampled runs.
"""
timings: list[JobTiming] = []
for run in workflow_runs(workflow, branch, status="success", limit=limit):
detail = read_run(run["id"], ("jobs",), soft=True)
if detail is None:
continue
for job in detail.get("jobs") or []:
seconds = _duration(job)
if seconds is None:
continue
name = job.get("name") or ""
timings.append(JobTiming(name, match_runner(name), seconds))
return timings
[docs]
@dataclass(frozen=True)
class RunnerReport:
"""Aggregated timings for one runner image."""
runner: str
"""The runner image, or {data}`UNATTRIBUTED`."""
job_count: int
"""How many finished jobs the image ran across the sample."""
median_seconds: float
"""Median whole-job wall-clock of those jobs."""
slowest_job: str
"""Name of the slowest of those jobs."""
slowest_seconds: float
"""Whole-job wall-clock of the slowest job."""
[docs]
def cells(self) -> tuple[str, str, str, str, str]:
"""The row of the report, in {data}`JOB_TIMINGS_HEADER_DEFS` order."""
return (
self.runner,
str(self.job_count),
format_duration(self.median_seconds),
self.slowest_job,
format_duration(self.slowest_seconds),
)
[docs]
def summarize(timings: Iterable[JobTiming]) -> list[RunnerReport]:
"""Aggregate per-job timings into one row per runner image.
Sorted slowest-median first: the question this answers is which image is
holding the matrix up, and that one belongs at the top rather than
alphabetically buried.
:param timings: Job timings, typically from {func}`fetch_job_timings`.
:return: One {class}`RunnerReport` per image seen.
"""
buckets: dict[str, list[JobTiming]] = {}
for timing in timings:
buckets.setdefault(timing.runner, []).append(timing)
reports = []
for runner, entries in buckets.items():
slowest = max(entries, key=lambda entry: entry.seconds)
reports.append(
RunnerReport(
runner=runner,
job_count=len(entries),
median_seconds=statistics.median(e.seconds for e in entries),
slowest_job=slowest.name,
slowest_seconds=slowest.seconds,
)
)
return sorted(reports, key=lambda report: report.median_seconds, reverse=True)
[docs]
def render_markdown(reports: Sequence[RunnerReport], workflow: str, runs: int) -> str:
"""Render a report as a Markdown table, for pasting into documentation.
Emitted on request rather than written by a sync job. Timings move on every
run, so a job regenerating a checked-in table would open a pull request
forever and never converge: this is a measurement to take when a decision
needs one, not a file to keep in sync.
:param reports: Aggregated rows from {func}`summarize`.
:param workflow: Workflow the sample came from.
:param runs: How many runs were sampled.
:return: A Markdown table, newline-terminated.
"""
table = render_markdown_table(
tuple(col.label for col in JOB_TIMINGS_HEADER_DEFS),
((f"`{runner}`", *rest) for runner, *rest in map(RunnerReport.cells, reports)),
align=("left", "right", "right", "left", "right"),
)
lines = [
f"Median whole-job wall-clock across the {runs} most recent successful",
f"`{workflow}` runs. Re-measure rather than trusting these: they are one",
"project's, on a shared fleet, and they drift.",
"",
table,
]
return "\n".join(lines) + "\n"