# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Global cache for tool executables, API responses and generated tool configs.
Four subtrees under the user-level cache directory:
**Binary cache** (`bin/`): platform-specific tool executables, keyed by
``{tool}/{version}/{platform}/{executable}``. Each cached binary has a
`.sha256` sidecar written after a verified archive download. Cache hits
verify the binary against this sidecar to detect local tampering.
**HTTP response cache** (`http/`): JSON API responses from PyPI and GitHub,
keyed by ``{namespace}/{key}.json``. Freshness is controlled by a per-caller
TTL (seconds); stale entries remain on disk until auto-purge removes them.
**Config cache** (`config/`): generated tool configuration files, keyed by
``{tool}/{digest}/{filename}``, where the digest names the content. Two runs
that generate different configs for one tool (two checkouts, two branches)
write two files, so neither run reads the config of the other. Passed to tools
via explicit `--config` flags so repomatic never writes to the user's
repository.
**Locks** (`locks/`): the lock files {func}`lock_path` names, which the tool
runner holds while a config file sits in the working directory. Neither listed
nor cleaned: a lock file deleted while a run holds it lets the next run start
without waiting.
```{note}
The cache module is intentionally a pure storage layer. It does not know
about checksums, registries, API semantics, or tool specifications. All
trust and freshness decisions belong to the caller.
```
"""
from __future__ import annotations
import hashlib
import logging
import os
import shutil
import time
from dataclasses import dataclass
from pathlib import Path
from typing import ClassVar
from click_extra import ColumnSpec, get_cache_dir
from .config import load_repomatic_config
from .file_ops import atomic_write
from .humanize import SECONDS_PER_DAY, format_age, format_file_size
TYPE_CHECKING = False
if TYPE_CHECKING:
from typing import Self
CACHE_LIST_HEADER_DEFS: tuple[ColumnSpec, ...] = (
ColumnSpec("type", "Type"),
ColumnSpec("name", "Name"),
ColumnSpec("detail", "Detail"),
ColumnSpec("size", "Size"),
ColumnSpec("age", "Age"),
)
"""Column definitions for the `repomatic cache show` table.
Lives beside the entry dataclasses it renders; the CLI derives its
`--sort-by` choices from it.
"""
CONFIG_DIGEST_LENGTH = 16
"""Hex digits of the content digest that names a cached config directory.
Sixty-four bits: enough to keep apart every config one cache ever holds.
"""
SIDECAR_SUFFIX = ".sha256"
"""Suffix of the digest sidecar stored beside each cached binary.
Part of the binary cache's on-disk layout: the listing skips sidecars and the
clean removes them along with their entry. Computing, writing, and verifying
the digest itself stays with the caller (`tooling/binaries.py`), per the module
note above.
"""
[docs]
def cache_dir() -> Path:
"""Resolve the cache root directory.
Precedence (highest to lowest):
1. `REPOMATIC_CACHE_DIR` environment variable.
2. `cache.dir` in `[tool.repomatic]`.
3. Platform-specific default, from {func}`click_extra.get_cache_dir`.
:return: Absolute path to the cache root (may not exist yet).
"""
# 1. Environment variable (highest priority).
env_override = os.environ.get("REPOMATIC_CACHE_DIR")
if env_override:
return Path(env_override).expanduser().resolve()
# 2. Config from [tool.repomatic].
config = load_repomatic_config()
if config.cache.dir:
return Path(config.cache.dir).expanduser().resolve()
# 3. Platform default.
return get_cache_dir("repomatic")
[docs]
def lock_path(name: str) -> Path:
"""Return the path of the lock file called *name*.
The file lives in the `locks/` subtree, which no listing or clean touches.
:param name: Lock name, unique to what the lock guards.
"""
return cache_dir() / "locks" / f"{name}.lock"
[docs]
@dataclass(frozen=True, kw_only=True)
class CachedFile:
"""One file of a cache subtree, with the facts every entry carries.
The three caches (binaries, HTTP responses, tool configs) differ only in
how they *name* an entry. Each subclass reads its identity fields off the
path of the entry, in {meth}`from_path`. The listing, the age filter and
the clean are the same for all three, so they live here.
"""
path: Path
"""Absolute path to the cached file."""
size: int
"""File size in bytes."""
mtime: float
"""File modification time (seconds since epoch)."""
kind: ClassVar[str]
"""The cache this entry belongs to, as the `repomatic cache show` table
spells it."""
subdir: ClassVar[str]
"""Name of the subtree under the cache root."""
scope_option: ClassVar[str]
"""The `repomatic cache clean` option that narrows a clean to one scope of
this cache."""
[docs]
@classmethod
def root(cls) -> Path:
"""Return the subtree of this cache under the cache root."""
return cache_dir() / cls.subdir
[docs]
@classmethod
def from_path(cls, path: Path, root: Path) -> Self | None:
"""Read an entry off the path of a file below *root*.
:return: The entry, or `None` when the file is not an entry of this
cache, like a digest sidecar.
"""
raise NotImplementedError
[docs]
@classmethod
def entries(cls) -> list[Self]:
"""List the entries of this cache, sorted by path."""
root = cls.root()
if not root.is_dir():
return []
found: list[Self] = []
for path in sorted(root.rglob("*")):
if path.is_file() and (entry := cls.from_path(path, root)) is not None:
found.append(entry)
return found
[docs]
@classmethod
def clear(
cls, scope: str | None = None, max_age_days: int | None = None
) -> tuple[int, int]:
"""Remove the entries of this cache that both filters select.
Empty directories left behind are pruned afterwards.
:param scope: Keep entries whose {attr}`scope` differs. `None` selects
every entry.
:param max_age_days: Keep entries younger than this. `None` selects
every entry.
:return: Tuple of (files_deleted, bytes_freed).
"""
files_deleted = 0
bytes_freed = 0
for entry in cls.entries():
if (scope is not None and entry.scope != scope) or entry.is_fresh(
max_age_days
):
continue
logging.debug(f"Purging cache entry: {entry.path}")
try:
entry.remove()
except OSError:
logging.debug(f"Failed to remove {entry.path}.")
continue
files_deleted += 1
bytes_freed += entry.size
_prune_empty_dirs(cls.root())
return files_deleted, bytes_freed
@property
def scope(self) -> str:
"""The name a `cache clean` filter matches this entry on.
Doubles as the table's subject column: the thing a reader identifies
the entry by (`--tool ruff`, `--namespace pypi`) is the same thing the
listing shows them, so one property serves both.
"""
raise NotImplementedError
@property
def detail(self) -> str:
"""What distinguishes this entry from its siblings in the same scope."""
raise NotImplementedError
[docs]
def remove(self) -> None:
"""Delete the files of this entry."""
self.path.unlink()
[docs]
def is_fresh(self, max_age_days: int | None) -> bool:
"""Whether this entry is younger than the age cutoff.
A `None` cutoff keeps nothing: age-unfiltered clears delete every
entry the caller's other filters matched.
"""
if max_age_days is None:
return False
return self.mtime >= time.time() - max_age_days * SECONDS_PER_DAY
[docs]
def as_row(self) -> tuple[str, str, str, str, str]:
"""Render this entry as one `repomatic cache show` table row."""
return (
self.kind,
self.scope,
self.detail,
format_file_size(self.size),
format_age(self.mtime),
)
[docs]
@dataclass(frozen=True, kw_only=True)
class BinaryCacheEntry(CachedFile):
"""A single cached binary with its metadata."""
tool: str
"""Tool name (registry key)."""
version: str
"""Pinned version string."""
platform: str
"""Platform key, like `linux-x64` or `macos-arm64`."""
executable: str
"""Executable filename."""
kind: ClassVar[str] = "binary"
subdir: ClassVar[str] = "bin"
scope_option: ClassVar[str] = "tool"
[docs]
@classmethod
def from_path(cls, path: Path, root: Path) -> Self | None:
"""Read the four levels of ``{tool}/{version}/{platform}/{executable}``.
A digest sidecar is part of its binary's entry, not an entry itself.
"""
parts = path.relative_to(root).parts
if len(parts) != 4 or path.name.endswith(SIDECAR_SUFFIX):
return None
tool, version, platform, executable = parts
stat = path.stat()
return cls(
path=path,
size=stat.st_size,
mtime=stat.st_mtime,
tool=tool,
version=version,
platform=platform,
executable=executable,
)
@property
def scope(self) -> str:
return self.tool
@property
def detail(self) -> str:
return f"{self.version} ({self.platform})"
[docs]
def remove(self) -> None:
"""Delete the binary and its digest sidecar."""
self.path.unlink()
binary_sidecar_path(self.path).unlink(missing_ok=True)
[docs]
@dataclass(frozen=True, kw_only=True)
class HttpCacheEntry(CachedFile):
"""A single cached HTTP response with its metadata."""
namespace: str
"""Cache namespace, like `pypi` or `github-release-tags`."""
key: str
"""Cache key within the namespace, like `requests` or `astral-sh/ruff`."""
kind: ClassVar[str] = "http"
subdir: ClassVar[str] = "http"
scope_option: ClassVar[str] = "namespace"
[docs]
@classmethod
def from_path(cls, path: Path, root: Path) -> Self | None:
"""Read ``{namespace}/{key}.json``, where the key can hold slashes."""
parts = path.relative_to(root).parts
if len(parts) < 2 or path.suffix != ".json":
return None
stat = path.stat()
return cls(
path=path,
size=stat.st_size,
mtime=stat.st_mtime,
namespace=parts[0],
key="/".join(parts[1:]).removesuffix(".json"),
)
@property
def scope(self) -> str:
return self.namespace
@property
def detail(self) -> str:
return self.key
[docs]
@dataclass(frozen=True, kw_only=True)
class ConfigCacheEntry(CachedFile):
"""A single cached tool configuration file with its metadata."""
tool: str
"""Tool name (registry key)."""
digest: str
"""Start of the SHA-256 of the content, which names the entry."""
filename: str
"""Config filename, like `yamllint.yaml` or `biome.json`."""
kind: ClassVar[str] = "config"
subdir: ClassVar[str] = "config"
scope_option: ClassVar[str] = "tool"
[docs]
@classmethod
def from_path(cls, path: Path, root: Path) -> Self | None:
"""Read ``{tool}/{digest}/{filename}``.
A file directly in the tool directory, with no digest level, is still
an entry: the clean has to reach it.
"""
parts = path.relative_to(root).parts
if len(parts) not in (2, 3):
return None
stat = path.stat()
return cls(
path=path,
size=stat.st_size,
mtime=stat.st_mtime,
tool=parts[0],
digest=parts[1] if len(parts) == 3 else "",
filename=parts[-1],
)
@property
def scope(self) -> str:
return self.tool
@property
def detail(self) -> str:
return f"{self.filename} ({self.digest})" if self.digest else self.filename
CACHE_KINDS: tuple[type[CachedFile], ...] = (
BinaryCacheEntry,
HttpCacheEntry,
ConfigCacheEntry,
)
"""Every cache, in the order `repomatic cache show` lists them."""
[docs]
def cached_binary_path(
name: str,
version: str,
platform_key: str,
executable: str,
) -> Path:
"""Construct the cache path for a binary (does not check existence).
:param name: Tool name.
:param version: Pinned version.
:param platform_key: Platform key, like `linux-x64`.
:param executable: Executable filename.
:return: Absolute path where the binary would be cached.
"""
return BinaryCacheEntry.root() / name / version / platform_key / executable
[docs]
def binary_sidecar_path(binary_path: Path) -> Path:
"""Return the digest sidecar path for a cached binary.
:param binary_path: Path to the cached binary.
:return: Path of the sidecar file next to it.
"""
return binary_path.with_suffix(binary_path.suffix + SIDECAR_SUFFIX)
[docs]
def get_cached_binary(
name: str,
version: str,
platform_key: str,
executable: str,
) -> Path | None:
"""Return the cached binary path if it exists and is executable.
Does **not** verify the checksum. The caller is responsible for integrity
checks since it owns the checksum value and the `skip_checksum` flag.
:param name: Tool name.
:param version: Pinned version.
:param platform_key: Platform key.
:param executable: Executable filename.
:return: Path to the cached binary, or `None` if not cached.
"""
path = cached_binary_path(name, version, platform_key, executable)
if path.is_file() and os.access(path, os.X_OK):
return path
return None
[docs]
def store_binary(
name: str,
version: str,
platform_key: str,
source: Path,
) -> Path | None:
"""Copy an extracted binary into the cache atomically.
Writes to a temporary file in the target directory, then renames to the
final name. This is atomic on POSIX (same-filesystem rename) and safe on
Windows (`Path.replace` overwrites atomically).
Triggers {func}`auto_purge` after a successful store.
:param name: Tool name.
:param version: Pinned version.
:param platform_key: Platform key.
:param source: Path to the extracted binary to cache.
:return: Path to the cached binary, or `None` when the cache is unwritable
(a read-only cache root, a restricted CI mount): callers fall back to
their staging copy, matching {func}`store_response` and
{func}`store_config`.
"""
dest = cached_binary_path(name, version, platform_key, source.name)
def fill(tmp_path: Path) -> None:
shutil.copy2(source, tmp_path)
tmp_path.chmod(0o755)
# copy2 preserves the source mtime, which for a binary extracted from
# a release archive is the upstream build date. Purging ages entries
# by mtime, so an old build date would get the entry deleted by the
# auto_purge() call below, before this store even returns. Stamp the
# store time instead.
os.utime(tmp_path)
try:
atomic_write(dest, f".{source.name}.", fill)
except OSError as exc:
logging.warning(f"Cannot cache {name} at {dest}: {exc}")
return None
logging.debug(f"Cached {name} {version} for {platform_key} at {dest}.")
auto_purge()
return dest
[docs]
def get_cached_response(
namespace: str,
key: str,
max_age_seconds: int,
) -> bytes | None:
"""Return a cached HTTP response if it exists and is fresh.
:param namespace: Cache namespace, like `pypi` or `github-release-tags`.
:param key: Cache key, may contain `/` for nested paths.
:param max_age_seconds: Maximum age in seconds. Entries with mtime older
than this are considered stale and ignored. `<= 0` disables the
cache (always returns `None`).
:return: Raw cached response bytes, or `None` if not cached or stale.
"""
if max_age_seconds <= 0:
return None
path = HttpCacheEntry.root() / namespace / f"{key}.json"
if not path.is_file():
return None
age = time.time() - path.stat().st_mtime
if age > max_age_seconds:
logging.debug(
f"Stale HTTP cache entry: {path} (age {age:.0f}s > {max_age_seconds}s)."
)
return None
logging.debug(f"HTTP cache hit: {path}.")
return path.read_bytes()
[docs]
def store_response(
namespace: str,
key: str,
data: bytes,
) -> Path | None:
"""Store an HTTP response in the cache atomically.
Uses the same write-to-temp-then-rename pattern as {func}`store_binary`.
Triggers {func}`auto_purge` after a successful store.
:param namespace: Cache namespace.
:param key: Cache key, may contain `/` for nested paths.
:param data: Raw response bytes to cache.
:return: Path to the cached response file, or `None` if the write
failed (permissions, read-only filesystem, sandbox restrictions).
"""
dest = HttpCacheEntry.root() / namespace / f"{key}.json"
try:
atomic_write(dest, ".response.", lambda tmp_path: tmp_path.write_bytes(data))
except OSError:
logging.debug(f"Failed to cache HTTP response: {namespace}/{key}.")
return None
logging.debug(f"Cached HTTP response: {namespace}/{key} at {dest}.")
auto_purge()
return dest
[docs]
def store_config(
tool_name: str,
filename: str,
content: str,
) -> Path | None:
"""Store a generated tool config in the cache atomically.
The digest of *content* names the directory of the file, so the same
content always lands on the same path, and different content never
overwrites it. A run storing a config refreshes its age, which keeps a
config in use out of reach of {func}`auto_purge`.
:param tool_name: Tool name (registry key).
:param filename: Config filename, like `yamllint.yaml`.
:param content: Config file content as text.
:return: Path to the cached config file, or `None` if the write
failed (permissions, read-only filesystem, sandbox restrictions).
"""
digest = hashlib.sha256(content.encode("UTF-8")).hexdigest()
dest = (
ConfigCacheEntry.root() / tool_name / digest[:CONFIG_DIGEST_LENGTH] / filename
)
try:
atomic_write(
dest,
f".{filename}.",
lambda tmp_path: tmp_path.write_text(content, encoding="UTF-8"),
)
except OSError:
logging.debug(f"Failed to cache config for {tool_name}.")
return None
logging.debug(f"Cached config for {tool_name} at {dest}.")
auto_purge()
return dest
[docs]
def cache_rows() -> tuple[list[tuple[str, str, str, str, str]], int]:
"""List every cached file across the three caches, as table rows.
Backs `repomatic cache show`: each entry renders itself
({meth}`CachedFile.as_row`), so the command stays a print call and a new
cache kind shows up in the listing by existing.
:return: `(rows, total_size)`, rows ordered as {data}`CACHE_KINDS`.
"""
entries = [entry for kind in CACHE_KINDS for entry in kind.entries()]
return [entry.as_row() for entry in entries], sum(e.size for e in entries)
[docs]
def clean(
*,
tool: str | None = None,
namespace: str | None = None,
max_age_days: int | None = None,
) -> tuple[int, int]:
"""Remove cached entries, as `repomatic cache clean` selects them.
A scope filter reaches only the caches it applies to: *tool* narrows the
binary and config caches and leaves HTTP responses alone, *namespace* the
reverse. With neither filter, every cache is cleaned.
:param tool: Only remove the binary and config entries of this tool.
:param namespace: Only remove the HTTP responses of this namespace.
:param max_age_days: Only remove entries older than this many days.
:return: Tuple of (files_deleted, bytes_freed) over every cache cleaned.
"""
scopes = {"tool": tool, "namespace": namespace}
scoped = any(scope is not None for scope in scopes.values())
files_deleted = 0
bytes_freed = 0
for kind in CACHE_KINDS:
scope = scopes[kind.scope_option]
if scoped and scope is None:
continue
deleted, freed = kind.clear(scope=scope, max_age_days=max_age_days)
files_deleted += deleted
bytes_freed += freed
return files_deleted, bytes_freed
def _prune_empty_dirs(root: Path) -> None:
"""Remove empty directories under *root*, bottom-up."""
if not root.is_dir():
return
# Walk bottom-up and attempt to remove each directory. rmdir() fails
# (OSError) if the directory is non-empty, which is the desired behavior.
for dirpath, _dirnames, _filenames in os.walk(root, topdown=False):
path = Path(dirpath)
if path == root:
continue
try:
path.rmdir()
except OSError:
pass
def _max_age_days() -> int:
"""Resolve the auto-purge TTL.
Precedence (highest to lowest):
1. `REPOMATIC_CACHE_MAX_AGE` environment variable.
2. `cache.max-age` in `[tool.repomatic]`.
3. `CacheConfig.max_age` field default.
:return: TTL in days. `0` means auto-purge is disabled.
"""
# 1. Environment variable (highest priority).
raw = os.environ.get("REPOMATIC_CACHE_MAX_AGE", "")
if raw.strip():
try:
return int(raw)
except ValueError:
logging.warning(
f"Invalid REPOMATIC_CACHE_MAX_AGE={raw!r}, using config default."
)
# 2 + 3. Config from [tool.repomatic] (falls back to field default).
config = load_repomatic_config()
return config.cache.max_age
_PURGED_ROOTS: set[Path] = set()
"""Cache roots already swept by {func}`auto_purge` in this process.
Every store triggers a purge, and a dependency sweep stores hundreds of
responses per run: without the guard each store re-walks every subtree to
re-delete nothing. One sweep per root per process is enough, since entries
only age on the scale of days.
"""
[docs]
def auto_purge() -> None:
"""Remove cached entries older than the configured TTL.
Called automatically after every store, and runs at most once per cache
root per process (see `_PURGED_ROOTS`). Purges every cache. Resolves the
TTL from `REPOMATIC_CACHE_MAX_AGE` env var, then `cache.max-age` in
`[tool.repomatic]`, then the `CacheConfig.max_age` field default. Set to
`0` to disable.
"""
root = cache_dir()
if root in _PURGED_ROOTS:
return
_PURGED_ROOTS.add(root)
days = _max_age_days()
if days <= 0:
return
deleted, freed = clean(max_age_days=days)
if deleted:
logging.debug(f"Auto-purged {deleted} cached entry(ies), freed {freed} bytes.")