# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""How a dependency is *declared*, as opposed to where it resolves from.
{mod}`~repomatic.deps.shippable` answers "can this ship": a git branch or a local
path breaks the install for whoever pulls the published artifact, so those
findings block a release. This module answers a narrower question that never
blocks anything: is the declaration written the way the project's own version
policy says to write it.
The split is what keeps both halves honest. A style finding that could stop a
release would eventually be silenced rather than fixed; a shippability finding
that only warned would ship a broken wheel.
Only rules decidable from `pyproject.toml` alone live here. Whether a floor is
*justified* by the APIs the code actually calls is the judgment call
`/repomatic-deps review` exists for, and it stays there: no amount of parsing
settles it, and a checker that guessed would train people to ignore it.
The rules, and what each one costs the reader when broken:
- An **upper bound** on a runtime dependency caps everyone downstream, and the
cap outlives whatever release prompted it. See
[Should You Use Upper Bound Version
Constraints?](https://iscinumpy.dev/post/bound-version-constraints/)
- A **bare dependency** pins nothing, so the install that passed CI and the one
a user gets can differ by a major version.
- An **unsorted list** makes every addition a merge conflict candidate and
hides duplicates.
- A **type stub outside the `typing` group** installs at runtime for users who
will never type-check.
- A **floor with no comment** cannot be audited: the next reader has no way to
tell a deliberate API minimum from a number a bot last touched.
- A **floor comment that runs long** has stopped justifying the floor and
started narrating how it got there. Each bump appends a paragraph about a
version no longer in force, and the one claim that matters (what breaks
below the floor that is declared) ends up buried in superseded history the
git log already keeps.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from itertools import pairwise
from packaging.utils import canonicalize_name
from .requirements import (
BUILD_TABLE,
iter_toml_lines,
parse_requirement,
read_project,
requirement_arrays,
)
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterator
from pathlib import Path
from packaging.requirements import Requirement
STUB_PREFIX = "types-"
"""Distribution-name prefix marking a [PEP 561](https://peps.python.org/pep-0561/)
stub-only package."""
STUB_GROUP = "typing"
"""Dependency group stub-only packages belong in.
They are build-time inputs to a type checker, so installing them anywhere a
user's runtime environment reaches is pure weight.
"""
UPPER_BOUND_OPERATORS = ("<", "<=", "==", "!=", "~=")
"""Specifier operators that cap a runtime dependency from above.
`~=` is included because it implies a ceiling: `~=1.2` is `>=1.2, ==1.*`.
Conditional markers (`python_version<'3.11'`) are not specifiers and never
reach this list.
"""
_ENTRY_RE = re.compile(r"""^\s*["'](?P<requirement>[^"']+)["']\s*,?\s*(?:#.*)?$""")
"""One dependency per line, the layout every check below assumes.
A list written inline on a single line parses fine but carries no per-entry
line to hang a comment on, so {func}`_entry_lines` reports it as unscannable
rather than guessing.
"""
[docs]
@dataclass(frozen=True)
class PolicyFinding:
"""One declaration that departs from the project's version policy.
Deliberately not a {class}`~repomatic.deps.shippable.DepFinding`: that type
carries a {class}`~repomatic.deps.shippable.SourceKind` because every one of
its findings is about where a package resolves from, and a style finding
has no answer to give there.
"""
package: str
"""Normalized name of the package the finding is about."""
location: str
"""The TOML path the declaration was read from."""
detail: str
"""The declaration as written, for the reader to go find."""
consequence: str
"""What it costs to leave this as it is."""
remedy: str
"""The next action."""
@property
def message(self) -> str:
"""The finding as a single annotation line."""
return (
f"{self.package}: {self.location} ({self.detail}). "
f"{self.consequence} {self.remedy}"
)
def _entry_lines(text: str, table: str, key: str) -> list[tuple[int, str]] | None:
"""Locate each entry of a one-per-line dependency array in the raw text.
Reading the file rather than the parsed document is what gives each entry
the line number the report points the reader to.
:param text: Full `pyproject.toml` contents.
:param table: Dotted path of the table holding the array.
:param key: Array key inside that table.
:return: `(line number, requirement string)` per entry, or `None` when the
array is written inline, where there is no per-entry line. The array's
own opening line is included as the first tuple with an empty
requirement, so a caller can ask whether a comment documents the whole
array rather than each entry.
"""
opening = re.compile(rf"^\s*{re.escape(key)}\s*=\s*\[\s*$")
lines = iter_toml_lines(text)
for number, current, line in lines:
if current != table or not opening.match(line):
continue
entries = [(number, "")]
for entry_number, _table, entry_line in lines:
if entry_line.strip().startswith("]"):
break
match = _ENTRY_RE.match(entry_line)
if match:
entries.append((entry_number, match.group("requirement")))
return entries
return None
def _comment_above(lines: list[str], number: int) -> list[str]:
"""The contiguous run of comment lines sitting above line *number*.
Read bottom-up from the adjacent line and stopping at the first line that
is not a comment, so a comment separated from the entry by a blank line is
not attached to it. Returned in reading order, each line stripped of its
`#` marker, which is what makes the run countable as prose.
:param lines: The file's lines.
:param number: The 1-based number of the line the comment documents.
:return: The comment lines, or an empty list when the entry carries none.
"""
comment: list[str] = []
cursor = number - 2
while cursor >= 0 and lines[cursor].strip().startswith("#"):
comment.append(lines[cursor].strip().removeprefix("#").strip())
cursor -= 1
comment.reverse()
return comment
def _check_specifier(
requirement: Requirement,
raw: str,
location: str,
runtime: bool,
floored: frozenset[str] = frozenset(),
) -> Iterator[PolicyFinding]:
"""Flag a missing specifier, and an upper bound on a runtime dependency.
:param floored: Canonical names carrying a specifier somewhere else in the
same file, plus the project's own name. A second mention needs no floor
of its own: a `docs` group pulling `click-extra[sphinx]` is selecting an
extra of a package the runtime list already constrains, and repeating
the floor there would just be a second number to keep in step. A
self-reference needs none either, and can carry none.
"""
if not str(requirement.specifier):
if canonicalize_name(requirement.name) in floored:
return
yield PolicyFinding(
package=canonicalize_name(requirement.name),
location=location,
detail=f"`{raw}`",
consequence=(
"Nothing constrains which version installs, so the release "
"this was tested against and the one a user gets can differ "
"by a major version."
),
remedy="Add a `>=` floor naming the oldest version the code works with.",
)
return
if not runtime:
return
capped = sorted(
str(specifier)
for specifier in requirement.specifier
if specifier.operator in UPPER_BOUND_OPERATORS
)
if capped:
yield PolicyFinding(
package=canonicalize_name(requirement.name),
location=location,
detail=f"`{raw}`",
consequence=(
f"The cap {', '.join(capped)} propagates to "
f"every project installing this one, and outlives whatever "
f"release prompted it."
),
remedy=(
"Drop the upper bound, per "
"https://iscinumpy.dev/post/bound-version-constraints/"
),
)
def _over_long(
package: str, location: str, detail: str, threshold: int
) -> PolicyFinding:
"""The finding for a floor comment that has outgrown its floor.
Shared by the per-entry comment and the block above the array, which fail
the same way and take the same remedy.
"""
return PolicyFinding(
package=package,
location=location,
detail=detail,
consequence=(
"A floor comment answers one question: what breaks below the "
f"version declared here. Past {threshold} words it is narrating "
"how the floor got there instead, and the answer is buried in "
"versions no longer in force."
),
remedy=(
"Cut it to the paragraph justifying the floor as it stands. "
"Superseded floors are in the git history, and belong in no "
"comment."
),
)
def _check_ordering(names: list[str], location: str) -> Iterator[PolicyFinding]:
"""Flag the first entry that breaks alphabetical order.
Only the first: a single misplaced entry makes every later one look
out of order too, and a report naming them all buries the one to move.
"""
for previous, current in pairwise(names):
if canonicalize_name(current) < canonicalize_name(previous):
yield PolicyFinding(
package=canonicalize_name(current),
location=location,
detail=f"`{current}` follows `{previous}`",
consequence=(
"An unsorted list turns every addition into a merge "
"conflict candidate and hides duplicates."
),
remedy="Sort the list alphabetically.",
)
return
[docs]
def scan_policy(
pyproject_path: Path, comment_word_threshold: int = 0
) -> list[PolicyFinding]:
"""Every declaration in *pyproject_path* that departs from version policy.
Entirely offline, reading only `pyproject.toml`, so it costs nothing to
run on every push.
:param pyproject_path: Path to the `pyproject.toml` file, whatever its
name.
:param comment_word_threshold: Word ceiling per floor comment. `0` (or
less) disables the check, which is what a caller with no configuration
to read gets.
:return: Findings sorted by location, then by package.
"""
data = read_project(pyproject_path)
if not data:
return []
text = pyproject_path.read_text(encoding="UTF-8")
lines = text.splitlines()
findings: list[PolicyFinding] = []
# The build front end alone reads `[build-system] requires`, so none of
# the rules below speak to it.
arrays = [array for array in requirement_arrays(data) if array.table != BUILD_TABLE]
# Every name floored anywhere in the file, so a second mention selecting an
# extra is not read as an unconstrained dependency. The project's own name
# joins them: an aggregate extra selecting the project's other extras
# (`orchard[toml,xml]` inside orchard) resolves to the very version being
# installed, so no other release exists for a floor to exclude, and a
# project can never floor itself in its own file anyway.
own_name = data.get("project", {}).get("name")
floored = frozenset(
canonicalize_name(requirement.name)
for array in arrays
for _index, _raw, requirement in array.requirements()
if str(requirement.specifier)
) | ({canonicalize_name(own_name)} if own_name else set())
for array in arrays:
location = array.location
parsed = [
(raw, requirement) for _index, raw, requirement in array.requirements()
]
if not parsed:
continue
for raw, requirement in parsed:
findings.extend(
_check_specifier(requirement, raw, location, array.published, floored)
)
findings.extend(_check_ordering([req.name for _raw, req in parsed], location))
# Stub packages belong in the typing group, wherever else they appear.
if array.key != STUB_GROUP:
for raw, requirement in parsed:
if not canonicalize_name(requirement.name).startswith(STUB_PREFIX):
continue
findings.append(
PolicyFinding(
package=canonicalize_name(requirement.name),
location=location,
detail=f"`{raw}`",
consequence=(
"A stub-only package is an input to a type "
"checker, so installing it here reaches "
"environments that never type-check."
),
remedy=f"Move it to the `{STUB_GROUP}` dependency group.",
)
)
# The comment check needs a line per entry, which an inline array
# cannot offer.
entry_lines = _entry_lines(text, array.table, array.key)
if entry_lines is None:
continue
# The first tuple is the array's own opening line. A comment above it
# documents the whole array, which is how a run of related entries
# (three `types-*` stubs, say) is usually justified in one block. It
# excuses a missing per-entry comment, and only that: an over-long
# comment is over-long wherever it sits, or a wall would escape the
# length check by moving up one line.
(array_line, _marker), *entries = entry_lines
block_comment = _comment_above(lines, array_line)
# The first entry with nothing of its own, so the block above the array
# is what documents it. Absent one, that block justifies no floor at
# all: a project restating its version policy there is writing a
# preamble, and length is not its measure.
leans_on_block: tuple[str, str] | None = None
for line_number, raw in entries:
floor = parse_requirement(raw)
if floor is None or not str(floor.specifier):
continue
package = canonicalize_name(floor.name)
comment = _comment_above(lines, line_number)
if not comment:
if block_comment:
if leans_on_block is None:
leans_on_block = (package, raw)
continue
findings.append(
PolicyFinding(
package=package,
location=location,
detail=f"`{raw}` at line {line_number}",
consequence=(
"An uncommented floor cannot be audited: nothing "
"distinguishes a deliberate API minimum from a "
"number a bot last touched."
),
remedy=(
"Add a comment above it naming the feature or fix "
"that version introduced."
),
)
)
continue
if comment_word_threshold <= 0:
continue
words = count_comment_words(comment)
if words > comment_word_threshold:
findings.append(
_over_long(
package,
location,
f"`{raw}` documented in {words} words",
comment_word_threshold,
)
)
if comment_word_threshold > 0 and leans_on_block is not None:
words = count_comment_words(block_comment)
if words > comment_word_threshold:
name, raw = leans_on_block
findings.append(
_over_long(
name,
location,
f"`{raw}` documented in a {words}-word block above the array",
comment_word_threshold,
)
)
return sorted(findings, key=lambda finding: (finding.location, finding.package))