From 1a755bdc83366eba7b88a2b2d5208613593e6d33 Mon Sep 17 00:00:00 2001 From: Nayjest Date: Mon, 6 Jul 2026 16:50:58 +0200 Subject: [PATCH] Add scan configuration: enable/disable categories, metrics, and components Lets a scan switch off any part of the methodology (component, metric, or category) via a ScanConfig embedded in every report, so scores stay reproducible and auditable. Disabled items are excluded from scoring exactly like missing data, with remaining weights renormalized. CLI gains --config/--disable-category/--disable-metric/--disable-component; HTML reports render a "Scan configuration" section. --- README.md | 31 ++++ docs/metrics.md | 44 +++++ docs/report-schema.md | 25 ++- src/scanner/cli.py | 97 ++++++++++- src/scanner/collect.py | 27 ++- src/scanner/metrics.py | 153 ++++++++++++++--- src/scanner/models.py | 57 ++++++- src/scanner/render.py | 31 ++++ src/scanner/templates/_macros.j2 | 29 ++++ src/scanner/templates/_style.css | 6 + src/scanner/templates/org.html.j2 | 2 + src/scanner/templates/report.html.j2 | 2 + tests/test_config.py | 235 +++++++++++++++++++++++++++ 13 files changed, 706 insertions(+), 33 deletions(-) create mode 100644 tests/test_config.py diff --git a/README.md b/README.md index f44329b..d21c1b1 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,39 @@ inspect-scan https://github.com/python # Store JSON + HTML under ./storage with standardized names inspect-scan pallets/flask --storage inspect-scan psf --storage D:/audit-reports + +# Enable/disable parts of the scoring methodology (see below) +inspect-scan pallets/flask --disable-category security --disable-metric popularity +inspect-scan pallets/flask --config scan-config.json --html report.html ``` +### Scan configuration (enabling / disabling metrics) + +Any part of the methodology — a component, a metric, or a whole category — can +be switched off for a scan. Disabled items are removed from scoring and the +remaining weights **renormalized** (never counted as zero), so scores stay on +the 1–100 scale. The configuration is embedded in the report (`config`) and +summarized in a **Scan configuration** section of the HTML. + +```bash +inspect-scan pallets/flask --disable-category security # drop a category +inspect-scan pallets/flask --disable-metric popularity # drop a metric +inspect-scan pallets/flask --disable-component documentation:Wiki # drop a component +inspect-scan pallets/flask --config scan-config.json # from a file (+ flags merge on top) +``` + +```jsonc +// scan-config.json +{ + "disabled_categories": ["security"], + "disabled_metrics": ["popularity"], + "disabled_components": { "documentation": ["Wiki"] } +} +``` + +Category/metric keys and component names are listed in +[docs/metrics.md](docs/metrics.md#configuration-enabling--disabling-metrics). + ### Report storage `--storage [DIR]` writes both report formats into a standardized layout: diff --git a/docs/metrics.md b/docs/metrics.md index 142830a..edc014c 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -229,6 +229,50 @@ Repository reports also embed the owning account's public profile in `data.owner` (both organizations and users); it feeds the `stewardship` metric above. +## Configuration (enabling / disabling metrics) + +A scan can switch off any part of the methodology — a **component**, a whole +**metric**, or a whole **category**. This is carried by a `ScanConfig` and +**embedded in every report** (`config` at the top level), so a score is always +reproducible and it is explicit what was and was not measured. + +Disabling something works *exactly like missing data*: the item is excluded and +the remaining weights are **renormalized**, so the score stays on the 1..100 +scale — it is never counted as a zero. Disabling every metric in a category (or +the category itself) drops that category from the overall, with its weight +renormalized away. + +```jsonc +// scan-config.json +{ + "disabled_categories": ["security"], // drop a whole category + "disabled_metrics": ["popularity"], // drop one metric + "disabled_components": { // drop components within a metric + "documentation": ["Wiki", "Topics"] + } +} +``` + +From the CLI, a config file and/or repeatable flags (flags merge on top of the +file): + +``` +inspect-scan owner/repo --config scan-config.json +inspect-scan owner/repo --disable-category security --disable-metric popularity +inspect-scan owner/repo --disable-component documentation:Wiki --html report.html +``` + +Category and metric **keys** are the identifiers in the tables above +(`security`, `popularity`, `security_posture`, …); component **names** are the +exact display names (`Wiki`, `Stars`, `README`, …). Unknown category/metric +keys are reported as warnings and ignored. The HTML report renders a **Scan +configuration** section listing what was disabled (or "Full methodology" when +nothing is), and each affected metric's `note` records the renormalization. + +Configuration selects *which* parts of the fixed methodology are active; it does +not change any formula, weight, or threshold — those remain versioned by +`metrics_version`. + ## Worked example pallets/flask (organization-owned): Vitality 75, Community & Adoption 96, diff --git a/docs/report-schema.md b/docs/report-schema.md index 750aab3..1f38fc5 100644 --- a/docs/report-schema.md +++ b/docs/report-schema.md @@ -1,6 +1,6 @@ # Report schema -**Schema version: 0.6.0** (`schema_version` field in every report). +**Schema version: 0.7.0** (`schema_version` field in every report). The schema is defined as Pydantic models in [`src/scanner/models.py`](../src/scanner/models.py); this document describes it for consumers. Any breaking structural change bumps `schema_version`. @@ -30,9 +30,10 @@ data/metrics layering, the `Metric` object shape, and the band scale. ```jsonc { "report_type": "repository", - "schema_version": "0.6.0", + "schema_version": "0.7.0", "generated_at": "2026-07-06T12:00:00Z", // UTC timestamp of the scan "source": { ... }, // what was scanned + "config": { ... }, // scan configuration (see below) "data": { ... }, // raw facts (data layer) "metrics": { ... }, // 1..100 scores (metrics layer) "warnings": ["..."] // non-fatal collection problems @@ -44,6 +45,23 @@ computing statistics, truncated file trees). A warning means the related fields are `null`/incomplete — affected metrics exclude those inputs rather than scoring them as zero. +## `config` — scan configuration + +Records which parts of the methodology were active for this scan, so the score +is reproducible. Empty collections mean the full methodology (everything +enabled). Disabled items are removed from scoring with the remaining weights +renormalized (see [metrics.md](metrics.md#configuration-enabling--disabling-metrics)). + +```jsonc +"config": { + "disabled_categories": ["security"], // category keys + "disabled_metrics": ["popularity"], // metric keys + "disabled_components": { // metric key -> component names + "documentation": ["Wiki"] + } +} +``` + ## `source` | Field | Type | Description | @@ -245,9 +263,10 @@ Produced when the scan target is an organization (`inspect-scan orgname`). ```jsonc { "report_type": "organization", - "schema_version": "0.6.0", + "schema_version": "0.7.0", "generated_at": "...", "source": { "url": "...", "host": "github.com", "login": "psf" }, + "config": { /* same ScanConfig shape as repository reports */ }, "data": { "info": { /* OrgInfo: login, name, description, blog, location, email, twitter_username, is_verified, public_repos, followers, diff --git a/src/scanner/cli.py b/src/scanner/cli.py index 4a632ee..b20b150 100644 --- a/src/scanner/cli.py +++ b/src/scanner/cli.py @@ -12,7 +12,8 @@ from .collect import scan_organization, scan_repository from .github import GitHubError, parse_target, resolve_token -from .models import OrgReport, Report +from .metrics import validate_config +from .models import OrgReport, Report, ScanConfig from .render import render_html, render_org_html from .storage import STORAGE_ENV_VAR, resolve_storage, store_report @@ -69,9 +70,88 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Emit compact single-line JSON instead of pretty-printed", ) + + config_group = parser.add_argument_group( + "scan configuration", + "Enable/disable parts of the scoring methodology. Disabled items are " + "removed from scoring and the remaining weights renormalized (never " + "counted as zero). The resulting configuration is embedded in the report.", + ) + config_group.add_argument( + "--config", + type=Path, + default=None, + metavar="PATH", + help="JSON file with a scan configuration " + '({"disabled_categories": [...], "disabled_metrics": [...], ' + '"disabled_components": {"metric_key": ["Component name"]}}). ' + "Command-line --disable-* flags are merged on top of it.", + ) + config_group.add_argument( + "--disable-category", + action="append", + default=[], + metavar="KEY", + help="Exclude a whole category from scoring (repeatable), e.g. security", + ) + config_group.add_argument( + "--disable-metric", + action="append", + default=[], + metavar="KEY", + help="Exclude a single metric from scoring (repeatable), e.g. popularity", + ) + config_group.add_argument( + "--disable-component", + action="append", + default=[], + metavar="METRIC:COMPONENT", + help="Exclude one component within a metric (repeatable), e.g. " + "popularity:Watchers", + ) return parser +def build_config(args: argparse.Namespace) -> ScanConfig: + """Assemble a ScanConfig from an optional --config file plus --disable-* flags.""" + config = ScanConfig() + if args.config is not None: + config = ScanConfig.model_validate_json(args.config.read_text(encoding="utf-8")) + + categories = list(config.disabled_categories) + for key in args.disable_category: + if key not in categories: + categories.append(key) + + metrics = list(config.disabled_metrics) + for key in args.disable_metric: + if key not in metrics: + metrics.append(key) + + components = {k: list(v) for k, v in config.disabled_components.items()} + for spec in args.disable_component: + metric_key, sep, component = spec.partition(":") + if not sep or not metric_key.strip() or not component.strip(): + raise ValueError( + f"--disable-component expects METRIC:COMPONENT, got {spec!r}" + ) + names = components.setdefault(metric_key.strip(), []) + if component.strip() not in names: + names.append(component.strip()) + + return ScanConfig( + disabled_categories=categories, + disabled_metrics=metrics, + disabled_components=components, + ) + + +def _config_requested(args: argparse.Namespace) -> bool: + return bool( + args.config or args.disable_category or args.disable_metric or args.disable_component + ) + + def _load_report_file(path: Path) -> AnyReport: raw = path.read_text(encoding="utf-8") kind = json.loads(raw).get("report_type", "repository") @@ -82,8 +162,19 @@ def _load_report_file(path: Path) -> AnyReport: def _load_or_scan(args: argparse.Namespace) -> AnyReport: target_path = Path(args.target) if args.target.lower().endswith(".json") and target_path.is_file(): + if _config_requested(args): + print( + "note: scan-configuration flags are ignored when re-rendering a " + "stored JSON report; the report keeps the configuration it was " + "scanned with", + file=sys.stderr, + ) return _load_report_file(target_path) + config = build_config(args) + for warning in validate_config(config): + print(f"warning: {warning}", file=sys.stderr) + kind, *ids = parse_target(args.target) token = resolve_token(args.token) if not token: @@ -93,8 +184,8 @@ def _load_or_scan(args: argparse.Namespace) -> AnyReport: file=sys.stderr, ) if kind == "org": - return scan_organization(ids[0], token=token) - return scan_repository(args.target, token=token) + return scan_organization(ids[0], token=token, config=config) + return scan_repository(args.target, token=token, config=config) def _render(report: AnyReport) -> str: diff --git a/src/scanner/collect.py b/src/scanner/collect.py index 6281e3e..7d3506c 100644 --- a/src/scanner/collect.py +++ b/src/scanner/collect.py @@ -29,6 +29,7 @@ Report, RepoInfo, RepoRef, + ScanConfig, SecuritySignals, TopRepo, ) @@ -100,8 +101,14 @@ TEST_FILE_PATTERNS = ("test_*.py", "*_test.py", "*_test.go", "*.test.js", "*.test.ts", "*.spec.js", "*.spec.ts", "*Test.java", "*Test.php") -def scan_repository(url: str, token: Optional[str] = None) -> Report: - """Scan a public GitHub repository and return a populated Report.""" +def scan_repository( + url: str, token: Optional[str] = None, config: Optional[ScanConfig] = None +) -> Report: + """Scan a public GitHub repository and return a populated Report. + + ``config`` selects which metrics/categories/components are scored; it + defaults to the full methodology and is embedded in the returned Report.""" + config = config or ScanConfig() owner, name = parse_repo_url(url) source = RepoRef(url=url, owner=owner, name=name) warnings: list[str] = [] @@ -137,8 +144,9 @@ def scan_repository(url: str, token: Optional[str] = None) -> Report: return Report( generated_at=datetime.now(timezone.utc), source=source, + config=config, data=data, - metrics=compute_metrics(data), + metrics=compute_metrics(data, config), warnings=warnings, ) @@ -399,8 +407,14 @@ def _security(paths: list[str], community: CommunityHealth) -> SecuritySignals: # --------------------------------------------------------------------------- -def scan_organization(login: str, token: Optional[str] = None) -> OrgReport: - """Scan a GitHub organization's public profile and repository portfolio.""" +def scan_organization( + login: str, token: Optional[str] = None, config: Optional[ScanConfig] = None +) -> OrgReport: + """Scan a GitHub organization's public profile and repository portfolio. + + ``config`` selects which metrics/categories/components are scored; it + defaults to the full methodology and is embedded in the returned report.""" + config = config or ScanConfig() warnings: list[str] = [] with GitHubClient(token=token) as gh: try: @@ -421,8 +435,9 @@ def scan_organization(login: str, token: Optional[str] = None) -> OrgReport: return OrgReport( generated_at=datetime.now(timezone.utc), source=OrgRef(url=f"https://github.com/{login}", login=data.info.login), + config=config, data=data, - metrics=compute_org_metrics(data), + metrics=compute_org_metrics(data, config), warnings=warnings, ) diff --git a/src/scanner/metrics.py b/src/scanner/metrics.py index fa0f244..1d884b1 100644 --- a/src/scanner/metrics.py +++ b/src/scanner/metrics.py @@ -31,10 +31,15 @@ OrgData, OrgMetrics, RepoData, + ScanConfig, ) METRICS_VERSION = "0.4.0" +# Detail string marking a component excluded because the scan configuration +# switched it off (as opposed to missing data). Used to phrase metric notes. +DISABLED_DETAIL = "disabled in scan configuration" + # Lower bound of each band, checked from the top down. BAND_THRESHOLDS: list[tuple[int, Band]] = [ (85, "excellent"), @@ -104,7 +109,16 @@ def _metric( return None earned = sum(c.points for c in scored) value = max(1, min(100, round(100 * earned / possible))) - excluded = [c.name for c in components if c.status == "excluded"] + excluded = [c for c in components if c.status == "excluded"] + disabled = [c.name for c in excluded if c.detail == DISABLED_DETAIL] + no_data = [c.name for c in excluded if c.detail != DISABLED_DETAIL] + note_parts: list[str] = [] + if no_data: + note_parts.append(f"Excluded from scoring (no data or not applicable): {', '.join(no_data)}.") + if disabled: + note_parts.append(f"Disabled in scan configuration: {', '.join(disabled)}.") + if excluded: + note_parts.append("Remaining weights renormalized.") return Metric( key=key, name=name, @@ -112,13 +126,39 @@ def _metric( band=band_for(value), components=components, inputs=inputs, - note=f"Excluded from scoring (no data or not applicable): {', '.join(excluded)}. " - "Remaining weights renormalized." - if excluded - else None, + note=" ".join(note_parts) if note_parts else None, ) +def _disable_components(metric: Metric, disabled_names: set[str]) -> Optional[Metric]: + """Re-score a metric with the named components switched off by configuration. + + Each named component becomes ``excluded`` and its weight is renormalized + away, mirroring how missing data is handled. Returns None if switching them + off leaves the metric with nothing scorable.""" + if not disabled_names: + return metric + updated: list[MetricComponent] = [] + changed = False + for c in metric.components: + if c.name in disabled_names and c.status != "excluded": + updated.append( + MetricComponent( + name=c.name, + points=0.0, + max_points=c.max_points, + status="excluded", + detail=DISABLED_DETAIL, + ) + ) + changed = True + else: + updated.append(c) + if not changed: + return metric + return _metric(metric.key, metric.name, updated, metric.inputs) + + # --------------------------------------------------------------------------- # Repository metrics # --------------------------------------------------------------------------- @@ -659,13 +699,19 @@ def _build( specs: list[CategorySpec], computed: dict[str, Optional[Metric]], overall_name: str, + config: ScanConfig, ) -> tuple[Optional[Metric], list[MetricCategory]]: - """Assemble categories and the overall score from computed metrics.""" + """Assemble categories and the overall score from computed metrics. + + Categories disabled by ``config`` are dropped entirely (not rendered, not + scored); categories with no scorable metric are dropped as before.""" categories: list[MetricCategory] = [] cat_values: dict[str, int] = {} cat_weights: dict[str, float] = {} for spec in specs: + if not config.category_enabled(spec.key): + continue present = [ computed[k] for k in spec.metrics if computed.get(k) is not None ] @@ -691,26 +737,53 @@ def _build( overall_value = _rollup(cat_values, cat_weights) if overall_value is None: return None, categories - dropped = [s.name for s in specs if s.key not in cat_values] + disabled = [s.name for s in specs if not config.category_enabled(s.key)] + no_data = [ + s.name for s in specs if config.category_enabled(s.key) and s.key not in cat_values + ] + note_parts: list[str] = [] + if no_data: + note_parts.append(f"Categories without data excluded: {', '.join(no_data)}.") + if disabled: + note_parts.append(f"Categories disabled in scan configuration: {', '.join(disabled)}.") + if disabled or no_data: + note_parts.append("Weights renormalized over the remaining categories.") overall = Metric( key="overall", name=overall_name, value=overall_value, band=band_for(overall_value), inputs={k: v for k, v in cat_values.items()}, - note=f"Categories without data excluded and weights renormalized: {', '.join(dropped)}" - if dropped - else None, + note=" ".join(note_parts) if note_parts else None, ) return overall, categories -def compute_metrics(data: RepoData) -> Metrics: +def _compute( + specs: list[CategorySpec], data: Any, config: ScanConfig +) -> dict[str, Optional[Metric]]: + """Run each metric function, honoring the scan configuration. + + Disabled categories and metrics are skipped (recorded as None); enabled + metrics have their configuration-disabled components switched off.""" computed: dict[str, Optional[Metric]] = {} - for spec in REPO_CATEGORIES: + for spec in specs: + category_on = config.category_enabled(spec.key) for key, (_, fn) in spec.metrics.items(): - computed[key] = fn(data) - overall, categories = _build(REPO_CATEGORIES, computed, "Overall health") + if not category_on or not config.metric_enabled(key): + computed[key] = None + continue + metric = fn(data) + if metric is not None: + metric = _disable_components(metric, config.disabled_component_names(key)) + computed[key] = metric + return computed + + +def compute_metrics(data: RepoData, config: Optional[ScanConfig] = None) -> Metrics: + config = config or ScanConfig() + computed = _compute(REPO_CATEGORIES, data, config) + overall, categories = _build(REPO_CATEGORIES, computed, "Overall health", config) return Metrics(metrics_version=METRICS_VERSION, overall=overall, categories=categories) @@ -842,10 +915,50 @@ def metric_org_reach(data: OrgData) -> Optional[Metric]: ] -def compute_org_metrics(data: OrgData) -> OrgMetrics: - computed: dict[str, Optional[Metric]] = {} - for spec in ORG_CATEGORIES: - for key, (_, fn) in spec.metrics.items(): - computed[key] = fn(data) - overall, categories = _build(ORG_CATEGORIES, computed, "Overall organization health") +def compute_org_metrics(data: OrgData, config: Optional[ScanConfig] = None) -> OrgMetrics: + config = config or ScanConfig() + computed = _compute(ORG_CATEGORIES, data, config) + overall, categories = _build(ORG_CATEGORIES, computed, "Overall organization health", config) return OrgMetrics(metrics_version=METRICS_VERSION, overall=overall, categories=categories) + + +# --------------------------------------------------------------------------- +# Configuration registry & validation +# --------------------------------------------------------------------------- + +# Repository and organization methodologies share the same key namespace for +# validation — a repository-only key is still "known" when scanning an org. +ALL_CATEGORIES: list[CategorySpec] = REPO_CATEGORIES + ORG_CATEGORIES + + +def known_category_keys() -> set[str]: + """Every category key the methodology defines (repositories + organizations).""" + return {c.key for c in ALL_CATEGORIES} + + +def known_metric_keys() -> set[str]: + """Every metric key the methodology defines (repositories + organizations).""" + return {k for c in ALL_CATEGORIES for k in c.metrics} + + +def validate_config(config: ScanConfig) -> list[str]: + """Return human-readable warnings for keys the methodology doesn't define. + + Category and metric keys are checked against the combined methodology. + Component names are scoped to their metric and are not strictly validated — + an unrecognized component name simply has no effect (only the parent metric + key is checked).""" + warnings: list[str] = [] + categories, metrics = known_category_keys(), known_metric_keys() + for key in config.disabled_categories: + if key not in categories: + warnings.append(f"unknown category '{key}' in scan configuration (ignored)") + for key in config.disabled_metrics: + if key not in metrics: + warnings.append(f"unknown metric '{key}' in scan configuration (ignored)") + for key in config.disabled_components: + if key not in metrics: + warnings.append( + f"disabled_components references unknown metric '{key}' (ignored)" + ) + return warnings diff --git a/src/scanner/models.py b/src/scanner/models.py index 62f5821..910293a 100644 --- a/src/scanner/models.py +++ b/src/scanner/models.py @@ -22,7 +22,7 @@ from pydantic import BaseModel, Field -SCHEMA_VERSION = "0.6.0" +SCHEMA_VERSION = "0.7.0" # --------------------------------------------------------------------------- # Data layer: raw observed facts @@ -435,6 +435,53 @@ def by_key(self, key: str) -> Optional[Metric]: return None +# --------------------------------------------------------------------------- +# Scan configuration +# --------------------------------------------------------------------------- + + +class ScanConfig(BaseModel): + """Which parts of the scoring methodology are active for a scan. + + Scoring is a three-level hierarchy — ``components → metrics → categories`` + — and any level can be switched off. Disabling something removes it from + scoring *exactly as if its data were unavailable*: the remaining weights + are renormalized so the score stays on the 1..100 scale. Disabling is + therefore transparent rather than a silent zero. + + Every report embeds the ``ScanConfig`` that produced it, so a score is + reproducible and it is explicit what was and was not measured. Empty + collections mean "everything enabled" — the default full methodology. + """ + + disabled_categories: list[str] = Field( + default_factory=list, description="Category keys excluded from scoring" + ) + disabled_metrics: list[str] = Field( + default_factory=list, description="Metric keys excluded from scoring" + ) + disabled_components: dict[str, list[str]] = Field( + default_factory=dict, + description="Metric key -> component names excluded within that metric", + ) + + def category_enabled(self, key: str) -> bool: + return key not in self.disabled_categories + + def metric_enabled(self, key: str) -> bool: + return key not in self.disabled_metrics + + def disabled_component_names(self, metric_key: str) -> set[str]: + return set(self.disabled_components.get(metric_key, ())) + + @property + def is_default(self) -> bool: + """True when nothing is disabled (the full methodology).""" + return not ( + self.disabled_categories or self.disabled_metrics or self.disabled_components + ) + + # --------------------------------------------------------------------------- # Top-level reports # --------------------------------------------------------------------------- @@ -447,6 +494,10 @@ class Report(BaseModel): schema_version: str = SCHEMA_VERSION generated_at: datetime source: RepoRef + config: ScanConfig = Field( + default_factory=ScanConfig, + description="The scan configuration that produced this report's metrics", + ) data: RepoData = Field(default_factory=RepoData) metrics: Optional[Metrics] = None warnings: list[str] = Field( @@ -462,6 +513,10 @@ class OrgReport(BaseModel): schema_version: str = SCHEMA_VERSION generated_at: datetime source: OrgRef + config: ScanConfig = Field( + default_factory=ScanConfig, + description="The scan configuration that produced this report's metrics", + ) data: OrgData metrics: Optional[OrgMetrics] = None warnings: list[str] = Field(default_factory=list) diff --git a/src/scanner/render.py b/src/scanner/render.py index a6725fb..0b2b868 100644 --- a/src/scanner/render.py +++ b/src/scanner/render.py @@ -279,6 +279,17 @@ def _effective_weights(specs) -> dict[str, float]: REPO_METRIC_WEIGHTS = _effective_weights(REPO_CATEGORIES) ORG_METRIC_WEIGHTS = _effective_weights(ORG_CATEGORIES) +# Display names for the scan-configuration summary. Category names come from the +# specs; metric names are prettified keys (disabled metrics are never computed, +# so a computed Metric.name isn't available for them). +_CATEGORY_NAMES: dict[str, str] = { + c.key: c.name for c in (*REPO_CATEGORIES, *ORG_CATEGORIES) +} + + +def _pretty_metric_name(key: str) -> str: + return key.replace("_", " ").capitalize() + _env = Environment( loader=FileSystemLoader(str(files("scanner") / "templates")), autoescape=True, @@ -370,6 +381,25 @@ def _category_views( return views +def _config_view(config) -> dict[str, Any]: + """Human-readable summary of the scan configuration for the report.""" + return { + "is_default": config.is_default, + "categories": [ + {"key": k, "name": _CATEGORY_NAMES.get(k, k)} + for k in config.disabled_categories + ], + "metrics": [ + {"key": k, "name": _pretty_metric_name(k)} for k in config.disabled_metrics + ], + "components": [ + {"metric": _pretty_metric_name(metric_key), "component": name} + for metric_key, names in config.disabled_components.items() + for name in names + ], + } + + def _shared_context( report: Union[Report, OrgReport], category_views: list[dict[str, Any]] ) -> dict[str, Any]: @@ -395,6 +425,7 @@ def _shared_context( "report_json": report.model_dump_json(indent=2).replace(" {%- endmacro %} +{% macro config_section(cfg) -%} +

Scan configuration

+
+ {% if cfg.is_default %} +

Full methodology. Every category, metric and component is enabled — nothing was excluded from scoring.

+ {% else %} +

This report was produced with a customized configuration. The items below were switched off; the remaining weights were renormalized, so scores stay on the standardized 1–100 scale.

+ {% if cfg.categories %} +
+

Disabled categories

+
{% for c in cfg.categories %}{{ c.name }}{% endfor %}
+
+ {% endif %} + {% if cfg.metrics %} +
+

Disabled metrics

+
{% for m in cfg.metrics %}{{ m.name }}{% endfor %}
+
+ {% endif %} + {% if cfg.components %} +
+

Disabled components

+
{% for c in cfg.components %}{{ c.metric }} · {{ c.component }}{% endfor %}
+
+ {% endif %} + {% endif %} +
+{%- endmacro %} + {% macro warnings_card(warnings) -%} {% if warnings %}
diff --git a/src/scanner/templates/_style.css b/src/scanner/templates/_style.css index b41528b..f46844e 100644 --- a/src/scanner/templates/_style.css +++ b/src/scanner/templates/_style.css @@ -246,3 +246,9 @@ pre.raw { .foot { margin-top: 40px; color: var(--faint); font-size: 12.5px; text-align: center; } .foot p { margin: 4px 0; } + +.cfg-card .cfg-note { margin: 0; color: var(--muted); font-size: 13.5px; max-width: 720px; } +.cfg-card .cfg-note .lucide { width: 15px; height: 15px; vertical-align: -2px; } +.cfg-card .cfg-full { color: #047857; } +.cfg-group { margin-top: 14px; } +.cfg-group h3 { margin: 0 0 8px; font-size: 12px; font-weight: 600; text-transform: uppercase; letter-spacing: .04em; color: var(--faint); } diff --git a/src/scanner/templates/org.html.j2 b/src/scanner/templates/org.html.j2 index ead3eed..b5c8a4f 100644 --- a/src/scanner/templates/org.html.j2 +++ b/src/scanner/templates/org.html.j2 @@ -36,6 +36,8 @@ {{ ui.radar_card("Each axis is a category. " ~ (overall.note if overall and overall.note else "")) }} + {{ ui.config_section(config_view) }} +

Metrics by category

{% for cat in category_views %} {{ ui.category_block(cat) }} diff --git a/src/scanner/templates/report.html.j2 b/src/scanner/templates/report.html.j2 index 91147fd..e51a15c 100644 --- a/src/scanner/templates/report.html.j2 +++ b/src/scanner/templates/report.html.j2 @@ -42,6 +42,8 @@ {% if ownership %}{{ ui.ownership_section(ownership) }}{% endif %} {% if ecosystem_packages %}{{ ui.ecosystem_section(ecosystem_packages) }}{% endif %} + {{ ui.config_section(config_view) }} +

Metrics by category

{% for cat in category_views %} {{ ui.category_block(cat) }} diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..93c736d --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,235 @@ +"""Scan-configuration behavior: enabling/disabling metrics, categories and +components, and how that flows into scores, reports and HTML.""" + +from datetime import datetime, timezone + +import pytest + +from scanner.cli import build_config, build_parser +from scanner.metrics import ( + compute_metrics, + known_category_keys, + known_metric_keys, + validate_config, +) +from scanner.models import ( + Activity, + CommunityHealth, + IssueMetrics, + Maintainership, + OwnerProfile, + Popularity, + QualitySignals, + RepoData, + RepoInfo, + RepoRef, + Report, + ScanConfig, +) +from scanner.render import render_html + + +def _rich_data() -> RepoData: + return RepoData( + owner=OwnerProfile(login="acme", type="Organization", is_verified=True, + followers=900, public_repos=30, account_age_days=2000), + repo=RepoInfo(homepage="https://x.io", topics=["a"], has_wiki=True, + primary_language="Python"), + popularity=Popularity(stars=3000, forks=400, watchers=100), + activity=Activity(days_since_last_push=2, active_weeks_last_year=45, + commits_last_year=300, releases_count=20, + days_since_latest_release=15, mean_days_between_releases=20.0), + maintainership=Maintainership(bus_factor=4, top_contributor_share=0.3, + contributors_sampled=30, + issues=IssueMetrics(closed_ratio=0.85, merged_prs=90, + closed_unmerged_prs=5)), + community=CommunityHealth(has_readme=True, has_license=True, has_contributing=True, + has_description=True), + quality_signals=QualitySignals(has_ci=True, has_tests=True, has_docs_dir=True, + has_linter_config=True), + ) + + +def _parse(argv): + return build_parser().parse_args(argv) + + +# --- model basics ------------------------------------------------------------- + + +def test_default_config_is_default(): + assert ScanConfig().is_default is True + assert ScanConfig(disabled_metrics=["popularity"]).is_default is False + + +def test_default_config_matches_no_config(): + data = _rich_data() + assert compute_metrics(data).overall.value == compute_metrics(data, ScanConfig()).overall.value + + +# --- disabling a category ----------------------------------------------------- + + +def test_disable_category_drops_and_renormalizes(): + data = _rich_data() + full = compute_metrics(data) + custom = compute_metrics(data, ScanConfig(disabled_categories=["security"])) + assert "security" in {c.key for c in full.categories} + assert "security" not in {c.key for c in custom.categories} + # security was the lowest category here, so removing it lifts the overall + assert custom.overall.value > full.overall.value + assert "security" not in custom.overall.inputs + assert "disabled in scan configuration" in (custom.overall.note or "").lower() + + +# --- disabling a metric ------------------------------------------------------- + + +def test_disable_metric_removed_from_category(): + data = _rich_data() + custom = compute_metrics(data, ScanConfig(disabled_metrics=["popularity"])) + community = custom.category("community") + assert community is not None + assert "popularity" not in {m.key for m in community.metrics} + assert custom.by_key("popularity") is None + + +def test_disable_every_metric_in_category_drops_category(): + data = _rich_data() + # Security has a single metric; disabling it removes the whole category. + custom = compute_metrics(data, ScanConfig(disabled_metrics=["security_posture"])) + assert "security" not in {c.key for c in custom.categories} + + +# --- disabling a component ---------------------------------------------------- + + +def test_disable_component_excludes_and_notes(): + data = _rich_data() + custom = compute_metrics(data, ScanConfig(disabled_components={"documentation": ["Wiki"]})) + doc = custom.by_key("documentation") + by = {c.name: c for c in doc.components} + assert by["Wiki"].status == "excluded" + assert by["Wiki"].detail == "disabled in scan configuration" + assert by["Wiki"].points == 0.0 + assert "Wiki" in (doc.note or "") + assert "renormalized" in (doc.note or "").lower() + + +def test_disable_component_changes_score_via_renormalization(): + # Documentation with only README + Wiki met; disabling Wiki renormalizes, + # so the same met README is now a larger share of the (smaller) total. + data = RepoData( + repo=RepoInfo(has_wiki=True), + community=CommunityHealth(has_readme=True), + ) + full = compute_metrics(data).by_key("documentation") + without_wiki = compute_metrics( + data, ScanConfig(disabled_components={"documentation": ["Wiki"]}) + ).by_key("documentation") + assert without_wiki.value != full.value + + +def test_disable_all_components_drops_metric(): + data = RepoData(popularity=Popularity(stars=100, forks=10, watchers=5)) + cfg = ScanConfig(disabled_components={"popularity": ["Stars", "Forks", "Watchers"]}) + metrics = compute_metrics(data, cfg) + assert metrics.by_key("popularity") is None + + +# --- validation --------------------------------------------------------------- + + +def test_known_keys_cover_repo_and_org(): + assert {"vitality", "security"} <= known_category_keys() + assert {"popularity", "security_posture", "portfolio_activity"} <= known_metric_keys() + + +def test_validate_config_flags_unknown_keys(): + warnings = validate_config(ScanConfig( + disabled_categories=["nope"], disabled_metrics=["also_nope"], + disabled_components={"missing_metric": ["x"]}, + )) + assert any("nope" in w for w in warnings) + assert any("also_nope" in w for w in warnings) + assert any("missing_metric" in w for w in warnings) + + +def test_validate_config_accepts_known_keys(): + assert validate_config(ScanConfig( + disabled_categories=["security"], disabled_metrics=["popularity"], + disabled_components={"documentation": ["Wiki"]}, + )) == [] + + +# --- report embedding --------------------------------------------------------- + + +def _report(config: ScanConfig) -> Report: + data = _rich_data() + return Report( + generated_at=datetime(2026, 7, 6, 12, 0, tzinfo=timezone.utc), + source=RepoRef(url="acme/widget", owner="acme", name="widget"), + config=config, + data=data, + metrics=compute_metrics(data, config), + ) + + +def test_report_embeds_and_roundtrips_config(): + cfg = ScanConfig(disabled_metrics=["popularity"], disabled_components={"documentation": ["Wiki"]}) + report = _report(cfg) + restored = Report.model_validate_json(report.model_dump_json()) + assert restored.config == cfg + + +# --- rendering ---------------------------------------------------------------- + + +def test_render_shows_full_methodology_by_default(): + html = render_html(_report(ScanConfig())) + assert "Scan configuration" in html + assert "Full methodology" in html + + +def test_render_lists_disabled_items(): + cfg = ScanConfig( + disabled_categories=["security"], + disabled_metrics=["popularity"], + disabled_components={"documentation": ["Wiki"]}, + ) + html = render_html(_report(cfg)) + assert "customized configuration" in html + assert "Disabled categories" in html and "Security" in html + assert "Disabled metrics" in html and "Popularity" in html + assert "Disabled components" in html and "Wiki" in html + + +# --- CLI config building ------------------------------------------------------ + + +def test_build_config_from_flags(): + args = _parse([ + "owner/name", + "--disable-category", "security", + "--disable-metric", "popularity", + "--disable-component", "documentation:Wiki", + ]) + cfg = build_config(args) + assert cfg.disabled_categories == ["security"] + assert cfg.disabled_metrics == ["popularity"] + assert cfg.disabled_components == {"documentation": ["Wiki"]} + + +def test_build_config_merges_file_and_flags(tmp_path): + path = tmp_path / "cfg.json" + path.write_text('{"disabled_metrics": ["popularity"]}', encoding="utf-8") + args = _parse(["owner/name", "--config", str(path), "--disable-metric", "stewardship"]) + cfg = build_config(args) + assert cfg.disabled_metrics == ["popularity", "stewardship"] + + +def test_build_config_rejects_malformed_component(): + args = _parse(["owner/name", "--disable-component", "no-colon-here"]) + with pytest.raises(ValueError): + build_config(args)