Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions docs/how-to/dashboards_and_quality_gates.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,22 @@ The documentation build writes ``metrics.json`` via ``score_metrics``, and the `

The dashboard charts and the CI gate both use the same computed metrics.

``metrics.json`` content
~~~~~~~~~~~~~~~~~~~~~~~~

The exported file contains a small, versioned set of fields that is kept
deliberately stable:

- ``overall_metrics`` – requirement coverage aggregated over all requirement types.
- ``metrics_by_type`` – the same coverage numbers per requirement type.
- ``tests`` – testcase linkage statistics, including broken test references.
- ``needs_overview`` – generic overview of the needs in the metrics scope
(respecting ``score_metrics_include_external_needs``): ``total``,
``external``, ``local`` and ``by_type`` counts.

Only data that is broadly useful is added here, as it is very hard to remove
fields later.

Inputs for Linkage Metrics
--------------------------

Expand Down
6 changes: 6 additions & 0 deletions scripts_bazel/tests/traceability_gate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ def _write_metrics_json(
"overall_metrics": _derive_overall_metrics(metrics_by_type),
"metrics_by_type": metrics_by_type,
"tests": tests,
"needs_overview": {
"total": 7,
"external": 0,
"local": 7,
"by_type": {"testcase": 3, "tool_req": 4},
},
}

out = tmp_path / "metrics.json"
Expand Down
40 changes: 39 additions & 1 deletion scripts_bazel/traceability_metrics_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
"generated_by",
"overall_metrics",
"metrics_by_type",
"tests"
"tests",
"needs_overview"
],
"properties": {
"schema_version": {
Expand Down Expand Up @@ -40,6 +41,43 @@
},
"tests": {
"$ref": "#/$defs/testMetrics"
},
"needs_overview": {
"type": "object",
"description": "Overview of all needs in the metrics scope, respecting the include_external setting.",
"additionalProperties": false,
"required": [
"total",
"external",
"local",
"by_type"
],
"properties": {
"total": {
"type": "integer",
"minimum": 0
},
"external": {
"type": "integer",
"minimum": 0
},
"local": {
"type": "integer",
"minimum": 0
},
"by_type": {
"type": "object",
"description": "Count of needs per need type.",
"additionalProperties": {
"type": "integer",
"minimum": 0
},
"propertyNames": {
"type": "string",
"minLength": 1
}
}
}
}
},
"$defs": {
Expand Down
72 changes: 72 additions & 0 deletions src/extensions/score_metrics/tests/test_traceability_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
# SPDX-License-Identifier: Apache-2.0
# *******************************************************************************

from __future__ import annotations

from typing import Any, cast

import pytest
Expand All @@ -23,6 +25,21 @@
from score_pytest.attribute_plugin import add_test_properties


class _OverviewNeedsView:
"""Minimal NeedsView-like test double with external filtering."""

def __init__(self, needs: list[dict[str, Any]]) -> None:
self._needs = needs

def values(self) -> list[dict[str, Any]]:
return self._needs

def filter_is_external(self, is_external: bool) -> _OverviewNeedsView:
return _OverviewNeedsView(
[n for n in self._needs if n.get("is_external", False) is is_external]
)


@add_test_properties(
partially_verifies=["tool_req__docs_test_linkage_metrics"],
test_type="requirements-based",
Expand Down Expand Up @@ -261,3 +278,58 @@ def test_calculate_test_metrics_counts_linked_tests_and_broken_refs() -> None:
assert result["broken_references"] == [
{"testcase": "TC_2", "missing_need": "REQ_404"}
]


@add_test_properties(
partially_verifies=["tool_req__docs_test_linkage_metrics"],
test_type="requirements-based",
derivation_technique="requirements-analysis",
)
def test_calculate_needs_overview_counts_types_and_external_split() -> None:
"""Count all needs with their type and external/local split."""
needs = _OverviewNeedsView(
[
{"id": "REQ_1", "type": "tool_req", "is_external": False},
{"id": "REQ_2", "type": "tool_req", "is_external": False},
{"id": "REQ_3", "type": "comp_req", "is_external": False},
{"id": "EXT_1", "type": "tool_req", "is_external": True},
{"id": "TC_1", "type": "testcase", "is_external": False},
]
)

result = metrics.calculate_needs_overview(needs, include_external=True)

assert result["total"] == 5
assert result["external"] == 1
assert result["local"] == 4
assert result["by_type"] == {
"comp_req": 1,
"testcase": 1,
"tool_req": 3,
}


@add_test_properties(
partially_verifies=["tool_req__docs_test_linkage_metrics"],
test_type="requirements-based",
derivation_technique="requirements-analysis",
)
def test_calculate_needs_overview_excludes_external_when_not_included() -> None:
"""Count only local needs when external needs are not in the metrics scope."""
needs = _OverviewNeedsView(
[
{"id": "REQ_1", "type": "tool_req", "is_external": False},
{"id": "REQ_2", "type": "comp_req", "is_external": False},
{"id": "EXT_1", "type": "tool_req", "is_external": True},
]
)

result = metrics.calculate_needs_overview(needs, include_external=False)

assert result["total"] == 2
assert result["external"] == 0
assert result["local"] == 2
assert result["by_type"] == {
"comp_req": 1,
"tool_req": 1,
}
35 changes: 35 additions & 0 deletions src/extensions/score_metrics/traceability_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,40 @@ def calculate_test_metrics(
}


def calculate_needs_overview(
all_needs: NeedsView, include_external: bool
) -> dict[str, Any]:
"""Summarize the needs in scope of the build.

Counts every need (not only requirements and tests) that is part of the
metrics scope, respecting the ``include_external`` setting. The result is
deliberately small and stable: total, external/local split and a count per
need type. It is a generic overview that is hard to derive from the other
sections, because they only cover requirements and testcases.
"""
if include_external:
scoped_needs = list(all_needs.values())
else:
scoped_needs = list(all_needs.filter_is_external(False).values())

total = 0
external = 0
by_type: dict[str, int] = {}
for need in scoped_needs:
total += 1
if need.get("is_external"):
external += 1
need_type = str(need.get("type", ""))
by_type[need_type] = by_type.get(need_type, 0) + 1

return {
"total": total,
"external": external,
"local": total - external,
"by_type": dict(sorted(by_type.items())),
}


def calculate_full_need_metrics(app: Sphinx, include_external: bool):
"""
Calculate all tracked metrics for requirements and tests.
Expand Down Expand Up @@ -188,6 +222,7 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool):
"overall_metrics": overall_metrics,
"metrics_by_type": metrics_by_type,
"tests": test_metrics,
"needs_overview": calculate_needs_overview(all_needs, include_external),
}
# Save the metrics in a Global Variable to enable access from other parts.
# Not a great solution but it is needed, as needpie filter functions for example
Expand Down
Loading