Check an AI summary sentence by sentence against the source

S1|The pilot processed 20 test cases.
S2|Eighteen cases completed successfully.
S3|Two cases stopped during input validation before the solver started.
S4|Among the 18 completed cases, the median runtime was 42 seconds.
S5|The test was performed on one workstation under the same software configuration.
S6|No production deployment or long-term reliability test was performed.

A1|The pilot processed 20 cases, and 18 completed successfully.
A2|Two cases failed because the solver crashed.
A3|The median runtime for completed cases was 42 seconds.
A4|The results show that the automation is reliable enough for production use.

summary_id,verdict,evidence,reason
A1,SUPPORTED,S1;S2,The case count and successful completions match the source.
A2,UNSUPPORTED,S3,The source attributes the stops to input validation before solver start.
A3,SUPPORTED,S4,The 42-second median is stated for the 18 completed cases.
A4,UNSUPPORTED,S6,Production readiness was not tested.

import csv
from pathlib import Path

SOURCE = Path("source.txt")
SUMMARY = Path("ai_summary.txt")
REVIEW = Path("summary_review.csv")
OUTPUT_DIR = Path("outputs") / "summary_source_check"
OUTPUT = OUTPUT_DIR / "validated_review.csv"
VALID_VERDICTS = {"SUPPORTED", "PARTIAL", "UNSUPPORTED"}


def read_id_text(path: Path) -> dict[str, str]:
    result = {}
    with path.open("r", encoding="utf-8") as stream:
        for line_number, line in enumerate(stream, start=1):
            line = line.rstrip("\n")
            if not line:
                continue
            item_id, separator, text = line.partition("|")
            if not separator or not item_id or not text:
                raise ValueError(f"Invalid line {line_number} in {path}.")
            if item_id in result:
                raise ValueError(f"Duplicate ID {item_id} in {path}.")
            result[item_id] = text
    return result


def main() -> None:
    for path in (SOURCE, SUMMARY, REVIEW):
        if not path.is_file():
            raise FileNotFoundError(f"Missing input: {path}")
    if OUTPUT_DIR.exists():
        raise FileExistsError(f"Output folder already exists: {OUTPUT_DIR}")

    source = read_id_text(SOURCE)
    summary = read_id_text(SUMMARY)

    reviewed = {}
    with REVIEW.open("r", encoding="utf-8-sig", newline="") as stream:
        reader = csv.DictReader(stream)
        required = {"summary_id", "verdict", "evidence", "reason"}
        if reader.fieldnames is None or not required.issubset(reader.fieldnames):
            raise ValueError("Review CSV is missing a required column.")

        for row in reader:
            summary_id = row["summary_id"].strip()
            verdict = row["verdict"].strip()
            evidence_ids = [x.strip() for x in row["evidence"].split(";") if x.strip()]

            if summary_id not in summary:
                raise ValueError(f"Unknown summary ID: {summary_id}")
            if summary_id in reviewed:
                raise ValueError(f"Duplicate review: {summary_id}")
            if verdict not in VALID_VERDICTS:
                raise ValueError(f"Invalid verdict for {summary_id}: {verdict}")
            if not evidence_ids:
                raise ValueError(f"No evidence listed for {summary_id}.")
            unknown = [item for item in evidence_ids if item not in source]
            if unknown:
                raise ValueError(f"Unknown source IDs for {summary_id}: {unknown}")
            if not row["reason"].strip():
                raise ValueError(f"Missing reason for {summary_id}.")

            reviewed[summary_id] = {
                "summary_id": summary_id,
                "summary_text": summary[summary_id],
                "verdict": verdict,
                "evidence": ";".join(evidence_ids),
                "reason": row["reason"].strip(),
            }

    missing = [item for item in summary if item not in reviewed]
    if missing:
        raise ValueError(f"Summary sentences not reviewed: {missing}")

    OUTPUT_DIR.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT_DIR.mkdir()
    fields = ["summary_id", "summary_text", "verdict", "evidence", "reason"]
    with OUTPUT.open("x", encoding="utf-8", newline="") as stream:
        writer = csv.DictWriter(stream, fieldnames=fields)
        writer.writeheader()
        for summary_id in summary:
            writer.writerow(reviewed[summary_id])

    counts = {verdict: 0 for verdict in VALID_VERDICTS}
    for row in reviewed.values():
        counts[row["verdict"]] += 1

    print(f"Summary sentences checked: {len(summary)}.")
    print(
        f"Supported: {counts['SUPPORTED']}; partial: {counts['PARTIAL']}; "
        f"unsupported: {counts['UNSUPPORTED']}."
    )
    print(f"Output: {OUTPUT.as_posix()}")


if __name__ == "__main__":
    main()

Summary sentences checked: 4.
Supported: 2; partial: 0; unsupported: 2.
Output: outputs/summary_source_check/validated_review.csv