#!/usr/bin/env python3 """Original aggregate-only audit of one frozen Anvisa device-catalogue file. The input is separately acquired and remains private. This program does not download data or emit source names, identifiers, corporate fields or raw rows. No clinical, legal-status or physical-device inference is performed. """ from __future__ import annotations import argparse import csv import hashlib import io import json import re import unicodedata from collections import Counter, defaultdict from datetime import datetime, timezone from pathlib import Path SOURCE_SHA = "16df7e01468ad666465d4b516d17d85c200547478a61bc7310c519ea83179e69" PROTOCOL_SHA = "368b90380ffbe843b294ce4086681c2efba4a47989f832499c51a10d04191879" HEADER = [ "NUMERO_REGISTRO_CADASTRO", "NUMERO_PROCESSO", "NOME_TECNICO", "CLASSE_RISCO", "NOME_COMERCIAL", "CNPJ_DETENTOR_REGISTRO_CADASTRO", "DETENTOR_REGISTRO_CADASTRO", "NOME_FABRICANTE", "NOME_PAIS_FABRIC", "DT_PUB_REGISTRO_CADASTRO", "VALIDADE_REGISTRO_CADASTRO", "DT_ATUALIZACAO_DADO", ] def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def normalized(value: str) -> str: return " ".join(unicodedata.normalize("NFC", value).split()).casefold() def text_status(value: bytes) -> tuple[str, str | None]: try: decoded = value.decode("cp1252", errors="strict") except UnicodeDecodeError: return "undefined_cp1252", None if not decoded.strip(): return "blank", None if '"' in decoded: return "quote_unknown", None return "known", decoded def load(input_path: Path) -> tuple[dict, set, set, set]: raw = input_path.read_bytes() if hashlib.sha256(raw).hexdigest() != SOURCE_SHA: raise ValueError("Input differs from the frozen source; stop, do not mix editions.") if not raw.endswith(b"\n"): raise ValueError("Unexpected final-record framing.") lines = raw.split(b"\n")[:-1] if lines[0].decode("ascii").split(";") != HEADER: raise ValueError("Header differs from the source-specific protocol.") fields = {name: i for i, name in enumerate(HEADER)} ids: set[str] = set() pairs: set[tuple[str, str]] = set() triples: set[tuple[str, str, str]] = set() id_names: dict[str, set] = defaultdict(set) id_technical: dict[str, set] = defaultdict(set) commercial_status = Counter() technical_status = Counter() commercial_unknown_ids: dict[str, set] = defaultdict(set) technical_unknown_ids: dict[str, set] = defaultdict(set) any_name_unknown: set[str] = set() any_technical_unknown: set[str] = set() id_lengths = Counter() id_problems = Counter() encoding_by_field = Counter() quotes_by_field = Counter() dates = Counter() unparsed_date_rows = 0 strict_record_failures = 0 exact_rows = set() suspicious_name_markers = Counter() name_whitespace_rows = 0 ids_with_any_usable_triple = set() records_with_pairs = 0 records_with_triples = 0 for physical_line in lines[1:]: if b"\r" in physical_line: raise ValueError("Unexpected CR inside this LF-framed frozen source.") cells = physical_line.split(b";") if len(cells) != len(HEADER): raise ValueError("Non-12-field physical record; no repair permitted.") exact_rows.add(physical_line) for key, value in zip(HEADER, cells): try: value.decode("cp1252", errors="strict") except UnicodeDecodeError: encoding_by_field[key] += 1 if b'"' in value: quotes_by_field[key] += 1 id_status, identifier = text_status(cells[fields["NUMERO_REGISTRO_CADASTRO"]]) if id_status != "known" or identifier is None: id_problems[id_status] += 1 continue if identifier != identifier.strip() or not re.fullmatch(r"[0-9]+", identifier): id_problems["whitespace_or_nonnumeric"] += 1 ids.add(identifier) id_lengths[str(len(identifier))] += 1 try: standard = list(csv.reader(io.StringIO(physical_line.decode("latin1")), delimiter=";", quotechar='"', strict=True)) record_accepted = (len(standard) == 1 and len(standard[0]) == len(HEADER) and [v.encode("latin1") for v in standard[0]] == cells) except csv.Error: record_accepted = False if not record_accepted: strict_record_failures += 1 commercial_kind, commercial = text_status(cells[fields["NOME_COMERCIAL"]]) technical_kind, technical = text_status(cells[fields["NOME_TECNICO"]]) if not record_accepted: commercial_kind, commercial = "record_syntax_unknown", None technical_kind, technical = "record_syntax_unknown", None commercial_status[commercial_kind] += 1 technical_status[technical_kind] += 1 if commercial_kind != "known": commercial_unknown_ids[commercial_kind].add(identifier) any_name_unknown.add(identifier) if technical_kind != "known": technical_unknown_ids[technical_kind].add(identifier) any_technical_unknown.add(identifier) if commercial is not None: records_with_pairs += 1 pairs.add((identifier, commercial)) id_names[identifier].add(commercial) if commercial != commercial.strip() or re.search(r"\s{2,}", commercial): name_whitespace_rows += 1 marker = commercial.strip().casefold() if marker in {"null", "none", "n/a", "na", "não informado", "nao informado", "sem nome"}: suspicious_name_markers[marker] += 1 if technical is not None: id_technical[identifier].add(technical) if commercial is not None and technical is not None: records_with_triples += 1 triples.add((identifier, commercial, technical)) ids_with_any_usable_triple.add(identifier) date_bytes = cells[fields["DT_ATUALIZACAO_DADO"]] try: date_text = date_bytes.decode("cp1252", errors="strict") if not re.fullmatch(r"[0-9/ :.-]+", date_text): raise ValueError("Unexpected metadata date syntax.") dates[date_text] += 1 except (ValueError, UnicodeDecodeError): unparsed_date_rows += 1 if id_problems: raise ValueError("Identifier form unresolved; review before outcome arithmetic: " + json.dumps(id_problems)) if suspicious_name_markers: raise ValueError("Placeholder-like names require explicit review before outcomes.") audit = { "source_sha256": SOURCE_SHA, "source_bytes": len(raw), "physical_records_including_header": len(lines), "data_rows": len(lines) - 1, "distinct_byte_exact_data_rows": len(exact_rows), "byte_exact_duplicate_rows": len(lines) - 1 - len(exact_rows), "field_count_each_record": len(HEADER), "distinct_exact_registration_ids": len(ids), "id_row_length_histogram": dict(sorted(id_lengths.items())), "identifier_form_problems": dict(id_problems), "encoding_failure_row_counts_by_field": dict(encoding_by_field), "ASCII_quote_row_counts_by_field": dict(quotes_by_field), "strict_standard_record_quarantined_rows": strict_record_failures, "commercial_row_status": dict(commercial_status), "technical_row_status": dict(technical_status), "commercial_unknown_distinct_ids_by_reason": {k: len(v) for k, v in commercial_unknown_ids.items()}, "technical_unknown_distinct_ids_by_reason": {k: len(v) for k, v in technical_unknown_ids.items()}, "ids_with_any_unknown_commercial_row": len(any_name_unknown), "ids_with_any_unknown_technical_row": len(any_technical_unknown), "ids_with_any_unknown_commercial_or_technical_row": len(any_name_unknown | any_technical_unknown), "ids_with_all_commercial_rows_interpretable": len(ids - any_name_unknown), "ids_with_all_commercial_and_technical_rows_interpretable": len(ids - any_name_unknown - any_technical_unknown), "ids_with_at_least_one_usable_commercial_name": len(id_names), "ids_with_at_least_one_usable_complete_triple": len(ids_with_any_usable_triple), "records_with_known_ID_name_pair": records_with_pairs, "distinct_known_ID_name_pairs": len(pairs), "records_with_known_complete_triple": records_with_triples, "distinct_known_complete_triples": len(triples), "ids_with_multiple_known_commercial_names": sum(len(v) > 1 for v in id_names.values()), "ids_with_multiple_known_technical_names": sum(len(v) > 1 for v in id_technical.values()), "known_commercial_rows_with_outer_or_repeated_whitespace": name_whitespace_rows, "placeholder_like_name_rows": dict(suspicious_name_markers), "data_update_field_literal_row_counts": dict(sorted(dates.items())), "data_update_date_order_and_timezone": "Unverified; retain the original literal. No ISO conversion or cutoff inference.", "unparsed_data_update_rows": unparsed_date_rows, "source_names_IDs_rows_or_corporate_values_emitted": False, } assert sum(commercial_status.values()) == audit["data_rows"] assert sum(technical_status.values()) == audit["data_rows"] assert len(pairs) <= records_with_pairs <= audit["data_rows"] assert len(triples) <= records_with_triples <= audit["data_rows"] return audit, ids, pairs, triples def grouped(pairs: set, ids: set, use_normalization: bool) -> dict: members = defaultdict(set) for identifier, name in pairs: members[normalized(name) if use_normalization else name].add(identifier) shared = {key for key, value in members.items() if len(value) >= 2} affected = set().union(*(members[key] for key in shared)) if shared else set() buckets = {key: {"labels": 0, "ID_name_memberships": 0} for key in ["1", "2", "3", "4", "5+"]} for value in members.values(): bucket = str(len(value)) if len(value) < 5 else "5+" buckets[bucket]["labels"] += 1 buckets[bucket]["ID_name_memberships"] += len(value) assert sum(v["labels"] for v in buckets.values()) == len(members) assert sum(v["ID_name_memberships"] for v in buckets.values()) == sum(map(len, members.values())) return { "name_key": "NFC_whitespace_casefold" if use_normalization else "literal_exact", "known_distinct_labels": len(members), "labels_attached_to_at_least_two_IDs": len(shared), "known_ID_name_memberships_after_keying": sum(map(len, members.values())), "memberships_in_shared_labels": sum(len(members[key]) for key in shared), "distinct_IDs_in_confirmed_shared_labels": len(affected), "ALL_source_ID_denominator": len(ids), "shared_label_percentage_of_known_label_universe": 100 * len(shared) / len(members), "confirmed_affected_ID_percentage_of_ALL_source_IDs_lower_bound": 100 * len(affected) / len(ids), "label_multiplicity_buckets": buckets, } def technical_comparison(triples: set, use_normalization: bool) -> dict: # T is the ORIGINAL complete distinct literal-triple population, fixed once. # Normalization changes lookup keys only; never deduplicate/reweight T. frame = [(i, normalized(n), normalized(t)) if use_normalization else (i, n, t) for i, n, t in triples] name_ids = defaultdict(set) combined_ids = defaultdict(set) for identifier, name, technical in frame: name_ids[name].add(identifier) combined_ids[(name, technical)].add(identifier) name_affected = sum(len(name_ids[name]) >= 2 for _, name, _ in frame) combined_affected = sum(len(combined_ids[(name, technical)]) >= 2 for _, name, technical in frame) assert combined_affected <= name_affected <= len(frame) return { "key_convention": "NFC_whitespace_casefold" if use_normalization else "literal_exact", "fixed_complete_distinct_triple_memberships": len(frame), "distinct_IDs_represented_in_fixed_T": len({i for i, _, _ in frame}), "triple_memberships_with_shared_commercial_key": name_affected, "triple_memberships_with_shared_commercial_and_technical_key": combined_affected, "triple_memberships_no_longer_shared_under_refined_key": name_affected - combined_affected, "shared_commercial_percentage_of_T": 100 * name_affected / len(frame), "shared_combined_percentage_of_same_T": 100 * combined_affected / len(frame), "interpretation": "Triple memberships, not physical products. Refinement is monotone by construction; no official-search or user-outcome test.", } def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--input", type=Path, required=True) ap.add_argument("--protocol", type=Path, required=True) ap.add_argument("--output", type=Path, required=True) ap.add_argument("--allow-analysis", action="store_true", help="Only after independent source/ID gate.") args = ap.parse_args() if digest(args.protocol) != PROTOCOL_SHA: raise ValueError("Protocol differs from the frozen pre-outcome amendment.") started = datetime.now(timezone.utc).isoformat() audit, ids, pairs, triples = load(args.input) result = {"started_at_utc": started, "source_audit": audit, "protocol_sha256": PROTOCOL_SHA, "script_sha256": digest(Path(__file__)), "outcome_arithmetic": args.allow_analysis} if args.allow_analysis: result["primary_literal"] = grouped(pairs, ids, False) result["secondary_conservative_normalization"] = grouped(pairs, ids, True) result["technical_comparison_literal"] = technical_comparison(triples, False) result["technical_comparison_normalized"] = technical_comparison(triples, True) result["completed_at_utc"] = datetime.now(timezone.utc).isoformat() args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps({"output": str(args.output), "sha256": digest(args.output), "outcome_arithmetic": args.allow_analysis}, ensure_ascii=False)) if __name__ == "__main__": main()