"""Derive a composition census from public aggregate JSON, offline and read-only. Usage: python analyze-composition.py reference-record.json > composition-analysis.json The method reads JSON metadata only: no ZIPs, client payloads, personal files, uploads, network requests, or binary execution. It copies the input inspection date; it does not claim a new inspection of the original archive. Counts and declared byte sizes must be integers from 0 through 2**53 - 1, supporting exact integer interchange with the JavaScript public renderer. """ import datetime import json import math from pathlib import Path import re import sys METHOD_VERSION = '1.1' MAX_SAFE_INTEGER = 2**53 - 1 LABEL = re.compile(r'(?:\.[a-z0-9]{1,10}|\(other/no extension\))') def nonnegative_integer(value, field): if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise ValueError(field + ' must be a nonnegative integer') if value > MAX_SAFE_INTEGER: raise ValueError(field + ' exceeds the supported maximum of 2**53 - 1') return value def finite_round(value): return round(value, 12) if value is not None else None def distribution(rows, field): """Weight a random file or random declared byte by its extension label.""" ranked = sorted(rows, key=lambda row: (-row[field], row['label'])) positive = [row for row in ranked if row[field] > 0] total = sum(row[field] for row in positive) entropy = None if total: entropy = max(0.0, -math.fsum((row[field] / total) * math.log2(row[field] / total) for row in sorted(positive, key=lambda row: row['label']))) def concentration(count): selected = positive[:count] weight = sum(row[field] for row in selected) return {'labels': [row['label'] for row in selected], 'weight': weight, 'share': finite_round(weight / total) if total else None} return {'totalWeight': total, 'supportLabels': len(positive), 'entropyBits': finite_round(entropy), 'effectiveLabels': finite_round(2 ** entropy) if entropy is not None else None, 'top1': concentration(1), 'top5': concentration(5)} def analyze(record): if not isinstance(record, dict): raise ValueError('input must be a JSON object') digest = record.get('zipSHA256') if not isinstance(digest, str) or not re.fullmatch(r'[a-f0-9]{64}', digest): raise ValueError('zipSHA256 must identify the reference with 64 lowercase hexadecimal characters') source_date = record.get('inspectedOn') if not isinstance(source_date, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', source_date): raise ValueError('inspectedOn must be a source inspection date in YYYY-MM-DD form') try: datetime.date.fromisoformat(source_date) except ValueError: raise ValueError('inspectedOn must be a valid date') from None files = nonnegative_integer(record.get('files'), 'files') byte_count = nonnegative_integer(record.get('uncompressedBytes'), 'uncompressedBytes') extensions = record.get('extensions') if not isinstance(extensions, dict): raise ValueError('extensions must be an object') rows = [] for label, values in sorted(extensions.items()): if not isinstance(label, str) or not LABEL.fullmatch(label): raise ValueError('extension labels must use the public census vocabulary') if not isinstance(values, dict): raise ValueError('extension records must be objects') count = nonnegative_integer(values.get('files'), 'extension file count') size = nonnegative_integer(values.get('bytes'), 'extension byte count') if count == 0 and size != 0: raise ValueError('a label without files cannot contain bytes') rows.append({'label': label, 'files': count, 'bytes': size}) if sum(row['files'] for row in rows) != files: raise ValueError('extension file counts do not reconcile with files') if sum(row['bytes'] for row in rows) != byte_count: raise ValueError('extension bytes do not reconcile with uncompressedBytes') for row in rows: row.update(fileShare=finite_round(row['files'] / files) if files else None, byteShare=finite_round(row['bytes'] / byte_count) if byte_count else None, meanBytesPerFile=finite_round(row['bytes'] / row['files']) if row['files'] else None) return { 'schemaVersion': 1, 'methodVersion': METHOD_VERSION, 'inputFile': 'reference-record.json', 'referenceZIP_SHA256': digest, 'sourceInspectionDate': source_date, 'scope': 'Completed derived observations from the published extension census; no new client-payload inspection.', 'inputTotals': {'files': files, 'uncompressedBytes': byte_count, 'labelCount': len(rows)}, 'reconciliation': {'fileCountMatches': True, 'byteCountMatches': True}, 'metrics': {'byFileCount': distribution(rows, 'files'), 'byByteWeight': distribution(rows, 'bytes')}, 'rows': rows, 'rankedByFiles': sorted(rows, key=lambda row: (-row['files'], row['label'])), 'rankedByBytes': sorted(rows, key=lambda row: (-row['bytes'], row['label'])), 'method': { 'inputKind': 'public aggregate JSON metadata only', 'clientPayloadsRead': False, 'networkConnections': False, 'binariesExecuted': False, 'supportedIntegerRange': 'Counts and declared byte sizes: 0 through 9007199254740991 (2**53 - 1), inclusive.', 'shares': 'Weight divided by the corresponding positive total; stored as fractions, not percentages.', 'entropy': 'H = -sum(p * log2(p)) over positive label weights; effectiveLabels = 2**H.', 'undefinedDistribution': 'Zero total weight produces null shares, entropy, and effective label count.', 'ties': 'Descending weight, then ascending label; tied leaders are ordered, not treated as uniquely dominant.', 'numericPrecision': 'Derived fractions, means, entropies, and effective counts rounded to 12 decimal places; integer totals remain exact.', }, 'limitations': [ 'Filename extension labels were not validated as file formats or versions.', 'Bytes are declared uncompressed member sizes, not storage allocation, compressed size, or observed memory use.', 'Label entropy describes this categorical distribution, not the entropy of payload bytes, software complexity, or cultural significance.', 'Label means do not reveal within-label variation, maximum member size, or outliers.', 'Reconciliation checks consistency of the published metadata; it does not reauthenticate its source archive.', 'Counts and declared byte sizes above 2**53 - 1 are rejected rather than approximated or converted to unbounded floating-point statistics.', ], } if __name__ == '__main__': if len(sys.argv) != 2: raise SystemExit('Usage: python analyze-composition.py reference-record.json > composition-analysis.json') try: with Path(sys.argv[1]).open(encoding='utf-8-sig') as stream: result = analyze(json.load(stream)) except (ValueError, TypeError) as error: raise SystemExit('Invalid composition metadata: ' + str(error)) from None print(json.dumps(result, indent=2, ensure_ascii=True, allow_nan=False))