"""MIT. Source-relative verification only; never executes model-generated code."""
import json
from pathlib import Path

LABELS = {'supported', 'contradicted', 'not_in_sources'}


def score(item, arguments, config):
    valid = (set(arguments) == {'claim_id', 'label', 'quote', 'reason'}
             and arguments.get('claim_id') == item['claim_id']
             and arguments.get('label') in LABELS
             and isinstance(arguments.get('quote'), str)
             and isinstance(arguments.get('reason'), str))
    label = arguments['label'] if valid else 'invalid'
    seeded = item['seeded_error']
    caught = valid and label in {'contradicted', 'not_in_sources'}
    quote = arguments.get('quote', '') if valid else ''
    quote_match = bool(quote.strip()) and any(quote in source for source in item['sources'])
    # Recall denominator includes only planted errors; supported claims have zero counts.
    return {'tp': int(seeded and caught), 'fp': int(not seeded and caught),
            'fn': int(seeded and not caught), 'secondary': {
                'seeded_errors': {'score': int(seeded)},
                'caught_errors': {'score': int(seeded and caught)},
                'false_alarms': {'score': int(not seeded and caught)},
                'false_alarm_rate': {'tp': int(not seeded and caught), 'fp': 0,
                                     'fn': int(not seeded and not caught)},
                'quote_match_rate': {'tp': int(quote_match), 'fp': int(bool(quote.strip()) and not quote_match), 'fn': 0},
                'label_accuracy': {'gold': item['gold'], 'prediction': label},
                'invalid_output_rate': {'score': int(not valid)},
            }}


def self_test():
    item = {'claim_id': 'C1', 'seeded_error': True, 'gold': 'contradicted', 'sources': ['The observatory opened in 2010.']}
    args = {'claim_id': 'C1', 'label': 'contradicted', 'quote': 'opened in 2010', 'reason': 'The claim gives another year.'}
    assert score(item, args, {})['tp'] == 1
    assert score(item, {}, {})['fn'] == 1
    assert score(item, dict(args, claim_id='C2'), {})['secondary']['invalid_output_rate']['score'] == 1
    supported = dict(item, seeded_error=False, gold='supported')
    assert score(supported, args, {})['secondary']['false_alarms']['score'] == 1
    assert score(item, dict(args, quote='invented quotation'), {})['secondary']['quote_match_rate']['fp'] == 1
    print('PASS: catch, miss, false alarm, quote mismatch and wrong-id checks')


if __name__ == '__main__':
    self_test()