Files

75 lines
4.2 KiB
Python

#!/usr/bin/env python3
"""Qualify artifacts compiled from installed PBS references using the existing runtime harness."""
import argparse
import importlib.util
import json
from pathlib import Path
import shutil
import sys
import tempfile
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--artifacts', type=Path, required=True)
parser.add_argument('--distribution', type=Path, required=True)
parser.add_argument('--runtime-headless', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
sys.dont_write_bytecode = True
spec = importlib.util.spec_from_file_location('pbs_corpus', Path(__file__).with_name('run.py'))
harness = importlib.util.module_from_spec(spec)
spec.loader.exec_module(harness)
artifacts = args.artifacts.resolve(strict=True)
distribution = args.distribution.resolve(strict=True)
binary = args.runtime_headless.resolve(strict=True)
output = args.output.resolve()
if output.exists():
parser.error('Output must be a new directory; evidence is never overwritten')
output.mkdir(parents=True)
compilation = json.loads((artifacts / 'compilation.json').read_text())
assert compilation['passed']
jars = {p.name: harness.sha(p) for p in (distribution / 'lib').glob('*.jar')}
assert jars == compilation['provenance']['installedJarSha256'], 'Compiled artifacts must match this distribution'
scratch = Path(tempfile.mkdtemp(prefix='pbs-installed-contracts-'))
observations = []
for case in compilation['cases']:
expected = case['expectation']['expectedRuntime']
if expected == 'not-run':
continue
assert case['compileStatus'] == 'compiled' and case['matchesExpectation'] and case['sourceUnchanged']
cartridge = artifacts / case['id']
assert harness.sha(cartridge / 'program.pbx') == case['bytecodeSha256']
assert harness.sha(cartridge / 'manifest.json') == case['manifestSha256']
saved = output / 'artifacts' / case['id']
shutil.copytree(cartridge, saved)
result = harness.execute(binary, str(saved), scratch / case['id'])
fault = result.get('fault')
if isinstance(fault, dict) and 'fault' in fault:
fault = fault['fault']
code = (fault or {}).get('trap', {}).get('code')
matched = result['observedRuntime'] == expected and result['workerTerminated']
if expected == 'fault':
matched &= code == case['expectation']['faultCode']
result.update(id=case['id'], expectedRuntime=expected, expectedFaultCode=case['expectation'].get('faultCode'),
observedFaultCode=code, matchesExpectation=matched)
harness.write(output / 'execution' / (case['id'] + '.json'), result)
observations.append({k: result[k] for k in ['id', 'expectedRuntime', 'observedRuntime', 'observedFaultCode', 'matchesExpectation', 'workerTerminated']})
print(case['id'], result['observedRuntime'], code, matched, flush=True)
shutil.copy2(artifacts / 'compilation.json', output / 'compilation.json')
provenance = {'installed': compilation['provenance'], 'distribution': str(distribution),
'runtime': str(binary), 'runtimeSha256': harness.sha(binary), 'runnerSha256': harness.sha(__file__),
'harnessSha256': harness.sha(Path(__file__).with_name('run.py')), 'scratch': str(scratch),
'runtimeWrites': False, 'gfxExecution': 'not-run; installed API compilation only'}
harness.write(output / 'provenance.json', provenance)
summary = {'compileClassifications': len(compilation['cases']), 'executions': observations,
'passed': bool(observations) and all(x['matchesExpectation'] for x in observations)}
harness.write(output / 'summary.json', summary)
harness.write(output / 'checksums.json', {str(p.relative_to(output)): harness.sha(p) for p in output.rglob('*') if p.is_file()})
print(json.dumps({'compileClassifications': summary['compileClassifications'], 'executionCount': len(observations), 'passed': summary['passed']}))
return 0 if summary['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())