75 lines
4.2 KiB
Python
75 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Qualify artifacts compiled from installed PBS references using the existing runtime harness."""
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--artifacts', type=Path, required=True)
|
|
parser.add_argument('--distribution', type=Path, required=True)
|
|
parser.add_argument('--runtime-headless', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
sys.dont_write_bytecode = True
|
|
spec = importlib.util.spec_from_file_location('pbs_corpus', Path(__file__).with_name('run.py'))
|
|
harness = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(harness)
|
|
artifacts = args.artifacts.resolve(strict=True)
|
|
distribution = args.distribution.resolve(strict=True)
|
|
binary = args.runtime_headless.resolve(strict=True)
|
|
output = args.output.resolve()
|
|
if output.exists():
|
|
parser.error('Output must be a new directory; evidence is never overwritten')
|
|
output.mkdir(parents=True)
|
|
compilation = json.loads((artifacts / 'compilation.json').read_text())
|
|
assert compilation['passed']
|
|
jars = {p.name: harness.sha(p) for p in (distribution / 'lib').glob('*.jar')}
|
|
assert jars == compilation['provenance']['installedJarSha256'], 'Compiled artifacts must match this distribution'
|
|
scratch = Path(tempfile.mkdtemp(prefix='pbs-installed-contracts-'))
|
|
observations = []
|
|
for case in compilation['cases']:
|
|
expected = case['expectation']['expectedRuntime']
|
|
if expected == 'not-run':
|
|
continue
|
|
assert case['compileStatus'] == 'compiled' and case['matchesExpectation'] and case['sourceUnchanged']
|
|
cartridge = artifacts / case['id']
|
|
assert harness.sha(cartridge / 'program.pbx') == case['bytecodeSha256']
|
|
assert harness.sha(cartridge / 'manifest.json') == case['manifestSha256']
|
|
saved = output / 'artifacts' / case['id']
|
|
shutil.copytree(cartridge, saved)
|
|
result = harness.execute(binary, str(saved), scratch / case['id'])
|
|
fault = result.get('fault')
|
|
if isinstance(fault, dict) and 'fault' in fault:
|
|
fault = fault['fault']
|
|
code = (fault or {}).get('trap', {}).get('code')
|
|
matched = result['observedRuntime'] == expected and result['workerTerminated']
|
|
if expected == 'fault':
|
|
matched &= code == case['expectation']['faultCode']
|
|
result.update(id=case['id'], expectedRuntime=expected, expectedFaultCode=case['expectation'].get('faultCode'),
|
|
observedFaultCode=code, matchesExpectation=matched)
|
|
harness.write(output / 'execution' / (case['id'] + '.json'), result)
|
|
observations.append({k: result[k] for k in ['id', 'expectedRuntime', 'observedRuntime', 'observedFaultCode', 'matchesExpectation', 'workerTerminated']})
|
|
print(case['id'], result['observedRuntime'], code, matched, flush=True)
|
|
shutil.copy2(artifacts / 'compilation.json', output / 'compilation.json')
|
|
provenance = {'installed': compilation['provenance'], 'distribution': str(distribution),
|
|
'runtime': str(binary), 'runtimeSha256': harness.sha(binary), 'runnerSha256': harness.sha(__file__),
|
|
'harnessSha256': harness.sha(Path(__file__).with_name('run.py')), 'scratch': str(scratch),
|
|
'runtimeWrites': False, 'gfxExecution': 'not-run; installed API compilation only'}
|
|
harness.write(output / 'provenance.json', provenance)
|
|
summary = {'compileClassifications': len(compilation['cases']), 'executions': observations,
|
|
'passed': bool(observations) and all(x['matchesExpectation'] for x in observations)}
|
|
harness.write(output / 'summary.json', summary)
|
|
harness.write(output / 'checksums.json', {str(p.relative_to(output)): harness.sha(p) for p in output.rglob('*') if p.is_file()})
|
|
print(json.dumps({'compileClassifications': summary['compileClassifications'], 'executionCount': len(observations), 'passed': summary['passed']}))
|
|
return 0 if summary['passed'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|