284 lines
8.7 KiB
Python
284 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# To run this in headless mode, set the following environment variables:
|
|
# * SDL_VIDEODRIVER=dummy
|
|
# * SDL_AUDIODRIVER=dummy
|
|
#
|
|
# Also make sure to specify the correct paths to the ScummVM binary, demos, and records if they are not in the default locations.
|
|
# You can also change them via environment variables:
|
|
# * SCUMMVM_BIN - path to the ScummVM binary
|
|
# * GAMES_DIR - path to the games directory that are automatically registered before running tests - defaults to none and is skipped then.
|
|
# it's assumed that you have already registered the games for playing back the event recorder files.
|
|
#
|
|
# Example usage:
|
|
# SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy SCUMMVM_BIN=./scummvm \
|
|
# python3 devtools/run_event_recorder_tests.py --xunit-output=event_recorder_tests.xml \
|
|
# --filter="*monkey*"
|
|
|
|
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import platform
|
|
import argparse
|
|
import time
|
|
import fnmatch
|
|
import re
|
|
import json
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
def generate_xunit_report(results, output_file, total_time):
|
|
testsuites = ET.Element("testsuites")
|
|
testsuite = ET.SubElement(testsuites, "testsuite", name="EventRecorderTest", tests=str(len(results)), time=str(total_time))
|
|
|
|
failures = 0
|
|
skipped = 0
|
|
for result in results:
|
|
# XUnit format: classname="TestSuite", name="TestCase"
|
|
testcase = ET.SubElement(testsuite, "testcase", name=result['test_case'], classname="EventRecorderTest", time=str(result['duration']))
|
|
if result.get('skipped'):
|
|
skipped += 1
|
|
ET.SubElement(testcase, "skipped", message=result['message'])
|
|
elif not result['success']:
|
|
failures += 1
|
|
failure = ET.SubElement(testcase, "failure", message=result['message'])
|
|
failure.text = result.get('output', '')
|
|
|
|
testsuite.set("failures", str(failures))
|
|
testsuite.set("skipped", str(skipped))
|
|
|
|
tree = ET.ElementTree(testsuites)
|
|
try:
|
|
tree.write(output_file, encoding="utf-8", xml_declaration=True)
|
|
# print(f"XUnit report generated at: {output_file}")
|
|
except Exception as e:
|
|
print(f"Error writing XUnit report: {e}")
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Run ScummVM Event Recorder tests.")
|
|
parser.add_argument("--xunit-output", help="Path to generate XUnit XML report", default=None)
|
|
parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose output", default=False)
|
|
parser.add_argument("--filter", help="Filter tests (glob pattern, e.g. *monkey*)", default="*")
|
|
parser.add_argument("--list", action="store_true", help="List tests", default=False)
|
|
args = parser.parse_args()
|
|
|
|
# Configuration
|
|
# Determine default binary name based on OS
|
|
bin_name = "scummvm.exe" if platform.system() == "Windows" else "./scummvm"
|
|
|
|
# Allow overriding via env vars
|
|
# Default to looking in current directory
|
|
scummvm_bin = os.getenv("SCUMMVM_BIN", bin_name)
|
|
if os.getenv("GAMES_DIR", None):
|
|
games_dir = Path(os.getenv("GAMES_DIR"))
|
|
else:
|
|
games_dir = None
|
|
|
|
# Check if ScummVM exists
|
|
if not Path(scummvm_bin).exists():
|
|
print(f"Error: ScummVM binary not found at {scummvm_bin}")
|
|
print("Please run this script from the root of the ScummVM source tree where the binary is built.")
|
|
sys.exit(127)
|
|
|
|
# Check if demos directory exists
|
|
if games_dir is not None:
|
|
if not games_dir.exists():
|
|
print(f"Error: Games directory not found at {games_dir}")
|
|
sys.exit(127)
|
|
|
|
# Register games
|
|
for game_path in sorted(games_dir.iterdir()):
|
|
if not game_path.is_dir():
|
|
continue
|
|
|
|
try:
|
|
detect_cmd = [
|
|
str(scummvm_bin),
|
|
"--add",
|
|
f"--path={game_path}"
|
|
]
|
|
subprocess.run(detect_cmd, capture_output=True)
|
|
except Exception as e:
|
|
print(f"Error detecting games in {game_path}: {e}")
|
|
continue
|
|
|
|
# Get available engines
|
|
available_engines = set()
|
|
try:
|
|
list_engines_cmd = [str(scummvm_bin), "--list-engines-json"]
|
|
result = subprocess.run(list_engines_cmd, capture_output=True, text=True)
|
|
if result.returncode == 0:
|
|
try:
|
|
engines_map = json.loads(result.stdout)
|
|
available_engines = set(engines_map.keys())
|
|
except json.JSONDecodeError:
|
|
print(f"Warning: Failed to parse engines JSON. Output was:\n{result.stdout}")
|
|
else:
|
|
print(f"Warning: Failed to list engines: {result.stderr}")
|
|
except Exception as e:
|
|
print(f"Warning: Error executing ScummVM to list engines: {e}")
|
|
|
|
# Collect tests
|
|
test_cases = []
|
|
try:
|
|
list_cmd = [str(scummvm_bin), "--list-records-json"]
|
|
result = subprocess.run(list_cmd, capture_output=True, text=True)
|
|
if result.returncode != 0:
|
|
print(f"Error listing records: {result.stderr}")
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"Error executing ScummVM to list records: {e}")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
json_output = result.stdout.strip()
|
|
records_map = json.loads(json_output)
|
|
except json.JSONDecodeError as e:
|
|
print(f"Error parsing records JSON: {e}\nOutput was:\n{result.stdout}")
|
|
sys.exit(1)
|
|
|
|
for target, info in records_map.items():
|
|
engine_id = info.get('engine')
|
|
for record_file in info.get('records', []):
|
|
# Test name: EventRecorderTest.<RecordFileName>
|
|
# We use the filename as the test case name
|
|
test_case_name = record_file
|
|
full_test_name = f"EventRecorderTest.{test_case_name}"
|
|
|
|
# Check filter
|
|
# Match against full name or just the test case name
|
|
if not fnmatch.fnmatch(full_test_name, args.filter) and not fnmatch.fnmatch(test_case_name, args.filter):
|
|
continue
|
|
|
|
test_cases.append({
|
|
'full_name': full_test_name,
|
|
'test_case': test_case_name,
|
|
'record_file': record_file,
|
|
'target': target,
|
|
'engine_id': engine_id
|
|
})
|
|
|
|
if args.list:
|
|
print("EventRecorderTest.")
|
|
for test in test_cases:
|
|
print(f" {test['test_case']}")
|
|
sys.exit(0)
|
|
|
|
total_tests = len(test_cases)
|
|
|
|
# Googletest compatible header
|
|
print(f"[==========] {total_tests} tests from 1 test suite ran.")
|
|
print(f"[----------] {total_tests} tests from EventRecorderTest")
|
|
|
|
start_total_time = time.time()
|
|
passed_tests = []
|
|
failed_tests = []
|
|
skipped_tests = []
|
|
results_for_xunit = []
|
|
|
|
for test in test_cases:
|
|
print(f"[ RUN ] {test['full_name']}")
|
|
sys.stdout.flush()
|
|
|
|
test_start_time = time.time()
|
|
success = False
|
|
message = ""
|
|
output = ""
|
|
|
|
if available_engines:
|
|
engine_id = test.get('engine_id')
|
|
if engine_id and engine_id not in available_engines:
|
|
print(f"[ SKIPPED ] {test['full_name']} (Engine '{engine_id}' not available)")
|
|
results_for_xunit.append({
|
|
'name': test['full_name'],
|
|
'test_case': test['test_case'],
|
|
'success': False,
|
|
'skipped': True,
|
|
'duration': 0,
|
|
'message': f"Engine '{engine_id}' not available",
|
|
'output': ""
|
|
})
|
|
skipped_tests.append(test['full_name'])
|
|
continue
|
|
|
|
try:
|
|
playback_cmd = [
|
|
str(scummvm_bin),
|
|
"--record-mode=fast_playback",
|
|
f"--record-file-name={test['record_file']}",
|
|
test['target']
|
|
]
|
|
|
|
# Run and capture output
|
|
if args.verbose:
|
|
print(f"Executing: {' '.join(playback_cmd)}")
|
|
playback_result = subprocess.run(playback_cmd, capture_output=True, text=True)
|
|
output = playback_result.stdout + "\n" + playback_result.stderr
|
|
|
|
if args.verbose:
|
|
if playback_result.stdout.strip():
|
|
print(playback_result.stdout)
|
|
if playback_result.stderr.strip():
|
|
print(playback_result.stderr)
|
|
|
|
if playback_result.returncode == 0:
|
|
success = True
|
|
else:
|
|
success = False
|
|
message = f"Exit Code: {playback_result.returncode}"
|
|
|
|
except Exception as e:
|
|
success = False
|
|
message = str(e)
|
|
output += f"\nException: {e}"
|
|
|
|
duration_sec = time.time() - test_start_time
|
|
duration_ms = int(duration_sec * 1000)
|
|
|
|
results_for_xunit.append({
|
|
'name': test['full_name'],
|
|
'test_case': test['test_case'],
|
|
'success': success,
|
|
'duration': duration_sec,
|
|
'message': message,
|
|
'output': output
|
|
})
|
|
|
|
if success:
|
|
print(f"[ OK ] {test['full_name']} ({duration_ms} ms)")
|
|
passed_tests.append(test['full_name'])
|
|
else:
|
|
print(f"[ FAILED ] {test['full_name']} ({duration_ms} ms)")
|
|
failed_tests.append(test['full_name'])
|
|
|
|
total_duration_sec = time.time() - start_total_time
|
|
total_duration_ms = int(total_duration_sec * 1000)
|
|
|
|
print(f"[----------] {total_tests} tests from EventRecorderTest ({total_duration_ms} ms total)")
|
|
print(f"[==========] {total_tests} tests from 1 test suite ran. ({total_duration_ms} ms total)")
|
|
print(f"[ PASSED ] {len(passed_tests)} tests.")
|
|
|
|
if args.xunit_output:
|
|
generate_xunit_report(results_for_xunit, args.xunit_output, total_duration_sec)
|
|
|
|
if failed_tests:
|
|
print(f"[ FAILED ] {len(failed_tests)} tests, listed below:")
|
|
for failed in failed_tests:
|
|
print(f"[ FAILED ] {failed}")
|
|
sys.exit(1)
|
|
|
|
if skipped_tests:
|
|
print(f"[ SKIPPED ] {len(skipped_tests)} tests.")
|
|
for skipped in skipped_tests:
|
|
print(f"[ SKIPPED ] {skipped}")
|
|
|
|
if total_tests == 0:
|
|
print("No tests were run.")
|
|
sys.exit(0)
|
|
|
|
sys.exit(0)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|