use std::{ env, path::Path, process::{Command, Output}, }; const BENCHMARKS: &[&str] = &[ "array-index-loop", "array-struct-field-loop", "branch-loop", "enum-struct-payload-loop", "json-quote-loop", "math-loop", "parse-loop", "string-eq-loop", "vec-i32-index-loop", "vec-string-eq-loop", ]; const IMPLEMENTATIONS: &[&str] = &["slovo", "c", "rust", "python", "clojure", "common_lisp"]; #[test] fn suite_catalog_is_byte_stable_and_lists_current_benchmarks() { let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); let python = python_command(); let first = run_suite_catalog(&repo, &python); let second = run_suite_catalog(&repo, &python); assert_success("first suite catalog run", &first); assert_success("second suite catalog run", &second); assert_eq!( first.stdout, second.stdout, "suite catalog JSON must be byte-stable across runs" ); let stdout = String::from_utf8_lossy(&first.stdout); for needle in [ r#""suite": "glagol-local-benchmark-suite""#, r#""benchmark_count": 10"#, r#""benchmark_metadata_files": 10"#, r#""required_files": 40"#, r#""missing_required_files": []"#, r#""implementation_slots": 60"#, r#""expected_implementation_slots": 60"#, r#""missing_implementation_slots": []"#, r#""status": "ok""#, r#""timing_scope": "local-machine comparison only""#, r#""timing_disclaimer": "Local timing comparison only; not a published benchmark result and not a cross-machine performance claim.""#, r#""cold-process""#, r#""hot-loop""#, r#""loop_count": 1000000"#, r#""hot_loop_count": 10000000"#, r#""checksum_metadata""#, r#""expected_checksum""#, r#""hot_expected_checksum""#, r#""required_files""#, r#""path": "benchmark.json""#, r#""path": "run.py""#, r#""path": "slovo.toml""#, r#""path": "src/main.slo""#, r#""status": "present""#, r#""implementation_slots""#, r#""loop_count_source": "stdin""#, ] { assert_contains(&stdout, needle); } for benchmark in BENCHMARKS { assert_contains(&stdout, &format!(r#""name": "{}""#, benchmark)); assert_contains(&stdout, &format!(r#""directory": "{}""#, benchmark)); } for implementation in IMPLEMENTATIONS { assert_contains(&stdout, &format!(r#""name": "{}""#, implementation)); } } fn run_suite_catalog(repo: &Path, python: &str) -> Output { Command::new(python) .arg("benchmarks/runner.py") .arg("--suite-list") .arg("--json") .current_dir(repo) .output() .unwrap_or_else(|err| { panic!( "run `{}` benchmarks/runner.py --suite-list --json: {}", python, err ) }) } fn python_command() -> String { if let Some(python) = env::var_os("PYTHON") { return python.to_string_lossy().into_owned(); } for candidate in ["python3", "python"] { if Command::new(candidate) .arg("--version") .output() .map(|output| output.status.success()) .unwrap_or(false) { return candidate.to_string(); } } panic!("benchmark suite catalog test requires python3 or python") } fn assert_success(context: &str, output: &Output) { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( output.status.success(), "{} failed\nstdout:\n{}\nstderr:\n{}", context, stdout, stderr ); assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr); } fn assert_contains(haystack: &str, needle: &str) { assert!( haystack.contains(needle), "suite catalog output missing `{}`\nstdout:\n{}", needle, haystack ); }