use std::{ env, ffi::OsStr, path::{Path, PathBuf}, process::{Command, Output}, }; const SCAFFOLD_FILES: &[&str] = &[ "slovo.toml", "README.md", "benchmark.json", "src/main.slo", "clojure", "common-lisp", "c", "rust", "python", "run.py", ]; #[test] fn benchmark_scaffolds_check_format_and_run_python_checksums() { let roots = benchmark_roots(); for root in roots { assert_benchmark_scaffold_checks_formats_and_lists_metadata(&root); } } fn assert_benchmark_scaffold_checks_formats_and_lists_metadata(root: &Path) { let benchmark_name = root .file_name() .and_then(|name| name.to_str()) .unwrap_or("benchmark"); for relative in SCAFFOLD_FILES { let path = root.join(relative); assert!(path.exists(), "missing scaffold path `{}`", path.display()); } let fmt = run_glagol([OsStr::new("fmt"), OsStr::new("--check"), root.as_os_str()]); assert_success_stdout(&format!("{benchmark_name} fmt --check"), fmt, ""); let check = run_glagol([OsStr::new("check"), root.as_os_str()]); assert_success_stdout(&format!("{benchmark_name} project check"), check, ""); let python = python_command(); let list = Command::new(&python) .arg("run.py") .arg("--list") .arg("--json") .current_dir(&root) .output() .unwrap_or_else(|err| panic!("run `{}` run.py --list --json: {}", python, err)); assert_success(&format!("{benchmark_name} runner --list --json"), &list); let stdout = String::from_utf8_lossy(&list.stdout); for needle in [ r#""loop_count": 1000000"#, r#""hot_loop_count": 10000000"#, r#""loop_count_source": "stdin""#, r#""timing_scope": "local-machine comparison only""#, r#""hot-loop""#, r#""cold-process""#, r#""name": "slovo""#, r#""name": "c""#, r#""name": "rust""#, r#""name": "python""#, r#""name": "clojure""#, r#""name": "common_lisp""#, ] { assert!( stdout.contains(needle), "runner metadata missing `{}`\nstdout:\n{}", needle, stdout ); } let run = Command::new(&python) .arg("run.py") .arg("--only") .arg("python") .arg("--repeats") .arg("1") .arg("--warmups") .arg("0") .arg("--json") .current_dir(&root) .output() .unwrap_or_else(|err| panic!("run `{}` run.py --only python: {}", python, err)); assert_success(&format!("{benchmark_name} runner python checksum"), &run); let stdout = String::from_utf8_lossy(&run.stdout); for needle in [ r#""name": "python""#, r#""status": "ok""#, r#""checksum": ""#, r#""warmups": 0"#, r#""repeats": 1"#, ] { assert!( stdout.contains(needle), "runner checksum execution missing `{}`\nstdout:\n{}", needle, stdout ); } let hot_run = Command::new(&python) .arg("run.py") .arg("--mode") .arg("hot-loop") .arg("--only") .arg("python") .arg("--repeats") .arg("1") .arg("--warmups") .arg("0") .arg("--json") .current_dir(root) .output() .unwrap_or_else(|err| { panic!( "run `{}` run.py --mode hot-loop --only python: {}", python, err ) }); assert_success( &format!("{benchmark_name} runner python hot-loop checksum"), &hot_run, ); let stdout = String::from_utf8_lossy(&hot_run.stdout); for needle in [ r#""name": "python""#, r#""status": "ok""#, r#""timing_mode": "hot-loop""#, r#""loop_count": 10000000"#, r#""normalized_median_ms": "#, ] { assert!( stdout.contains(needle), "runner hot-loop execution missing `{}`\nstdout:\n{}", needle, stdout ); } } fn benchmark_roots() -> Vec { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../benchmarks"); vec![ root.join("math-loop"), root.join("branch-loop"), root.join("parse-loop"), root.join("array-index-loop"), root.join("string-eq-loop"), root.join("array-struct-field-loop"), root.join("enum-struct-payload-loop"), root.join("vec-i32-index-loop"), root.join("vec-string-eq-loop"), ] } fn run_glagol(args: I) -> Output where I: IntoIterator, S: AsRef, { Command::new(env!("CARGO_BIN_EXE_glagol")) .args(args) .current_dir(Path::new(env!("CARGO_MANIFEST_DIR"))) .output() .expect("run glagol") } fn python_command() -> String { if let Some(python) = env::var_os("PYTHON") { return python.to_string_lossy().into_owned(); } for candidate in ["python3", "python"] { if Command::new(candidate) .arg("--version") .output() .map(|output| output.status.success()) .unwrap_or(false) { return candidate.to_string(); } } panic!("benchmark runner list-mode test requires python3 or python") } fn assert_success(context: &str, output: &Output) { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( output.status.success(), "{} failed\nstdout:\n{}\nstderr:\n{}", context, stdout, stderr ); assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr); } fn assert_success_stdout(context: &str, output: Output, expected: &str) { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( output.status.success(), "{} failed\nstdout:\n{}\nstderr:\n{}", context, stdout, stderr ); assert_eq!(stdout, expected, "{} stdout mismatch", context); assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr); }