135 lines
3.9 KiB
Rust
135 lines
3.9 KiB
Rust
use std::{
|
|
env,
|
|
path::Path,
|
|
process::{Command, Output},
|
|
};
|
|
|
|
const BENCHMARKS: &[&str] = &[
|
|
"array-index-loop",
|
|
"array-struct-field-loop",
|
|
"branch-loop",
|
|
"enum-struct-payload-loop",
|
|
"json-quote-loop",
|
|
"math-loop",
|
|
"parse-loop",
|
|
"string-eq-loop",
|
|
"vec-i32-index-loop",
|
|
"vec-string-eq-loop",
|
|
];
|
|
|
|
const IMPLEMENTATIONS: &[&str] = &["slovo", "c", "rust", "python", "clojure", "common_lisp"];
|
|
|
|
#[test]
|
|
fn suite_catalog_is_byte_stable_and_lists_current_benchmarks() {
|
|
let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
|
|
let python = python_command();
|
|
|
|
let first = run_suite_catalog(&repo, &python);
|
|
let second = run_suite_catalog(&repo, &python);
|
|
|
|
assert_success("first suite catalog run", &first);
|
|
assert_success("second suite catalog run", &second);
|
|
assert_eq!(
|
|
first.stdout, second.stdout,
|
|
"suite catalog JSON must be byte-stable across runs"
|
|
);
|
|
|
|
let stdout = String::from_utf8_lossy(&first.stdout);
|
|
for needle in [
|
|
r#""suite": "glagol-local-benchmark-suite""#,
|
|
r#""benchmark_count": 10"#,
|
|
r#""benchmark_metadata_files": 10"#,
|
|
r#""required_files": 40"#,
|
|
r#""missing_required_files": []"#,
|
|
r#""implementation_slots": 60"#,
|
|
r#""expected_implementation_slots": 60"#,
|
|
r#""missing_implementation_slots": []"#,
|
|
r#""status": "ok""#,
|
|
r#""timing_scope": "local-machine comparison only""#,
|
|
r#""timing_disclaimer": "Local timing comparison only; not a published benchmark result and not a cross-machine performance claim.""#,
|
|
r#""cold-process""#,
|
|
r#""hot-loop""#,
|
|
r#""loop_count": 1000000"#,
|
|
r#""hot_loop_count": 10000000"#,
|
|
r#""checksum_metadata""#,
|
|
r#""expected_checksum""#,
|
|
r#""hot_expected_checksum""#,
|
|
r#""required_files""#,
|
|
r#""path": "benchmark.json""#,
|
|
r#""path": "run.py""#,
|
|
r#""path": "slovo.toml""#,
|
|
r#""path": "src/main.slo""#,
|
|
r#""status": "present""#,
|
|
r#""implementation_slots""#,
|
|
r#""loop_count_source": "stdin""#,
|
|
] {
|
|
assert_contains(&stdout, needle);
|
|
}
|
|
|
|
for benchmark in BENCHMARKS {
|
|
assert_contains(&stdout, &format!(r#""name": "{}""#, benchmark));
|
|
assert_contains(&stdout, &format!(r#""directory": "{}""#, benchmark));
|
|
}
|
|
|
|
for implementation in IMPLEMENTATIONS {
|
|
assert_contains(&stdout, &format!(r#""name": "{}""#, implementation));
|
|
}
|
|
}
|
|
|
|
fn run_suite_catalog(repo: &Path, python: &str) -> Output {
|
|
Command::new(python)
|
|
.arg("benchmarks/runner.py")
|
|
.arg("--suite-list")
|
|
.arg("--json")
|
|
.current_dir(repo)
|
|
.output()
|
|
.unwrap_or_else(|err| {
|
|
panic!(
|
|
"run `{}` benchmarks/runner.py --suite-list --json: {}",
|
|
python, err
|
|
)
|
|
})
|
|
}
|
|
|
|
fn python_command() -> String {
|
|
if let Some(python) = env::var_os("PYTHON") {
|
|
return python.to_string_lossy().into_owned();
|
|
}
|
|
|
|
for candidate in ["python3", "python"] {
|
|
if Command::new(candidate)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|output| output.status.success())
|
|
.unwrap_or(false)
|
|
{
|
|
return candidate.to_string();
|
|
}
|
|
}
|
|
|
|
panic!("benchmark suite catalog test requires python3 or python")
|
|
}
|
|
|
|
fn assert_success(context: &str, output: &Output) {
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
|
|
assert!(
|
|
output.status.success(),
|
|
"{} failed\nstdout:\n{}\nstderr:\n{}",
|
|
context,
|
|
stdout,
|
|
stderr
|
|
);
|
|
assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr);
|
|
}
|
|
|
|
fn assert_contains(haystack: &str, needle: &str) {
|
|
assert!(
|
|
haystack.contains(needle),
|
|
"suite catalog output missing `{}`\nstdout:\n{}",
|
|
needle,
|
|
haystack
|
|
);
|
|
}
|