slovo/compiler/tests/benchmark_math_loop_scaffold.rs

use std::{
    env,
    ffi::OsStr,
    path::{Path, PathBuf},
    process::{Command, Output},
};

const SCAFFOLD_FILES: &[&str] = &[
    "slovo.toml",
    "README.md",
    "benchmark.json",
    "src/main.slo",
    "clojure",
    "common-lisp",
    "c",
    "rust",
    "python",
    "run.py",
];

#[test]
fn benchmark_scaffolds_check_format_and_run_python_checksums() {
    let roots = benchmark_roots();

    for root in roots {
        assert_benchmark_scaffold_checks_formats_and_lists_metadata(&root);
    }
}

fn assert_benchmark_scaffold_checks_formats_and_lists_metadata(root: &Path) {
    let benchmark_name = root
        .file_name()
        .and_then(|name| name.to_str())
        .unwrap_or("benchmark");

    for relative in SCAFFOLD_FILES {
        let path = root.join(relative);
        assert!(path.exists(), "missing scaffold path `{}`", path.display());
    }

    let fmt = run_glagol([OsStr::new("fmt"), OsStr::new("--check"), root.as_os_str()]);
    assert_success_stdout(&format!("{benchmark_name} fmt --check"), fmt, "");

    let check = run_glagol([OsStr::new("check"), root.as_os_str()]);
    assert_success_stdout(&format!("{benchmark_name} project check"), check, "");

    let python = python_command();
    let list = Command::new(&python)
        .arg("run.py")
        .arg("--list")
        .arg("--json")
        .current_dir(&root)
        .output()
        .unwrap_or_else(|err| panic!("run `{}` run.py --list --json: {}", python, err));
    assert_success(&format!("{benchmark_name} runner --list --json"), &list);

    let stdout = String::from_utf8_lossy(&list.stdout);
    for needle in [
        r#""loop_count": 1000000"#,
        r#""hot_loop_count": 10000000"#,
        r#""loop_count_source": "stdin""#,
        r#""timing_scope": "local-machine comparison only""#,
        r#""hot-loop""#,
        r#""cold-process""#,
        r#""name": "slovo""#,
        r#""name": "c""#,
        r#""name": "rust""#,
        r#""name": "python""#,
        r#""name": "clojure""#,
        r#""name": "common_lisp""#,
    ] {
        assert!(
            stdout.contains(needle),
            "runner metadata missing `{}`\nstdout:\n{}",
            needle,
            stdout
        );
    }

    let run = Command::new(&python)
        .arg("run.py")
        .arg("--only")
        .arg("python")
        .arg("--repeats")
        .arg("1")
        .arg("--warmups")
        .arg("0")
        .arg("--json")
        .current_dir(&root)
        .output()
        .unwrap_or_else(|err| panic!("run `{}` run.py --only python: {}", python, err));
    assert_success(&format!("{benchmark_name} runner python checksum"), &run);

    let stdout = String::from_utf8_lossy(&run.stdout);
    for needle in [
        r#""name": "python""#,
        r#""status": "ok""#,
        r#""checksum": ""#,
        r#""warmups": 0"#,
        r#""repeats": 1"#,
    ] {
        assert!(
            stdout.contains(needle),
            "runner checksum execution missing `{}`\nstdout:\n{}",
            needle,
            stdout
        );
    }

    let hot_run = Command::new(&python)
        .arg("run.py")
        .arg("--mode")
        .arg("hot-loop")
        .arg("--only")
        .arg("python")
        .arg("--repeats")
        .arg("1")
        .arg("--warmups")
        .arg("0")
        .arg("--json")
        .current_dir(root)
        .output()
        .unwrap_or_else(|err| {
            panic!(
                "run `{}` run.py --mode hot-loop --only python: {}",
                python, err
            )
        });
    assert_success(
        &format!("{benchmark_name} runner python hot-loop checksum"),
        &hot_run,
    );

    let stdout = String::from_utf8_lossy(&hot_run.stdout);
    for needle in [
        r#""name": "python""#,
        r#""status": "ok""#,
        r#""timing_mode": "hot-loop""#,
        r#""loop_count": 10000000"#,
        r#""normalized_median_ms": "#,
    ] {
        assert!(
            stdout.contains(needle),
            "runner hot-loop execution missing `{}`\nstdout:\n{}",
            needle,
            stdout
        );
    }
}

fn benchmark_roots() -> Vec<PathBuf> {
    let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../benchmarks");
    vec![
        root.join("math-loop"),
        root.join("branch-loop"),
        root.join("parse-loop"),
        root.join("array-index-loop"),
        root.join("string-eq-loop"),
        root.join("array-struct-field-loop"),
        root.join("enum-struct-payload-loop"),
        root.join("vec-i32-index-loop"),
        root.join("vec-string-eq-loop"),
    ]
}

fn run_glagol<I, S>(args: I) -> Output
where
    I: IntoIterator<Item = S>,
    S: AsRef<OsStr>,
{
    Command::new(env!("CARGO_BIN_EXE_glagol"))
        .args(args)
        .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")))
        .output()
        .expect("run glagol")
}

fn python_command() -> String {
    if let Some(python) = env::var_os("PYTHON") {
        return python.to_string_lossy().into_owned();
    }

    for candidate in ["python3", "python"] {
        if Command::new(candidate)
            .arg("--version")
            .output()
            .map(|output| output.status.success())
            .unwrap_or(false)
        {
            return candidate.to_string();
        }
    }

    panic!("benchmark runner list-mode test requires python3 or python")
}

fn assert_success(context: &str, output: &Output) {
    let stdout = String::from_utf8_lossy(&output.stdout);
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(
        output.status.success(),
        "{} failed\nstdout:\n{}\nstderr:\n{}",
        context,
        stdout,
        stderr
    );
    assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr);
}

fn assert_success_stdout(context: &str, output: Output, expected: &str) {
    let stdout = String::from_utf8_lossy(&output.stdout);
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(
        output.status.success(),
        "{} failed\nstdout:\n{}\nstderr:\n{}",
        context,
        stdout,
        stderr
    );
    assert_eq!(stdout, expected, "{} stdout mismatch", context);
    assert!(stderr.is_empty(), "{} wrote stderr:\n{}", context, stderr);
}