slovo/compiler/tests/reliability_hardening.rs
2026-05-22 08:38:43 +02:00

834 lines
25 KiB
Rust

use std::{
env,
ffi::OsStr,
fs,
path::{Path, PathBuf},
process::{Command, Output},
sync::atomic::{AtomicUsize, Ordering},
};
static NEXT_ID: AtomicUsize = AtomicUsize::new(0);
#[test]
fn promoted_fixtures_format_idempotently_and_remain_checkable() {
let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
let mut fixtures = vec![
manifest.join("../tests/comments.slo"),
manifest.join("../tests/formatter-stability-v1.fmt"),
manifest.join("../tests/top-level-test.slo"),
manifest.join("../tests/owned-string-concat.slo"),
manifest.join("../tests/vec-i32.slo"),
manifest.join("../tests/standard-runtime.slo"),
manifest.join("../tests/standard-io-host-env.slo"),
manifest.join("../tests/enum-basic.slo"),
];
let generated = [
(
"generated-branches",
r#"
(module main)
(fn clamp ((value i32)) -> i32
(if (< value 0) 0 value))
(test "clamp positive"
(= (clamp 7) 7))
"#,
),
(
"generated-vec-string-time",
r#"
(module main)
(fn main () -> i32
(std.time.sleep_ms 0)
(+ (std.vec.i32.len (std.vec.i32.append (std.vec.i32.empty) 1))
(std.string.len (std.string.concat "a" "bc"))))
"#,
),
];
for (name, source) in generated {
fixtures.push(write_fixture(name, source));
}
for fixture in fixtures {
let formatted = run_glagol(["fmt".as_ref(), fixture.as_os_str()]);
assert_success(&format!("format {}", fixture.display()), &formatted);
assert!(
formatted.stderr.is_empty(),
"formatter wrote stderr for `{}`:\n{}",
fixture.display(),
String::from_utf8_lossy(&formatted.stderr)
);
let formatted_source =
String::from_utf8(formatted.stdout).expect("formatted source is UTF-8");
let formatted_fixture = write_fixture("exp9-formatted", &formatted_source);
let second = run_glagol(["fmt".as_ref(), formatted_fixture.as_os_str()]);
assert_success(
&format!("format second pass {}", fixture.display()),
&second,
);
assert_eq!(
String::from_utf8(second.stdout).expect("second formatted source is UTF-8"),
formatted_source,
"formatter was not idempotent for `{}`",
fixture.display()
);
let fmt_check = run_glagol([
"fmt".as_ref(),
"--check".as_ref(),
formatted_fixture.as_os_str(),
]);
assert_success(&format!("fmt --check {}", fixture.display()), &fmt_check);
let check = run_glagol(["check".as_ref(), formatted_fixture.as_os_str()]);
assert_success_stdout(&format!("check formatted {}", fixture.display()), check, "");
}
}
#[test]
fn bounded_malformed_inputs_emit_structured_diagnostics_without_panic() {
let cases = [
("unclosed-top-list", "(module main"),
(
"unbalanced-function",
"(module main)\n\n(fn main () -> i32\n (+ 1 2)\n",
),
(
"unknown-top-level",
"(module main)\n\n(exp9 unknown form)\n",
),
(
"malformed-if",
"(module main)\n\n(fn main () -> i32\n (if true 1))\n",
),
(
"bounded-parens",
"((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((\n",
),
(
"nul-byte",
"(module main)\n\n(fn main () -> i32\n (print_string \"a\0b\")\n 0)\n",
),
(
"mutated-promoted-call",
"(module main)\n\n(fn main () -> i32\n (std.vec.i32.append (std.vec.i32.empty)))\n",
),
];
for (name, source) in cases {
let fixture = write_fixture(name, source);
let output = run_glagol([
"--json-diagnostics".as_ref(),
"check".as_ref(),
fixture.as_os_str(),
]);
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert_eq!(
output.status.code(),
Some(1),
"malformed case `{}` exit code drifted\nstdout:\n{}\nstderr:\n{}",
name,
stdout,
stderr
);
assert!(stdout.is_empty(), "malformed case `{}` wrote stdout", name);
assert_no_panic_text(name, &stderr);
assert_json_diagnostic_shape(name, &stderr);
assert!(
stderr.contains(r#""severity":"error""#),
"malformed case `{}` did not report an error diagnostic:\n{}",
name,
stderr
);
}
}
#[test]
fn json_diagnostic_schema_and_spans_are_sane_across_pipeline_snapshots() {
let cases = [
DiagnosticCase {
name: "parse",
args: &["--json-diagnostics", "check"],
source: "(module main",
expected_code: "UnclosedList",
},
DiagnosticCase {
name: "lower",
args: &["--json-diagnostics", "check"],
source: "(module main)\n\n(unknown top level)\n",
expected_code: "UnknownTopLevelForm",
},
DiagnosticCase {
name: "type-check",
args: &["--json-diagnostics", "check"],
source: r#"
(module main)
(fn id ((value i32)) -> i32
value)
(fn main () -> i32
(id true))
"#,
expected_code: "TypeMismatch",
},
DiagnosticCase {
name: "formatter",
args: &["--json-diagnostics", "fmt"],
source: r#"
(module main) ; trailing comments are outside the formatter subset
(fn main () -> i32
0)
"#,
expected_code: "UnsupportedFormatterComment",
},
DiagnosticCase {
name: "test-runner",
args: &["--json-diagnostics", "test"],
source: r#"
(module main)
(test "false"
false)
"#,
expected_code: "TestFailed",
},
DiagnosticCase {
name: "related-span",
args: &["--json-diagnostics", "check"],
source: r#"
(module main)
(fn dup () -> i32
1)
(fn dup () -> i32
2)
"#,
expected_code: "DuplicateFunction",
},
];
for case in cases {
let fixture = write_fixture(case.name, case.source);
let mut args = case.args.iter().map(OsStr::new).collect::<Vec<_>>();
args.push(fixture.as_os_str());
let output = run_glagol(args);
let stderr = String::from_utf8_lossy(&output.stderr);
assert_eq!(
output.status.code(),
Some(1),
"diagnostic case `{}` exit code drifted\nstdout:\n{}\nstderr:\n{}",
case.name,
String::from_utf8_lossy(&output.stdout),
stderr
);
assert_json_diagnostic_shape(case.name, &stderr);
assert!(
stderr.contains(&format!(r#""code":"{}""#, case.expected_code)),
"diagnostic case `{}` did not contain expected code `{}`:\n{}",
case.name,
case.expected_code,
stderr
);
assert_span_bounds(case.name, case.source, &stderr);
}
}
#[test]
fn project_workspace_graph_and_manifest_output_are_deterministic() {
let workspace = write_workspace("exp9-workspace");
let first_manifest = temp_path("exp9-workspace-manifest-a", "slo");
let second_manifest = temp_path("exp9-workspace-manifest-b", "slo");
let first = run_glagol([
"check".as_ref(),
"--manifest".as_ref(),
first_manifest.as_os_str(),
workspace.as_os_str(),
]);
assert_success("workspace check first", &first);
let second = run_glagol([
"check".as_ref(),
"--manifest".as_ref(),
second_manifest.as_os_str(),
workspace.as_os_str(),
]);
assert_success("workspace check second", &second);
let first_manifest = fs::read_to_string(&first_manifest).expect("read first manifest");
let second_manifest = fs::read_to_string(&second_manifest).expect("read second manifest");
assert_manifest_schema("first workspace manifest", &first_manifest);
assert_manifest_schema("second workspace manifest", &second_manifest);
assert_eq!(
project_block(&first_manifest),
project_block(&second_manifest),
"workspace project manifest block was not deterministic"
);
let block = project_block(&first_manifest);
assert_order(
block,
r#"(member "packages/app")"#,
r#"(member "packages/mathlib")"#,
);
assert_order(block, r#"(name "mathlib")"#, r#"(name "app")"#);
assert_order(block, r#"(name "mathlib.math")"#, r#"(name "app.main")"#);
assert!(
block.contains(r#"(package_dependency"#)
&& block.contains(r#"(from "app")"#)
&& block.contains(r#"(to "mathlib")"#)
&& block.contains(r#"(import_edge"#)
&& block.contains(r#"(from "app.main")"#)
&& block.contains(r#"(to "mathlib.math")"#),
"workspace graph manifest missed dependency/import edges:\n{}",
block
);
let filtered = run_glagol([
"test".as_ref(),
workspace.as_os_str(),
"--filter".as_ref(),
"app imports".as_ref(),
]);
assert_success_stdout(
"workspace filtered test",
filtered,
"test \"mathlib local\" ... skipped\ntest \"app imports mathlib\" ... ok\n1 test(s) passed (total_discovered 2, selected 1, passed 1, failed 0, skipped 1, filter \"app imports\")\n",
);
}
#[test]
fn project_manifest_boundary_failures_stay_structured_and_non_panicking() {
let cases = [
(
"invalid-manifest",
write_project_with_manifest(
"exp9-invalid-manifest",
"[project]\nname = \"Bad_Name\"\nsource_root = \"src\"\nentry = \"main\"\n",
&[("main", "(module main)\n\n(fn main () -> i32\n 0)\n")],
),
"ProjectManifestInvalid",
),
(
"workspace-path-escape",
write_workspace_with_manifest(
"exp9-path-escape",
"[workspace]\nmembers = [\"../outside\"]\n",
),
"WorkspaceMemberPathEscape",
),
(
"duplicate-package",
write_workspace_duplicate_packages("exp9-duplicate-package"),
"DuplicatePackageName",
),
(
"package-cycle",
write_workspace_package_cycle("exp9-package-cycle"),
"PackageDependencyCycle",
),
];
for (name, root, expected_code) in cases {
let output = run_glagol([
"--json-diagnostics".as_ref(),
"check".as_ref(),
root.as_os_str(),
]);
let stderr = String::from_utf8_lossy(&output.stderr);
assert_eq!(
output.status.code(),
Some(1),
"project boundary `{}` exit code drifted\nstdout:\n{}\nstderr:\n{}",
name,
String::from_utf8_lossy(&output.stdout),
stderr
);
assert_no_panic_text(name, &stderr);
assert_json_diagnostic_shape(name, &stderr);
assert!(
stderr.contains(&format!(r#""code":"{}""#, expected_code)),
"project boundary `{}` did not report `{}`:\n{}",
name,
expected_code,
stderr
);
}
}
#[test]
fn promoted_runtime_api_smoke_and_traps_are_stable_through_test_runner() {
let fixture = write_fixture(
"exp9-runtime-smoke",
r#"
(module main)
(enum Color Red Blue)
(fn joined () -> string
(std.string.concat "slo" "vo"))
(fn vector_len () -> i32
(std.vec.i32.len (std.vec.i32.append (std.vec.i32.empty) 9)))
(fn sleep_zero_then_one () -> i32
(std.time.sleep_ms 0)
1)
(test "string concat length"
(= (std.string.len (joined)) 5))
(test "vec append length"
(= (vector_len) 1))
(test "env missing remains empty"
(= (std.env.get "GLAGOL_EXP9_MISSING") ""))
(test "time smoke"
(= (sleep_zero_then_one) 1))
(test "enum equality"
(= (Color.Red) (Color.Red)))
"#,
);
let output = run_glagol(["test".as_ref(), fixture.as_os_str()]);
assert_success_stdout(
"promoted runtime smoke",
output,
concat!(
"test \"string concat length\" ... ok\n",
"test \"vec append length\" ... ok\n",
"test \"env missing remains empty\" ... ok\n",
"test \"time smoke\" ... ok\n",
"test \"enum equality\" ... ok\n",
"5 test(s) passed\n",
),
);
let traps = [
(
"vec-index-trap",
r#"
(module main)
(test "vec index trap"
(= (std.vec.i32.index (std.vec.i32.empty) 0) 0))
"#,
"slovo runtime error: vector index out of bounds",
),
(
"sleep-negative-trap",
r#"
(module main)
(test "negative sleep trap"
(= (do_negative_sleep) 0))
(fn do_negative_sleep () -> i32
(std.time.sleep_ms -1)
0)
"#,
"slovo runtime error: sleep_ms negative duration",
),
(
"process-arg-trap",
r#"
(module main)
(test "process arg trap"
(= (std.string.len (std.process.arg 99)) 0))
"#,
"slovo runtime error: process argument index out of bounds",
),
];
for (name, source, expected_trap) in traps {
let fixture = write_fixture(name, source);
let output = run_glagol(["test".as_ref(), fixture.as_os_str()]);
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert_eq!(
output.status.code(),
Some(1),
"runtime trap `{}` exit code drifted\nstdout:\n{}\nstderr:\n{}",
name,
stdout,
stderr
);
assert!(
stdout.is_empty(),
"runtime trap `{}` wrote stdout:\n{}",
name,
stdout
);
assert!(
stderr.contains("TestRuntimeTrap") && stderr.contains(expected_trap),
"runtime trap `{}` diagnostic drifted:\n{}",
name,
stderr
);
}
}
struct DiagnosticCase<'a> {
name: &'a str,
args: &'a [&'a str],
source: &'a str,
expected_code: &'a str,
}
fn run_glagol<I, S>(args: I) -> Output
where
I: IntoIterator<Item = S>,
S: AsRef<OsStr>,
{
Command::new(env!("CARGO_BIN_EXE_glagol"))
.args(args)
.current_dir(Path::new(env!("CARGO_MANIFEST_DIR")))
.output()
.expect("run glagol")
}
fn write_fixture(name: &str, source: &str) -> PathBuf {
let path = temp_path(&format!("{}.slo", name), "slo");
fs::write(&path, source).unwrap_or_else(|err| panic!("write `{}`: {}", path.display(), err));
path
}
fn temp_path(name: &str, extension: &str) -> PathBuf {
env::temp_dir().join(format!(
"glagol-exp9-{}-{}-{}.{}",
std::process::id(),
NEXT_ID.fetch_add(1, Ordering::Relaxed),
sanitize_name(name),
extension
))
}
fn sanitize_name(name: &str) -> String {
name.chars()
.map(|ch| {
if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' {
ch
} else {
'-'
}
})
.collect()
}
fn unique_dir(name: &str) -> PathBuf {
env::temp_dir().join(format!(
"glagol-exp9-{}-{}-{}",
std::process::id(),
NEXT_ID.fetch_add(1, Ordering::Relaxed),
sanitize_name(name)
))
}
fn write_project_with_manifest(name: &str, manifest: &str, modules: &[(&str, &str)]) -> PathBuf {
let root = unique_dir(name);
let src = root.join("src");
fs::create_dir_all(&src).expect("create project src");
fs::write(root.join("slovo.toml"), manifest).expect("write project manifest");
for (module, source) in modules {
fs::write(src.join(format!("{}.slo", module)), source).expect("write module source");
}
root
}
fn write_workspace(name: &str) -> PathBuf {
let root = write_workspace_with_manifest(
name,
"[workspace]\nmembers = [\"packages/app\", \"packages/mathlib\"]\n",
);
write_package(
&root,
"packages/mathlib",
"[package]\nname = \"mathlib\"\nversion = \"0.1.0\"\nsource_root = \"src\"\n",
&[(
"math",
r#"(module math (export add_one))
(fn add_one ((value i32)) -> i32
(+ value 1))
(test "mathlib local"
(= (add_one 1) 2))
"#,
)],
);
write_package(
&root,
"packages/app",
"[package]\nname = \"app\"\nversion = \"0.1.0\"\nsource_root = \"src\"\nentry = \"main\"\n\n[dependencies]\nmathlib = { path = \"../mathlib\" }\n",
&[(
"main",
r#"(module main)
(import mathlib.math (add_one))
(fn main () -> i32
(add_one 41))
(test "app imports mathlib"
(= (add_one 41) 42))
"#,
)],
);
root
}
fn write_workspace_with_manifest(name: &str, manifest: &str) -> PathBuf {
let root = unique_dir(name);
fs::create_dir_all(&root).expect("create workspace root");
fs::write(root.join("slovo.toml"), manifest).expect("write workspace manifest");
root
}
fn write_workspace_duplicate_packages(name: &str) -> PathBuf {
let root = write_workspace_with_manifest(
name,
"[workspace]\nmembers = [\"packages/a\", \"packages/b\"]\n",
);
for member in ["packages/a", "packages/b"] {
write_package(
&root,
member,
"[package]\nname = \"dup\"\nversion = \"0.1.0\"\nsource_root = \"src\"\n",
&[("main", "(module main)\n\n(fn main () -> i32\n 0)\n")],
);
}
root
}
fn write_workspace_package_cycle(name: &str) -> PathBuf {
let root = write_workspace_with_manifest(
name,
"[workspace]\nmembers = [\"packages/app\", \"packages/mathlib\"]\n",
);
write_package(
&root,
"packages/app",
"[package]\nname = \"app\"\nversion = \"0.1.0\"\nsource_root = \"src\"\n\n[dependencies]\nmathlib = { path = \"../mathlib\" }\n",
&[("main", "(module main)\n\n(fn main () -> i32\n 0)\n")],
);
write_package(
&root,
"packages/mathlib",
"[package]\nname = \"mathlib\"\nversion = \"0.1.0\"\nsource_root = \"src\"\n\n[dependencies]\napp = { path = \"../app\" }\n",
&[("math", "(module math)\n\n(fn value () -> i32\n 1)\n")],
);
root
}
fn write_package(root: &Path, member: &str, manifest: &str, modules: &[(&str, &str)]) {
let package_root = root.join(member);
let src = package_root.join("src");
fs::create_dir_all(&src).expect("create package src");
fs::write(package_root.join("slovo.toml"), manifest).expect("write package manifest");
for (module, source) in modules {
fs::write(src.join(format!("{}.slo", module)), source).expect("write package module");
}
}
fn assert_success(context: &str, output: &Output) {
assert!(
output.status.success(),
"{} failed\nstdout:\n{}\nstderr:\n{}",
context,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
fn assert_success_stdout(context: &str, output: Output, expected: &str) {
assert_success(context, &output);
assert_eq!(
String::from_utf8_lossy(&output.stdout),
expected,
"{} stdout mismatch",
context
);
assert!(
output.stderr.is_empty(),
"{} wrote stderr:\n{}",
context,
String::from_utf8_lossy(&output.stderr)
);
}
fn assert_no_panic_text(context: &str, stderr: &str) {
assert!(
!stderr.contains("panicked at")
&& !stderr.contains("thread 'main' panicked")
&& !stderr.contains("internal compiler error"),
"{} exposed panic text:\n{}",
context,
stderr
);
}
fn assert_json_diagnostic_shape(context: &str, stderr: &str) {
let lines = stderr
.lines()
.filter(|line| !line.trim().is_empty())
.collect::<Vec<_>>();
assert!(!lines.is_empty(), "{} did not emit diagnostics", context);
for line in lines {
assert!(
line.starts_with('{') && line.ends_with('}'),
"{} emitted non-JSON diagnostic line:\n{}",
context,
stderr
);
assert!(
line.contains(r#""schema":"slovo.diagnostic""#)
&& line.contains(r#""version":1"#)
&& line.contains(r#""severity":"#)
&& line.contains(r#""code":"#)
&& line.contains(r#""message":"#)
&& line.contains(r#""file":"#)
&& line.contains(r#""span":"#),
"{} JSON diagnostic missed required fields:\n{}",
context,
line
);
}
}
fn assert_span_bounds(context: &str, source: &str, stderr: &str) {
let source_len = source.len();
let line_count = source.lines().count().max(1);
let spans = json_span_objects(stderr);
assert!(
!spans.is_empty(),
"{} did not include any concrete spans:\n{}",
context,
stderr
);
for span in spans {
let byte_start = json_number(span, "byte_start").expect("span byte_start");
let byte_end = json_number(span, "byte_end").expect("span byte_end");
assert!(
byte_start <= byte_end && byte_end <= source_len,
"{} span byte bounds invalid for source length {}: {}",
context,
source_len,
span
);
if let Some(line_start) = json_number(span, "line_start") {
let column_start = json_number(span, "column_start").expect("span column_start");
let line_end = json_number(span, "line_end").expect("span line_end");
let column_end = json_number(span, "column_end").expect("span column_end");
assert!(
line_start >= 1
&& line_start <= line_end
&& line_end <= line_count + 1
&& column_start >= 1
&& column_end >= 1,
"{} span line/column bounds invalid for {} lines: {}",
context,
line_count,
span
);
}
}
}
fn json_span_objects(json_lines: &str) -> Vec<&str> {
let mut spans = Vec::new();
let needle = r#""span":{"#;
for line in json_lines.lines() {
let mut search_start = 0;
while let Some(relative) = line[search_start..].find(needle) {
let object_start = search_start + relative + r#""span":"#.len();
let mut depth = 0usize;
let mut object_end = None;
for (offset, ch) in line[object_start..].char_indices() {
match ch {
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
object_end = Some(object_start + offset + 1);
break;
}
}
_ => {}
}
}
if let Some(end) = object_end {
spans.push(&line[object_start..end]);
search_start = end;
} else {
break;
}
}
}
spans
}
fn json_number(object: &str, field: &str) -> Option<usize> {
let needle = format!(r#""{}":"#, field);
let start = object.find(&needle)? + needle.len();
let end = object[start..]
.find(|ch: char| !ch.is_ascii_digit())
.map(|relative| start + relative)
.unwrap_or(object.len());
object[start..end].parse().ok()
}
fn assert_manifest_schema(context: &str, manifest: &str) {
assert!(
manifest.contains(" (schema slovo.artifact-manifest)\n")
&& manifest.contains(" (version 1)\n")
&& manifest.contains(" (diagnostics-schema-version 1)\n")
&& manifest.contains(" (diagnostics-encoding sexpr)\n"),
"{} manifest schema fields drifted:\n{}",
context,
manifest
);
}
fn project_block(manifest: &str) -> &str {
let start = manifest
.find(" (project\n")
.expect("manifest did not contain project block");
manifest[start..]
.strip_suffix("\n)\n")
.expect("manifest did not end with artifact-manifest close")
}
fn assert_order(haystack: &str, before: &str, after: &str) {
let before_index = haystack
.find(before)
.unwrap_or_else(|| panic!("missing `{}` in:\n{}", before, haystack));
let after_index = haystack
.find(after)
.unwrap_or_else(|| panic!("missing `{}` in:\n{}", after, haystack));
assert!(
before_index < after_index,
"`{}` did not appear before `{}` in:\n{}",
before,
after,
haystack
);
}