tests: WIP introduce performance tests

Signed-off-by: Matej Hrica <mhrica@redhat.com>

Author: mtjhrc

tests/Cargo.lock

@@ -43,4 +43,12 @@ make test
 ```
 
 ## Adding tests
-To add a test you need to add a new rust module in the `test_cases` directory, implement the  required host and guest side methods (see existing tests) and register the test in the `test_cases/src/lib.rs` to be ran.
+To add a test you need to add a new rust module in the `test_cases` directory, implement the  required host and guest side methods (see existing tests) and register the test in the `test_cases/src/lib.rs` to be ran.
+
+## Rootfs images
+
+Some tests (e.g. the iperf3 performance tests) need a full Linux rootfs with extra packages installed. These are built automatically via podman and stored in podman's local image store (tagged as `libkrun-test-<name>`). Podman's layer cache handles rebuild efficiency.
+
+Container image definitions are registered in the `rootfs_image()` function in `test_cases/src/lib.rs`. Tests refer to images by name only. Tests that need a rootfs will be skipped if podman is not installed.
+
+To clean up images: `podman rmi $(podman images --filter reference='libkrun-test-*' -q)`

tests/README.md

@@ -65,6 +65,9 @@ if [ -n "${KRUN_TEST_BASE_DIR}" ]; then
 	RUNNER_ARGS="${RUNNER_ARGS} --base-dir ${KRUN_TEST_BASE_DIR}"
 fi
 
+# Build rootfs images before entering the network namespace (needs internet + podman)
+target/debug/runner build-images
+
 if [ "$OS" != "Darwin" ] && [ -z "${KRUN_NO_UNSHARE}" ] && which unshare 2>&1 >/dev/null; then
 	unshare --user --map-root-user --net -- /bin/sh -c "ifconfig lo 127.0.0.1 && exec target/debug/runner ${RUNNER_ARGS}"
 else

tests/run.sh

@@ -8,14 +8,7 @@ use std::panic::catch_unwind;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use tempdir::TempDir;
-use test_cases::{test_cases, ShouldRun, Test, TestCase, TestSetup};
-
-#[derive(Clone)]
-enum TestOutcome {
-    Pass,
-    Fail,
-    Skip(&'static str),
-}
+use test_cases::{test_cases, rootfs_images, Report, ShouldRun, Test, TestCase, TestOutcome, TestSetup};
 
 struct TestResult {
     name: String,
@@ -88,21 +81,32 @@ fn run_single_test(
         .context("Failed to start subprocess for test")?;
 
     let test_name = test_case.name.to_string();
-    let result = catch_unwind(|| {
+    let outcome = match catch_unwind(|| {
         let test = get_test(&test_name).unwrap();
-        test.check(child);
-    });
+        test.check(child)
+    }) {
+        Ok(outcome) => outcome,
+        Err(_) => TestOutcome::Fail,
+    };
 
-    let outcome = if result.is_ok() {
-        eprintln!("OK");
-        if !keep_all {
-            let _ = fs::remove_dir_all(&test_dir);
+    match &outcome {
+        TestOutcome::Pass => {
+            eprintln!("OK");
+            if !keep_all {
+                let _ = fs::remove_dir_all(&test_dir);
+            }
         }
-        TestOutcome::Pass
-    } else {
-        eprintln!("FAIL");
-        TestOutcome::Fail
-    };
+        TestOutcome::Fail => {
+            eprintln!("FAIL");
+        }
+        TestOutcome::Skip(reason) => {
+            eprintln!("SKIP ({})", reason);
+        }
+        TestOutcome::Report(report) => {
+            eprintln!("REPORT");
+            eprintln!("{:2}", report.text());
+        }
+    }
 
     Ok(TestResult {
         name: test_case.name.to_string(),
@@ -116,6 +120,7 @@ fn write_github_summary(
     num_pass: usize,
     num_fail: usize,
     num_skip: usize,
+    num_report: usize,
 ) -> anyhow::Result<()> {
     let summary_path = env::var("GITHUB_STEP_SUMMARY")
         .context("GITHUB_STEP_SUMMARY environment variable not set")?;
@@ -128,22 +133,30 @@ fn write_github_summary(
 
     let num_ran = num_pass + num_fail;
     let status = if num_fail == 0 { "✅" } else { "❌" };
-    let skip_msg = if num_skip > 0 {
-        format!(" ({num_skip} skipped)")
-    } else {
+    let mut extra = Vec::new();
+    if num_skip > 0 {
+        extra.push(format!("{num_skip} skipped"));
+    }
+    if num_report > 0 {
+        extra.push(format!("{num_report} reports"));
+    }
+    let extra_msg = if extra.is_empty() {
         String::new()
+    } else {
+        format!(" ({})", extra.join(", "))
     };
 
     writeln!(
         file,
-        "## {status} Integration Tests - {num_pass}/{num_ran} passed{skip_msg}\n"
+        "## {status} Integration Tests - {num_pass}/{num_ran} passed{extra_msg}\n"
     )?;
 
     for result in results {
         let (icon, status_text) = match &result.outcome {
             TestOutcome::Pass => ("✅", String::new()),
             TestOutcome::Fail => ("❌", String::new()),
             TestOutcome::Skip(reason) => ("⏭️", format!(" - {}", reason)),
+            TestOutcome::Report(_) => ("📊", String::new()),
         };
 
         writeln!(file, "<details>")?;
@@ -153,7 +166,9 @@ fn write_github_summary(
             result.name, status_text
         )?;
 
-        if let Some(log_path) = &result.log_path {
+        if let TestOutcome::Report(report) = &result.outcome {
+            writeln!(file, "{}", report.gh_markdown())?;
+        } else if let Some(log_path) = &result.log_path {
             let log_content = fs::read_to_string(log_path).unwrap_or_default();
             writeln!(file, "```")?;
             // Limit log size to avoid huge summaries (2 MiB limit)
@@ -227,28 +242,39 @@ fn run_tests(
         .iter()
         .filter(|r| matches!(r.outcome, TestOutcome::Skip(_)))
         .count();
+    let num_report = results
+        .iter()
+        .filter(|r| matches!(r.outcome, TestOutcome::Report(_)))
+        .count();
     let num_ran = num_pass + num_fail;
 
     // Write GitHub Actions summary if requested
     if github_summary {
-        write_github_summary(&results, num_pass, num_fail, num_skip)?;
+        write_github_summary(&results, num_pass, num_fail, num_skip, num_report)?;
     }
 
-    let skip_msg = if num_skip > 0 {
-        format!(" ({num_skip} skipped)")
-    } else {
+    let mut extra = Vec::new();
+    if num_skip > 0 {
+        extra.push(format!("{num_skip} skipped"));
+    }
+    if num_report > 0 {
+        extra.push(format!("{num_report} reports"));
+    }
+    let extra_msg = if extra.is_empty() {
         String::new()
+    } else {
+        format!(" ({})", extra.join(", "))
     };
 
     if num_fail > 0 {
         eprintln!("(See test artifacts at: {})", base_dir.display());
-        println!("\nFAIL - {num_pass}/{num_ran} passed{skip_msg}");
+        println!("\nFAIL - {num_pass}/{num_ran} passed{extra_msg}");
         anyhow::bail!("")
     } else {
         if keep_all {
             eprintln!("(See test artifacts at: {})", base_dir.display());
         }
-        eprintln!("\nOK - {num_pass}/{num_ran} passed{skip_msg}");
+        eprintln!("\nOK - {num_pass}/{num_ran} passed{extra_msg}");
     }
 
     Ok(())
@@ -276,6 +302,8 @@ enum CliCommand {
         #[arg(long)]
         tmp_dir: PathBuf,
     },
+    /// Build all registered rootfs images (requires network; run before unshare)
+    BuildImages,
 }
 
 impl Default for CliCommand {
@@ -295,6 +323,17 @@ struct Cli {
     command: Option<CliCommand>,
 }
 
+fn build_images() -> anyhow::Result<()> {
+    use test_cases::rootfs;
+
+    for (name, _) in rootfs_images() {
+        eprint!("Building rootfs image {name}...");
+        rootfs::build_rootfs(name)?;
+        eprintln!(" done");
+    }
+    Ok(())
+}
+
 fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
     let command = cli.command.unwrap_or_default();
@@ -307,5 +346,6 @@ fn main() -> anyhow::Result<()> {
             keep_all,
             github_summary,
         } => run_tests(&test_case, base_dir, keep_all, github_summary),
+        CliCommand::BuildImages => build_images(),
     }
 }

tests/runner/src/main.rs

@@ -3,7 +3,7 @@ name = "test_cases"
 edition = "2021"
 
 [features]
-host = ["krun-sys"]
+host = ["krun-sys", "serde", "serde_json"]
 guest = []
 
 [lib]
@@ -14,4 +14,6 @@ krun-sys = { path = "../../krun-sys", optional = true }
 macros = { path = "../macros" }
 nix = { version = "0.29.0", features = ["socket", "ioctl"] }
 anyhow = "1.0.95"
-tempdir = "0.3.7"
+serde = { version = "1", features = ["derive"], optional = true }
+serde_json = { version = "1", optional = true }
+tempdir = "0.3.7"

tests/test_cases/Cargo.toml

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo:rerun-if-env-changed=IPERF_DURATION");
+}

tests/test_cases/build.rs

@@ -50,3 +50,30 @@ pub fn setup_fs_and_enter(ctx: u32, test_setup: TestSetup) -> anyhow::Result<()>
     }
     unreachable!()
 }
+
+/// Like setup_fs_and_enter, but uses an existing rootfs directory (e.g. a Fedora rootfs with
+/// extra packages installed). Copies the guest-agent into it and enters the VM.
+pub fn setup_existing_rootfs_and_enter(
+    ctx: u32,
+    test_setup: TestSetup,
+    rootfs_dir: &Path,
+) -> anyhow::Result<()> {
+    anyhow::ensure!(rootfs_dir.is_dir(), "rootfs directory not found: {}", rootfs_dir.display());
+    let path_str = CString::new(rootfs_dir.as_os_str().as_bytes()).context("CString::new")?;
+    copy_guest_agent(rootfs_dir)?;
+    unsafe {
+        krun_call!(krun_set_root(ctx, path_str.as_ptr()))?;
+        krun_call!(krun_set_workdir(ctx, c"/".as_ptr()))?;
+        let test_case_cstr = CString::new(test_setup.test_case).context("CString::new")?;
+        let argv = [test_case_cstr.as_ptr(), null()];
+        let envp = [null()];
+        krun_call!(krun_set_exec(
+            ctx,
+            c"/guest-agent".as_ptr(),
+            argv.as_ptr(),
+            envp.as_ptr(),
+        ))?;
+        krun_call!(krun_start_enter(ctx))?;
+    }
+    unreachable!()
+}