Composefs test parity with ostree

.github/workflows/ci.yml

@@ -157,7 +157,7 @@ jobs:
       matrix:
         # No fedora-44 due to https://bugzilla.redhat.com/show_bug.cgi?id=2429501
         test_os: [fedora-43, centos-9, centos-10]
-        variant: [ostree, composefs-sealeduki-sdboot]
+        variant: [ostree, composefs-sealeduki-sdboot, composefs-sdboot, composefs-grub]
         exclude:
           # centos-9 UKI is experimental/broken (https://github.com/bootc-dev/bootc/issues/1812)
           - test_os: centos-9
@@ -178,7 +178,18 @@ jobs:
         run: |
           BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
           echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
-          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+
+          case "${{ matrix.variant }}" in
+            composefs-grub|composefs-sdboot)
+              echo "BOOTC_variant=composefs" >> $GITHUB_ENV
+            ;;
+
+            *)
+              echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+            ;;
+          esac
+
+
 
           if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then
             BUILDROOTBASE=$(just pullspec-for-os buildroot-base ${{ matrix.test_os }})
@@ -207,11 +218,12 @@ jobs:
 
       - name: Run TMT integration tests
         run: |
-          if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then
-            just test-composefs
+          if [[ "${{ matrix.variant }}" = composefs* ]]; then
+            just "test-${{ matrix.variant }}"
           else
             just test-tmt integration
           fi
+
           just clean-local-images
 
       - name: Archive TMT logs

Dockerfile

@@ -34,7 +34,8 @@ WORKDIR /src
 # We aren't using the full recommendations there, just the simple bits.
 # First we download all of our Rust dependencies
 # Note: Local path dependencies (from [patch] sections) are auto-detected and bind-mounted by the Justfile
-RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch
+RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome \
+    rm -rf /var/roothome/.cargo/registry; cargo fetch
 
 # We always do a "from scratch" build
 # https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/
@@ -143,13 +144,15 @@ RUN --network=none --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp
 # Perform all filesystem transformations except generating the sealed UKI (if configured)
 FROM base as base-penultimate
 ARG variant
-# Switch to a signed systemd-boot, if configured
+# Switch to systemd-boot (signed or unsigned), if configured
 RUN --network=none --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     --mount=type=bind,from=sdboot-signed,src=/,target=/run/sdboot-signed <<EORUN
 set -xeuo pipefail
 if test "${variant}" = "composefs-sealeduki-sdboot"; then
   /run/packaging/switch-to-sdboot /run/sdboot-signed
+elif test "${variant}" = "composefs"; then
+  /run/packaging/install-unsigned-sdboot
 fi
 EORUN
 # Configure the rootfs

Justfile

@@ -105,9 +105,32 @@ test-container: build build-units
 
 # Build and test sealed composefs images
 [group('core')]
-test-composefs:
+test-composefs-sealeduki-sdboot:
     just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot
 
+[group('core')]
+test-composefs bootloader:
+    just variant=composefs test-tmt --composefs-backend --bootloader {{bootloader}} \
+        readonly \
+        bib-build \
+        download-only \
+        image-pushpull-upgrade \
+        image-upgrade-reboot \
+        install-outside-container \
+        install-to-filesystem-var-mount \
+        soft-reboot \
+        usroverlay
+
+# Build and test composefs images booted using Type1 boot entries and systemd-boot as the bootloader
+[group('core')]
+test-composefs-sdboot:
+    just test-composefs systemd
+
+# Build and test composefs images booted using Type1 boot entries and grub as the bootloader
+[group('core')]
+test-composefs-grub:
+    just test-composefs grub
+
 # Run cargo fmt and clippy checks in container
 [group('core')]
 validate:
@@ -220,6 +243,7 @@ clean-local-images:
     podman image prune -f
     podman rmi {{fedora-coreos}} -f
 
+
 # Build packages (RPM) into target/packages/
 [group('maintenance')]
 package:

contrib/packaging/install-unsigned-sdboot

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Install unsigned systemd-boot RPM (already downloaded by tools stage)
+set -xeuo pipefail
+
+# Uninstall bootupd if present (we're switching to sd-boot managed differently)
+if rpm -q bootupd &>/dev/null; then
+  rpm -e bootupd
+  rm -vrf /usr/lib/bootupd/updates
+fi
+
+# Install the unsigned systemd-boot RPM that was downloaded by the tools stage
+# The RPM is available in /run/sdboot-signed/out (copied from tools stage)
+rpm -Uvh /run/sdboot-signed/out/*.rpm

crates/tests-integration/src/container.rs

@@ -52,8 +52,8 @@ pub(crate) fn test_bootc_container_inspect() -> Result<()> {
         .expect("kernel.unified should be a boolean");
     if let Some(variant) = std::env::var("BOOTC_variant").ok() {
         match variant.as_str() {
-            "ostree" => {
-                assert!(!unified, "Expected unified=false for ostree variant");
+            v @ "ostree" | v @ "composefs" => {
+                assert!(!unified, "Expected unified=false for variant {v}");
                 // For traditional kernels, version should look like a uname (contains digits)
                 assert!(
                     version.chars().any(|c| c.is_ascii_digit()),
@@ -159,7 +159,7 @@ fn test_variant_base_crosscheck() -> Result<()> {
         // TODO add this to `bootc status` or so?
         let boot_efi = Utf8Path::new("/boot/EFI");
         match variant.as_str() {
-            "ostree" => {
+            "composefs" | "ostree" => {
                 assert!(!boot_efi.try_exists()?);
             }
             "composefs-sealeduki-sdboot" => {

crates/xtask/src/tmt.rs

@@ -29,6 +29,8 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image";
 // Distro identifiers
 const DISTRO_CENTOS_9: &str = "centos-9";
 
+const COMPOSEFS_KERNEL_ARGS: [&str; 1] = ["--karg=enforcing=0"];
+
 // Import the argument types from xtask.rs
 use crate::{RunTmtArgs, TmtProvisionArgs};
 
@@ -430,6 +432,17 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
                     opts.push("--filesystem=xfs".to_string());
                 }
             }
+
+            if args.composefs_backend {
+                opts.push("--filesystem=ext4".into());
+                opts.push("--composefs-backend".into());
+                opts.extend(COMPOSEFS_KERNEL_ARGS.map(|x| x.into()));
+            }
+
+            if let Some(b) = &args.bootloader {
+                opts.push(format!("--bootloader={b}"));
+            }
+
             opts
         };
 

crates/xtask/src/xtask.rs

@@ -5,13 +5,14 @@
 //! end up as a lot of nontrivial bash code.
 
 use std::borrow::Cow;
+use std::fmt::Display;
 use std::fs::File;
 use std::io::{BufRead, BufReader, BufWriter, Write};
 use std::process::Command;
 
 use anyhow::{Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
-use clap::{Args, Parser, Subcommand};
+use clap::{Args, Parser, Subcommand, ValueEnum};
 use fn_error_context::context;
 use xshell::{Shell, cmd};
 
@@ -76,6 +77,25 @@ pub(crate) struct LocalRustDepsArgs {
     pub(crate) format: String,
 }
 
+/// Bootloader passed as --bootloader param for composefs builds
+// TODO: Find a better way to share this Enum between this and crates/lib
+#[derive(Debug, Clone, ValueEnum)]
+pub enum Bootloader {
+    /// grub as bootloader
+    Grub,
+    /// systemd-boot as bootloader
+    Systemd,
+}
+
+impl Display for Bootloader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Bootloader::Grub => f.write_str("grub"),
+            Bootloader::Systemd => f.write_str("systemd"),
+        }
+    }
+}
+
 /// Arguments for run-tmt command
 #[derive(Debug, Args)]
 pub(crate) struct RunTmtArgs {
@@ -101,6 +121,12 @@ pub(crate) struct RunTmtArgs {
     /// Preserve VMs after test completion (useful for debugging)
     #[arg(long)]
     pub(crate) preserve_vm: bool,
+
+    #[arg(long)]
+    pub(crate) composefs_backend: bool,
+
+    #[arg(long, requires = "composefs_backend")]
+    pub(crate) bootloader: Option<Bootloader>,
 }
 
 /// Arguments for tmt-provision command

tmt/tests/booted/readonly/001-test-status.nu

@@ -14,7 +14,7 @@ assert ($opts | any { |o| $o == "ro" }) "/sysroot should be mounted read-only"
 
 let st = bootc status --json | from json
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 assert equal $st.apiVersion org.containers.bootc/v1
 

tmt/tests/booted/readonly/010-test-bootc-container-store.nu

@@ -5,7 +5,7 @@ tap begin "verify bootc-owned container storage"
 
 # Detect composefs by checking if composefs field is present
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 if $is_composefs {
     print "# TODO composefs: skipping test - /usr/lib/bootc/storage doesn't exist with composefs"

tmt/tests/booted/readonly/011-test-ostree-ext-cli.nu

@@ -8,7 +8,7 @@ tap begin "verify bootc wrapping ostree-ext"
 # Parse the status and get the booted image
 let st = bootc status --json | from json
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 if $is_composefs {
     print "# TODO composefs: skipping test - ostree-container commands don't work with composefs"
 } else {

tmt/tests/booted/readonly/011-test-resolvconf.nu

@@ -6,7 +6,7 @@ tap begin "verify there's not an empty /etc/resolv.conf in the image"
 let st = bootc status --json | from json
 
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 if $is_composefs {
     print "# TODO composefs: skipping test - ostree commands don't work with composefs"
 } else {

tmt/tests/booted/readonly/012-test-unit-status.nu

@@ -6,7 +6,7 @@ tap begin "verify our systemd units"
 
 # Detect composefs by checking if composefs field is present
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 if $is_composefs {
     print "# TODO composefs: skipping test - bootc-status-updated.path watches /ostree/bootc which doesn't exist with composefs"

tmt/tests/booted/readonly/015-test-fsck.nu

@@ -5,7 +5,7 @@ tap begin "Run fsck"
 
 # Detect composefs by checking if composefs field is present
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 if $is_composefs {
     print "# TODO composefs: skipping test - fsck requires ostree-booted host"

tmt/tests/booted/readonly/017-test-bound-storage.nu

@@ -10,7 +10,7 @@ if not (bootc_testlib have_hostexports) {
 
 bootc status
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 if $is_composefs {
     # TODO we don't have imageDigest yet in status
     exit 0

tmt/tests/booted/readonly/030-test-composefs.nu

@@ -9,7 +9,7 @@ def parse_cmdline []  {
 
 # Detect composefs by checking if composefs field is present
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 let expecting_composefs = ($env.BOOTC_variant? | default "" | find "composefs") != null
 if $expecting_composefs {
     assert $is_composefs

tmt/tests/booted/tap.nu

@@ -13,3 +13,8 @@ export def ok [] {
 export def fail [] {
   print "not ok"
 }
+
+export def is_composefs [] {
+    let st = bootc status --json | from json
+    $st.status.booted.composefs? != null
+}

tmt/tests/booted/test-image-pushpull-upgrade.nu

@@ -22,7 +22,7 @@ const quoted_karg = '"thisarg=quoted with spaces"'
 bootc status
 let st = bootc status --json | from json
 let booted = $st.status.booted.image
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 # Parse the kernel commandline into a list.
 # This is not a proper parser, but good enough
@@ -54,7 +54,11 @@ RUN echo test content > /usr/share/blah.txt
     let v = podman run --rm localhost/bootc-derived cat /usr/share/blah.txt | str trim
     assert equal $v "test content"
 
-    let orig_root_mtime = ls -Dl /ostree/bootc | get modified
+    mut orig_root_mtime = null;
+
+    if not $is_composefs {
+        $orig_root_mtime = ls -Dl /ostree/bootc | get modified
+    }
 
     # Now, fetch it back into the bootc storage!
     # We also test the progress API here
@@ -68,24 +72,25 @@ RUN echo test content > /usr/share/blah.txt
     systemd-run -u test-cat-progress -- /bin/bash -c $"exec cat ($progress_fifo) > ($progress_json)"
     # nushell doesn't do fd passing right now either, so run via bash
     bash -c $"bootc switch --progress-fd 3 --transport containers-storage localhost/bootc-derived 3>($progress_fifo)"
-    # Now, let's do some checking of the progress json
-    let progress = open --raw $progress_json | from json -o
-    sanity_check_switch_progress_json $progress
 
-    # Check that /run/reboot-required exists and is not empty
-    let rr_meta = (ls /run/reboot-required | first)
-    assert ($rr_meta.size > 0b)
+    if not $is_composefs {
+        # Now, let's do some checking of the progress json
+        let progress = open --raw $progress_json | from json -o
+        sanity_check_switch_progress_json $progress
 
-    # Verify that we logged to the journal
-    journalctl _MESSAGE_ID=3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8b7
+        # Check that /run/reboot-required exists and is not empty
+        let rr_meta = (ls /run/reboot-required | first)
+        assert ($rr_meta.size > 0b)
 
-    # The mtime should change on modification
-    let new_root_mtime = ls -Dl /ostree/bootc | get modified
-    assert ($new_root_mtime > $orig_root_mtime)
+        # Verify that we logged to the journal
+        journalctl _MESSAGE_ID=3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8b7
 
-    # Test for https://github.com/ostreedev/ostree/issues/3544
-    # Add a quoted karg using rpm-ostree if available
-    if not $is_composefs {
+        # The mtime should change on modification
+        let new_root_mtime = ls -Dl /ostree/bootc | get modified
+        assert ($new_root_mtime > $orig_root_mtime)
+
+        # Test for https://github.com/ostreedev/ostree/issues/3544
+        # Add a quoted karg using rpm-ostree if available
         # Check rpm-ostree and rpm-ostreed service status before run rpm-ostree
         # And collect info for flaky error "error: System transaction in progress"
         rpm-ostree status

tmt/tests/booted/test-install-to-filesystem-var-mount.sh

@@ -114,6 +114,17 @@ echo "Filesystem layout:"
 mount | grep /var/mnt/target || true
 df -h /var/mnt/target /var/mnt/target/boot /var/mnt/target/boot/efi /var/mnt/target/var
 
+COMPOSEFS_BACKEND=()
+
+is_composefs=$(bootc status --json | jq '.status.booted.composefs')
+
+if [[ $is_composefs != "null" ]]; then
+    COMPOSEFS_BACKEND+=("--composefs-backend")
+    COMPOSEFS_BACKEND+=("--filesystem=ext4")
+fi
+
+echo "${COMPOSEFS_BACKEND[@]}"
+
 # Run bootc install to-filesystem from within the container image under test
 podman run \
     --rm --privileged \
@@ -124,6 +135,7 @@ podman run \
     "$TARGET_IMAGE" \
     bootc install to-filesystem \
         --disable-selinux \
+        "${COMPOSEFS_BACKEND[@]}" \
         --karg=root=UUID="$ROOT_UUID" \
         --root-mount-spec=UUID="$ROOT_UUID" \
         --boot-mount-spec=UUID="$BOOT_UUID" \