From 5f046899b58e98d29a1dd3e1f55bb43c66c9ef4c Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Tue, 7 Oct 2025 05:34:16 -0700 Subject: [PATCH] xtask: teach integration-test vm to consume kernel debs directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundle handling of Debian kernel archives into xtask so callers can pipe the raw `.deb` paths straight into `cargo xtask integration-test vm …`. The driver now extracts each archive into `/kernel-archives`, locates the matching `vmlinuz-*`, `lib/modules/*`, and config files, and feeds those into the initramfs build without requiring the user to pre-run dpkg/tar. With this in place we drop `.github/scripts/find_kernels.py`, simplify AGENTS.md/CI instructions to use `find test/.tmp -name '*.deb'`, remove the gnu-tar requirement we no longer need, and add `tar` as a workspace dependency for the extractor. --- .github/scripts/find_kernels.py | 33 ------- .github/workflows/ci.yml | 24 ++--- AGENTS.md | 7 +- Brewfile | 1 - Cargo.toml | 1 + xtask/Cargo.toml | 1 + xtask/src/run.rs | 161 ++++++++++++++++++++++---------- 7 files changed, 122 insertions(+), 106 deletions(-) delete mode 100755 .github/scripts/find_kernels.py diff --git a/.github/scripts/find_kernels.py b/.github/scripts/find_kernels.py deleted file mode 100755 index c077e767..00000000 --- a/.github/scripts/find_kernels.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -import os -import glob -import sys -from typing import List - -def find_kernels(directory: str) -> List[str]: - return glob.glob(f"{directory}/**/vmlinuz-*", recursive=True) - -def find_modules_directory(directory: str, kernel: str) -> str: - matches = glob.glob(f"{directory}/**/modules/{kernel}", recursive=True) - if len(matches) != 1: - raise RuntimeError(f"Expected to find exactly one modules directory. Found {len(matches)}.") - return matches[0] - -def main() -> None: - images = find_kernels('test/.tmp') - modules = [] - - for image in images: - image_name = os.path.basename(image).replace('vmlinuz-', '') - module_dir = find_modules_directory('test/.tmp', image_name) - modules.append(module_dir) - - for image, module in zip(images, modules): - sys.stdout.write(image) - sys.stdout.write(':') - sys.stdout.write(module) - sys.stdout.write('\0') - -if __name__ == "__main__": - main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47fc257..5819d588 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,7 +253,6 @@ jobs: # Dependencies are tracked in `Brewfile`. brew bundle echo $(brew --prefix curl)/bin >> $GITHUB_PATH - echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH echo $(brew --prefix llvm)/bin >> $GITHUB_PATH # https://github.com/actions/setup-python/issues/577 @@ -316,17 +315,6 @@ jobs: set -euxo pipefail rm -rf test/.tmp/boot test/.tmp/lib - - name: Extract debian kernels - run: | - set -euxo pipefail - # The wildcard '**/boot/*' extracts kernel images and config. - # The wildcard '**/modules/*' extracts kernel modules. - # Modules are required since not all parts of the kernel we want to - # test are built-in. - find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \ - sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp \ - --wildcards --extract '**/boot/*' '**/modules/*' --file -" - - name: Run local integration tests if: runner.os == 'Linux' run: cargo xtask integration-test local @@ -341,9 +329,9 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm || true # kvm is not available on arm64. - .github/scripts/find_kernels.py | xargs -t -0 \ - cargo xtask integration-test vm --cache-dir test/.tmp \ - --github-api-token ${{ secrets.GITHUB_TOKEN }} \ + find test/.tmp -name '*.deb' -print0 | xargs -t -0 \ + cargo xtask integration-test vm --cache-dir test/.tmp \ + --github-api-token ${{ secrets.GITHUB_TOKEN }} - name: Run virtualized integration tests if: runner.os == 'macOS' @@ -352,9 +340,9 @@ jobs: CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: x86_64-linux-musl-gcc run: | set -euxo pipefail - .github/scripts/find_kernels.py | xargs -t -0 \ - cargo xtask integration-test vm --cache-dir test/.tmp \ - --github-api-token ${{ secrets.GITHUB_TOKEN }} \ + find test/.tmp -name '*.deb' -print0 | xargs -t -0 \ + cargo xtask integration-test vm --cache-dir test/.tmp \ + --github-api-token ${{ secrets.GITHUB_TOKEN }} # Provides a single status check for the entire build workflow. # This is used for merge automation, like Mergify, since GH actions diff --git a/AGENTS.md b/AGENTS.md index e424466a..0087076e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,19 +17,14 @@ ```sh .github/scripts/download_kernel_images.sh \ test/.tmp/debian-kernels/ [VERSIONS]... - - find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \ - sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp \ - --wildcards --extract '**/boot/*' '**/modules/*' --file -" ``` - You might need to use gtar rather than tar on mac. - Run: ```sh - .github/scripts/find_kernels.py | xargs -0 -t sh -c \ + find test/.tmp -name '*.deb' -print0 | xargs -0 -t sh -c \ 'cargo xtask integration-test vm --cache-dir test/.tmp "$@" -- [ARGS]...' _ ``` diff --git a/Brewfile b/Brewfile index 2b7525a5..5a0f6c90 100644 --- a/Brewfile +++ b/Brewfile @@ -2,7 +2,6 @@ brew "curl" brew "dpkg" -brew "gnu-tar" brew "llvm" brew "lynx" brew "pkg-config" diff --git a/Cargo.toml b/Cargo.toml index e24bb95c..81409567 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,6 +98,7 @@ rustup-toolchain = { version = "0.1.5", default-features = false } rustversion = { version = "1.0.0", default-features = false } scopeguard = { version = "1.2.0", default-features = false } syn = { version = "2", default-features = false } +tar = { version = "0.4.44", default-features = false } tempfile = { version = "3", default-features = false } test-case = { version = "3.1.0", default-features = false } test-log = { version = "0.2.13", default-features = false } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 7ce79c6e..04df81f6 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -27,5 +27,6 @@ quote = { workspace = true } rustdoc-json = { workspace = true } rustup-toolchain = { workspace = true } syn = { workspace = true } +tar = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } diff --git a/xtask/src/run.rs b/xtask/src/run.rs index 872fe861..7174e58d 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -1,10 +1,10 @@ use std::{ - ffi::OsString, + ffi::{OsStr, OsString}, fmt::Write as _, fs::{self, OpenOptions}, io::{BufRead as _, BufReader, Write as _}, ops::Deref as _, - path::{Path, PathBuf}, + path::{self, Path, PathBuf}, process::{Child, ChildStdin, Command, Output, Stdio}, sync::{Arc, Mutex}, thread, @@ -38,48 +38,12 @@ enum Environment { #[clap(long)] github_api_token: Option, - /// The kernel image and modules to use. - /// - /// Format: : - /// - /// You can download some images with: - /// - /// wget --accept-regex '.*/linux-image-[0-9\.-]+-cloud-.*-unsigned*' \ - /// --recursive http://ftp.us.debian.org/debian/pool/main/l/linux/ - /// - /// You can then extract the images and kernel modules with: - /// - /// find . -name '*.deb' -print0 \ - /// | xargs -0 -I {} sh -c "dpkg --fsys-tarfile {} \ - /// | tar --wildcards --extract '**/boot/*' '**/modules/*' --file -" - /// - /// `**/boot/*` is used to extract the kernel image and config. - /// - /// `**/modules/*` is used to extract the kernel modules. - /// - /// Modules are required since not all parts of the kernel we want to - /// test are built-in. - #[clap(required = true, value_parser=parse_image_and_modules)] - image_and_modules: Vec<(PathBuf, PathBuf)>, + /// Debian kernel archives (.deb) to boot in the VM. + #[clap(required = true)] + kernel_archives: Vec, }, } -pub(crate) fn parse_image_and_modules(s: &str) -> Result<(PathBuf, PathBuf), std::io::Error> { - let mut parts = s.split(':'); - let image = parts - .next() - .ok_or(std::io::ErrorKind::InvalidInput) - .map(PathBuf::from)?; - let modules = parts - .next() - .ok_or(std::io::ErrorKind::InvalidInput) - .map(PathBuf::from)?; - if parts.next().is_some() { - return Err(std::io::ErrorKind::InvalidInput.into()); - } - Ok((image, modules)) -} - #[derive(Parser)] pub(crate) struct Options { #[clap(subcommand)] @@ -212,7 +176,7 @@ pub(crate) fn run(opts: Options) -> Result<()> { Environment::VM { cache_dir, github_api_token, - image_and_modules, + kernel_archives, } => { // The user has asked us to run the tests on a VM. This is involved; strap in. // @@ -320,13 +284,114 @@ pub(crate) fn run(opts: Options) -> Result<()> { } } + let extraction_root = tempfile::tempdir().context("tempdir failed")?; let mut errors = Vec::new(); - for (kernel_image, modules_dir) in image_and_modules { + for (index, archive) in kernel_archives.iter().enumerate() { + let archive_dir = extraction_root + .path() + .join(format!("kernel-archive-{index}")); + fs::create_dir_all(&archive_dir) + .with_context(|| format!("failed to create {}", archive_dir.display()))?; + + let mut dpkg = Command::new("dpkg-deb"); + dpkg.arg("--fsys-tarfile") + .arg(archive) + .stdout(Stdio::piped()); + let mut dpkg_child = dpkg + .spawn() + .with_context(|| format!("failed to spawn {dpkg:?}"))?; + let Child { stdout, .. } = &mut dpkg_child; + let stdout = stdout.take().unwrap(); + let mut archive_reader = tar::Archive::new(stdout); + archive_reader.unpack(&archive_dir).with_context(|| { + format!( + "failed to unpack archive {} to {}", + archive.display(), + archive_dir.display() + ) + })?; + let status = dpkg_child + .wait() + .with_context(|| format!("failed to wait for {dpkg:?}"))?; + if !status.success() { + bail!("{dpkg:?} exited with {status}"); + } + + let mut kernel_images = Vec::new(); + for entry in WalkDir::new(&archive_dir) { + let entry = entry.with_context(|| { + format!("failed to read entry in {}", archive_dir.display()) + })?; + if !entry.file_type().is_file() { + continue; + } + let path = entry.into_path(); + if let Some(file_name) = path.file_name() { + match file_name.as_encoded_bytes() { + // "vmlinuz-" + [ + b'v', + b'm', + b'l', + b'i', + b'n', + b'u', + b'z', + b'-', + kernel_version @ .., + ] => { + let kernel_version = + unsafe { OsStr::from_encoded_bytes_unchecked(kernel_version) } + .to_os_string(); + kernel_images.push((path, kernel_version)) + } + _ => {} + } + } + } + let (kernel_image, kernel_version) = match kernel_images.as_slice() { + [kernel_image] => kernel_image, + [] => bail!("no kernel images in {}", archive.display()), + kernel_images => bail!( + "multiple kernel images in {}: {:?}", + archive.display(), + kernel_images + ), + }; + + let mut modules_dirs = Vec::new(); + for entry in WalkDir::new(&archive_dir) { + let entry = entry.with_context(|| { + format!("failed to read entry in {}", archive_dir.display()) + })?; + if !entry.file_type().is_dir() { + continue; + } + let path = entry.into_path(); + let mut components = path.components().rev(); + if components.next() != Some(path::Component::Normal(kernel_version)) { + continue; + } + if components.next() != Some(path::Component::Normal(OsStr::new("modules"))) { + continue; + } + modules_dirs.push(path); + } + let modules_dir = match modules_dirs.as_slice() { + [modules_dir] => modules_dir, + [] => bail!("no modules directories in {}", archive.display()), + modules_dirs => bail!( + "multiple modules directories in {}: {:?}", + archive.display(), + modules_dirs + ), + }; + // Guess the guest architecture. let mut file = Command::new("file"); let output = file .arg("--brief") - .arg(&kernel_image) + .arg(kernel_image) .output() .with_context(|| format!("failed to run {file:?}"))?; let Output { status, .. } = &output; @@ -441,7 +506,7 @@ pub(crate) fn run(opts: Options) -> Result<()> { .arg("run") .args(test_distro_args) .args(["--bin", "depmod", "--", "-b"]) - .arg(&modules_dir) + .arg(modules_dir) .output() .with_context(|| format!("failed to run {cargo:?}"))?; let Output { status, .. } = &output; @@ -452,12 +517,12 @@ pub(crate) fn run(opts: Options) -> Result<()> { // Now our modules.alias file is built, we can recursively // walk the modules directory and add all the files to the // initramfs. - for entry in WalkDir::new(&modules_dir) { + for entry in WalkDir::new(modules_dir) { let entry = entry.context("read_dir failed")?; let path = entry.path(); let metadata = entry.metadata().context("metadata failed")?; let out_path = Path::new("/lib/modules").join( - path.strip_prefix(&modules_dir).with_context(|| { + path.strip_prefix(modules_dir).with_context(|| { format!( "strip prefix {} failed for {}", path.display(), @@ -528,7 +593,7 @@ pub(crate) fn run(opts: Options) -> Result<()> { .arg("-append") .arg(kernel_args) .arg("-kernel") - .arg(&kernel_image) + .arg(kernel_image) .arg("-initrd") .arg(&initrd_image); let mut qemu_child = qemu