From 51d97a4303c1e78d3a553e189a175df8300c534a Mon Sep 17 00:00:00 2001 From: Friday Ortiz Date: Wed, 15 Oct 2025 13:24:08 -0400 Subject: [PATCH] test,xtask: include debug symbols for attaching breakpoints in tests For some reason, the aarch64 6.1 debian kernel was not compiled with CONFIG_KALLSYMS_ALL=y, and the locations of globals are not available in kallsyms. To attach breakpoints to these symbols in the test pipeline, we need to read them from System.map and apply the kaslr offset to get their real address. The System.map file is not provided in the kernel package by default, so we need to extract it from the corresponding debug package. - .github: pull the corresponding debug packages down as well as regular kernels - test: attach the perf_event_bp test breakpoint to the modprobe_path address in kallsyms if present, or by applying the kaslr offset to the System.map address if not found - xtask: preferentially extract the System.map file from the debug package, if available --- .github/scripts/download_kernel_images.sh | 17 ++ test/integration-test/Cargo.toml | 1 + .../src/tests/perf_event_bp.rs | 48 ++++- xtask/src/run.rs | 193 +++++++++++++++--- 4 files changed, 222 insertions(+), 37 deletions(-) diff --git a/.github/scripts/download_kernel_images.sh b/.github/scripts/download_kernel_images.sh index da827460..c86c6ca7 100755 --- a/.github/scripts/download_kernel_images.sh +++ b/.github/scripts/download_kernel_images.sh @@ -25,6 +25,23 @@ for VERSION in "${VERSIONS[@]}"; do exit 1 } FILES+=("$match") + + # The debug package contains the actual System.map. Debian has transitioned + # between -dbg and -dbgsym suffixes, so try both. + DEBUG_REGEX_BASE="linux-image-${VERSION//./\\.}\\.[0-9]+(-[0-9]+)?(\+bpo|\+deb[0-9]+)?-cloud-${ARCHITECTURE}-" + debug_match="" + for debug_suffix in dbg dbgsym; do + regex="${DEBUG_REGEX_BASE}${debug_suffix}_.*\\.deb" + debug_match=$(printf '%s\n' "$URLS" | grep -E "$regex" | sort -V | tail -n1 || true) + if [[ -n "$debug_match" ]]; then + break + fi + done + if [[ -z "$debug_match" ]]; then + printf 'Failed to locate debug package for VERSION=%s (tried dbg/dbgsym)\n' "$VERSION" >&2 + exit 1 + fi + FILES+=("$debug_match") done # Note: `--etag-{compare,save}` are not idempotent until curl 8.9.0 which included diff --git a/test/integration-test/Cargo.toml b/test/integration-test/Cargo.toml index 916db163..502cfcfc 100644 --- a/test/integration-test/Cargo.toml +++ b/test/integration-test/Cargo.toml @@ -23,6 +23,7 @@ aya-log = { path = "../../aya-log", version = "^0.2.1", default-features = false aya-obj = { path = "../../aya-obj", version = "^0.2.1", default-features = false } epoll = { workspace = true } futures = { workspace = true, features = ["alloc"] } +glob = { workspace = true } integration-common = { path = "../integration-common", features = ["user"] } libc = { workspace = true } log = { workspace = true } diff --git a/test/integration-test/src/tests/perf_event_bp.rs b/test/integration-test/src/tests/perf_event_bp.rs index 56d0179c..90268270 100644 --- a/test/integration-test/src/tests/perf_event_bp.rs +++ b/test/integration-test/src/tests/perf_event_bp.rs @@ -14,12 +14,43 @@ use aya::{ }, util::online_cpus, }; -use log::info; +use glob::glob; +use log::{debug, info}; + +fn find_system_map_symbol(sym: &str) -> Option { + for e in fs::read_dir("/boot").unwrap() { + let e = e.unwrap(); + debug!("found /boot/{:}", e.path().to_str().unwrap()); + } + let map = glob("/boot/System.map*") + .expect("failed to read /boot/System.map*") + .next() + .expect("no matching System.map-* file found") + .unwrap(); + let file = File::open(&map).expect("failed to open System.map"); + let reader = BufReader::new(file); + + for line in reader.lines().map_while(Result::ok) { + // Format: " []" + let mut parts = line.split_whitespace(); + let addr_str = parts.next()?; + let _type = parts.next()?; + let name = parts.next()?; + if name == sym + && let Ok(addr) = u64::from_str_radix(addr_str, 16) + { + debug!("found symbol {sym} at address {addr:#x}"); + return Some(addr); + } + } + + None +} // Parse /proc/kallsyms and return the address for the given symbol name, if // found. fn find_kallsyms_symbol(sym: &str) -> Option { - let file = File::open("/proc/kallsyms").ok()?; + let file = File::open("/proc/kallsyms").expect("failed to open /proc/kallsyms"); let reader = BufReader::new(file); for line in reader.lines().map_while(Result::ok) { @@ -41,9 +72,18 @@ fn find_kallsyms_symbol(sym: &str) -> Option { #[test_log::test] fn perf_event_bp() { let mut bpf = Ebpf::load(crate::PERF_EVENT_BP).unwrap(); + let attach_addr = if let Some(addr) = find_kallsyms_symbol("modprobe_path") { + addr + } else { + let kaslr_offset: i64 = (i128::from(find_kallsyms_symbol("_text").unwrap()) + - (i128::from(find_system_map_symbol("_text").unwrap()))) + .try_into() + .unwrap(); - let attach_addr = find_kallsyms_symbol("modprobe_path").unwrap(); - + find_system_map_symbol("modprobe_path") + .unwrap() + .wrapping_add_signed(kaslr_offset) + }; let prog: &mut aya::programs::PerfEvent = bpf .program_mut("perf_event_bp") .unwrap() diff --git a/xtask/src/run.rs b/xtask/src/run.rs index 79433217..f656bcdf 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeMap, ffi::{OsStr, OsString}, fmt::Write as _, fs::{self, OpenOptions}, @@ -18,6 +19,42 @@ use xtask::{AYA_BUILD_INTEGRATION_BPF, Errors}; const GEN_INIT_CPIO_PATCH: &str = include_str!("../patches/gen_init_cpio.c.macos.diff"); +#[derive(Default)] +struct KernelPackageGroup { + kernel: Option, + debug: Option, +} + +fn extract_deb(archive: &Path, dest: &Path) -> Result<()> { + fs::create_dir_all(dest).with_context(|| format!("failed to create {}", dest.display()))?; + + let mut dpkg = Command::new("dpkg-deb"); + dpkg.arg("--fsys-tarfile") + .arg(archive) + .stdout(Stdio::piped()); + let mut dpkg_child = dpkg + .spawn() + .with_context(|| format!("failed to spawn {dpkg:?}"))?; + let Child { stdout, .. } = &mut dpkg_child; + let stdout = stdout.take().unwrap(); + let mut archive_reader = tar::Archive::new(stdout); + archive_reader.unpack(dest).with_context(|| { + format!( + "failed to unpack archive {} to {}", + archive.display(), + dest.display() + ) + })?; + let status = dpkg_child + .wait() + .with_context(|| format!("failed to wait for {dpkg:?}"))?; + if !status.success() { + bail!("{dpkg:?} exited with {status}"); + } + + Ok(()) +} + #[derive(Parser)] enum Environment { /// Runs the integration tests locally. @@ -285,40 +322,88 @@ pub(crate) fn run(opts: Options) -> Result<()> { } let extraction_root = tempfile::tempdir().context("tempdir failed")?; - let mut errors = Vec::new(); - for (index, archive) in kernel_archives.iter().enumerate() { - let archive_dir = extraction_root - .path() - .join(format!("kernel-archive-{index}")); - fs::create_dir_all(&archive_dir) - .with_context(|| format!("failed to create {}", archive_dir.display()))?; - let mut dpkg = Command::new("dpkg-deb"); - dpkg.arg("--fsys-tarfile") - .arg(archive) - .stdout(Stdio::piped()); - let mut dpkg_child = dpkg - .spawn() - .with_context(|| format!("failed to spawn {dpkg:?}"))?; - let Child { stdout, .. } = &mut dpkg_child; - let stdout = stdout.take().unwrap(); - let mut archive_reader = tar::Archive::new(stdout); - archive_reader.unpack(&archive_dir).with_context(|| { - format!( - "failed to unpack archive {} to {}", - archive.display(), - archive_dir.display() - ) + let mut package_groups: BTreeMap = BTreeMap::new(); + for archive in &kernel_archives { + let file_name = archive.file_name().ok_or_else(|| { + anyhow!("archive path missing filename: {}", archive.display()) })?; - let status = dpkg_child - .wait() - .with_context(|| format!("failed to wait for {dpkg:?}"))?; - if !status.success() { - bail!("{dpkg:?} exited with {status}"); + let file_name = file_name.to_string_lossy(); + let (package_name, _) = file_name + .split_once('_') + .ok_or_else(|| anyhow!("unexpected archive filename: {file_name}"))?; + let (base, is_debug) = if let Some(base) = package_name.strip_suffix("-dbg") { + (base, true) + } else if let Some(base) = package_name.strip_suffix("-dbgsym") { + (base, true) + } else if let Some(base) = package_name.strip_suffix("-unsigned") { + (base, false) + } else { + (package_name, false) + }; + let entry = package_groups.entry(OsString::from(base)).or_default(); + if is_debug { + entry.debug = Some(archive.clone()); + } else { + entry.kernel = Some(archive.clone()); } + } + + let mut errors = Vec::new(); + for (index, (base, group)) in package_groups.into_iter().enumerate() { + let KernelPackageGroup { kernel, debug } = group; + let base_display = base.to_string_lossy(); + let kernel_archive = + kernel.ok_or_else(|| anyhow!("missing kernel package for {base_display}"))?; + + let archive_dir = extraction_root + .path() + .join(format!("kernel-archive-{index}-image")); + extract_deb(&kernel_archive, &archive_dir)?; + + let debug_maps = if let Some(debug_archive) = debug { + let debug_dir = extraction_root + .path() + .join(format!("kernel-archive-{index}-debug")); + extract_deb(&debug_archive, &debug_dir)?; + WalkDir::new(&debug_dir) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_type().is_file()) + .filter_map(|entry| { + let path = entry.into_path(); + let is_system_map = path + .file_name() + .map(|file_name| { + matches!( + file_name.as_encoded_bytes(), + [ + b'S', + b'y', + b's', + b't', + b'e', + b'm', + b'.', + b'm', + b'a', + b'p', + b'-', + .. + ] + ) + }) + .unwrap_or(false); + if is_system_map { Some(path) } else { None } + }) + .collect::>() + } else { + Vec::new() + }; let mut kernel_images = Vec::new(); let mut configs = Vec::new(); + let mut kernel_maps = Vec::new(); for entry in WalkDir::new(&archive_dir) { let entry = entry.with_context(|| { format!("failed to read entry in {}", archive_dir.display()) @@ -350,22 +435,59 @@ pub(crate) fn run(opts: Options) -> Result<()> { [b'c', b'o', b'n', b'f', b'i', b'g', b'-', ..] => { configs.push(path); } + // "System.map-" + [ + b'S', + b'y', + b's', + b't', + b'e', + b'm', + b'.', + b'm', + b'a', + b'p', + b'-', + .., + ] => { + kernel_maps.push(path); + } _ => {} } } } let (kernel_image, kernel_version) = match kernel_images.as_slice() { [kernel_image] => kernel_image, - [] => bail!("no kernel images in {}", archive.display()), + [] => bail!("no kernel images in {}", kernel_archive.display()), kernel_images => bail!( "multiple kernel images in {}: {:?}", - archive.display(), + kernel_archive.display(), kernel_images ), }; let config = match configs.as_slice() { [config] => config, - configs => bail!("multiple configs in {}: {:?}", archive.display(), configs), + configs => bail!( + "multiple configs in {}: {:?}", + kernel_archive.display(), + configs + ), + }; + let system_map = match debug_maps.as_slice() { + [system_map] => system_map, + [] => match kernel_maps.as_slice() { + [system_map] => system_map, + kernel_maps => bail!( + "multiple kernel System.maps in {}: {:?}", + kernel_archive.display(), + kernel_maps + ), + }, + system_maps => bail!( + "multiple debug System.maps in {}: {:?}", + kernel_archive.display(), + system_maps + ), }; let mut modules_dirs = Vec::new(); @@ -388,10 +510,10 @@ pub(crate) fn run(opts: Options) -> Result<()> { } let modules_dir = match modules_dirs.as_slice() { [modules_dir] => modules_dir, - [] => bail!("no modules directories in {}", archive.display()), + [] => bail!("no modules directories in {}", kernel_archive.display()), modules_dirs => bail!( "multiple modules directories in {}: {:?}", - archive.display(), + kernel_archive.display(), modules_dirs ), }; @@ -505,6 +627,11 @@ pub(crate) fn run(opts: Options) -> Result<()> { write_file(&Path::new("/boot").join(name), config, "644 0 0"); } + write_file(Path::new("/boot/System.map"), system_map, "644 0 0"); + if let Some(name) = system_map.file_name() { + write_file(&Path::new("/boot").join(name), system_map, "644 0 0"); + } + test_distro.iter().for_each(|(name, path)| { if name == "init" { write_file(Path::new("/init"), path, "755 0 0");