test,xtask: include debug symbols for attaching breakpoints in tests

For some reason, the aarch64 6.1 debian kernel was not compiled with
CONFIG_KALLSYMS_ALL=y, and the locations of globals are not available in
kallsyms. To attach breakpoints to these symbols in the test pipeline, we need
to read them from System.map and apply the kaslr offset to get their real
address. The System.map file is not provided in the kernel package by default,
so we need to extract it from the corresponding debug package.

- .github: pull the corresponding debug packages down as well as regular
  kernels
- test: attach the perf_event_bp test breakpoint to the modprobe_path address
  in kallsyms if present, or by applying the kaslr offset to the System.map
  address if not found
- xtask: preferentially extract the System.map file from the debug package, if
  available

.github/scripts/download_kernel_images.sh

@@ -25,6 +25,23 @@ for VERSION in "${VERSIONS[@]}"; do
     exit 1
   }
   FILES+=("$match")
+
+  # The debug package contains the actual System.map. Debian has transitioned
+  # between -dbg and -dbgsym suffixes, so try both.
+  DEBUG_REGEX_BASE="linux-image-${VERSION//./\\.}\\.[0-9]+(-[0-9]+)?(\+bpo|\+deb[0-9]+)?-cloud-${ARCHITECTURE}-"
+  debug_match=""
+  for debug_suffix in dbg dbgsym; do
+    regex="${DEBUG_REGEX_BASE}${debug_suffix}_.*\\.deb"
+    debug_match=$(printf '%s\n' "$URLS" | grep -E "$regex" | sort -V | tail -n1 || true)
+    if [[ -n "$debug_match" ]]; then
+      break
+    fi
+  done
+  if [[ -z "$debug_match" ]]; then
+    printf 'Failed to locate debug package for VERSION=%s (tried dbg/dbgsym)\n' "$VERSION" >&2
+    exit 1
+  fi
+  FILES+=("$debug_match")
 done
 
 # Note: `--etag-{compare,save}` are not idempotent until curl 8.9.0 which included

test/integration-test/Cargo.toml

@@ -23,6 +23,7 @@ aya-log = { path = "../../aya-log", version = "^0.2.1", default-features = false
 aya-obj = { path = "../../aya-obj", version = "^0.2.1", default-features = false }
 epoll = { workspace = true }
 futures = { workspace = true, features = ["alloc"] }
+glob = { workspace = true }
 integration-common = { path = "../integration-common", features = ["user"] }
 libc = { workspace = true }
 log = { workspace = true }

test/integration-test/src/tests/perf_event_bp.rs

@@ -14,12 +14,43 @@ use aya::{
     },
     util::online_cpus,
 };
-use log::info;
+use glob::glob;
+use log::{debug, info};
+
+fn find_system_map_symbol(sym: &str) -> Option<u64> {
+    for e in fs::read_dir("/boot").unwrap() {
+        let e = e.unwrap();
+        debug!("found /boot/{:}", e.path().to_str().unwrap());
+    }
+    let map = glob("/boot/System.map*")
+        .expect("failed to read /boot/System.map*")
+        .next()
+        .expect("no matching System.map-* file found")
+        .unwrap();
+    let file = File::open(&map).expect("failed to open System.map");
+    let reader = BufReader::new(file);
+
+    for line in reader.lines().map_while(Result::ok) {
+        // Format: "<addr> <type> <symbol> [<module>]"
+        let mut parts = line.split_whitespace();
+        let addr_str = parts.next()?;
+        let _type = parts.next()?;
+        let name = parts.next()?;
+        if name == sym
+            && let Ok(addr) = u64::from_str_radix(addr_str, 16)
+        {
+            debug!("found symbol {sym} at address {addr:#x}");
+            return Some(addr);
+        }
+    }
+
+    None
+}
 
 // Parse /proc/kallsyms and return the address for the given symbol name, if
 // found.
 fn find_kallsyms_symbol(sym: &str) -> Option<u64> {
-    let file = File::open("/proc/kallsyms").ok()?;
+    let file = File::open("/proc/kallsyms").expect("failed to open /proc/kallsyms");
     let reader = BufReader::new(file);
 
     for line in reader.lines().map_while(Result::ok) {
@@ -41,9 +72,18 @@ fn find_kallsyms_symbol(sym: &str) -> Option<u64> {
 #[test_log::test]
 fn perf_event_bp() {
     let mut bpf = Ebpf::load(crate::PERF_EVENT_BP).unwrap();
+    let attach_addr = if let Some(addr) = find_kallsyms_symbol("modprobe_path") {
+        addr
+    } else {
+        let kaslr_offset: i64 = (i128::from(find_kallsyms_symbol("_text").unwrap())
+            - (i128::from(find_system_map_symbol("_text").unwrap())))
+        .try_into()
+        .unwrap();
 
-    let attach_addr = find_kallsyms_symbol("modprobe_path").unwrap();
+        find_system_map_symbol("modprobe_path")
-
+            .unwrap()
+            .wrapping_add_signed(kaslr_offset)
+    };
     let prog: &mut aya::programs::PerfEvent = bpf
         .program_mut("perf_event_bp")
         .unwrap()

xtask/src/run.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::BTreeMap,
     ffi::{OsStr, OsString},
     fmt::Write as _,
     fs::{self, OpenOptions},
@@ -18,6 +19,42 @@ use xtask::{AYA_BUILD_INTEGRATION_BPF, Errors};
 
 const GEN_INIT_CPIO_PATCH: &str = include_str!("../patches/gen_init_cpio.c.macos.diff");
 
+#[derive(Default)]
+struct KernelPackageGroup {
+    kernel: Option<PathBuf>,
+    debug: Option<PathBuf>,
+}
+
+fn extract_deb(archive: &Path, dest: &Path) -> Result<()> {
+    fs::create_dir_all(dest).with_context(|| format!("failed to create {}", dest.display()))?;
+
+    let mut dpkg = Command::new("dpkg-deb");
+    dpkg.arg("--fsys-tarfile")
+        .arg(archive)
+        .stdout(Stdio::piped());
+    let mut dpkg_child = dpkg
+        .spawn()
+        .with_context(|| format!("failed to spawn {dpkg:?}"))?;
+    let Child { stdout, .. } = &mut dpkg_child;
+    let stdout = stdout.take().unwrap();
+    let mut archive_reader = tar::Archive::new(stdout);
+    archive_reader.unpack(dest).with_context(|| {
+        format!(
+            "failed to unpack archive {} to {}",
+            archive.display(),
+            dest.display()
+        )
+    })?;
+    let status = dpkg_child
+        .wait()
+        .with_context(|| format!("failed to wait for {dpkg:?}"))?;
+    if !status.success() {
+        bail!("{dpkg:?} exited with {status}");
+    }
+
+    Ok(())
+}
+
 #[derive(Parser)]
 enum Environment {
     /// Runs the integration tests locally.
@@ -285,40 +322,88 @@ pub(crate) fn run(opts: Options) -> Result<()> {
             }
 
             let extraction_root = tempfile::tempdir().context("tempdir failed")?;
+
+            let mut package_groups: BTreeMap<OsString, KernelPackageGroup> = BTreeMap::new();
+            for archive in &kernel_archives {
+                let file_name = archive.file_name().ok_or_else(|| {
+                    anyhow!("archive path missing filename: {}", archive.display())
+                })?;
+                let file_name = file_name.to_string_lossy();
+                let (package_name, _) = file_name
+                    .split_once('_')
+                    .ok_or_else(|| anyhow!("unexpected archive filename: {file_name}"))?;
+                let (base, is_debug) = if let Some(base) = package_name.strip_suffix("-dbg") {
+                    (base, true)
+                } else if let Some(base) = package_name.strip_suffix("-dbgsym") {
+                    (base, true)
+                } else if let Some(base) = package_name.strip_suffix("-unsigned") {
+                    (base, false)
+                } else {
+                    (package_name, false)
+                };
+                let entry = package_groups.entry(OsString::from(base)).or_default();
+                if is_debug {
+                    entry.debug = Some(archive.clone());
+                } else {
+                    entry.kernel = Some(archive.clone());
+                }
+            }
+
             let mut errors = Vec::new();
-            for (index, archive) in kernel_archives.iter().enumerate() {
+            for (index, (base, group)) in package_groups.into_iter().enumerate() {
+                let KernelPackageGroup { kernel, debug } = group;
+                let base_display = base.to_string_lossy();
+                let kernel_archive =
+                    kernel.ok_or_else(|| anyhow!("missing kernel package for {base_display}"))?;
+
                 let archive_dir = extraction_root
                     .path()
-                    .join(format!("kernel-archive-{index}"));
+                    .join(format!("kernel-archive-{index}-image"));
-                fs::create_dir_all(&archive_dir)
+                extract_deb(&kernel_archive, &archive_dir)?;
-                    .with_context(|| format!("failed to create {}", archive_dir.display()))?;
 
-                let mut dpkg = Command::new("dpkg-deb");
+                let debug_maps = if let Some(debug_archive) = debug {
-                dpkg.arg("--fsys-tarfile")
+                    let debug_dir = extraction_root
-                    .arg(archive)
+                        .path()
-                    .stdout(Stdio::piped());
+                        .join(format!("kernel-archive-{index}-debug"));
-                let mut dpkg_child = dpkg
+                    extract_deb(&debug_archive, &debug_dir)?;
-                    .spawn()
+                    WalkDir::new(&debug_dir)
-                    .with_context(|| format!("failed to spawn {dpkg:?}"))?;
+                        .into_iter()
-                let Child { stdout, .. } = &mut dpkg_child;
+                        .filter_map(|entry| entry.ok())
-                let stdout = stdout.take().unwrap();
+                        .filter(|entry| entry.file_type().is_file())
-                let mut archive_reader = tar::Archive::new(stdout);
+                        .filter_map(|entry| {
-                archive_reader.unpack(&archive_dir).with_context(|| {
+                            let path = entry.into_path();
-                    format!(
+                            let is_system_map = path
-                        "failed to unpack archive {} to {}",
+                                .file_name()
-                        archive.display(),
+                                .map(|file_name| {
-                        archive_dir.display()
+                                    matches!(
+                                        file_name.as_encoded_bytes(),
+                                        [
+                                            b'S',
+                                            b'y',
+                                            b's',
+                                            b't',
+                                            b'e',
+                                            b'm',
+                                            b'.',
+                                            b'm',
+                                            b'a',
+                                            b'p',
+                                            b'-',
+                                            ..
+                                        ]
                                     )
-                })?;
+                                })
-                let status = dpkg_child
+                                .unwrap_or(false);
-                    .wait()
+                            if is_system_map { Some(path) } else { None }
-                    .with_context(|| format!("failed to wait for {dpkg:?}"))?;
+                        })
-                if !status.success() {
+                        .collect::<Vec<_>>()
-                    bail!("{dpkg:?} exited with {status}");
+                } else {
-                }
+                    Vec::new()
+                };
 
                 let mut kernel_images = Vec::new();
                 let mut configs = Vec::new();
+                let mut kernel_maps = Vec::new();
                 for entry in WalkDir::new(&archive_dir) {
                     let entry = entry.with_context(|| {
                         format!("failed to read entry in {}", archive_dir.display())
@@ -350,22 +435,59 @@ pub(crate) fn run(opts: Options) -> Result<()> {
                             [b'c', b'o', b'n', b'f', b'i', b'g', b'-', ..] => {
                                 configs.push(path);
                             }
+                            // "System.map-"
+                            [
+                                b'S',
+                                b'y',
+                                b's',
+                                b't',
+                                b'e',
+                                b'm',
+                                b'.',
+                                b'm',
+                                b'a',
+                                b'p',
+                                b'-',
+                                ..,
+                            ] => {
+                                kernel_maps.push(path);
+                            }
                             _ => {}
                         }
                     }
                 }
                 let (kernel_image, kernel_version) = match kernel_images.as_slice() {
                     [kernel_image] => kernel_image,
-                    [] => bail!("no kernel images in {}", archive.display()),
+                    [] => bail!("no kernel images in {}", kernel_archive.display()),
                     kernel_images => bail!(
                         "multiple kernel images in {}: {:?}",
-                        archive.display(),
+                        kernel_archive.display(),
                         kernel_images
                     ),
                 };
                 let config = match configs.as_slice() {
                     [config] => config,
-                    configs => bail!("multiple configs in {}: {:?}", archive.display(), configs),
+                    configs => bail!(
+                        "multiple configs in {}: {:?}",
+                        kernel_archive.display(),
+                        configs
+                    ),
+                };
+                let system_map = match debug_maps.as_slice() {
+                    [system_map] => system_map,
+                    [] => match kernel_maps.as_slice() {
+                        [system_map] => system_map,
+                        kernel_maps => bail!(
+                            "multiple kernel System.maps in {}: {:?}",
+                            kernel_archive.display(),
+                            kernel_maps
+                        ),
+                    },
+                    system_maps => bail!(
+                        "multiple debug System.maps in {}: {:?}",
+                        kernel_archive.display(),
+                        system_maps
+                    ),
                 };
 
                 let mut modules_dirs = Vec::new();
@@ -388,10 +510,10 @@ pub(crate) fn run(opts: Options) -> Result<()> {
                 }
                 let modules_dir = match modules_dirs.as_slice() {
                     [modules_dir] => modules_dir,
-                    [] => bail!("no modules directories in {}", archive.display()),
+                    [] => bail!("no modules directories in {}", kernel_archive.display()),
                     modules_dirs => bail!(
                         "multiple modules directories in {}: {:?}",
-                        archive.display(),
+                        kernel_archive.display(),
                         modules_dirs
                     ),
                 };
@@ -505,6 +627,11 @@ pub(crate) fn run(opts: Options) -> Result<()> {
                     write_file(&Path::new("/boot").join(name), config, "644 0 0");
                 }
 
+                write_file(Path::new("/boot/System.map"), system_map, "644 0 0");
+                if let Some(name) = system_map.file_name() {
+                    write_file(&Path::new("/boot").join(name), system_map, "644 0 0");
+                }
+
                 test_distro.iter().for_each(|(name, path)| {
                     if name == "init" {
                         write_file(Path::new("/init"), path, "755 0 0");