xtask: ensure libbpf submodule is initialized

Libbpf is used by xtasks, in the command, ensure that the submodules
are initialized. This eases the user-experience so that users don't
need to think about the submodule, while retaining all the benefits
of using a submodule vs forcing the user to manually check out libbpf
and stick it in some pre-defined place.

We use the symbol pointing to libbpf in xtask in the build script
to avoid repeating this constant.

Also, we install git in the vm so that we can init the submodule
when we build in the vm.

test/integration-test/Cargo.toml

@@ -26,3 +26,4 @@ tokio = { version = "1.24", default-features = false, features = [
 [build-dependencies]
 cargo_metadata = { version = "0.15.4", default-features = false }
 which = { version = "4.4.0", default-features = false }
+xtask = { path = "../../xtask" }

test/integration-test/build.rs

@@ -13,6 +13,7 @@ use cargo_metadata::{
     Artifact, CompilerMessage, Dependency, Message, Metadata, MetadataCommand, Package, Target,
 };
 use which::which;
+use xtask::{exec, LIBBPF_DIR};
 
 fn main() {
     const AYA_BUILD_INTEGRATION_BPF: &str = "AYA_BUILD_INTEGRATION_BPF";
@@ -69,7 +70,8 @@ fn main() {
             .unwrap()
             .parent()
             .unwrap()
-            .join("libbpf");
+            .join(LIBBPF_DIR);
+        println!("cargo:rerun-if-changed={}", libbpf_dir.to_str().unwrap());
 
         let libbpf_headers_dir = out_dir.join("libbpf_headers");
 
@@ -77,21 +79,14 @@ fn main() {
         includedir.push("INCLUDEDIR=");
         includedir.push(&libbpf_headers_dir);
 
-        let mut cmd = Command::new("make");
+        exec(
-        cmd.arg("-C")
+            Command::new("make")
-            .arg(libbpf_dir.join("src"))
+                .arg("-C")
-            .arg(includedir)
+                .arg(libbpf_dir.join("src"))
-            .arg("install_headers");
+                .arg(includedir)
-        let status = cmd
+                .arg("install_headers"),
-            .status()
+        )
-            .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
+        .unwrap();
-        match status.code() {
-            Some(code) => match code {
-                0 => {}
-                code => panic!("{cmd:?} exited with code {code}"),
-            },
-            None => panic!("{cmd:?} terminated by signal"),
-        }
 
         let bpf_dir = manifest_dir.join("bpf");
 
@@ -110,24 +105,18 @@ fn main() {
         for (src, dst) in c_bpf_probes {
             let src = bpf_dir.join(src);
             println!("cargo:rerun-if-changed={}", src.to_str().unwrap());
-            let mut cmd = Command::new("clang");
+
-            cmd.arg("-I")
+            exec(
-                .arg(&libbpf_headers_dir)
+                Command::new("clang")
-                .args(["-g", "-O2", "-target", target, "-c"])
+                    .arg("-I")
-                .arg(&target_arch)
+                    .arg(&libbpf_headers_dir)
-                .arg(src)
+                    .args(["-g", "-O2", "-target", target, "-c"])
-                .arg("-o")
+                    .arg(&target_arch)
-                .arg(dst);
+                    .arg(src)
-            let status = cmd
+                    .arg("-o")
-                .status()
+                    .arg(dst),
-                .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
+            )
-            match status.code() {
+            .unwrap();
-                Some(code) => match code {
-                    0 => {}
-                    code => panic!("{cmd:?} exited with code {code}"),
-                },
-                None => panic!("{cmd:?} terminated by signal"),
-            }
         }
 
         let target = format!("{target}-unknown-none");

test/run.sh

@@ -192,7 +192,7 @@ EOF
     exec_vm sudo dnf config-manager --set-enabled updates-testing
     exec_vm sudo dnf config-manager --set-enabled updates-testing-modular
     echo "Installing dependencies"
-    exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel
+    exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel git
     exec_vm 'curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
         -y --profile minimal --default-toolchain nightly --component rust-src --component clippy'
     exec_vm 'echo source ~/.cargo/env >> ~/.bashrc'

xtask/src/codegen/aya.rs

@@ -1,19 +1,18 @@
 use anyhow::anyhow;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use aya_tool::{bindgen, write_to_file};
 
 use crate::codegen::{Architecture, SysrootOptions};
 
-pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
-    codegen_internal_btf_bindings()?;
+    codegen_internal_btf_bindings(libbpf_dir)?;
-    codegen_bindings(opts)
+    codegen_bindings(opts, libbpf_dir)
 }
 
-fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> {
+fn codegen_internal_btf_bindings(libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let dir = PathBuf::from("aya-obj");
     let generated = dir.join("src/generated");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let mut bindgen = bindgen::user_builder()
         .clang_arg(format!(
@@ -52,7 +51,7 @@ fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> {
     Ok(())
 }
 
-fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+fn codegen_bindings(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let SysrootOptions {
         x86_64_sysroot,
         aarch64_sysroot,
@@ -162,7 +161,6 @@ fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
 
     let dir = PathBuf::from("aya-obj");
     let generated = dir.join("src/generated");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let builder = || {
         bindgen::user_builder()

xtask/src/codegen/aya_bpf_bindings.rs

@@ -1,7 +1,7 @@
 use anyhow::anyhow;
 use proc_macro2::TokenStream;
 use quote::ToTokens;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use aya_tool::{bindgen, write_to_file_fmt};
 use syn::{parse_str, Item};
@@ -11,7 +11,7 @@ use crate::codegen::{
     Architecture, SysrootOptions,
 };
 
-pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let SysrootOptions {
         x86_64_sysroot,
         aarch64_sysroot,
@@ -20,7 +20,6 @@ pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
     } = opts;
 
     let dir = PathBuf::from("bpf/aya-bpf-bindings");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let builder = || {
         let mut bindgen = bindgen::bpf_builder()

xtask/src/codegen/mod.rs

@@ -2,7 +2,7 @@ mod aya;
 mod aya_bpf_bindings;
 mod helpers;
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clap::Parser;
 
@@ -86,19 +86,19 @@ enum Command {
     AyaBpfBindings,
 }
 
-pub fn codegen(opts: Options) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: Options, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let Options {
         sysroot_options,
         command,
     } = opts;
     match command {
         Some(command) => match command {
-            Command::Aya => aya::codegen(&sysroot_options),
+            Command::Aya => aya::codegen(&sysroot_options, libbpf_dir),
-            Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options),
+            Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir),
         },
         None => {
-            aya::codegen(&sysroot_options)?;
+            aya::codegen(&sysroot_options, libbpf_dir)?;
-            aya_bpf_bindings::codegen(&sysroot_options)
+            aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir)
         }
     }
 }

xtask/src/docs.rs

@@ -1,22 +1,10 @@
-use anyhow::{anyhow, Context as _, Result};
+use anyhow::{Context as _, Result};
-use cargo_metadata::{Metadata, MetadataCommand};
+use cargo_metadata::Metadata;
 use indoc::{indoc, writedoc};
 use std::{ffi::OsString, fs, io::Write as _, process::Command};
+use xtask::exec;
 
-pub fn exec(cmd: &mut Command) -> Result<()> {
+pub fn docs(metadata: Metadata) -> Result<()> {
-    let status = cmd
-        .status()
-        .with_context(|| format!("failed to run {cmd:?}"))?;
-    match status.code() {
-        Some(code) => match code {
-            0 => Ok(()),
-            code => Err(anyhow!("{cmd:?} exited with code {code}")),
-        },
-        None => Err(anyhow!("{cmd:?} terminated by signal")),
-    }
-}
-
-pub fn docs() -> Result<()> {
     const PACKAGE_TO_DESCRIPTION: &[(&str, &str)] = &[
         ("aya", "User-space BPF program loading and manipulation"),
         ("aya-bpf", "Kernel-space BPF program implementation toolkit"),
@@ -31,7 +19,7 @@ pub fn docs() -> Result<()> {
         workspace_root,
         target_directory,
         ..
-    } = MetadataCommand::new().exec().context("cargo metadata")?;
+    } = metadata;
 
     exec(
         Command::new("cargo")

xtask/src/lib.rs

@@ -0,0 +1,17 @@
+use anyhow::{anyhow, Context as _, Result};
+use std::process::Command;
+
+pub fn exec(cmd: &mut Command) -> Result<()> {
+    let status = cmd
+        .status()
+        .with_context(|| format!("failed to run {cmd:?}"))?;
+    match status.code() {
+        Some(code) => match code {
+            0 => Ok(()),
+            code => Err(anyhow!("{cmd:?} exited with code {code}")),
+        },
+        None => Err(anyhow!("{cmd:?} terminated by signal")),
+    }
+}
+
+pub const LIBBPF_DIR: &str = "libbpf";

xtask/src/main.rs

@@ -2,29 +2,48 @@ mod codegen;
 mod docs;
 mod run;
 
+use anyhow::{Context as _, Result};
+use cargo_metadata::{Metadata, MetadataCommand};
 use clap::Parser;
+use std::process::Command;
+use xtask::{exec, LIBBPF_DIR};
 
 #[derive(Parser)]
 pub struct XtaskOptions {
     #[clap(subcommand)]
-    command: Command,
+    command: Subcommand,
 }
 
 #[derive(Parser)]
-enum Command {
+enum Subcommand {
     Codegen(codegen::Options),
     Docs,
     BuildIntegrationTest(run::BuildOptions),
     IntegrationTest(run::Options),
 }
 
-fn main() -> anyhow::Result<()> {
+fn main() -> Result<()> {
     let XtaskOptions { command } = Parser::parse();
 
+    let metadata = MetadataCommand::new()
+        .no_deps()
+        .exec()
+        .context("failed to run cargo metadata")?;
+    let Metadata { workspace_root, .. } = &metadata;
+
+    // Initialize the submodules.
+    exec(Command::new("git").arg("-C").arg(workspace_root).args([
+        "submodule",
+        "update",
+        "--init",
+    ]))?;
+    let libbpf_dir = workspace_root.join(LIBBPF_DIR);
+    let libbpf_dir = libbpf_dir.as_std_path();
+
     match command {
-        Command::Codegen(opts) => codegen::codegen(opts),
+        Subcommand::Codegen(opts) => codegen::codegen(opts, libbpf_dir),
-        Command::Docs => docs::docs(),
+        Subcommand::Docs => docs::docs(metadata),
-        Command::BuildIntegrationTest(opts) => {
+        Subcommand::BuildIntegrationTest(opts) => {
             let binaries = run::build(opts)?;
             let mut stdout = std::io::stdout();
             for (_name, binary) in binaries {
@@ -35,6 +54,6 @@ fn main() -> anyhow::Result<()> {
             }
             Ok(())
         }
-        Command::IntegrationTest(opts) => run::run(opts),
+        Subcommand::IntegrationTest(opts) => run::run(opts),
     }
 }