Merge pull request #664 from ajwerner/always-initialize-submodules

xtask: ensure libbpf submodule is initialized

.github/workflows/gen.yml

@@ -8,10 +8,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
-      - name: libbpf-version
+      - name: update libbpf
-        working-directory: libbpf
+        working-directory: xtask/libbpf
-        run: echo "LIBBPF_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+        run: |
+          set -e
+          git fetch origin
+          git checkout origin/HEAD
+          echo "LIBBPF_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
 
       - uses: dtolnay/rust-toolchain@master
         with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "libbpf"]
-	path = libbpf
+	path = xtask/libbpf
 	url = https://github.com/libbpf/libbpf

test/integration-test/Cargo.toml

@@ -26,3 +26,4 @@ tokio = { version = "1.24", default-features = false, features = [
 [build-dependencies]
 cargo_metadata = { version = "0.15.4", default-features = false }
 which = { version = "4.4.0", default-features = false }
+xtask = { path = "../../xtask" }

test/integration-test/build.rs

@@ -13,6 +13,7 @@ use cargo_metadata::{
     Artifact, CompilerMessage, Dependency, Message, Metadata, MetadataCommand, Package, Target,
 };
 use which::which;
+use xtask::{exec, LIBBPF_DIR};
 
 fn main() {
     const AYA_BUILD_INTEGRATION_BPF: &str = "AYA_BUILD_INTEGRATION_BPF";
@@ -69,7 +70,8 @@ fn main() {
             .unwrap()
             .parent()
             .unwrap()
-            .join("libbpf");
+            .join(LIBBPF_DIR);
+        println!("cargo:rerun-if-changed={}", libbpf_dir.to_str().unwrap());
 
         let libbpf_headers_dir = out_dir.join("libbpf_headers");
 
@@ -77,21 +79,14 @@ fn main() {
         includedir.push("INCLUDEDIR=");
         includedir.push(&libbpf_headers_dir);
 
-        let mut cmd = Command::new("make");
+        exec(
-        cmd.arg("-C")
+            Command::new("make")
-            .arg(libbpf_dir.join("src"))
+                .arg("-C")
-            .arg(includedir)
+                .arg(libbpf_dir.join("src"))
-            .arg("install_headers");
+                .arg(includedir)
-        let status = cmd
+                .arg("install_headers"),
-            .status()
+        )
-            .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
+        .unwrap();
-        match status.code() {
-            Some(code) => match code {
-                0 => {}
-                code => panic!("{cmd:?} exited with code {code}"),
-            },
-            None => panic!("{cmd:?} terminated by signal"),
-        }
 
         let bpf_dir = manifest_dir.join("bpf");
 
@@ -110,24 +105,18 @@ fn main() {
         for (src, dst) in c_bpf_probes {
             let src = bpf_dir.join(src);
             println!("cargo:rerun-if-changed={}", src.to_str().unwrap());
-            let mut cmd = Command::new("clang");
+
-            cmd.arg("-I")
+            exec(
-                .arg(&libbpf_headers_dir)
+                Command::new("clang")
-                .args(["-g", "-O2", "-target", target, "-c"])
+                    .arg("-I")
-                .arg(&target_arch)
+                    .arg(&libbpf_headers_dir)
-                .arg(src)
+                    .args(["-g", "-O2", "-target", target, "-c"])
-                .arg("-o")
+                    .arg(&target_arch)
-                .arg(dst);
+                    .arg(src)
-            let status = cmd
+                    .arg("-o")
-                .status()
+                    .arg(dst),
-                .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
+            )
-            match status.code() {
+            .unwrap();
-                Some(code) => match code {
-                    0 => {}
-                    code => panic!("{cmd:?} exited with code {code}"),
-                },
-                None => panic!("{cmd:?} terminated by signal"),
-            }
         }
 
         let target = format!("{target}-unknown-none");

test/run.sh

@@ -192,7 +192,7 @@ EOF
     exec_vm sudo dnf config-manager --set-enabled updates-testing
     exec_vm sudo dnf config-manager --set-enabled updates-testing-modular
     echo "Installing dependencies"
-    exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel
+    exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel git
     exec_vm 'curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
         -y --profile minimal --default-toolchain nightly --component rust-src --component clippy'
     exec_vm 'echo source ~/.cargo/env >> ~/.bashrc'

xtask/src/codegen/aya.rs

@@ -1,19 +1,18 @@
 use anyhow::anyhow;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use aya_tool::{bindgen, write_to_file};
 
 use crate::codegen::{Architecture, SysrootOptions};
 
-pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
-    codegen_internal_btf_bindings()?;
+    codegen_internal_btf_bindings(libbpf_dir)?;
-    codegen_bindings(opts)
+    codegen_bindings(opts, libbpf_dir)
 }
 
-fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> {
+fn codegen_internal_btf_bindings(libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let dir = PathBuf::from("aya-obj");
     let generated = dir.join("src/generated");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let mut bindgen = bindgen::user_builder()
         .clang_arg(format!(
@@ -52,7 +51,7 @@ fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> {
     Ok(())
 }
 
-fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+fn codegen_bindings(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let SysrootOptions {
         x86_64_sysroot,
         aarch64_sysroot,
@@ -162,7 +161,6 @@ fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
 
     let dir = PathBuf::from("aya-obj");
     let generated = dir.join("src/generated");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let builder = || {
         bindgen::user_builder()

xtask/src/codegen/aya_bpf_bindings.rs

@@ -1,7 +1,7 @@
 use anyhow::anyhow;
 use proc_macro2::TokenStream;
 use quote::ToTokens;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use aya_tool::{bindgen, write_to_file_fmt};
 use syn::{parse_str, Item};
@@ -11,7 +11,7 @@ use crate::codegen::{
     Architecture, SysrootOptions,
 };
 
-pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let SysrootOptions {
         x86_64_sysroot,
         aarch64_sysroot,
@@ -20,7 +20,6 @@ pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> {
     } = opts;
 
     let dir = PathBuf::from("bpf/aya-bpf-bindings");
-    let libbpf_dir = PathBuf::from("libbpf");
 
     let builder = || {
         let mut bindgen = bindgen::bpf_builder()

xtask/src/codegen/mod.rs

@@ -2,7 +2,7 @@ mod aya;
 mod aya_bpf_bindings;
 mod helpers;
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use clap::Parser;
 
@@ -86,19 +86,19 @@ enum Command {
     AyaBpfBindings,
 }
 
-pub fn codegen(opts: Options) -> Result<(), anyhow::Error> {
+pub fn codegen(opts: Options, libbpf_dir: &Path) -> Result<(), anyhow::Error> {
     let Options {
         sysroot_options,
         command,
     } = opts;
     match command {
         Some(command) => match command {
-            Command::Aya => aya::codegen(&sysroot_options),
+            Command::Aya => aya::codegen(&sysroot_options, libbpf_dir),
-            Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options),
+            Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir),
         },
         None => {
-            aya::codegen(&sysroot_options)?;
+            aya::codegen(&sysroot_options, libbpf_dir)?;
-            aya_bpf_bindings::codegen(&sysroot_options)
+            aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir)
         }
     }
 }

xtask/src/docs.rs

@@ -1,22 +1,10 @@
-use anyhow::{anyhow, Context as _, Result};
+use anyhow::{Context as _, Result};
-use cargo_metadata::{Metadata, MetadataCommand};
+use cargo_metadata::Metadata;
 use indoc::{indoc, writedoc};
 use std::{ffi::OsString, fs, io::Write as _, process::Command};
+use xtask::exec;
 
-pub fn exec(cmd: &mut Command) -> Result<()> {
+pub fn docs(metadata: Metadata) -> Result<()> {
-    let status = cmd
-        .status()
-        .with_context(|| format!("failed to run {cmd:?}"))?;
-    match status.code() {
-        Some(code) => match code {
-            0 => Ok(()),
-            code => Err(anyhow!("{cmd:?} exited with code {code}")),
-        },
-        None => Err(anyhow!("{cmd:?} terminated by signal")),
-    }
-}
-
-pub fn docs() -> Result<()> {
     const PACKAGE_TO_DESCRIPTION: &[(&str, &str)] = &[
         ("aya", "User-space BPF program loading and manipulation"),
         ("aya-bpf", "Kernel-space BPF program implementation toolkit"),
@@ -31,7 +19,7 @@ pub fn docs() -> Result<()> {
         workspace_root,
         target_directory,
         ..
-    } = MetadataCommand::new().exec().context("cargo metadata")?;
+    } = metadata;
 
     exec(
         Command::new("cargo")

xtask/src/lib.rs

@@ -0,0 +1,17 @@
+use anyhow::{anyhow, Context as _, Result};
+use std::process::Command;
+
+pub fn exec(cmd: &mut Command) -> Result<()> {
+    let status = cmd
+        .status()
+        .with_context(|| format!("failed to run {cmd:?}"))?;
+    match status.code() {
+        Some(code) => match code {
+            0 => Ok(()),
+            code => Err(anyhow!("{cmd:?} exited with code {code}")),
+        },
+        None => Err(anyhow!("{cmd:?} terminated by signal")),
+    }
+}
+
+pub const LIBBPF_DIR: &str = "xtask/libbpf";

xtask/src/main.rs

@@ -2,29 +2,48 @@ mod codegen;
 mod docs;
 mod run;
 
+use anyhow::{Context as _, Result};
+use cargo_metadata::{Metadata, MetadataCommand};
 use clap::Parser;
+use std::process::Command;
+use xtask::{exec, LIBBPF_DIR};
 
 #[derive(Parser)]
 pub struct XtaskOptions {
     #[clap(subcommand)]
-    command: Command,
+    command: Subcommand,
 }
 
 #[derive(Parser)]
-enum Command {
+enum Subcommand {
     Codegen(codegen::Options),
     Docs,
     BuildIntegrationTest(run::BuildOptions),
     IntegrationTest(run::Options),
 }
 
-fn main() -> anyhow::Result<()> {
+fn main() -> Result<()> {
     let XtaskOptions { command } = Parser::parse();
 
+    let metadata = MetadataCommand::new()
+        .no_deps()
+        .exec()
+        .context("failed to run cargo metadata")?;
+    let Metadata { workspace_root, .. } = &metadata;
+
+    // Initialize the submodules.
+    exec(Command::new("git").arg("-C").arg(workspace_root).args([
+        "submodule",
+        "update",
+        "--init",
+    ]))?;
+    let libbpf_dir = workspace_root.join(LIBBPF_DIR);
+    let libbpf_dir = libbpf_dir.as_std_path();
+
     match command {
-        Command::Codegen(opts) => codegen::codegen(opts),
+        Subcommand::Codegen(opts) => codegen::codegen(opts, libbpf_dir),
-        Command::Docs => docs::docs(),
+        Subcommand::Docs => docs::docs(metadata),
-        Command::BuildIntegrationTest(opts) => {
+        Subcommand::BuildIntegrationTest(opts) => {
             let binaries = run::build(opts)?;
             let mut stdout = std::io::stdout();
             for (_name, binary) in binaries {
@@ -35,6 +54,6 @@ fn main() -> anyhow::Result<()> {
             }
             Ok(())
         }
-        Command::IntegrationTest(opts) => run::run(opts),
+        Subcommand::IntegrationTest(opts) => run::run(opts),
     }
 }