From 15b3c459ae50e7c04ec98a529a4f5d7008a7f33d Mon Sep 17 00:00:00 2001 From: Andrew Werner Date: Mon, 17 Jul 2023 09:52:14 -0400 Subject: [PATCH] xtask: ensure libbpf submodule is initialized Libbpf is used by xtasks, in the command, ensure that the submodules are initialized. This eases the user-experience so that users don't need to think about the submodule, while retaining all the benefits of using a submodule vs forcing the user to manually check out libbpf and stick it in some pre-defined place. We use the symbol pointing to libbpf in xtask in the build script to avoid repeating this constant. Also, we install git in the vm so that we can init the submodule when we build in the vm. --- test/integration-test/Cargo.toml | 1 + test/integration-test/build.rs | 57 +++++++++++---------------- test/run.sh | 2 +- xtask/src/codegen/aya.rs | 14 +++---- xtask/src/codegen/aya_bpf_bindings.rs | 5 +-- xtask/src/codegen/mod.rs | 12 +++--- xtask/src/docs.rs | 22 +++-------- xtask/src/lib.rs | 17 ++++++++ xtask/src/main.rs | 33 ++++++++++++---- 9 files changed, 87 insertions(+), 76 deletions(-) create mode 100644 xtask/src/lib.rs diff --git a/test/integration-test/Cargo.toml b/test/integration-test/Cargo.toml index 298757c1..bf82ab18 100644 --- a/test/integration-test/Cargo.toml +++ b/test/integration-test/Cargo.toml @@ -26,3 +26,4 @@ tokio = { version = "1.24", default-features = false, features = [ [build-dependencies] cargo_metadata = { version = "0.15.4", default-features = false } which = { version = "4.4.0", default-features = false } +xtask = { path = "../../xtask" } diff --git a/test/integration-test/build.rs b/test/integration-test/build.rs index 8f5596f0..1ac66724 100644 --- a/test/integration-test/build.rs +++ b/test/integration-test/build.rs @@ -13,6 +13,7 @@ use cargo_metadata::{ Artifact, CompilerMessage, Dependency, Message, Metadata, MetadataCommand, Package, Target, }; use which::which; +use xtask::{exec, LIBBPF_DIR}; fn main() { const AYA_BUILD_INTEGRATION_BPF: &str = "AYA_BUILD_INTEGRATION_BPF"; @@ -69,7 +70,8 @@ fn main() { .unwrap() .parent() .unwrap() - .join("libbpf"); + .join(LIBBPF_DIR); + println!("cargo:rerun-if-changed={}", libbpf_dir.to_str().unwrap()); let libbpf_headers_dir = out_dir.join("libbpf_headers"); @@ -77,21 +79,14 @@ fn main() { includedir.push("INCLUDEDIR="); includedir.push(&libbpf_headers_dir); - let mut cmd = Command::new("make"); - cmd.arg("-C") - .arg(libbpf_dir.join("src")) - .arg(includedir) - .arg("install_headers"); - let status = cmd - .status() - .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}")); - match status.code() { - Some(code) => match code { - 0 => {} - code => panic!("{cmd:?} exited with code {code}"), - }, - None => panic!("{cmd:?} terminated by signal"), - } + exec( + Command::new("make") + .arg("-C") + .arg(libbpf_dir.join("src")) + .arg(includedir) + .arg("install_headers"), + ) + .unwrap(); let bpf_dir = manifest_dir.join("bpf"); @@ -110,24 +105,18 @@ fn main() { for (src, dst) in c_bpf_probes { let src = bpf_dir.join(src); println!("cargo:rerun-if-changed={}", src.to_str().unwrap()); - let mut cmd = Command::new("clang"); - cmd.arg("-I") - .arg(&libbpf_headers_dir) - .args(["-g", "-O2", "-target", target, "-c"]) - .arg(&target_arch) - .arg(src) - .arg("-o") - .arg(dst); - let status = cmd - .status() - .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}")); - match status.code() { - Some(code) => match code { - 0 => {} - code => panic!("{cmd:?} exited with code {code}"), - }, - None => panic!("{cmd:?} terminated by signal"), - } + + exec( + Command::new("clang") + .arg("-I") + .arg(&libbpf_headers_dir) + .args(["-g", "-O2", "-target", target, "-c"]) + .arg(&target_arch) + .arg(src) + .arg("-o") + .arg(dst), + ) + .unwrap(); } let target = format!("{target}-unknown-none"); diff --git a/test/run.sh b/test/run.sh index 9def9a22..4f4f285e 100755 --- a/test/run.sh +++ b/test/run.sh @@ -192,7 +192,7 @@ EOF exec_vm sudo dnf config-manager --set-enabled updates-testing exec_vm sudo dnf config-manager --set-enabled updates-testing-modular echo "Installing dependencies" - exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel + exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel git exec_vm 'curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ -y --profile minimal --default-toolchain nightly --component rust-src --component clippy' exec_vm 'echo source ~/.cargo/env >> ~/.bashrc' diff --git a/xtask/src/codegen/aya.rs b/xtask/src/codegen/aya.rs index c9f48b48..950b862c 100644 --- a/xtask/src/codegen/aya.rs +++ b/xtask/src/codegen/aya.rs @@ -1,19 +1,18 @@ use anyhow::anyhow; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use aya_tool::{bindgen, write_to_file}; use crate::codegen::{Architecture, SysrootOptions}; -pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> { - codegen_internal_btf_bindings()?; - codegen_bindings(opts) +pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> { + codegen_internal_btf_bindings(libbpf_dir)?; + codegen_bindings(opts, libbpf_dir) } -fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> { +fn codegen_internal_btf_bindings(libbpf_dir: &Path) -> Result<(), anyhow::Error> { let dir = PathBuf::from("aya-obj"); let generated = dir.join("src/generated"); - let libbpf_dir = PathBuf::from("libbpf"); let mut bindgen = bindgen::user_builder() .clang_arg(format!( @@ -52,7 +51,7 @@ fn codegen_internal_btf_bindings() -> Result<(), anyhow::Error> { Ok(()) } -fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> { +fn codegen_bindings(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> { let SysrootOptions { x86_64_sysroot, aarch64_sysroot, @@ -162,7 +161,6 @@ fn codegen_bindings(opts: &SysrootOptions) -> Result<(), anyhow::Error> { let dir = PathBuf::from("aya-obj"); let generated = dir.join("src/generated"); - let libbpf_dir = PathBuf::from("libbpf"); let builder = || { bindgen::user_builder() diff --git a/xtask/src/codegen/aya_bpf_bindings.rs b/xtask/src/codegen/aya_bpf_bindings.rs index d0af3221..d594a5fe 100644 --- a/xtask/src/codegen/aya_bpf_bindings.rs +++ b/xtask/src/codegen/aya_bpf_bindings.rs @@ -1,7 +1,7 @@ use anyhow::anyhow; use proc_macro2::TokenStream; use quote::ToTokens; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use aya_tool::{bindgen, write_to_file_fmt}; use syn::{parse_str, Item}; @@ -11,7 +11,7 @@ use crate::codegen::{ Architecture, SysrootOptions, }; -pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> { +pub fn codegen(opts: &SysrootOptions, libbpf_dir: &Path) -> Result<(), anyhow::Error> { let SysrootOptions { x86_64_sysroot, aarch64_sysroot, @@ -20,7 +20,6 @@ pub fn codegen(opts: &SysrootOptions) -> Result<(), anyhow::Error> { } = opts; let dir = PathBuf::from("bpf/aya-bpf-bindings"); - let libbpf_dir = PathBuf::from("libbpf"); let builder = || { let mut bindgen = bindgen::bpf_builder() diff --git a/xtask/src/codegen/mod.rs b/xtask/src/codegen/mod.rs index 6722c660..9645be6d 100644 --- a/xtask/src/codegen/mod.rs +++ b/xtask/src/codegen/mod.rs @@ -2,7 +2,7 @@ mod aya; mod aya_bpf_bindings; mod helpers; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use clap::Parser; @@ -86,19 +86,19 @@ enum Command { AyaBpfBindings, } -pub fn codegen(opts: Options) -> Result<(), anyhow::Error> { +pub fn codegen(opts: Options, libbpf_dir: &Path) -> Result<(), anyhow::Error> { let Options { sysroot_options, command, } = opts; match command { Some(command) => match command { - Command::Aya => aya::codegen(&sysroot_options), - Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options), + Command::Aya => aya::codegen(&sysroot_options, libbpf_dir), + Command::AyaBpfBindings => aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir), }, None => { - aya::codegen(&sysroot_options)?; - aya_bpf_bindings::codegen(&sysroot_options) + aya::codegen(&sysroot_options, libbpf_dir)?; + aya_bpf_bindings::codegen(&sysroot_options, libbpf_dir) } } } diff --git a/xtask/src/docs.rs b/xtask/src/docs.rs index e10c80a2..0fb13d35 100644 --- a/xtask/src/docs.rs +++ b/xtask/src/docs.rs @@ -1,22 +1,10 @@ -use anyhow::{anyhow, Context as _, Result}; -use cargo_metadata::{Metadata, MetadataCommand}; +use anyhow::{Context as _, Result}; +use cargo_metadata::Metadata; use indoc::{indoc, writedoc}; use std::{ffi::OsString, fs, io::Write as _, process::Command}; +use xtask::exec; -pub fn exec(cmd: &mut Command) -> Result<()> { - let status = cmd - .status() - .with_context(|| format!("failed to run {cmd:?}"))?; - match status.code() { - Some(code) => match code { - 0 => Ok(()), - code => Err(anyhow!("{cmd:?} exited with code {code}")), - }, - None => Err(anyhow!("{cmd:?} terminated by signal")), - } -} - -pub fn docs() -> Result<()> { +pub fn docs(metadata: Metadata) -> Result<()> { const PACKAGE_TO_DESCRIPTION: &[(&str, &str)] = &[ ("aya", "User-space BPF program loading and manipulation"), ("aya-bpf", "Kernel-space BPF program implementation toolkit"), @@ -31,7 +19,7 @@ pub fn docs() -> Result<()> { workspace_root, target_directory, .. - } = MetadataCommand::new().exec().context("cargo metadata")?; + } = metadata; exec( Command::new("cargo") diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs new file mode 100644 index 00000000..db50c7fd --- /dev/null +++ b/xtask/src/lib.rs @@ -0,0 +1,17 @@ +use anyhow::{anyhow, Context as _, Result}; +use std::process::Command; + +pub fn exec(cmd: &mut Command) -> Result<()> { + let status = cmd + .status() + .with_context(|| format!("failed to run {cmd:?}"))?; + match status.code() { + Some(code) => match code { + 0 => Ok(()), + code => Err(anyhow!("{cmd:?} exited with code {code}")), + }, + None => Err(anyhow!("{cmd:?} terminated by signal")), + } +} + +pub const LIBBPF_DIR: &str = "libbpf"; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 47b425e4..0e2ad36f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,29 +2,48 @@ mod codegen; mod docs; mod run; +use anyhow::{Context as _, Result}; +use cargo_metadata::{Metadata, MetadataCommand}; use clap::Parser; +use std::process::Command; +use xtask::{exec, LIBBPF_DIR}; #[derive(Parser)] pub struct XtaskOptions { #[clap(subcommand)] - command: Command, + command: Subcommand, } #[derive(Parser)] -enum Command { +enum Subcommand { Codegen(codegen::Options), Docs, BuildIntegrationTest(run::BuildOptions), IntegrationTest(run::Options), } -fn main() -> anyhow::Result<()> { +fn main() -> Result<()> { let XtaskOptions { command } = Parser::parse(); + let metadata = MetadataCommand::new() + .no_deps() + .exec() + .context("failed to run cargo metadata")?; + let Metadata { workspace_root, .. } = &metadata; + + // Initialize the submodules. + exec(Command::new("git").arg("-C").arg(workspace_root).args([ + "submodule", + "update", + "--init", + ]))?; + let libbpf_dir = workspace_root.join(LIBBPF_DIR); + let libbpf_dir = libbpf_dir.as_std_path(); + match command { - Command::Codegen(opts) => codegen::codegen(opts), - Command::Docs => docs::docs(), - Command::BuildIntegrationTest(opts) => { + Subcommand::Codegen(opts) => codegen::codegen(opts, libbpf_dir), + Subcommand::Docs => docs::docs(metadata), + Subcommand::BuildIntegrationTest(opts) => { let binaries = run::build(opts)?; let mut stdout = std::io::stdout(); for (_name, binary) in binaries { @@ -35,6 +54,6 @@ fn main() -> anyhow::Result<()> { } Ok(()) } - Command::IntegrationTest(opts) => run::run(opts), + Subcommand::IntegrationTest(opts) => run::run(opts), } }