diff --git a/Cargo.toml b/Cargo.toml index a49fa346..02f6db9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "aya", + "aya-build", "aya-log", "aya-log-common", "aya-log-parser", @@ -26,6 +27,7 @@ resolver = "2" default-members = [ "aya", + "aya-build", "aya-log", "aya-log-common", "aya-log-parser", diff --git a/aya-build/Cargo.toml b/aya-build/Cargo.toml new file mode 100644 index 00000000..8f7c7a68 --- /dev/null +++ b/aya-build/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "aya-build" +version = "0.1.0" +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +cargo_metadata = { workspace = true } diff --git a/aya-build/src/lib.rs b/aya-build/src/lib.rs new file mode 100644 index 00000000..f84c5a24 --- /dev/null +++ b/aya-build/src/lib.rs @@ -0,0 +1,148 @@ +use std::{ + env, fs, + io::{BufRead as _, BufReader}, + path::PathBuf, + process::{Child, Command, Stdio}, +}; + +use anyhow::{anyhow, Context as _, Result}; +// Re-export `cargo_metadata` to having to encode the version downstream and risk mismatches. +pub use cargo_metadata; +use cargo_metadata::{Artifact, CompilerMessage, Message, Package, Target}; + +/// Build binary artifacts produced by `packages`. +/// +/// This would be better expressed as one or more [artifact-dependencies][bindeps] but issues such +/// as: +/// +/// * +/// * +/// * +/// +/// prevent their use for the time being. +/// +/// [bindeps]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html?highlight=feature#artifact-dependencies +pub fn build_ebpf(packages: impl IntoIterator) -> Result<()> { + let out_dir = env::var_os("OUT_DIR").ok_or(anyhow!("OUT_DIR not set"))?; + let out_dir = PathBuf::from(out_dir); + + let endian = + env::var_os("CARGO_CFG_TARGET_ENDIAN").ok_or(anyhow!("CARGO_CFG_TARGET_ENDIAN not set"))?; + let target = if endian == "big" { + "bpfeb" + } else if endian == "little" { + "bpfel" + } else { + return Err(anyhow!("unsupported endian={endian:?}")); + }; + + let arch = + env::var_os("CARGO_CFG_TARGET_ARCH").ok_or(anyhow!("CARGO_CFG_TARGET_ARCH not set"))?; + + let target = format!("{target}-unknown-none"); + + for Package { + name, + manifest_path, + .. + } in packages + { + let dir = manifest_path + .parent() + .ok_or(anyhow!("no parent for {manifest_path}"))?; + + // We have a build-dependency on `name`, so cargo will automatically rebuild us if `name`'s + // *library* target or any of its dependencies change. Since we depend on `name`'s *binary* + // targets, that only gets us half of the way. This stanza ensures cargo will rebuild us on + // changes to the binaries too, which gets us the rest of the way. + println!("cargo:rerun-if-changed={dir}"); + + let mut cmd = Command::new("cargo"); + cmd.args([ + "+nightly", + "build", + "--package", + &name, + "-Z", + "build-std=core", + "--bins", + "--message-format=json", + "--release", + "--target", + &target, + ]); + + cmd.env("CARGO_CFG_BPF_TARGET_ARCH", &arch); + + // Workaround to make sure that the correct toolchain is used. + for key in ["RUSTC", "RUSTC_WORKSPACE_WRAPPER"] { + cmd.env_remove(key); + } + + // Workaround for https://github.com/rust-lang/cargo/issues/6412 where cargo flocks itself. + let target_dir = out_dir.join(name); + cmd.arg("--target-dir").arg(&target_dir); + + let mut child = cmd + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn {cmd:?}"))?; + let Child { stdout, stderr, .. } = &mut child; + + // Trampoline stdout to cargo warnings. + let stderr = stderr.take().expect("stderr"); + let stderr = BufReader::new(stderr); + let stderr = std::thread::spawn(move || { + for line in stderr.lines() { + let line = line.expect("read line"); + println!("cargo:warning={line}"); + } + }); + + let stdout = stdout.take().expect("stdout"); + let stdout = BufReader::new(stdout); + let mut executables = Vec::new(); + for message in Message::parse_stream(stdout) { + #[allow(clippy::collapsible_match)] + match message.expect("valid JSON") { + Message::CompilerArtifact(Artifact { + executable, + target: Target { name, .. }, + .. + }) => { + if let Some(executable) = executable { + executables.push((name, executable.into_std_path_buf())); + } + } + Message::CompilerMessage(CompilerMessage { message, .. }) => { + for line in message.rendered.unwrap_or_default().split('\n') { + println!("cargo:warning={line}"); + } + } + Message::TextLine(line) => { + println!("cargo:warning={line}"); + } + _ => {} + } + } + + let status = child + .wait() + .with_context(|| format!("failed to wait for {cmd:?}"))?; + if !status.success() { + return Err(anyhow!("{cmd:?} failed: {status:?}")); + } + + match stderr.join().map_err(std::panic::resume_unwind) { + Ok(()) => {} + } + + for (name, binary) in executables { + let dst = out_dir.join(name); + let _: u64 = fs::copy(&binary, &dst) + .with_context(|| format!("failed to copy {binary:?} to {dst:?}"))?; + } + } + Ok(()) +} diff --git a/test/integration-test/Cargo.toml b/test/integration-test/Cargo.toml index 3a2d8014..7caeb3a9 100644 --- a/test/integration-test/Cargo.toml +++ b/test/integration-test/Cargo.toml @@ -30,7 +30,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } xdpilone = { workspace = true } [build-dependencies] -cargo_metadata = { workspace = true } +aya-build = { path = "../../aya-build" } # TODO(https://github.com/rust-lang/cargo/issues/12375): this should be an artifact dependency, but # it's not possible to tell cargo to use `-Z build-std` to build it. We cargo-in-cargo in the build # script to build this, but we want to teach cargo about the dependecy so that cache invalidation diff --git a/test/integration-test/build.rs b/test/integration-test/build.rs index a65ecd78..6f71f8c2 100644 --- a/test/integration-test/build.rs +++ b/test/integration-test/build.rs @@ -2,25 +2,13 @@ use std::{ env, ffi::OsString, fs, - io::{BufRead as _, BufReader}, path::PathBuf, process::{Child, Command, Output, Stdio}, }; -use cargo_metadata::{ - Artifact, CompilerMessage, Message, Metadata, MetadataCommand, Package, Target, TargetKind, -}; +use aya_build::cargo_metadata::{Metadata, MetadataCommand, Package, Target, TargetKind}; use xtask::{exec, AYA_BUILD_INTEGRATION_BPF, LIBBPF_DIR}; -/// This crate has a runtime dependency on artifacts produced by the `integration-ebpf` crate. This -/// would be better expressed as one or more [artifact-dependencies][bindeps] but issues such as: -/// -/// * https://github.com/rust-lang/cargo/issues/12374 -/// * https://github.com/rust-lang/cargo/issues/12375 -/// * https://github.com/rust-lang/cargo/issues/12385 -/// -/// prevent their use for the time being. -/// /// This file, along with the xtask crate, allows analysis tools such as `cargo check`, `cargo /// clippy`, and even `cargo build` to work as users expect. Prior to this file's existence, this /// crate's undeclared dependency on artifacts from `integration-ebpf` would cause build (and `cargo check`, @@ -34,11 +22,11 @@ use xtask::{exec, AYA_BUILD_INTEGRATION_BPF, LIBBPF_DIR}; /// occur on metadata-only actions such as `cargo check` or `cargo clippy` of this crate. This means /// that naively attempting to `cargo test --no-run` this crate will produce binaries that fail at /// runtime because the stubs are inadequate for actually running the tests. -/// -/// [bindeps]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html?highlight=feature#artifact-dependencies fn main() { println!("cargo:rerun-if-env-changed={}", AYA_BUILD_INTEGRATION_BPF); + // TODO(https://github.com/rust-lang/cargo/issues/4001): generalize this and move it to + // aya-build if we can determine that we're in a check build. let build_integration_bpf = env::var(AYA_BUILD_INTEGRATION_BPF) .as_deref() .map(str::parse) @@ -177,100 +165,7 @@ fn main() { } } - let target = format!("{target}-unknown-none"); - - let Package { manifest_path, .. } = integration_ebpf_package; - let integration_ebpf_dir = manifest_path.parent().unwrap(); - - // We have a build-dependency on `integration-ebpf`, so cargo will automatically rebuild us - // if `integration-ebpf`'s *library* target or any of its dependencies change. Since we - // depend on `integration-ebpf`'s *binary* targets, that only gets us half of the way. This - // stanza ensures cargo will rebuild us on changes to the binaries too, which gets us the - // rest of the way. - println!("cargo:rerun-if-changed={}", integration_ebpf_dir.as_str()); - - let mut cmd = Command::new("cargo"); - cmd.args([ - "+nightly", - "build", - "--package", - "integration-ebpf", - "-Z", - "build-std=core", - "--bins", - "--message-format=json", - "--release", - "--target", - &target, - ]); - - cmd.env("CARGO_CFG_BPF_TARGET_ARCH", arch); - - // Workaround to make sure that the correct toolchain is used. - for key in ["RUSTC", "RUSTC_WORKSPACE_WRAPPER"] { - cmd.env_remove(key); - } - - // Workaround for https://github.com/rust-lang/cargo/issues/6412 where cargo flocks itself. - let ebpf_target_dir = out_dir.join("integration-ebpf"); - cmd.arg("--target-dir").arg(&ebpf_target_dir); - - let mut child = cmd - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}")); - let Child { stdout, stderr, .. } = &mut child; - - // Trampoline stdout to cargo warnings. - let stderr = stderr.take().unwrap(); - let stderr = BufReader::new(stderr); - let stderr = std::thread::spawn(move || { - for line in stderr.lines() { - let line = line.unwrap(); - println!("cargo:warning={line}"); - } - }); - - let stdout = stdout.take().unwrap(); - let stdout = BufReader::new(stdout); - let mut executables = Vec::new(); - for message in Message::parse_stream(stdout) { - #[allow(clippy::collapsible_match)] - match message.expect("valid JSON") { - Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { - if let Some(executable) = executable { - executables.push((name, executable.into_std_path_buf())); - } - } - Message::CompilerMessage(CompilerMessage { message, .. }) => { - for line in message.rendered.unwrap_or_default().split('\n') { - println!("cargo:warning={line}"); - } - } - Message::TextLine(line) => { - println!("cargo:warning={line}"); - } - _ => {} - } - } - - let status = child - .wait() - .unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}")); - assert_eq!(status.code(), Some(0), "{cmd:?} failed: {status:?}"); - - stderr.join().map_err(std::panic::resume_unwind).unwrap(); - - for (name, binary) in executables { - let dst = out_dir.join(name); - let _: u64 = fs::copy(&binary, &dst) - .unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}")); - } + aya_build::build_ebpf([integration_ebpf_package]).unwrap(); } else { for (src, build_btf) in C_BPF { let dst = out_dir.join(src).with_extension("o"); diff --git a/xtask/public-api/aya-build.txt b/xtask/public-api/aya-build.txt new file mode 100644 index 00000000..9a55017f --- /dev/null +++ b/xtask/public-api/aya-build.txt @@ -0,0 +1,3 @@ +pub mod aya_build +pub use aya_build::cargo_metadata +pub fn aya_build::build_ebpf(packages: impl core::iter::traits::collect::IntoIterator) -> anyhow::Result<()>