Extract aya-build for building eBPF crates

We'll use this in the template and book to avoid duplicating all the
code.
reviewable/pr805/r5
Tamir Duberstein 2 months ago committed by Michal Rostecki
parent e0c4948e36
commit 2b2af44915

@ -1,6 +1,7 @@
[workspace]
members = [
"aya",
"aya-build",
"aya-log",
"aya-log-common",
"aya-log-parser",
@ -26,6 +27,7 @@ resolver = "2"
default-members = [
"aya",
"aya-build",
"aya-log",
"aya-log-common",
"aya-log-parser",

@ -0,0 +1,12 @@
[package]
name = "aya-build"
version = "0.1.0"
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
edition.workspace = true
[dependencies]
anyhow = { workspace = true }
cargo_metadata = { workspace = true }

@ -0,0 +1,148 @@
use std::{
env, fs,
io::{BufRead as _, BufReader},
path::PathBuf,
process::{Child, Command, Stdio},
};
use anyhow::{anyhow, Context as _, Result};
// Re-export `cargo_metadata` to having to encode the version downstream and risk mismatches.
pub use cargo_metadata;
use cargo_metadata::{Artifact, CompilerMessage, Message, Package, Target};
/// Build binary artifacts produced by `packages`.
///
/// This would be better expressed as one or more [artifact-dependencies][bindeps] but issues such
/// as:
///
/// * <https://github.com/rust-lang/cargo/issues/12374>
/// * <https://github.com/rust-lang/cargo/issues/12375>
/// * <https://github.com/rust-lang/cargo/issues/12385>
///
/// prevent their use for the time being.
///
/// [bindeps]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html?highlight=feature#artifact-dependencies
pub fn build_ebpf(packages: impl IntoIterator<Item = Package>) -> Result<()> {
let out_dir = env::var_os("OUT_DIR").ok_or(anyhow!("OUT_DIR not set"))?;
let out_dir = PathBuf::from(out_dir);
let endian =
env::var_os("CARGO_CFG_TARGET_ENDIAN").ok_or(anyhow!("CARGO_CFG_TARGET_ENDIAN not set"))?;
let target = if endian == "big" {
"bpfeb"
} else if endian == "little" {
"bpfel"
} else {
return Err(anyhow!("unsupported endian={endian:?}"));
};
let arch =
env::var_os("CARGO_CFG_TARGET_ARCH").ok_or(anyhow!("CARGO_CFG_TARGET_ARCH not set"))?;
let target = format!("{target}-unknown-none");
for Package {
name,
manifest_path,
..
} in packages
{
let dir = manifest_path
.parent()
.ok_or(anyhow!("no parent for {manifest_path}"))?;
// We have a build-dependency on `name`, so cargo will automatically rebuild us if `name`'s
// *library* target or any of its dependencies change. Since we depend on `name`'s *binary*
// targets, that only gets us half of the way. This stanza ensures cargo will rebuild us on
// changes to the binaries too, which gets us the rest of the way.
println!("cargo:rerun-if-changed={dir}");
let mut cmd = Command::new("cargo");
cmd.args([
"+nightly",
"build",
"--package",
&name,
"-Z",
"build-std=core",
"--bins",
"--message-format=json",
"--release",
"--target",
&target,
]);
cmd.env("CARGO_CFG_BPF_TARGET_ARCH", &arch);
// Workaround to make sure that the correct toolchain is used.
for key in ["RUSTC", "RUSTC_WORKSPACE_WRAPPER"] {
cmd.env_remove(key);
}
// Workaround for https://github.com/rust-lang/cargo/issues/6412 where cargo flocks itself.
let target_dir = out_dir.join(name);
cmd.arg("--target-dir").arg(&target_dir);
let mut child = cmd
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.with_context(|| format!("failed to spawn {cmd:?}"))?;
let Child { stdout, stderr, .. } = &mut child;
// Trampoline stdout to cargo warnings.
let stderr = stderr.take().expect("stderr");
let stderr = BufReader::new(stderr);
let stderr = std::thread::spawn(move || {
for line in stderr.lines() {
let line = line.expect("read line");
println!("cargo:warning={line}");
}
});
let stdout = stdout.take().expect("stdout");
let stdout = BufReader::new(stdout);
let mut executables = Vec::new();
for message in Message::parse_stream(stdout) {
#[allow(clippy::collapsible_match)]
match message.expect("valid JSON") {
Message::CompilerArtifact(Artifact {
executable,
target: Target { name, .. },
..
}) => {
if let Some(executable) = executable {
executables.push((name, executable.into_std_path_buf()));
}
}
Message::CompilerMessage(CompilerMessage { message, .. }) => {
for line in message.rendered.unwrap_or_default().split('\n') {
println!("cargo:warning={line}");
}
}
Message::TextLine(line) => {
println!("cargo:warning={line}");
}
_ => {}
}
}
let status = child
.wait()
.with_context(|| format!("failed to wait for {cmd:?}"))?;
if !status.success() {
return Err(anyhow!("{cmd:?} failed: {status:?}"));
}
match stderr.join().map_err(std::panic::resume_unwind) {
Ok(()) => {}
}
for (name, binary) in executables {
let dst = out_dir.join(name);
let _: u64 = fs::copy(&binary, &dst)
.with_context(|| format!("failed to copy {binary:?} to {dst:?}"))?;
}
}
Ok(())
}

@ -30,7 +30,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }
xdpilone = { workspace = true }
[build-dependencies]
cargo_metadata = { workspace = true }
aya-build = { path = "../../aya-build" }
# TODO(https://github.com/rust-lang/cargo/issues/12375): this should be an artifact dependency, but
# it's not possible to tell cargo to use `-Z build-std` to build it. We cargo-in-cargo in the build
# script to build this, but we want to teach cargo about the dependecy so that cache invalidation

@ -2,25 +2,13 @@ use std::{
env,
ffi::OsString,
fs,
io::{BufRead as _, BufReader},
path::PathBuf,
process::{Child, Command, Output, Stdio},
};
use cargo_metadata::{
Artifact, CompilerMessage, Message, Metadata, MetadataCommand, Package, Target, TargetKind,
};
use aya_build::cargo_metadata::{Metadata, MetadataCommand, Package, Target, TargetKind};
use xtask::{exec, AYA_BUILD_INTEGRATION_BPF, LIBBPF_DIR};
/// This crate has a runtime dependency on artifacts produced by the `integration-ebpf` crate. This
/// would be better expressed as one or more [artifact-dependencies][bindeps] but issues such as:
///
/// * https://github.com/rust-lang/cargo/issues/12374
/// * https://github.com/rust-lang/cargo/issues/12375
/// * https://github.com/rust-lang/cargo/issues/12385
///
/// prevent their use for the time being.
///
/// This file, along with the xtask crate, allows analysis tools such as `cargo check`, `cargo
/// clippy`, and even `cargo build` to work as users expect. Prior to this file's existence, this
/// crate's undeclared dependency on artifacts from `integration-ebpf` would cause build (and `cargo check`,
@ -34,11 +22,11 @@ use xtask::{exec, AYA_BUILD_INTEGRATION_BPF, LIBBPF_DIR};
/// occur on metadata-only actions such as `cargo check` or `cargo clippy` of this crate. This means
/// that naively attempting to `cargo test --no-run` this crate will produce binaries that fail at
/// runtime because the stubs are inadequate for actually running the tests.
///
/// [bindeps]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html?highlight=feature#artifact-dependencies
fn main() {
println!("cargo:rerun-if-env-changed={}", AYA_BUILD_INTEGRATION_BPF);
// TODO(https://github.com/rust-lang/cargo/issues/4001): generalize this and move it to
// aya-build if we can determine that we're in a check build.
let build_integration_bpf = env::var(AYA_BUILD_INTEGRATION_BPF)
.as_deref()
.map(str::parse)
@ -177,100 +165,7 @@ fn main() {
}
}
let target = format!("{target}-unknown-none");
let Package { manifest_path, .. } = integration_ebpf_package;
let integration_ebpf_dir = manifest_path.parent().unwrap();
// We have a build-dependency on `integration-ebpf`, so cargo will automatically rebuild us
// if `integration-ebpf`'s *library* target or any of its dependencies change. Since we
// depend on `integration-ebpf`'s *binary* targets, that only gets us half of the way. This
// stanza ensures cargo will rebuild us on changes to the binaries too, which gets us the
// rest of the way.
println!("cargo:rerun-if-changed={}", integration_ebpf_dir.as_str());
let mut cmd = Command::new("cargo");
cmd.args([
"+nightly",
"build",
"--package",
"integration-ebpf",
"-Z",
"build-std=core",
"--bins",
"--message-format=json",
"--release",
"--target",
&target,
]);
cmd.env("CARGO_CFG_BPF_TARGET_ARCH", arch);
// Workaround to make sure that the correct toolchain is used.
for key in ["RUSTC", "RUSTC_WORKSPACE_WRAPPER"] {
cmd.env_remove(key);
}
// Workaround for https://github.com/rust-lang/cargo/issues/6412 where cargo flocks itself.
let ebpf_target_dir = out_dir.join("integration-ebpf");
cmd.arg("--target-dir").arg(&ebpf_target_dir);
let mut child = cmd
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}"));
let Child { stdout, stderr, .. } = &mut child;
// Trampoline stdout to cargo warnings.
let stderr = stderr.take().unwrap();
let stderr = BufReader::new(stderr);
let stderr = std::thread::spawn(move || {
for line in stderr.lines() {
let line = line.unwrap();
println!("cargo:warning={line}");
}
});
let stdout = stdout.take().unwrap();
let stdout = BufReader::new(stdout);
let mut executables = Vec::new();
for message in Message::parse_stream(stdout) {
#[allow(clippy::collapsible_match)]
match message.expect("valid JSON") {
Message::CompilerArtifact(Artifact {
executable,
target: Target { name, .. },
..
}) => {
if let Some(executable) = executable {
executables.push((name, executable.into_std_path_buf()));
}
}
Message::CompilerMessage(CompilerMessage { message, .. }) => {
for line in message.rendered.unwrap_or_default().split('\n') {
println!("cargo:warning={line}");
}
}
Message::TextLine(line) => {
println!("cargo:warning={line}");
}
_ => {}
}
}
let status = child
.wait()
.unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}"));
assert_eq!(status.code(), Some(0), "{cmd:?} failed: {status:?}");
stderr.join().map_err(std::panic::resume_unwind).unwrap();
for (name, binary) in executables {
let dst = out_dir.join(name);
let _: u64 = fs::copy(&binary, &dst)
.unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}"));
}
aya_build::build_ebpf([integration_ebpf_package]).unwrap();
} else {
for (src, build_btf) in C_BPF {
let dst = out_dir.join(src).with_extension("o");

@ -0,0 +1,3 @@
pub mod aya_build
pub use aya_build::cargo_metadata
pub fn aya_build::build_ebpf(packages: impl core::iter::traits::collect::IntoIterator<Item = cargo_metadata::Package>) -> anyhow::Result<()>
Loading…
Cancel
Save