From 6ac1320707ba5b3e2d12104733ed65e16c5d1119 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 10 Jul 2023 13:42:11 -0400 Subject: [PATCH] integration-test: build "fake" by default Use the environment variable AYA_BUILD_INTEGRATION_BPF to indicate to the build script that it should *actually* build bpf, otherwise emitting empty files. This allows metadata builds to skip costly build steps without sacrificing ergonomics; all compile-time tools such as cargo clippy work out of the box. Cargo even gives each of these builds (depending on the value of the environment variable) its own cache key, so they do not invalidate each other when the user alternates between metadata and real builds. This allows the lint action to move out of the VM. --- .github/workflows/lint.yml | 2 +- test/integration-test/build.rs | 257 +++++++++++++++++++-------------- test/run.sh | 3 - xtask/src/run.rs | 2 +- 4 files changed, 150 insertions(+), 114 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 713f36fa..fae57e42 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: run: cargo fmt --all -- --check - name: Run clippy - run: cargo clippy --all-targets --workspace --exclude integration-test -- --deny warnings + run: cargo clippy --all-targets --workspace -- --deny warnings - name: Run miri run: cargo miri test --all-targets diff --git a/test/integration-test/build.rs b/test/integration-test/build.rs index 948f62b2..c36d20a4 100644 --- a/test/integration-test/build.rs +++ b/test/integration-test/build.rs @@ -2,51 +2,34 @@ use std::{ env, ffi::OsString, fmt::Write as _, - fs::copy, + fs, io::BufReader, path::PathBuf, process::{Child, Command, Stdio}, }; -use cargo_metadata::{Artifact, CompilerMessage, Message, Target}; +use cargo_metadata::{ + Artifact, CompilerMessage, Message, Metadata, MetadataCommand, Package, Target, +}; fn main() { + const AYA_BUILD_INTEGRATION_BPF: &str = "AYA_BUILD_INTEGRATION_BPF"; + + println!("cargo:rerun-if-env-changed={}", AYA_BUILD_INTEGRATION_BPF); + + let build_integration_bpf = match env::var_os(AYA_BUILD_INTEGRATION_BPF) { + None => false, + Some(s) => { + let s = s.to_str().unwrap(); + s.parse::().unwrap() + } + }; + let manifest_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap(); let manifest_dir = PathBuf::from(manifest_dir); let out_dir = env::var_os("OUT_DIR").unwrap(); let out_dir = PathBuf::from(out_dir); - let libbpf_dir = manifest_dir - .parent() - .unwrap() - .parent() - .unwrap() - .join("libbpf"); - - let libbpf_headers_dir = out_dir.join("libbpf_headers"); - - let mut includedir = OsString::new(); - includedir.push("INCLUDEDIR="); - includedir.push(&libbpf_headers_dir); - - let mut cmd = Command::new("make"); - cmd.arg("-C") - .arg(libbpf_dir.join("src")) - .arg(includedir) - .arg("install_headers"); - let status = cmd - .status() - .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}")); - match status.code() { - Some(code) => match code { - 0 => {} - code => panic!("{cmd:?} exited with code {code}"), - }, - None => panic!("{cmd:?} terminated by signal"), - } - - let bpf_dir = manifest_dir.join("bpf"); - let endian = env::var_os("CARGO_CFG_TARGET_ENDIAN").unwrap(); let target = if endian == "big" { "bpfeb" @@ -56,34 +39,36 @@ fn main() { panic!("unsupported endian={:?}", endian) }; - let mut target_arch = OsString::new(); - target_arch.push("-D__TARGET_ARCH_"); - - let arch = env::var_os("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "x86_64" { - target_arch.push("x86"); - } else if arch == "aarch64" { - target_arch.push("arm64"); - } else { - target_arch.push(arch); - }; - - for (src, dst) in [ + const C_BPF_PROBES: &[(&str, &str)] = &[ ("ext.bpf.c", "ext.bpf.o"), ("main.bpf.c", "main.bpf.o"), ("multimap-btf.bpf.c", "multimap-btf.bpf.o"), ("text_64_64_reloc.c", "text_64_64_reloc.o"), - ] { - let src = bpf_dir.join(src); - let out = out_dir.join(dst); - let mut cmd = Command::new("clang"); - cmd.arg("-I") - .arg(&libbpf_headers_dir) - .args(["-g", "-O2", "-target", target, "-c"]) - .arg(&target_arch) - .arg(src) - .arg("-o") - .arg(out); + ]; + + let c_bpf_probes = C_BPF_PROBES + .iter() + .map(|(src, dst)| (src, out_dir.join(dst))); + + if build_integration_bpf { + let libbpf_dir = manifest_dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("libbpf"); + + let libbpf_headers_dir = out_dir.join("libbpf_headers"); + + let mut includedir = OsString::new(); + includedir.push("INCLUDEDIR="); + includedir.push(&libbpf_headers_dir); + + let mut cmd = Command::new("make"); + cmd.arg("-C") + .arg(libbpf_dir.join("src")) + .arg(includedir) + .arg("install_headers"); let status = cmd .status() .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}")); @@ -94,64 +79,118 @@ fn main() { }, None => panic!("{cmd:?} terminated by signal"), } - } - let ebpf_dir = manifest_dir.parent().unwrap().join("integration-ebpf"); - let target = format!("{target}-unknown-none"); - - let mut cmd = Command::new("cargo"); - cmd.current_dir(&ebpf_dir).args([ - "build", - "-Z", - "build-std=core", - "--release", - "--message-format=json", - "--target", - &target, - ]); - let mut child = cmd - .stdout(Stdio::piped()) - .spawn() - .unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}")); - let Child { stdout, .. } = &mut child; - let stdout = stdout.take().unwrap(); - let reader = BufReader::new(stdout); - let mut executables = Vec::new(); - let mut compiler_messages = String::new(); - for message in Message::parse_stream(reader) { - #[allow(clippy::collapsible_match)] - match message.expect("valid JSON") { - Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { - if let Some(executable) = executable { - executables.push((name, executable.into_std_path_buf())); - } + let bpf_dir = manifest_dir.join("bpf"); + + let mut target_arch = OsString::new(); + target_arch.push("-D__TARGET_ARCH_"); + + let arch = env::var_os("CARGO_CFG_TARGET_ARCH").unwrap(); + if arch == "x86_64" { + target_arch.push("x86"); + } else if arch == "aarch64" { + target_arch.push("arm64"); + } else { + target_arch.push(arch); + }; + + for (src, dst) in c_bpf_probes { + let src = bpf_dir.join(src); + let mut cmd = Command::new("clang"); + cmd.arg("-I") + .arg(&libbpf_headers_dir) + .args(["-g", "-O2", "-target", target, "-c"]) + .arg(&target_arch) + .arg(src) + .arg("-o") + .arg(dst); + let status = cmd + .status() + .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}")); + match status.code() { + Some(code) => match code { + 0 => {} + code => panic!("{cmd:?} exited with code {code}"), + }, + None => panic!("{cmd:?} terminated by signal"), } - Message::CompilerMessage(CompilerMessage { message, .. }) => { - writeln!(&mut compiler_messages, "{message}").unwrap() + } + + let ebpf_dir = manifest_dir.parent().unwrap().join("integration-ebpf"); + let target = format!("{target}-unknown-none"); + + let mut cmd = Command::new("cargo"); + cmd.current_dir(&ebpf_dir).args([ + "build", + "-Z", + "build-std=core", + "--release", + "--message-format=json", + "--target", + &target, + ]); + let mut child = cmd + .stdout(Stdio::piped()) + .spawn() + .unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}")); + let Child { stdout, .. } = &mut child; + let stdout = stdout.take().unwrap(); + let reader = BufReader::new(stdout); + let mut executables = Vec::new(); + let mut compiler_messages = String::new(); + for message in Message::parse_stream(reader) { + #[allow(clippy::collapsible_match)] + match message.expect("valid JSON") { + Message::CompilerArtifact(Artifact { + executable, + target: Target { name, .. }, + .. + }) => { + if let Some(executable) = executable { + executables.push((name, executable.into_std_path_buf())); + } + } + Message::CompilerMessage(CompilerMessage { message, .. }) => { + writeln!(&mut compiler_messages, "{message}").unwrap() + } + _ => {} } - _ => {} } - } - let status = child - .wait() - .unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}")); + let status = child + .wait() + .unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}")); - match status.code() { - Some(code) => match code { - 0 => {} - code => panic!("{cmd:?} exited with status code {code}:\n{compiler_messages}"), - }, - None => panic!("{cmd:?} terminated by signal"), - } + match status.code() { + Some(code) => match code { + 0 => {} + code => panic!("{cmd:?} exited with status code {code}:\n{compiler_messages}"), + }, + None => panic!("{cmd:?} terminated by signal"), + } - for (name, binary) in executables { - let dst = out_dir.join(name); - let _: u64 = copy(&binary, &dst) - .unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}")); + for (name, binary) in executables { + let dst = out_dir.join(name); + let _: u64 = fs::copy(&binary, &dst) + .unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}")); + } + } else { + for (_src, dst) in c_bpf_probes { + fs::write(&dst, []).unwrap_or_else(|err| panic!("failed to create {dst:?}: {err}")); + } + + let Metadata { packages, .. } = MetadataCommand::new().no_deps().exec().unwrap(); + for Package { name, targets, .. } in packages { + if name != "integration-ebpf" { + continue; + } + for Target { name, kind, .. } in targets { + if kind != ["bin"] { + continue; + } + let dst = out_dir.join(name); + fs::write(&dst, []).unwrap_or_else(|err| panic!("failed to create {dst:?}: {err}")); + } + } } } diff --git a/test/run.sh b/test/run.sh index 33e2b6e0..23c0dd1a 100755 --- a/test/run.sh +++ b/test/run.sh @@ -242,9 +242,6 @@ trap cleanup_vm EXIT exec_vm "rm -rf aya/*" rsync_vm "--exclude=target --exclude=.tmp $AYA_SOURCE_DIR" -# need to build or linting will fail trying to include object files; don't run the tests though. -exec_vm "cd aya; cargo xtask integration-test -- filter-that-matches-nothing" -exec_vm "cd aya; cargo clippy --all-targets -p integration-test -- --deny warnings" exec_vm "cd aya; cargo xtask integration-test" # we rm and sync but it doesn't seem to work reliably - I guess we could sleep a diff --git a/xtask/src/run.rs b/xtask/src/run.rs index 36e4c40f..396d9ece 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -35,7 +35,7 @@ pub struct Options { pub fn build(opts: BuildOptions) -> Result> { let BuildOptions { release, target } = opts; let mut cmd = Command::new("cargo"); - cmd.args([ + cmd.env("AYA_BUILD_INTEGRATION_BPF", "true").args([ "build", "--tests", "--message-format=json",