integration-test: build "fake" by default

Use the environment variable AYA_BUILD_INTEGRATION_BPF to indicate to
the build script that it should *actually* build bpf, otherwise emitting
empty files.

This allows metadata builds to skip costly build steps without
sacrificing ergonomics; all compile-time tools such as cargo clippy work
out of the box.

Cargo even gives each of these builds (depending on the value of the
environment variable) its own cache key, so they do not invalidate each
other when the user alternates between metadata and real builds.

This allows the lint action to move out of the VM.

.github/workflows/lint.yml

@@ -29,7 +29,7 @@ jobs:
         run: cargo fmt --all -- --check
 
       - name: Run clippy
-        run: cargo clippy --all-targets --workspace --exclude integration-test -- --deny warnings
+        run: cargo clippy --all-targets --workspace -- --deny warnings
 
       - name: Run miri
         run: cargo miri test --all-targets

test/integration-test/build.rs

@@ -2,51 +2,34 @@ use std::{
     env,
     ffi::OsString,
     fmt::Write as _,
-    fs::copy,
+    fs,
     io::BufReader,
     path::PathBuf,
     process::{Child, Command, Stdio},
 };
 
-use cargo_metadata::{Artifact, CompilerMessage, Message, Target};
+use cargo_metadata::{
+    Artifact, CompilerMessage, Message, Metadata, MetadataCommand, Package, Target,
+};
 
 fn main() {
+    const AYA_BUILD_INTEGRATION_BPF: &str = "AYA_BUILD_INTEGRATION_BPF";
+
+    println!("cargo:rerun-if-env-changed={}", AYA_BUILD_INTEGRATION_BPF);
+
+    let build_integration_bpf = match env::var_os(AYA_BUILD_INTEGRATION_BPF) {
+        None => false,
+        Some(s) => {
+            let s = s.to_str().unwrap();
+            s.parse::<bool>().unwrap()
+        }
+    };
+
     let manifest_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap();
     let manifest_dir = PathBuf::from(manifest_dir);
     let out_dir = env::var_os("OUT_DIR").unwrap();
     let out_dir = PathBuf::from(out_dir);
 
-    let libbpf_dir = manifest_dir
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap()
-        .join("libbpf");
-
-    let libbpf_headers_dir = out_dir.join("libbpf_headers");
-
-    let mut includedir = OsString::new();
-    includedir.push("INCLUDEDIR=");
-    includedir.push(&libbpf_headers_dir);
-
-    let mut cmd = Command::new("make");
-    cmd.arg("-C")
-        .arg(libbpf_dir.join("src"))
-        .arg(includedir)
-        .arg("install_headers");
-    let status = cmd
-        .status()
-        .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
-    match status.code() {
-        Some(code) => match code {
-            0 => {}
-            code => panic!("{cmd:?} exited with code {code}"),
-        },
-        None => panic!("{cmd:?} terminated by signal"),
-    }
-
-    let bpf_dir = manifest_dir.join("bpf");
-
     let endian = env::var_os("CARGO_CFG_TARGET_ENDIAN").unwrap();
     let target = if endian == "big" {
         "bpfeb"
@@ -56,34 +39,36 @@ fn main() {
         panic!("unsupported endian={:?}", endian)
     };
 
-    let mut target_arch = OsString::new();
+    const C_BPF_PROBES: &[(&str, &str)] = &[
-    target_arch.push("-D__TARGET_ARCH_");
-
-    let arch = env::var_os("CARGO_CFG_TARGET_ARCH").unwrap();
-    if arch == "x86_64" {
-        target_arch.push("x86");
-    } else if arch == "aarch64" {
-        target_arch.push("arm64");
-    } else {
-        target_arch.push(arch);
-    };
-
-    for (src, dst) in [
         ("ext.bpf.c", "ext.bpf.o"),
         ("main.bpf.c", "main.bpf.o"),
         ("multimap-btf.bpf.c", "multimap-btf.bpf.o"),
         ("text_64_64_reloc.c", "text_64_64_reloc.o"),
-    ] {
+    ];
-        let src = bpf_dir.join(src);
+
-        let out = out_dir.join(dst);
+    let c_bpf_probes = C_BPF_PROBES
-        let mut cmd = Command::new("clang");
+        .iter()
-        cmd.arg("-I")
+        .map(|(src, dst)| (src, out_dir.join(dst)));
-            .arg(&libbpf_headers_dir)
+
-            .args(["-g", "-O2", "-target", target, "-c"])
+    if build_integration_bpf {
-            .arg(&target_arch)
+        let libbpf_dir = manifest_dir
-            .arg(src)
+            .parent()
-            .arg("-o")
+            .unwrap()
-            .arg(out);
+            .parent()
+            .unwrap()
+            .join("libbpf");
+
+        let libbpf_headers_dir = out_dir.join("libbpf_headers");
+
+        let mut includedir = OsString::new();
+        includedir.push("INCLUDEDIR=");
+        includedir.push(&libbpf_headers_dir);
+
+        let mut cmd = Command::new("make");
+        cmd.arg("-C")
+            .arg(libbpf_dir.join("src"))
+            .arg(includedir)
+            .arg("install_headers");
         let status = cmd
             .status()
             .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
@@ -94,64 +79,118 @@ fn main() {
             },
             None => panic!("{cmd:?} terminated by signal"),
         }
-    }
 
-    let ebpf_dir = manifest_dir.parent().unwrap().join("integration-ebpf");
+        let bpf_dir = manifest_dir.join("bpf");
-    let target = format!("{target}-unknown-none");
+
-
+        let mut target_arch = OsString::new();
-    let mut cmd = Command::new("cargo");
+        target_arch.push("-D__TARGET_ARCH_");
-    cmd.current_dir(&ebpf_dir).args([
+
-        "build",
+        let arch = env::var_os("CARGO_CFG_TARGET_ARCH").unwrap();
-        "-Z",
+        if arch == "x86_64" {
-        "build-std=core",
+            target_arch.push("x86");
-        "--release",
+        } else if arch == "aarch64" {
-        "--message-format=json",
+            target_arch.push("arm64");
-        "--target",
+        } else {
-        &target,
+            target_arch.push(arch);
-    ]);
+        };
-    let mut child = cmd
+
-        .stdout(Stdio::piped())
+        for (src, dst) in c_bpf_probes {
-        .spawn()
+            let src = bpf_dir.join(src);
-        .unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}"));
+            let mut cmd = Command::new("clang");
-    let Child { stdout, .. } = &mut child;
+            cmd.arg("-I")
-    let stdout = stdout.take().unwrap();
+                .arg(&libbpf_headers_dir)
-    let reader = BufReader::new(stdout);
+                .args(["-g", "-O2", "-target", target, "-c"])
-    let mut executables = Vec::new();
+                .arg(&target_arch)
-    let mut compiler_messages = String::new();
+                .arg(src)
-    for message in Message::parse_stream(reader) {
+                .arg("-o")
-        #[allow(clippy::collapsible_match)]
+                .arg(dst);
-        match message.expect("valid JSON") {
+            let status = cmd
-            Message::CompilerArtifact(Artifact {
+                .status()
-                executable,
+                .unwrap_or_else(|err| panic!("failed to run {cmd:?}: {err}"));
-                target: Target { name, .. },
+            match status.code() {
-                ..
+                Some(code) => match code {
-            }) => {
+                    0 => {}
-                if let Some(executable) = executable {
+                    code => panic!("{cmd:?} exited with code {code}"),
-                    executables.push((name, executable.into_std_path_buf()));
+                },
-                }
+                None => panic!("{cmd:?} terminated by signal"),
             }
-            Message::CompilerMessage(CompilerMessage { message, .. }) => {
+        }
-                writeln!(&mut compiler_messages, "{message}").unwrap()
+
+        let ebpf_dir = manifest_dir.parent().unwrap().join("integration-ebpf");
+        let target = format!("{target}-unknown-none");
+
+        let mut cmd = Command::new("cargo");
+        cmd.current_dir(&ebpf_dir).args([
+            "build",
+            "-Z",
+            "build-std=core",
+            "--release",
+            "--message-format=json",
+            "--target",
+            &target,
+        ]);
+        let mut child = cmd
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap_or_else(|err| panic!("failed to spawn {cmd:?}: {err}"));
+        let Child { stdout, .. } = &mut child;
+        let stdout = stdout.take().unwrap();
+        let reader = BufReader::new(stdout);
+        let mut executables = Vec::new();
+        let mut compiler_messages = String::new();
+        for message in Message::parse_stream(reader) {
+            #[allow(clippy::collapsible_match)]
+            match message.expect("valid JSON") {
+                Message::CompilerArtifact(Artifact {
+                    executable,
+                    target: Target { name, .. },
+                    ..
+                }) => {
+                    if let Some(executable) = executable {
+                        executables.push((name, executable.into_std_path_buf()));
+                    }
+                }
+                Message::CompilerMessage(CompilerMessage { message, .. }) => {
+                    writeln!(&mut compiler_messages, "{message}").unwrap()
+                }
+                _ => {}
             }
-            _ => {}
         }
-    }
 
-    let status = child
+        let status = child
-        .wait()
+            .wait()
-        .unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}"));
+            .unwrap_or_else(|err| panic!("failed to wait for {cmd:?}: {err}"));
 
-    match status.code() {
+        match status.code() {
-        Some(code) => match code {
+            Some(code) => match code {
-            0 => {}
+                0 => {}
-            code => panic!("{cmd:?} exited with status code {code}:\n{compiler_messages}"),
+                code => panic!("{cmd:?} exited with status code {code}:\n{compiler_messages}"),
-        },
+            },
-        None => panic!("{cmd:?} terminated by signal"),
+            None => panic!("{cmd:?} terminated by signal"),
-    }
+        }
 
-    for (name, binary) in executables {
+        for (name, binary) in executables {
-        let dst = out_dir.join(name);
+            let dst = out_dir.join(name);
-        let _: u64 = copy(&binary, &dst)
+            let _: u64 = fs::copy(&binary, &dst)
-            .unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}"));
+                .unwrap_or_else(|err| panic!("failed to copy {binary:?} to {dst:?}: {err}"));
+        }
+    } else {
+        for (_src, dst) in c_bpf_probes {
+            fs::write(&dst, []).unwrap_or_else(|err| panic!("failed to create {dst:?}: {err}"));
+        }
+
+        let Metadata { packages, .. } = MetadataCommand::new().no_deps().exec().unwrap();
+        for Package { name, targets, .. } in packages {
+            if name != "integration-ebpf" {
+                continue;
+            }
+            for Target { name, kind, .. } in targets {
+                if kind != ["bin"] {
+                    continue;
+                }
+                let dst = out_dir.join(name);
+                fs::write(&dst, []).unwrap_or_else(|err| panic!("failed to create {dst:?}: {err}"));
+            }
+        }
     }
 }

test/run.sh

@@ -242,9 +242,6 @@ trap cleanup_vm EXIT
 exec_vm "rm -rf aya/*"
 rsync_vm "--exclude=target --exclude=.tmp $AYA_SOURCE_DIR"
 
-# need to build or linting will fail trying to include object files; don't run the tests though.
-exec_vm "cd aya; cargo xtask integration-test -- filter-that-matches-nothing"
-exec_vm "cd aya; cargo clippy --all-targets -p integration-test -- --deny warnings"
 exec_vm "cd aya; cargo xtask integration-test"
 
 # we rm and sync but it doesn't seem to work reliably - I guess we could sleep a

xtask/src/run.rs

@@ -35,7 +35,7 @@ pub struct Options {
 pub fn build(opts: BuildOptions) -> Result<Vec<(String, PathBuf)>> {
     let BuildOptions { release, target } = opts;
     let mut cmd = Command::new("cargo");
-    cmd.args([
+    cmd.env("AYA_BUILD_INTEGRATION_BPF", "true").args([
         "build",
         "--tests",
         "--message-format=json",