From 657fb4452563d323721c3671e63f4f6d7cf9e334 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 21 Jan 2022 18:17:22 +0000 Subject: [PATCH 1/4] test: Add compile_c_ebpf to library This allows for C code to be compiled and used in tests! It does require libbpf on the host machine though... Also moved removal of the `user` and `ebpf` dirs created by rust-script into `lib.sh` and parameterized bits of the XDP smoke test for ease of copying Signed-off-by: Dave Tucker --- test/cases/000_smoke/000_xdp/test.sh | 10 ++-- test/cases/_lib/compile-ebpf.ers | 85 ++++++++++++++++++++++++++++ test/cases/_lib/lib.sh | 12 ++++ 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 test/cases/_lib/compile-ebpf.ers diff --git a/test/cases/000_smoke/000_xdp/test.sh b/test/cases/000_smoke/000_xdp/test.sh index 974e033c..eba23581 100755 --- a/test/cases/000_smoke/000_xdp/test.sh +++ b/test/cases/000_smoke/000_xdp/test.sh @@ -11,8 +11,8 @@ set -e NAME=pass clean_up() { - rm -rf ebpf user ${NAME}.o ${NAME} - exec_vm rm -f pass pass.o + rm -rf ${NAME}.o ${NAME} + exec_vm rm -f ${NAME} ${NAME}.o } trap clean_up EXIT @@ -21,9 +21,9 @@ trap clean_up EXIT compile_ebpf "$(pwd)/${NAME}.ebpf.rs" compile_user "$(pwd)/${NAME}.rs" -scp_vm pass.o -scp_vm pass +scp_vm ${NAME}.o +scp_vm ${NAME} -exec_vm sudo ./pass +exec_vm sudo ./${NAME} exit 0 \ No newline at end of file diff --git a/test/cases/_lib/compile-ebpf.ers b/test/cases/_lib/compile-ebpf.ers new file mode 100644 index 00000000..66993d20 --- /dev/null +++ b/test/cases/_lib/compile-ebpf.ers @@ -0,0 +1,85 @@ +//! ```cargo +//! [dependencies] +//! libbpf-sys = { version = "0.6.1-1" } +//! anyhow = "1" +//! ``` + +use std::{ + env, + fs::{self, OpenOptions}, + io::Write, + path::Path, + process::Command, + string::String, +}; +use anyhow::{bail, Context, Result}; +static CLANG_DEFAULT: &str = "/usr/bin/clang"; + +/// Extract vendored libbpf headers from libbpf-sys. +fn extract_libbpf_headers>(include_path: P) -> Result<()> { + let dir = include_path.as_ref().join("bpf"); + fs::create_dir_all(&dir)?; + for (filename, contents) in libbpf_sys::API_HEADERS.iter() { + let path = dir.as_path().join(filename); + let mut file = OpenOptions::new().write(true).create(true).open(path)?; + file.write_all(contents.as_bytes())?; + } + + Ok(()) +} + +/// Build eBPF programs with clang and libbpf headers. +fn build_ebpf>(in_file: P, out_file: P, include_path: P) -> Result<()> { + extract_libbpf_headers(include_path.clone())?; + let clang = match env::var("CLANG") { + Ok(val) => val, + Err(_) => String::from(CLANG_DEFAULT), + }; + let arch = match std::env::consts::ARCH { + "x86_64" => "x86", + "aarch64" => "arm64", + _ => std::env::consts::ARCH, + }; + let mut cmd = Command::new(clang); + cmd.arg(format!("-I{}", include_path.as_ref().to_string_lossy())) + .arg("-g") + .arg("-O2") + .arg("-target") + .arg("bpf") + .arg("-c") + .arg(format!("-D__TARGET_ARCH_{}", arch)) + .arg(in_file.as_ref().as_os_str()) + .arg("-o") + .arg(out_file.as_ref().as_os_str()); + + let output = cmd.output().context("Failed to execute clang")?; + if !output.status.success() { + bail!( + "Failed to compile eBPF programs\n \ + stdout=\n \ + {}\n \ + stderr=\n \ + {}\n", + String::from_utf8(output.stdout).unwrap(), + String::from_utf8(output.stderr).unwrap() + ); + } + + Ok(()) +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + if args.len() != 3 { + bail!("requires 2 arguments. src and dst") + } + let path = env::current_dir()?; + let src = Path::new(&args[1]); + let dst = Path::new(&args[2]); + + let include_path = path.join("include"); + fs::create_dir_all(include_path.clone())?; + build_ebpf(src, dst, &include_path)?; + + Ok(()) +} \ No newline at end of file diff --git a/test/cases/_lib/lib.sh b/test/cases/_lib/lib.sh index f6a7e7e1..384aea6d 100644 --- a/test/cases/_lib/lib.sh +++ b/test/cases/_lib/lib.sh @@ -54,6 +54,17 @@ EOF cargo build -q --manifest-path "${dir}/ebpf/Cargo.toml" mv "${dir}/ebpf/target/bpfel-unknown-none/debug/${artifact}" "${dir}/${base}.o" rm -rf "${dir}/.cargo" + rm -rf "${dir}/ebpf" +} + +# compile a C BPF file +compile_c_ebpf() { + file=$(basename "$1") + dir=$(dirname "$1") + base=$(echo "${file}" | cut -f1 -d '.') + + rust-script "${RT_PROJECT_ROOT}/_lib/compile-ebpf.ers" "${1}" "${dir}/${base}.o" + rm -rf "${dir}/include" } # compiles the userspace program by using rust-script to create a temporary @@ -75,6 +86,7 @@ members = [] EOF cargo build -q --release --manifest-path "${dir}/user/Cargo.toml" --target=x86_64-unknown-linux-musl mv "${dir}/user/target/x86_64-unknown-linux-musl/release/${artifact}" "${dir}/${base}" + rm -rf "${dir}/user" } download_images() { From 4bad464c923aefe3a9385c6f1c9c0056ebf05c0c Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 21 Jan 2022 18:20:30 +0000 Subject: [PATCH 2/4] test: Add smoke test for BPF_PROG_TYPE_EXT Signed-off-by: Dave Tucker --- test/cases/000_smoke/010_ext/ext.bpf.c | 11 +++++++++ test/cases/000_smoke/010_ext/ext.rs | 24 +++++++++++++++++++ test/cases/000_smoke/010_ext/main.bpf.c | 11 +++++++++ test/cases/000_smoke/010_ext/test.sh | 31 +++++++++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 test/cases/000_smoke/010_ext/ext.bpf.c create mode 100755 test/cases/000_smoke/010_ext/ext.rs create mode 100644 test/cases/000_smoke/010_ext/main.bpf.c create mode 100755 test/cases/000_smoke/010_ext/test.sh diff --git a/test/cases/000_smoke/010_ext/ext.bpf.c b/test/cases/000_smoke/010_ext/ext.bpf.c new file mode 100644 index 00000000..f0da9445 --- /dev/null +++ b/test/cases/000_smoke/010_ext/ext.bpf.c @@ -0,0 +1,11 @@ +#include +#include +#include + +SEC("xdp/drop") +int xdp_drop(struct xdp_md *ctx) +{ + return XDP_DROP; +} + +char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/test/cases/000_smoke/010_ext/ext.rs b/test/cases/000_smoke/010_ext/ext.rs new file mode 100755 index 00000000..909a848b --- /dev/null +++ b/test/cases/000_smoke/010_ext/ext.rs @@ -0,0 +1,24 @@ +//! ```cargo +//! [dependencies] +//! aya = { path = "../../../../aya" } +//! ``` + +use aya::{ + Bpf, BpfLoader, + programs::{Extension, ProgramFd, Xdp, XdpFlags}, +}; +use std::convert::TryInto; + +fn main() { + println!("Loading Root XDP program"); + let mut bpf = Bpf::load_file("main.o").unwrap(); + let pass: &mut Xdp = bpf.program_mut("pass").unwrap().try_into().unwrap(); + pass.load().unwrap(); + pass.attach("lo", XdpFlags::default()).unwrap(); + + println!("Loading Extension Program"); + let mut bpf = BpfLoader::new().extension("drop").load_file("ext.o").unwrap(); + let drop_: &mut Extension = bpf.program_mut("drop").unwrap().try_into().unwrap(); + drop_.load(pass.fd().unwrap(), "xdp_pass").unwrap(); + println!("Success..."); +} diff --git a/test/cases/000_smoke/010_ext/main.bpf.c b/test/cases/000_smoke/010_ext/main.bpf.c new file mode 100644 index 00000000..fbe4600b --- /dev/null +++ b/test/cases/000_smoke/010_ext/main.bpf.c @@ -0,0 +1,11 @@ +#include +#include +#include + +SEC("xdp/pass") +int xdp_pass(struct xdp_md *ctx) +{ + return XDP_PASS; +} + +char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/test/cases/000_smoke/010_ext/test.sh b/test/cases/000_smoke/010_ext/test.sh new file mode 100755 index 00000000..d8d1559f --- /dev/null +++ b/test/cases/000_smoke/010_ext/test.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# SUMMARY: Check that a simple XDP program an be loaded +# LABELS: + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +NAME=ext + +clean_up() { + rm -rf main.o ${NAME}.o ${NAME} + exec_vm rm -f main.o ${NAME}.o ${NAME} +} + +trap clean_up EXIT + +# Test code goes here +compile_c_ebpf "$(pwd)/main.bpf.c" +compile_c_ebpf "$(pwd)/${NAME}.bpf.c" +compile_user "$(pwd)/${NAME}.rs" + +scp_vm main.o +scp_vm ${NAME}.o +scp_vm ${NAME} + +exec_vm sudo ./${NAME} + +exit 0 \ No newline at end of file From 61c3675b92d83e464d1c1fbe3682f35c2239069e Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 21 Jan 2022 18:51:50 +0000 Subject: [PATCH 3/4] test: Add a check for minimum kernel version Signed-off-by: Dave Tucker --- test/cases/000_smoke/010_ext/test.sh | 2 ++ test/cases/_lib/lib.sh | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/test/cases/000_smoke/010_ext/test.sh b/test/cases/000_smoke/010_ext/test.sh index d8d1559f..695849a5 100755 --- a/test/cases/000_smoke/010_ext/test.sh +++ b/test/cases/000_smoke/010_ext/test.sh @@ -18,6 +18,8 @@ clean_up() { trap clean_up EXIT # Test code goes here +min_kernel_version 5.9 + compile_c_ebpf "$(pwd)/main.bpf.c" compile_c_ebpf "$(pwd)/${NAME}.bpf.c" compile_user "$(pwd)/${NAME}.rs" diff --git a/test/cases/_lib/lib.sh b/test/cases/_lib/lib.sh index 384aea6d..2df42b8f 100644 --- a/test/cases/_lib/lib.sh +++ b/test/cases/_lib/lib.sh @@ -9,6 +9,9 @@ AYA_TMPDIR="${RT_PROJECT_ROOT}/_tmp" # Directory for VM images AYA_IMGDIR="${RT_PROJECT_ROOT}/_images" +# Cancel Exit Code +RT_CANCEL=253 + # Test Architecture if [ -z "${AYA_TEST_ARCH}" ]; then AYA_TEST_ARCH="$(uname -m)" @@ -239,3 +242,20 @@ cleanup_vm() { stop_vm fi } + +# Check that host machine meets minimum kernel requirement +# Must be in format {major}.{minor} +min_kernel_version() { + target_major=$(echo "$1" | cut -d '.' -f1) + target_minor=$(echo "$1" | cut -d '.' -f2) + + vm_kernel=$(exec_vm uname -r) + vm_major=$(echo "${vm_kernel}" | cut -d '.' -f1) + vm_minor=$(echo "${vm_kernel}" | cut -d '.' -f2) + + if [ "${vm_major}" -lt "${target_major}" ] || [ "${vm_minor}" -lt "${target_minor}" ]; then + echo "Test not supported on kernel ${vm_major}.${vm_minor}" + return ${RT_CANCEL} + fi + return 0 +} \ No newline at end of file From ce93774e0a75c133eff98b66ef038bde404d328c Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 21 Jan 2022 18:21:40 +0000 Subject: [PATCH 4/4] ci: try running regression tests in a container Signed-off-by: Dave Tucker --- .github/workflows/build-aya.yml | 34 +++--------------------- .github/workflows/images.yml | 46 +++++++++++++++++++++++++++++++++ images/Dockerfile.rtf | 38 +++++++++++++++++++++++++++ test/README.md | 8 +++++- 4 files changed, 94 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/images.yml create mode 100644 images/Dockerfile.rtf diff --git a/.github/workflows/build-aya.yml b/.github/workflows/build-aya.yml index ab7546cd..6f8c5235 100644 --- a/.github/workflows/build-aya.yml +++ b/.github/workflows/build-aya.yml @@ -16,7 +16,6 @@ env: jobs: build: runs-on: ubuntu-20.04 - steps: - uses: actions/checkout@v2 - uses: Swatinem/rust-cache@v1 @@ -30,41 +29,14 @@ jobs: test: runs-on: ubuntu-20.04 needs: build + container: + image: ghcr.io/aya-rs/aya-test-rtf:main steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - components: rustfmt, clippy, rust-src - override: true - target: x86_64-unknown-linux-musl - - - uses: Swatinem/rust-cache@v1 - - - name: Set up Go 1.17 - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Set GOPATH - run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - env: - GOPATH: ${{runner.workspace}} - - - name: Install prereqs - run: | - go install github.com/linuxkit/rtf@latest - cargo install bpf-linker - cargo install rust-script - cargo install sccache - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update - sudo apt-get install -qy qemu-utils qemu-system-x86 cloud-image-utils genisoimage - - name: Run regression tests run: | + ln -s /root/.rustup ${HOME}/.rustup cd test rtf -vvv run \ No newline at end of file diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 00000000..00e8d458 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,46 @@ +name: Aya test image + +on: + schedule: + - cron: "42 2 * * 0" + push: + branches: + - 'main' + paths: + - 'images/**' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: aya-rs/aya-test-rtf + +jobs: + rtf: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v2 + + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + path: ./images + file: Dockerfile.rtf + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/images/Dockerfile.rtf b/images/Dockerfile.rtf new file mode 100644 index 00000000..72f28cf1 --- /dev/null +++ b/images/Dockerfile.rtf @@ -0,0 +1,38 @@ +FROM fedora:35 + +# Rust Nightly +RUN curl https://sh.rustup.rs -sSf | sh -s -- \ + --default-toolchain nightly \ + --component rustfmt \ + --component clippy \ + --component rust-src \ + --target x86_64-unknown-linux-musl \ + -y + +ENV PATH "/root/.cargo/bin:$PATH" + +# Pre-requisites +RUN dnf install \ + --setopt=install_weak_deps=False --best -qy \ + golang \ + qemu-system-x86 \ + cloud-utils \ + genisoimage \ + libbpf-devel \ + clang \ + openssl-devel \ + musl-libc \ + git && dnf clean all \ + && rm -rf /var/cache/yum + +RUN cargo install \ + bpf-linker \ + rust-script \ + sccache + +RUN go install github.com/linuxkit/rtf@latest +ENV PATH "/root/go/bin:$PATH" +ENV RUSTC_WRAPPER "sccache" + +ENTRYPOINT ["rtf"] +CMD ["-vvv", "run"] diff --git a/test/README.md b/test/README.md index 3cef36c7..3cc6c0be 100644 --- a/test/README.md +++ b/test/README.md @@ -7,13 +7,19 @@ common usage behaviours work on real Linux distros This assumes you have a working Rust and Go toolchain on the host machine -1. `rustup toolcahin add x86_64-unknown-linux-musl` +1. `rustup toolchain add x86_64-unknown-linux-musl` 1. Install [`rtf`](https://github.com/linuxkit/rtf): `go install github.com/linuxkit/rtf@latest` 1. Install rust-script: `cargo install rust-script` 1. Install `qemu` and `cloud-init-utils` package - or any package that provides `cloud-localds` It is not required, but the tests run significantly faster if you use `sccache` +You may also use the docker image to run the tests: + +``` +docker run -it --rm --device /dev/kvm -v/home/dave/dev/aya-rs/aya:/src -w /src/test ghcr.io/aya-rs/aya-test-rtf:main +``` + ## Usage To read more about how to use `rtf`, see the [documentation](https://github.com/linuxkit/rtf/blob/master/docs/USER_GUIDE.md)