diff --git a/.cargo/config b/.cargo/config index dbd1a2b4..8f7da22a 100644 --- a/.cargo/config +++ b/.cargo/config @@ -7,4 +7,7 @@ build-bpfeb = "build -Zbuild-std=core --target=bpfeb-unknown-none" linker = "arm-linux-gnueabi-gcc" [target.armv7-unknown-linux-gnueabihf] -linker = "arm-linux-gnueabihf-gcc" \ No newline at end of file +linker = "arm-linux-gnueabihf-gcc" + +[target.aarch64-unknown-linux-musl] +linker = "aarch64-linux-musl-gcc" diff --git a/.github/workflows/build-aya.yml b/.github/workflows/build-aya.yml index 991298fa..076e3ac5 100644 --- a/.github/workflows/build-aya.yml +++ b/.github/workflows/build-aya.yml @@ -14,7 +14,7 @@ env: CARGO_TERM_COLOR: always jobs: - build: + build-test: strategy: matrix: arch: @@ -43,42 +43,3 @@ jobs: RUST_BACKTRACE: full run: | cross test --verbose --target ${{matrix.arch}} - - test: - runs-on: ubuntu-20.04 - needs: build - - steps: - - uses: actions/checkout@v2 - - uses: actions/checkout@v2 - with: - repository: libbpf/libbpf - path: libbpf - - - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - components: rustfmt, clippy, rust-src - target: x86_64-unknown-linux-musl - override: true - - - uses: Swatinem/rust-cache@v1 - - - name: Install Pre-requisites - run: | - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" | sudo tee -a /etc/apt/sources.list - sudo apt-get update - sudo apt-get -qy install linux-tools-common qemu-system-x86 cloud-image-utils openssh-client libelf-dev gcc-multilib llvm-15 clang-15 - sudo update-alternatives --install /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-15 200 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 200 - cargo install bpf-linker - - - name: Lint integration tests - run: | - cargo xtask build-integration-test-ebpf --libbpf-dir ./libbpf - cargo clippy -p integration-test -- --deny warnings - - - name: Run integration tests - run: | - (cd test && ./run.sh ../libbpf) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..f06de720 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,43 @@ +name: integration-tests + +on: + push: + branches: + - main + + pull_request: + branches: + - main + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + runs-on: macos-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + with: + repository: libbpf/libbpf + path: libbpf + + - name: Install Pre-requisites + run: | + brew install qemu gnu-getopt coreutils cdrtools + + - name: Cache tmp files + uses: actions/cache@v3 + with: + path: | + .tmp/*.qcow2 + .tmp/test_rsa + .tmp/test_rsa.pub + # FIXME: we should invalidate the cache on new bpf-linker releases. + # For now we must manually delete the cache when we release a new + # bpf-linker version. + key: tmp-files-${{ hashFiles('test/run.sh') }} + + - name: Run integration tests + run: test/run.sh ./libbpf diff --git a/test/cloud-localds b/test/cloud-localds new file mode 100755 index 00000000..3a28129f --- /dev/null +++ b/test/cloud-localds @@ -0,0 +1,264 @@ +#!/bin/bash + +VERBOSITY=0 +TEMP_D="" +DEF_DISK_FORMAT="raw" +DEF_FILESYSTEM="iso9660" +CR=" +" + +error() { echo "$@" 1>&2; } +fail() { [ $# -eq 0 ] || error "$@"; exit 1; } + +Usage() { + cat < my-meta-data + * ${0##*/} my-seed.img my-user-data my-meta-data + * kvm -net nic -net user,hostfwd=tcp::2222-:22 \\ + -drive file=disk1.img,if=virtio -drive file=my-seed.img,if=virtio + * ssh -p 2222 ubuntu@localhost +EOF +} + +bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } +cleanup() { + [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" +} + +debug() { + local level=${1}; shift; + [ "${level}" -gt "${VERBOSITY}" ] && return + error "${@}" +} + +has_cmd() { + command -v "$1" >/dev/null 2>&1 +} + +short_opts="hH:i:d:f:m:N:o:V:v" +long_opts="disk-format:,dsmode:,filesystem:,help,hostname:,interfaces:," +long_opts="${long_opts}network-config:,output:,vendor-data:,verbose" +getopt_out=$(getopt -n "${0##*/}" \ + -o "${short_opts}" -l "${long_opts}" -- "$@") && + eval set -- "${getopt_out}" || + bad_Usage + +## <> +output="" +userdata="" +metadata="" +vendordata="" +filesystem="" +diskformat=$DEF_DISK_FORMAT +interfaces=_unset +dsmode="" +hostname="" +ncname="network-config" + + +while [ $# -ne 0 ]; do + cur=${1}; next=${2}; + case "$cur" in + -h|--help) Usage ; exit 0;; + -d|--disk-format) diskformat=$next; shift;; + -f|--filesystem) filesystem=$next; shift;; + -H|--hostname) hostname=$next; shift;; + -i|--interfaces) interfaces=$next; shift;; + -N|--network-config) netcfg=$next; shift;; + -m|--dsmode) dsmode=$next; shift;; + -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; + -V|--vendor-data) vendordata="$next";; + --) shift; break;; + esac + shift; +done + +## check arguments here +## how many args do you expect? +echo $1 +echo $2 +echo $3 +[ $# -ge 2 ] || bad_Usage "must provide output, userdata" +[ $# -le 3 ] || bad_Usage "confused by additional args" + +output=$1 +userdata=$2 +metadata=$3 + +if [ -n "$metadata" ]; then + [ "$interfaces" = "_unset" -a -z "$dsmode" -a -z "$hostname" ] || + fail "metadata is incompatible with:" \ + "--interfaces, --hostname, --dsmode" +fi + +case "$diskformat" in + tar|tar-seed-local|tar-seed-net) + if [ "${filesystem:-tar}" != "tar" ]; then + fail "diskformat=tar is incompatible with filesystem" + fi + filesystem="$diskformat" + ;; + tar*) + fail "supported 'tar' formats are tar, tar-seed-local, tar-seed-net" +esac + +if [ -z "$filesystem" ]; then + filesystem="$DEF_FILESYSTEM" +fi +if [ "$filesystem" = "iso" ]; then + filesystem="iso9660" +fi + +case "$filesystem" in + tar*) + has_cmd tar || + fail "missing 'tar'. Required for --filesystem=$filesystem";; + vfat) + has_cmd mkfs.vfat || + fail "missing 'mkfs.vfat'. Required for --filesystem=vfat." + has_cmd mcopy || + fail "missing 'mcopy'. Required for --filesystem=vfat." + ;; + iso9660) + has_cmd mkisofs || + fail "missing 'mkisofs'. Required for --filesystem=iso9660." + ;; + *) fail "unknown filesystem $filesystem";; +esac + +case "$diskformat" in + tar*|raw) :;; + *) has_cmd "qemu-img" || + fail "missing 'qemu-img'. Required for --disk-format=$diskformat." +esac + +[ "$interfaces" = "_unset" -o -r "$interfaces" ] || + fail "$interfaces: not a readable file" + +TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") || + fail "failed to make tempdir" +trap cleanup EXIT + +files=( "${TEMP_D}/user-data" "${TEMP_D}/meta-data" ) +if [ -n "$metadata" ]; then + cp "$metadata" "$TEMP_D/meta-data" || fail "$metadata: failed to copy" +else + instance_id="iid-local01" + iface_data="" + [ "$interfaces" != "_unset" ] && + iface_data=$(sed ':a;N;$!ba;s/\n/\\n/g' "$interfaces") + + # write json formatted user-data (json is a subset of yaml) + mdata="" + for kv in "instance-id:$instance_id" "local-hostname:$hostname" \ + "interfaces:${iface_data}" "dsmode:$dsmode"; do + key=${kv%%:*} + val=${kv#*:} + [ -n "$val" ] || continue + mdata="${mdata:+${mdata},${CR}}\"$key\": \"$val\"" + done + printf "{\n%s\n}\n" "$mdata" > "${TEMP_D}/meta-data" +fi + +if [ -n "$netcfg" ]; then + cp "$netcfg" "${TEMP_D}/$ncname" || + fail "failed to copy network config" + files[${#files[@]}]="$TEMP_D/$ncname" +fi + +if [ -n "$vendordata" ]; then + cp "$vendordata" "${TEMP_D}/vendor-data" || + fail "failed to copy vendor data" + files[${#files[@]}]="$TEMP_D/vendor-data" +fi + +files_rel=( ) +for f in "${files[@]}"; do + files_rel[${#files_rel[@]}]="${f#${TEMP_D}/}" +done + +if [ "$userdata" = "-" ]; then + cat > "$TEMP_D/user-data" || fail "failed to read from stdin" +else + cp "$userdata" "$TEMP_D/user-data" || fail "$userdata: failed to copy" +fi + +## alternatively, create a vfat filesystem with same files +img="$TEMP_D/seed-data" +tar_opts=( --owner=root --group=root ) + +case "$filesystem" in + tar) + tar "${tar_opts[@]}" -C "${TEMP_D}" -cf "$img" "${files_rel[@]}" || + fail "failed to create tarball of ${files_rel[*]}" + ;; + tar-seed-local|tar-seed-net) + if [ "$filesystem" = "tar-seed-local" ]; then + path="var/lib/cloud/seed/nocloud" + else + path="var/lib/cloud/seed/nocloud-net" + fi + mkdir -p "${TEMP_D}/${path}" || + fail "failed making path for seed files" + mv "${files[@]}" "${TEMP_D}/$path" || + fail "failed moving files" + tar "${tar_opts[@]}" -C "${TEMP_D}" -cf "$img" "${path}" || + fail "failed to create tarball with $path" + ;; + iso9660) + mkisofs -output "$img" -volid cidata \ + -joliet -rock "${files[@]}" > "$TEMP_D/err" 2>&1 || + { cat "$TEMP_D/err" 1>&2; fail "failed to mkisofs"; } + ;; + vfat) + truncate -s 128K "$img" || fail "failed truncate image" + out=$(mkfs.vfat -n cidata "$img" 2>&1) || + { error "failed: mkfs.vfat -n cidata $img"; error "$out"; } + mcopy -oi "$img" "${files[@]}" :: || + fail "failed to copy user-data, meta-data to img" + ;; +esac + +[ "$output" = "-" ] && output="$TEMP_D/final" +if [ "${diskformat#tar}" != "$diskformat" -o "$diskformat" = "raw" ]; then + cp "$img" "$output" || + fail "failed to copy image to $output" +else + qemu-img convert -f raw -O "$diskformat" "$img" "$output" || + fail "failed to convert to disk format $diskformat" +fi + +[ "$output" != "$TEMP_D/final" ] || { cat "$output" && output="-"; } || + fail "failed to write to -" + +debug 1 "wrote ${output} with filesystem=$filesystem and diskformat=$diskformat" +# vi: ts=4 noexpandtab diff --git a/test/run.sh b/test/run.sh index 66a282b1..94600529 100755 --- a/test/run.sh +++ b/test/run.sh @@ -2,15 +2,40 @@ set -e +if [ "$(uname -s)" = "Darwin" ]; then + export PATH="$(dirname $(brew list gnu-getopt | grep "bin/getopt$")):$PATH" +fi + +AYA_SOURCE_DIR="$(realpath $(dirname $0)/..)" +LIBBPF_DIR=$1 + # Temporary directory for tests to use. -AYA_TMPDIR="$(pwd)/.tmp" +AYA_TMPDIR="${AYA_SOURCE_DIR}/.tmp" # Directory for VM images AYA_IMGDIR=${AYA_TMPDIR} -# Test Architecture -if [ -z "${AYA_TEST_ARCH}" ]; then - AYA_TEST_ARCH="$(uname -m)" +if [ -z "${AYA_BUILD_TARGET}" ]; then + AYA_BUILD_TARGET=$(rustc -vV | sed -n 's|host: ||p') +fi + +AYA_HOST_ARCH=$(uname -m) +if [ "${AYA_HOST_ARCH}" = "arm64" ]; then + AYA_HOST_ARCH="aarch64" +fi + +if [ -z "${AYA_GUEST_ARCH}" ]; then + AYA_GUEST_ARCH="${AYA_HOST_ARCH}" +fi + +if [ "${AYA_GUEST_ARCH}" = "aarch64" ]; then + if [ -z "${AARCH64_UEFI}" ]; then + AARCH64_UEFI="$(brew list qemu -1 -v | grep edk2-aarch64-code.fd)" + fi +fi + +if [ -z "$AYA_MUSL_TARGET" ]; then + AYA_MUSL_TARGET=${AYA_GUEST_ARCH}-unknown-linux-musl fi # Test Image @@ -27,19 +52,19 @@ download_images() { mkdir -p "${AYA_IMGDIR}" case $1 in fedora37) - if [ ! -f "${AYA_IMGDIR}/fedora37.${AYA_TEST_ARCH}.qcow2" ]; then - IMAGE="Fedora-Cloud-Base-37-1.7.${AYA_TEST_ARCH}.qcow2" - IMAGE_URL="https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/${AYA_TEST_ARCH}/images" + if [ ! -f "${AYA_IMGDIR}/fedora37.${AYA_GUEST_ARCH}.qcow2" ]; then + IMAGE="Fedora-Cloud-Base-37-1.7.${AYA_GUEST_ARCH}.qcow2" + IMAGE_URL="https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/${AYA_GUEST_ARCH}/images" echo "Downloading: ${IMAGE}, this may take a while..." - curl -o "${AYA_IMGDIR}/fedora37.${AYA_TEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" + curl -o "${AYA_IMGDIR}/fedora37.${AYA_GUEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" fi ;; centos8) - if [ ! -f "${AYA_IMGDIR}/centos8.${AYA_TEST_ARCH}.qcow2" ]; then - IMAGE="CentOS-8-GenericCloud-8.4.2105-20210603.0.${AYA_TEST_ARCH}.qcow2" - IMAGE_URL="https://cloud.centos.org/centos/8/${AYA_TEST_ARCH}/images" + if [ ! -f "${AYA_IMGDIR}/centos8.${AYA_GUEST_ARCH}.qcow2" ]; then + IMAGE="CentOS-8-GenericCloud-8.4.2105-20210603.0.${AYA_GUEST_ARCH}.qcow2" + IMAGE_URL="https://cloud.centos.org/centos/8/${AYA_GUEST_ARCH}/images" echo "Downloading: ${IMAGE}, this may take a while..." - curl -o "${AYA_IMGDIR}/centos8.${AYA_TEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" + curl -o "${AYA_IMGDIR}/centos8.${AYA_GUEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" fi ;; *) @@ -60,11 +85,6 @@ EOF if [ ! -f "${AYA_TMPDIR}/test_rsa" ]; then ssh-keygen -t rsa -b 4096 -f "${AYA_TMPDIR}/test_rsa" -N "" -C "" -q pub_key=$(cat "${AYA_TMPDIR}/test_rsa.pub") - cat > "${AYA_TMPDIR}/user-data.yaml" < "${AYA_TMPDIR}/user-data.yaml" < actual_cores + nr_cpus=8 fi fi ;; *) - echo "${AYA_TEST_ARCH} is not supported" + echo "${AYA_GUEST_ARCH} is not supported" return 1 ;; esac - qemu-img create -F qcow2 -f qcow2 -o backing_file="${AYA_IMGDIR}/${AYA_TEST_IMAGE}.${AYA_TEST_ARCH}.qcow2" "${AYA_TMPDIR}/vm.qcow2" || return 1 + if [ ! -f "${AYA_IMGDIR}/vm.qcow2" ]; then + echo "Creating VM image" + qemu-img create -F qcow2 -f qcow2 -o backing_file="${AYA_IMGDIR}/${AYA_TEST_IMAGE}.${AYA_GUEST_ARCH}.qcow2" "${AYA_IMGDIR}/vm.qcow2" || return 1 + CACHED_VM=0 + else + echo "Reusing existing VM image" + CACHED_VM=1 + fi $QEMU \ -machine "${machine}" \ -cpu "${cpu}" \ - -m 2G \ + -m 3G \ + -smp "${nr_cpus}" \ -display none \ -monitor none \ -daemonize \ -pidfile "${AYA_TMPDIR}/vm.pid" \ -device virtio-net-pci,netdev=net0 \ -netdev user,id=net0,hostfwd=tcp::2222-:22 \ - -drive if=virtio,format=qcow2,file="${AYA_TMPDIR}/vm.qcow2" \ + $uefi \ + -drive if=virtio,format=qcow2,file="${AYA_IMGDIR}/vm.qcow2" \ -drive if=virtio,format=raw,file="${AYA_TMPDIR}/seed.img" || return 1 trap cleanup_vm EXIT @@ -142,7 +182,11 @@ EOF echo "VM launched" exec_vm uname -a echo "Installing dependencies" - exec_vm sudo dnf install -qy bpftool + exec_vm sudo dnf install -qy bpftool llvm llvm-devel clang clang-devel zlib-devel + exec_vm 'curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ + -y --profile minimal --default-toolchain nightly --component rust-src --component clippy' + exec_vm 'echo source ~/.cargo/env >> ~/.bashrc' + exec_vm cargo install bpf-linker --no-default-features --features system-llvm } scp_vm() { @@ -154,6 +198,10 @@ scp_vm() { "${AYA_SSH_USER}@localhost:${remote}" } +rsync_vm() { + rsync -a -e "ssh -p 2222 -F ${AYA_TMPDIR}/ssh_config -i ${AYA_TMPDIR}/test_rsa" $1 $AYA_SSH_USER@localhost: +} + exec_vm() { ssh -q -F "${AYA_TMPDIR}/ssh_config" \ -i "${AYA_TMPDIR}/test_rsa" \ @@ -168,26 +216,34 @@ stop_vm() { kill -9 "$(cat "${AYA_TMPDIR}/vm.pid")" rm "${AYA_TMPDIR}/vm.pid" fi - rm -f "${AYA_TMPDIR}/vm.qcow2" } cleanup_vm() { + stop_vm if [ "$?" != "0" ]; then - stop_vm + rm -f "${AYA_IMGDIR}/vm.qcow2" fi } -if [ -z "$1" ]; then +if [ -z "$LIBBPF_DIR" ]; then echo "path to libbpf required" exit 1 fi start_vm -trap stop_vm EXIT - -cargo xtask build-integration-test --musl --libbpf-dir "$1" -scp_vm ../target/x86_64-unknown-linux-musl/debug/integration-test -exec_vm sudo ./integration-test --skip relocations - -# Relocation tests build the eBPF programs themself. We run them outside VM. -sudo -E ../target/x86_64-unknown-linux-musl/debug/integration-test relocations +trap cleanup_vm EXIT + +# make sure we always use fresh aya and libbpf (also see comment at the end) +exec_vm "rm -rf aya/* libbpf" +rsync_vm "--exclude=target --exclude=.tmp $AYA_SOURCE_DIR" +rsync_vm "$LIBBPF_DIR" + +# need to build or linting will fail trying to include object files +exec_vm "cd aya; cargo xtask build-integration-test --libbpf-dir ~/libbpf" +exec_vm "cd aya; cargo clippy -p integration-test -- --deny warnings" +exec_vm "cd aya; cargo xtask integration-test --libbpf-dir ~/libbpf" + +# we rm and sync but it doesn't seem to work reliably - I guess we could sleep a +# few seconds after but ain't nobody got time for that. Instead we also rm +# before rsyncing. +exec_vm "rm -rf aya/* libbpf; sync" diff --git a/xtask/src/build_test.rs b/xtask/src/build_test.rs index 5d727ac0..b45943a1 100644 --- a/xtask/src/build_test.rs +++ b/xtask/src/build_test.rs @@ -5,9 +5,9 @@ use crate::build_ebpf; #[derive(Parser)] pub struct Options { - /// Whether to compile for the musl libc target - #[clap(short, long)] - pub musl: bool, + /// Target triple for which the code is compiled + #[clap(long)] + pub musl_target: Option, #[clap(flatten)] pub ebpf_options: build_ebpf::BuildEbpfOptions, @@ -16,9 +16,12 @@ pub struct Options { pub fn build_test(opts: Options) -> anyhow::Result<()> { build_ebpf::build_ebpf(opts.ebpf_options)?; - let mut args = vec!["build", "-p", "integration-test", "--verbose"]; - if opts.musl { - args.push("--target=x86_64-unknown-linux-musl"); + let mut args = ["build", "-p", "integration-test", "--verbose"] + .iter() + .map(|s| s.to_string()) + .collect::>(); + if let Some(target) = opts.musl_target { + args.push(format!("--target={target}")); } let status = Command::new("cargo") .args(&args)