From e836865088e4769e21462cff8908480aaa0f4c10 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 7 Aug 2023 16:57:22 -0400 Subject: [PATCH] integration-test: Implement running on VMs This allows integration tests to be run using qemu at an arbitrary kernel version. Note that before this change we were only testing fedora 38 which used 6.2. Now we're testing 6.1 kernels. The tests all pass. Older kernel versions were attempted, but the tests don't all pass. Later work can add more kernel versions to test. --- .github/workflows/ci.yml | 118 +++++---- Cargo.toml | 3 + bpf/aya-log-ebpf/src/lib.rs | 19 +- init/Cargo.toml | 10 + init/src/main.rs | 161 ++++++++++++ test/README.md | 22 +- test/cloud-localds | 264 -------------------- test/run.sh | 241 ------------------ xtask/src/main.rs | 12 - xtask/src/run.rs | 472 +++++++++++++++++++++++++++++++----- 10 files changed, 680 insertions(+), 642 deletions(-) create mode 100644 init/Cargo.toml create mode 100644 init/src/main.rs delete mode 100755 test/cloud-localds delete mode 100755 test/run.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2ab4cee..fb0c9566 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,8 +139,14 @@ jobs: --target ${{ matrix.target }} \ -Z build-std=core - build-integration-test: - runs-on: ubuntu-22.04 + run-integration-test: + strategy: + fail-fast: false + matrix: + runner: + - macos-12 + - ubuntu-22.04 + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v3 with: @@ -150,13 +156,12 @@ jobs: with: toolchain: nightly components: rust-src + targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl - uses: Swatinem/rust-cache@v2 - - name: bpf-linker - run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git - - - name: Install dependencies + - name: Install prerequisites + if: runner.os == 'Linux' # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit # enum values which was added in clang 15[1]. # @@ -171,63 +176,78 @@ jobs: set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list - sudo apt-get update - sudo apt-get -y install clang gcc-multilib llvm + sudo apt update + sudo apt -y install clang gcc-multilib llvm locate qemu-system-{arm,x86} - - name: Build + - name: bpf-linker + if: runner.os == 'Linux' + run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git + + - name: Install prerequisites + if: runner.os == 'macOS' + # The clang shipped on macOS doesn't support BPF, so we need LLVM from brew. + # + # We also need LLVM for bpf-linker, see comment below. run: | set -euxo pipefail - mkdir -p integration-test-binaries - # See https://doc.rust-lang.org/cargo/reference/profiles.html for the - # names of the builtin profiles. Note that dev builds "debug" targets. - cargo xtask build-integration-test --cargo-arg=--profile=dev | xargs -I % cp % integration-test-binaries/dev - cargo xtask build-integration-test --cargo-arg=--profile=release | xargs -I % cp % integration-test-binaries/release - - - uses: actions/upload-artifact@v3 - with: - name: integration-test-binaries - path: integration-test-binaries + brew install qemu dpkg pkg-config llvm + echo /usr/local/opt/llvm/bin >> $GITHUB_PATH - run-integration-test: - runs-on: macos-latest - needs: ["build-integration-test"] - steps: - - uses: actions/checkout@v3 - with: - sparse-checkout: | - test/run.sh - test/cloud-localds + - name: bpf-linker + if: runner.os == 'macOS' + # NB: rustc doesn't ship libLLVM.so on macOS, so disable proxying (default feature). + run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features - - name: Install Pre-requisites + - name: Download debian kernels + if: runner.arch == 'ARM64' run: | - brew install qemu gnu-getopt coreutils cdrtools - - - name: Cache tmp files - uses: actions/cache@v3 - with: - path: | - .tmp/*.qcow2 - .tmp/test_rsa - .tmp/test_rsa.pub - key: tmp-files-${{ hashFiles('test/run.sh') }} - - - uses: actions/download-artifact@v3 - with: - name: integration-test-binaries - path: integration-test-binaries - - - name: Run integration tests + set -euxo pipefail + mkdir -p test/.tmp/debian-kernels/arm64 + # NB: a 4.19 kernel image for arm64 was not available. + # TODO(https://github.com/aya-rs/aya/pull/725): enable tests on kernels before 6.0. + # linux-image-5.10.0-23-cloud-arm64-unsigned_5.10.179-3_arm64.deb \ + printf '%s\0' \ + linux-image-6.1.0-10-cloud-arm64-unsigned_6.1.38-2_arm64.deb \ + linux-image-6.4.0-1-cloud-arm64-unsigned_6.4.4-2_arm64.deb \ + | xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/arm64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{} + + - name: Download debian kernels + if: runner.arch == 'X64' run: | set -euxo pipefail - find integration-test-binaries -type f -exec chmod +x {} \; - test/run.sh integration-test-binaries + mkdir -p test/.tmp/debian-kernels/amd64 + # TODO(https://github.com/aya-rs/aya/pull/725): enable tests on kernels before 6.0. + # linux-image-4.19.0-21-cloud-amd64-unsigned_4.19.249-2_amd64.deb \ + # linux-image-5.10.0-23-cloud-amd64-unsigned_5.10.179-3_amd64.deb \ + printf '%s\0' \ + linux-image-6.1.0-10-cloud-amd64-unsigned_6.1.38-2_amd64.deb \ + linux-image-6.4.0-1-cloud-amd64-unsigned_6.4.4-2_amd64.deb \ + | xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/amd64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{} + + - name: Alias gtar as tar + if: runner.os == 'macOS' + # macOS tar doesn't support --wildcards which we use below. + run: mkdir tar-is-gtar && ln -s "$(which gtar)" tar-is-gtar/tar && echo "$PWD"/tar-is-gtar >> $GITHUB_PATH + + - name: Extract debian kernels + run: | + set -euxo pipefail + find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \ + sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -" + + - name: Run integration tests + run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm # Provides a single status check for the entire build workflow. # This is used for merge automation, like Mergify, since GH actions # has no concept of "when all status checks pass". # https://docs.mergify.com/conditions/#validating-all-status-checks build-workflow-complete: - needs: ["lint", "build-test-aya", "build-test-aya-bpf", "run-integration-test"] + needs: + - lint + - build-test-aya + - build-test-aya-bpf + - run-integration-test runs-on: ubuntu-latest steps: - name: Build Complete diff --git a/Cargo.toml b/Cargo.toml index 0a95461b..4a4bdbca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "aya-log-parser", "aya-obj", "aya-tool", + "init", "test/integration-test", "xtask", @@ -29,6 +30,7 @@ default-members = [ "aya-log-parser", "aya-obj", "aya-tool", + "init", # test/integration-test is omitted; including it in this list causes `cargo test` to run its # tests, and that doesn't work unless they've been built with `cargo xtask`. "xtask", @@ -72,6 +74,7 @@ lazy_static = { version = "1", default-features = false } libc = { version = "0.2.105", default-features = false } log = { version = "0.4", default-features = false } netns-rs = { version = "0.1", default-features = false } +nix = { version = "0.26.2", default-features = false } num_enum = { version = "0.6", default-features = false } object = { version = "0.31", default-features = false } parking_lot = { version = "0.12.0", default-features = false } diff --git a/bpf/aya-log-ebpf/src/lib.rs b/bpf/aya-log-ebpf/src/lib.rs index 2933509c..5962c239 100644 --- a/bpf/aya-log-ebpf/src/lib.rs +++ b/bpf/aya-log-ebpf/src/lib.rs @@ -1,10 +1,9 @@ #![no_std] #![warn(clippy::cast_lossless, clippy::cast_sign_loss)] -use aya_bpf::{ - macros::map, - maps::{PerCpuArray, PerfEventByteArray}, -}; +#[cfg(target_arch = "bpf")] +use aya_bpf::macros::map; +use aya_bpf::maps::{PerCpuArray, PerfEventByteArray}; pub use aya_log_common::{write_record_header, Level, WriteToBuf, LOG_BUF_CAPACITY}; pub use aya_log_ebpf_macros::{debug, error, info, log, trace, warn}; @@ -15,11 +14,19 @@ pub struct LogBuf { } #[doc(hidden)] -#[map] +// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't +// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so +// because the integration-test crate depends on this crate transitively. See comment in +// test/integration-test/Cargo.toml. +#[cfg_attr(target_arch = "bpf", map)] pub static mut AYA_LOG_BUF: PerCpuArray = PerCpuArray::with_max_entries(1, 0); #[doc(hidden)] -#[map] +// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't +// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so +// because the integration-test crate depends on this crate transitively. See comment in +// test/integration-test/Cargo.toml. +#[cfg_attr(target_arch = "bpf", map)] pub static mut AYA_LOGS: PerfEventByteArray = PerfEventByteArray::new(0); #[doc(hidden)] diff --git a/init/Cargo.toml b/init/Cargo.toml new file mode 100644 index 00000000..6adf153e --- /dev/null +++ b/init/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "init" +version = "0.1.0" +authors = ["Tamir Duberstein "] +edition = "2021" +publish = false + +[dependencies] +anyhow = { workspace = true, features = ["std"] } +nix = { workspace = true, features = ["fs", "mount", "reboot"] } diff --git a/init/src/main.rs b/init/src/main.rs new file mode 100644 index 00000000..948e5504 --- /dev/null +++ b/init/src/main.rs @@ -0,0 +1,161 @@ +use anyhow::Context as _; + +#[derive(Debug)] +struct Errors(Vec); + +impl std::fmt::Display for Errors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self(errors) = self; + for (i, error) in errors.iter().enumerate() { + if i != 0 { + writeln!(f)?; + } + write!(f, "{:?}", error)?; + } + Ok(()) + } +} + +impl std::error::Error for Errors {} + +fn run() -> anyhow::Result<()> { + const RXRXRX: nix::sys::stat::Mode = nix::sys::stat::Mode::empty() + .union(nix::sys::stat::Mode::S_IRUSR) + .union(nix::sys::stat::Mode::S_IXUSR) + .union(nix::sys::stat::Mode::S_IRGRP) + .union(nix::sys::stat::Mode::S_IXGRP) + .union(nix::sys::stat::Mode::S_IROTH) + .union(nix::sys::stat::Mode::S_IXOTH); + + struct Mount { + source: &'static str, + target: &'static str, + fstype: &'static str, + flags: nix::mount::MsFlags, + data: Option<&'static str>, + target_mode: Option, + } + + for Mount { + source, + target, + fstype, + flags, + data, + target_mode, + } in [ + Mount { + source: "proc", + target: "/proc", + fstype: "proc", + flags: nix::mount::MsFlags::empty(), + data: None, + target_mode: Some(RXRXRX), + }, + Mount { + source: "sysfs", + target: "/sys", + fstype: "sysfs", + flags: nix::mount::MsFlags::empty(), + data: None, + target_mode: Some(RXRXRX), + }, + Mount { + source: "debugfs", + target: "/sys/kernel/debug", + fstype: "debugfs", + flags: nix::mount::MsFlags::empty(), + data: None, + target_mode: None, + }, + Mount { + source: "bpffs", + target: "/sys/fs/bpf", + fstype: "bpf", + flags: nix::mount::MsFlags::empty(), + data: None, + target_mode: None, + }, + ] { + match target_mode { + None => { + // Must exist. + let nix::sys::stat::FileStat { st_mode, .. } = nix::sys::stat::stat(target) + .with_context(|| format!("stat({target}) failed"))?; + let s_flag = nix::sys::stat::SFlag::from_bits_truncate(st_mode); + + if !s_flag.contains(nix::sys::stat::SFlag::S_IFDIR) { + anyhow::bail!("{target} is not a directory"); + } + } + Some(target_mode) => { + // Must not exist. + nix::unistd::mkdir(target, target_mode) + .with_context(|| format!("mkdir({target}) failed"))?; + } + } + nix::mount::mount(Some(source), target, Some(fstype), flags, data).with_context(|| { + format!("mount({source}, {target}, {fstype}, {flags:?}, {data:?}) failed") + })?; + } + + // By contract we run everything in /bin and assume they're rust test binaries. + // + // If the user requested command line arguments, they're named init.arg={}. + + // Read kernel parameters from /proc/cmdline. They're space separated on a single line. + let cmdline = std::fs::read_to_string("/proc/cmdline") + .with_context(|| "read_to_string(/proc/cmdline) failed")?; + let args = cmdline + .split_whitespace() + .filter_map(|parameter| { + parameter + .strip_prefix("init.arg=") + .map(std::ffi::OsString::from) + }) + .collect::>(); + + // Iterate files in /bin. + let read_dir = std::fs::read_dir("/bin").context("read_dir(/bin) failed")?; + let errors = read_dir + .filter_map(|entry| { + match (|| { + let entry = entry.context("read_dir(/bin) failed")?; + let path = entry.path(); + let status = std::process::Command::new(&path) + .args(&args) + .status() + .with_context(|| format!("failed to execute {}", path.display()))?; + + if status.code() == Some(0) { + Ok(()) + } else { + Err(anyhow::anyhow!("{} failed: {status:?}", path.display())) + } + })() { + Ok(()) => None, + Err(err) => Some(err), + } + }) + .collect::>(); + if errors.is_empty() { + Ok(()) + } else { + Err(Errors(errors).into()) + } +} + +fn main() { + match run() { + Ok(()) => { + println!("init: success"); + } + Err(err) => { + println!("{err:?}"); + println!("init: failure"); + } + } + let _: std::convert::Infallible = + nix::sys::reboot::reboot(nix::sys::reboot::RebootMode::RB_POWER_OFF) + .expect("reboot failed"); +} diff --git a/test/README.md b/test/README.md index a9521c0b..82e41da0 100644 --- a/test/README.md +++ b/test/README.md @@ -3,21 +3,15 @@ Aya Integration Tests The aya integration test suite is a set of tests to ensure that common usage behaviours work on real Linux distros -## Prerequisites - -### Linux - -To run locally all you need is: -1. Rust nightly -1. `cargo install bpf-linker` +## Prerequisites -### Other OSs +You'll need: -1. A POSIX shell -1. `rustup target add x86_64-unknown-linux-musl` +1. `rustup toolchain install nightly` +1. `rustup target add {aarch64,x86_64}-unknown-linux-musl` 1. `cargo install bpf-linker` -1. Install `qemu` and `cloud-init-utils` package - or any package that provides `cloud-localds` +1. (virtualized only) `qemu` ## Usage @@ -26,15 +20,13 @@ From the root of this repository: ### Native ``` -cargo xtask integration-test +cargo xtask integration-test local ``` ### Virtualized ``` -mkdir -p integration-test-binaries -cargo xtask build-integration-test | xargs -I % cp % integration-test-binaries -./test/run.sh integration-test-binaries +cargo xtask integration-test vm ``` ### Writing an integration test diff --git a/test/cloud-localds b/test/cloud-localds deleted file mode 100755 index 3a28129f..00000000 --- a/test/cloud-localds +++ /dev/null @@ -1,264 +0,0 @@ -#!/bin/bash - -VERBOSITY=0 -TEMP_D="" -DEF_DISK_FORMAT="raw" -DEF_FILESYSTEM="iso9660" -CR=" -" - -error() { echo "$@" 1>&2; } -fail() { [ $# -eq 0 ] || error "$@"; exit 1; } - -Usage() { - cat < my-meta-data - * ${0##*/} my-seed.img my-user-data my-meta-data - * kvm -net nic -net user,hostfwd=tcp::2222-:22 \\ - -drive file=disk1.img,if=virtio -drive file=my-seed.img,if=virtio - * ssh -p 2222 ubuntu@localhost -EOF -} - -bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } -cleanup() { - [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" -} - -debug() { - local level=${1}; shift; - [ "${level}" -gt "${VERBOSITY}" ] && return - error "${@}" -} - -has_cmd() { - command -v "$1" >/dev/null 2>&1 -} - -short_opts="hH:i:d:f:m:N:o:V:v" -long_opts="disk-format:,dsmode:,filesystem:,help,hostname:,interfaces:," -long_opts="${long_opts}network-config:,output:,vendor-data:,verbose" -getopt_out=$(getopt -n "${0##*/}" \ - -o "${short_opts}" -l "${long_opts}" -- "$@") && - eval set -- "${getopt_out}" || - bad_Usage - -## <> -output="" -userdata="" -metadata="" -vendordata="" -filesystem="" -diskformat=$DEF_DISK_FORMAT -interfaces=_unset -dsmode="" -hostname="" -ncname="network-config" - - -while [ $# -ne 0 ]; do - cur=${1}; next=${2}; - case "$cur" in - -h|--help) Usage ; exit 0;; - -d|--disk-format) diskformat=$next; shift;; - -f|--filesystem) filesystem=$next; shift;; - -H|--hostname) hostname=$next; shift;; - -i|--interfaces) interfaces=$next; shift;; - -N|--network-config) netcfg=$next; shift;; - -m|--dsmode) dsmode=$next; shift;; - -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; - -V|--vendor-data) vendordata="$next";; - --) shift; break;; - esac - shift; -done - -## check arguments here -## how many args do you expect? -echo $1 -echo $2 -echo $3 -[ $# -ge 2 ] || bad_Usage "must provide output, userdata" -[ $# -le 3 ] || bad_Usage "confused by additional args" - -output=$1 -userdata=$2 -metadata=$3 - -if [ -n "$metadata" ]; then - [ "$interfaces" = "_unset" -a -z "$dsmode" -a -z "$hostname" ] || - fail "metadata is incompatible with:" \ - "--interfaces, --hostname, --dsmode" -fi - -case "$diskformat" in - tar|tar-seed-local|tar-seed-net) - if [ "${filesystem:-tar}" != "tar" ]; then - fail "diskformat=tar is incompatible with filesystem" - fi - filesystem="$diskformat" - ;; - tar*) - fail "supported 'tar' formats are tar, tar-seed-local, tar-seed-net" -esac - -if [ -z "$filesystem" ]; then - filesystem="$DEF_FILESYSTEM" -fi -if [ "$filesystem" = "iso" ]; then - filesystem="iso9660" -fi - -case "$filesystem" in - tar*) - has_cmd tar || - fail "missing 'tar'. Required for --filesystem=$filesystem";; - vfat) - has_cmd mkfs.vfat || - fail "missing 'mkfs.vfat'. Required for --filesystem=vfat." - has_cmd mcopy || - fail "missing 'mcopy'. Required for --filesystem=vfat." - ;; - iso9660) - has_cmd mkisofs || - fail "missing 'mkisofs'. Required for --filesystem=iso9660." - ;; - *) fail "unknown filesystem $filesystem";; -esac - -case "$diskformat" in - tar*|raw) :;; - *) has_cmd "qemu-img" || - fail "missing 'qemu-img'. Required for --disk-format=$diskformat." -esac - -[ "$interfaces" = "_unset" -o -r "$interfaces" ] || - fail "$interfaces: not a readable file" - -TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") || - fail "failed to make tempdir" -trap cleanup EXIT - -files=( "${TEMP_D}/user-data" "${TEMP_D}/meta-data" ) -if [ -n "$metadata" ]; then - cp "$metadata" "$TEMP_D/meta-data" || fail "$metadata: failed to copy" -else - instance_id="iid-local01" - iface_data="" - [ "$interfaces" != "_unset" ] && - iface_data=$(sed ':a;N;$!ba;s/\n/\\n/g' "$interfaces") - - # write json formatted user-data (json is a subset of yaml) - mdata="" - for kv in "instance-id:$instance_id" "local-hostname:$hostname" \ - "interfaces:${iface_data}" "dsmode:$dsmode"; do - key=${kv%%:*} - val=${kv#*:} - [ -n "$val" ] || continue - mdata="${mdata:+${mdata},${CR}}\"$key\": \"$val\"" - done - printf "{\n%s\n}\n" "$mdata" > "${TEMP_D}/meta-data" -fi - -if [ -n "$netcfg" ]; then - cp "$netcfg" "${TEMP_D}/$ncname" || - fail "failed to copy network config" - files[${#files[@]}]="$TEMP_D/$ncname" -fi - -if [ -n "$vendordata" ]; then - cp "$vendordata" "${TEMP_D}/vendor-data" || - fail "failed to copy vendor data" - files[${#files[@]}]="$TEMP_D/vendor-data" -fi - -files_rel=( ) -for f in "${files[@]}"; do - files_rel[${#files_rel[@]}]="${f#${TEMP_D}/}" -done - -if [ "$userdata" = "-" ]; then - cat > "$TEMP_D/user-data" || fail "failed to read from stdin" -else - cp "$userdata" "$TEMP_D/user-data" || fail "$userdata: failed to copy" -fi - -## alternatively, create a vfat filesystem with same files -img="$TEMP_D/seed-data" -tar_opts=( --owner=root --group=root ) - -case "$filesystem" in - tar) - tar "${tar_opts[@]}" -C "${TEMP_D}" -cf "$img" "${files_rel[@]}" || - fail "failed to create tarball of ${files_rel[*]}" - ;; - tar-seed-local|tar-seed-net) - if [ "$filesystem" = "tar-seed-local" ]; then - path="var/lib/cloud/seed/nocloud" - else - path="var/lib/cloud/seed/nocloud-net" - fi - mkdir -p "${TEMP_D}/${path}" || - fail "failed making path for seed files" - mv "${files[@]}" "${TEMP_D}/$path" || - fail "failed moving files" - tar "${tar_opts[@]}" -C "${TEMP_D}" -cf "$img" "${path}" || - fail "failed to create tarball with $path" - ;; - iso9660) - mkisofs -output "$img" -volid cidata \ - -joliet -rock "${files[@]}" > "$TEMP_D/err" 2>&1 || - { cat "$TEMP_D/err" 1>&2; fail "failed to mkisofs"; } - ;; - vfat) - truncate -s 128K "$img" || fail "failed truncate image" - out=$(mkfs.vfat -n cidata "$img" 2>&1) || - { error "failed: mkfs.vfat -n cidata $img"; error "$out"; } - mcopy -oi "$img" "${files[@]}" :: || - fail "failed to copy user-data, meta-data to img" - ;; -esac - -[ "$output" = "-" ] && output="$TEMP_D/final" -if [ "${diskformat#tar}" != "$diskformat" -o "$diskformat" = "raw" ]; then - cp "$img" "$output" || - fail "failed to copy image to $output" -else - qemu-img convert -f raw -O "$diskformat" "$img" "$output" || - fail "failed to convert to disk format $diskformat" -fi - -[ "$output" != "$TEMP_D/final" ] || { cat "$output" && output="-"; } || - fail "failed to write to -" - -debug 1 "wrote ${output} with filesystem=$filesystem and diskformat=$diskformat" -# vi: ts=4 noexpandtab diff --git a/test/run.sh b/test/run.sh deleted file mode 100755 index 26c98918..00000000 --- a/test/run.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -if [ "$(uname -s)" = "Darwin" ]; then - PATH="$(dirname "$(brew list gnu-getopt | grep "bin/getopt$")"):$PATH" - export PATH -fi - -AYA_SOURCE_DIR="$(realpath "$(dirname "$0")"/..)" - -# Temporary directory for tests to use. -AYA_TMPDIR="${AYA_SOURCE_DIR}/.tmp" - -# Directory for VM images -AYA_IMGDIR=${AYA_TMPDIR} - -if [ -z "${AYA_BUILD_TARGET}" ]; then - AYA_BUILD_TARGET=$(rustc -vV | sed -n 's|host: ||p') -fi - -AYA_HOST_ARCH=$(uname -m) -if [ "${AYA_HOST_ARCH}" = "arm64" ]; then - AYA_HOST_ARCH="aarch64" -fi - -if [ -z "${AYA_GUEST_ARCH}" ]; then - AYA_GUEST_ARCH="${AYA_HOST_ARCH}" -fi - -if [ "${AYA_GUEST_ARCH}" = "aarch64" ]; then - if [ -z "${AARCH64_UEFI}" ]; then - AARCH64_UEFI="$(brew list qemu -1 -v | grep edk2-aarch64-code.fd)" - fi -fi - -if [ -z "$AYA_MUSL_TARGET" ]; then - AYA_MUSL_TARGET=${AYA_GUEST_ARCH}-unknown-linux-musl -fi - -# Test Image -if [ -z "${AYA_TEST_IMAGE}" ]; then - AYA_TEST_IMAGE="fedora38" -fi - -case "${AYA_TEST_IMAGE}" in - fedora*) AYA_SSH_USER="fedora";; - centos*) AYA_SSH_USER="centos";; -esac - -download_images() { - mkdir -p "${AYA_IMGDIR}" - case $1 in - fedora37) - if [ ! -f "${AYA_IMGDIR}/fedora37.${AYA_GUEST_ARCH}.qcow2" ]; then - IMAGE="Fedora-Cloud-Base-37-1.7.${AYA_GUEST_ARCH}.qcow2" - IMAGE_URL="https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/${AYA_GUEST_ARCH}/images" - echo "Downloading: ${IMAGE}, this may take a while..." - curl -o "${AYA_IMGDIR}/fedora37.${AYA_GUEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" - fi - ;; - fedora38) - if [ ! -f "${AYA_IMGDIR}/fedora38.${AYA_GUEST_ARCH}.qcow2" ]; then - IMAGE="Fedora-Cloud-Base-38_Beta-1.3.${AYA_GUEST_ARCH}.qcow2" - IMAGE_URL="https://fr2.rpmfind.net/linux/fedora/linux/releases/test/38_Beta/Cloud/${AYA_GUEST_ARCH}/images" - echo "Downloading: ${IMAGE}, this may take a while..." - curl -o "${AYA_IMGDIR}/fedora38.${AYA_GUEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" - fi - ;; - centos8) - if [ ! -f "${AYA_IMGDIR}/centos8.${AYA_GUEST_ARCH}.qcow2" ]; then - IMAGE="CentOS-8-GenericCloud-8.4.2105-20210603.0.${AYA_GUEST_ARCH}.qcow2" - IMAGE_URL="https://cloud.centos.org/centos/8/${AYA_GUEST_ARCH}/images" - echo "Downloading: ${IMAGE}, this may take a while..." - curl -o "${AYA_IMGDIR}/centos8.${AYA_GUEST_ARCH}.qcow2" -sSL "${IMAGE_URL}/${IMAGE}" - fi - ;; - *) - echo "$1 is not a recognized image name" - return 1 - ;; - esac -} - -start_vm() { - download_images "${AYA_TEST_IMAGE}" - # prepare config - cat > "${AYA_TMPDIR}/metadata.yaml" < "${AYA_TMPDIR}/ssh_config" < "${AYA_TMPDIR}/user-data.yaml" < actual_cores - nr_cpus=8 - fi - fi - ;; - *) - echo "${AYA_GUEST_ARCH} is not supported" - return 1 - ;; - esac - - if [ ! -f "${AYA_IMGDIR}/vm.qcow2" ]; then - echo "Creating VM image" - qemu-img create -F qcow2 -f qcow2 -o backing_file="${AYA_IMGDIR}/${AYA_TEST_IMAGE}.${AYA_GUEST_ARCH}.qcow2" "${AYA_IMGDIR}/vm.qcow2" || return 1 - else - echo "Reusing existing VM image" - fi - $QEMU \ - -machine "${machine}" \ - -cpu "${cpu}" \ - -m 3G \ - -smp "${nr_cpus}" \ - -display none \ - -monitor none \ - -daemonize \ - -pidfile "${AYA_TMPDIR}/vm.pid" \ - -device virtio-net-pci,netdev=net0 \ - -netdev user,id=net0,hostfwd=tcp::2222-:22 \ - "${uefi[@]}" \ - -drive if=virtio,format=qcow2,file="${AYA_IMGDIR}/vm.qcow2" \ - -drive if=virtio,format=raw,file="${AYA_TMPDIR}/seed.img" || return 1 - - trap cleanup_vm EXIT - echo "Waiting for SSH on port 2222..." - retry=0 - max_retries=300 - while ! ssh -q -F "${AYA_TMPDIR}/ssh_config" -o ConnectTimeout=1 -i "${AYA_TMPDIR}/test_rsa" "${AYA_SSH_USER}"@localhost -p 2222 echo "Hello VM"; do - retry=$((retry+1)) - if [ ${retry} -gt ${max_retries} ]; then - echo "Unable to connect to VM" - return 1 - fi - sleep 1 - done - - echo "VM launched" - exec_vm uname -a - echo "Enabling testing repositories" - exec_vm sudo dnf config-manager --set-enabled updates-testing - exec_vm sudo dnf config-manager --set-enabled updates-testing-modular -} - -scp_vm() { - local=$1 - remote=$(basename "$1") - scp -q -F "${AYA_TMPDIR}/ssh_config" \ - -i "${AYA_TMPDIR}/test_rsa" \ - -P 2222 "${local}" \ - "${AYA_SSH_USER}@localhost:${remote}" -} - -rsync_vm() { - rsync -a -e "ssh -p 2222 -F ${AYA_TMPDIR}/ssh_config -i ${AYA_TMPDIR}/test_rsa" "$1" "$AYA_SSH_USER"@localhost: -} - -exec_vm() { - ssh -q -F "${AYA_TMPDIR}/ssh_config" \ - -i "${AYA_TMPDIR}/test_rsa" \ - -p 2222 \ - "${AYA_SSH_USER}"@localhost \ - "$@" -} - -stop_vm() { - if [ -f "${AYA_TMPDIR}/vm.pid" ]; then - echo "Stopping VM forcefully" - kill -9 "$(cat "${AYA_TMPDIR}/vm.pid")" - rm "${AYA_TMPDIR}/vm.pid" - fi -} - -cleanup_vm() { - if ! stop_vm; then - rm -f "${AYA_IMGDIR}/vm.qcow2" - fi -} - -start_vm -trap cleanup_vm EXIT - -# make sure we always use fresh sources (also see comment at the end) -rsync_vm "$*" - -exec_vm "find $* -type f -executable -print0 | xargs -0 -I {} sudo {} --test-threads=1" - -# we rm and sync but it doesn't seem to work reliably - I guess we could sleep a -# few seconds after but ain't nobody got time for that. Instead we also rm -# before rsyncing. -exec_vm "rm -rf $*; sync" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 3f93d692..01105c00 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -19,7 +19,6 @@ pub struct XtaskOptions { enum Subcommand { Codegen(codegen::Options), Docs, - BuildIntegrationTest(run::BuildOptions), IntegrationTest(run::Options), PublicApi(public_api::Options), } @@ -45,17 +44,6 @@ fn main() -> Result<()> { match command { Subcommand::Codegen(opts) => codegen::codegen(opts, libbpf_dir), Subcommand::Docs => docs::docs(metadata), - Subcommand::BuildIntegrationTest(opts) => { - let binaries = run::build(opts)?; - let mut stdout = std::io::stdout(); - for (_name, binary) in binaries { - use std::{io::Write as _, os::unix::ffi::OsStrExt as _}; - - stdout.write_all(binary.as_os_str().as_bytes())?; - stdout.write_all("\n".as_bytes())?; - } - Ok(()) - } Subcommand::IntegrationTest(opts) => run::run(opts), Subcommand::PublicApi(opts) => public_api::public_api(opts, metadata), } diff --git a/xtask/src/run.rs b/xtask/src/run.rs index 6742a26b..f0063c75 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -1,47 +1,66 @@ use std::{ + env::consts::{ARCH, OS}, ffi::OsString, fmt::Write as _, - io::BufReader, - path::PathBuf, - process::{Child, Command, Stdio}, + fs::{copy, create_dir_all, metadata, File}, + io::{BufRead as _, BufReader, ErrorKind, Write as _}, + path::{Path, PathBuf}, + process::{Child, Command, Output, Stdio}, }; use anyhow::{anyhow, bail, Context as _, Result}; use cargo_metadata::{Artifact, CompilerMessage, Message, Target}; use clap::Parser; -use xtask::AYA_BUILD_INTEGRATION_BPF; +use xtask::{exec, AYA_BUILD_INTEGRATION_BPF}; -#[derive(Debug, Parser)] -pub struct BuildOptions { - /// Arguments to pass to `cargo build`. - #[clap(long)] - pub cargo_arg: Vec, +#[derive(Parser)] +pub enum Environment { + /// Runs the integration tests locally. + Local { + /// The command used to wrap your application. + #[clap(short, long, default_value = "sudo -E")] + runner: String, + }, + /// Runs the integration tests in a VM. + VM { + /// The kernel images to use. + /// + /// You can download some images with: + /// + /// wget --accept-regex '.*/linux-image-[0-9\.-]+-cloud-.*-unsigned*' \ + /// --recursive ftp://ftp.us.debian.org/debian/pool/main/l/linux/ + /// + /// You can then extract them with: + /// + /// find . -name '*.deb' -print0 \ + /// | xargs -0 -I {} sh -c "dpkg --fsys-tarfile {} \ + /// | tar --wildcards --extract '*vmlinuz*' --file -" + #[clap(required = true)] + kernel_image: Vec, + }, } -#[derive(Debug, Parser)] +#[derive(Parser)] pub struct Options { - #[command(flatten)] - pub build_options: BuildOptions, - /// The command used to wrap your application. - #[clap(short, long, default_value = "sudo -E")] - pub runner: String, + #[clap(subcommand)] + pub environment: Environment, /// Arguments to pass to your application. - #[clap(last = true)] + #[clap(global = true, last = true)] pub run_args: Vec, } -/// Build the project -pub fn build(opts: BuildOptions) -> Result> { - let BuildOptions { cargo_arg } = opts; +pub fn build(target: Option<&str>, f: F) -> Result> +where + F: FnOnce(&mut Command) -> &mut Command, +{ + // Always use rust-lld and -Zbuild-std in case we're cross-compiling. let mut cmd = Command::new("cargo"); - cmd.env(AYA_BUILD_INTEGRATION_BPF, "true") - .args([ - "build", - "--tests", - "--message-format=json", - "--package=integration-test", - ]) - .args(cargo_arg); + cmd.args(["build", "--message-format=json"]); + if let Some(target) = target { + let config = format!("target.{target}.linker = \"rust-lld\""); + cmd.args(["--target", target, "--config", &config]); + } + f(&mut cmd); let mut child = cmd .stdout(Stdio::piped()) @@ -83,40 +102,383 @@ pub fn build(opts: BuildOptions) -> Result> { Ok(executables) } +#[derive(Debug)] +struct Errors(Vec); + +impl std::fmt::Display for Errors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self(errors) = self; + for (i, error) in errors.iter().enumerate() { + if i != 0 { + writeln!(f)?; + } + write!(f, "{:?}", error)?; + } + Ok(()) + } +} + +impl std::error::Error for Errors {} + /// Build and run the project pub fn run(opts: Options) -> Result<()> { let Options { - build_options, - runner, + environment, run_args, } = opts; - let binaries = build(build_options).context("error while building userspace application")?; - let mut args = runner.trim().split_terminator(' '); - let runner = args.next().ok_or(anyhow!("no first argument"))?; - let args = args.collect::>(); - - let mut failures = String::new(); - for (name, binary) in binaries { - let mut cmd = Command::new(runner); - let cmd = cmd - .args(args.iter()) - .arg(binary) - .args(run_args.iter()) - .arg("--test-threads=1"); - - println!("{name} running {cmd:?}"); - - let status = cmd - .status() - .with_context(|| format!("failed to run {cmd:?}"))?; - if status.code() != Some(0) { - writeln!(&mut failures, "{name} failed: {status:?}").context("String write failed")? - } + #[allow(clippy::type_complexity)] + fn binaries(target: Option<&str>) -> Result)>> { + ["dev", "release"] + .into_iter() + .map(|profile| { + let binaries = build(target, |cmd| { + cmd.env(AYA_BUILD_INTEGRATION_BPF, "true").args([ + "--package", + "integration-test", + "--tests", + "--profile", + profile, + ]) + })?; + anyhow::Ok((profile, binaries)) + }) + .collect() } - if failures.is_empty() { - Ok(()) - } else { - Err(anyhow!("failures:\n{}", failures)) + + // Use --test-threads=1 to prevent tests from interacting with shared + // kernel state due to the lack of inter-test isolation. + let default_args = [OsString::from("--test-threads=1")]; + let run_args = default_args.iter().chain(run_args.iter()); + + match environment { + Environment::Local { runner } => { + let mut args = runner.trim().split_terminator(' '); + let runner = args.next().ok_or(anyhow!("no first argument"))?; + let args = args.collect::>(); + + let binaries = binaries(None)?; + + let mut failures = String::new(); + for (profile, binaries) in binaries { + for (name, binary) in binaries { + let mut cmd = Command::new(runner); + let cmd = cmd.args(args.iter()).arg(binary).args(run_args.clone()); + + println!("{profile}:{name} running {cmd:?}"); + + let status = cmd + .status() + .with_context(|| format!("failed to run {cmd:?}"))?; + if status.code() != Some(0) { + writeln!(&mut failures, "{profile}:{name} failed: {status:?}") + .context("String write failed")? + } + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(anyhow!("failures:\n{}", failures)) + } + } + Environment::VM { kernel_image } => { + let cache_dir = Path::new("test/.tmp"); + create_dir_all(cache_dir).context("failed to create cache dir")?; + let gen_init_cpio = cache_dir.join("gen_init_cpio"); + if !gen_init_cpio + .try_exists() + .context("failed to check existence of gen_init_cpio")? + { + let mut curl = Command::new("curl"); + curl.args([ + "-sfSL", + "https://raw.githubusercontent.com/torvalds/linux/master/usr/gen_init_cpio.c", + ]); + let mut curl_child = curl + .stdout(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn {curl:?}"))?; + let Child { stdout, .. } = &mut curl_child; + let curl_stdout = stdout.take().unwrap(); + + let mut clang = Command::new("clang"); + exec( + clang + .args(["-g", "-O2", "-x", "c", "-", "-o"]) + .arg(&gen_init_cpio) + .stdin(curl_stdout), + ) + .with_context(|| format!("failed to run {clang:?}"))?; + + let output = curl_child + .wait_with_output() + .with_context(|| format!("failed to wait for {curl:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{curl:?} failed: {output:?}") + } + } + + let mut errors = Vec::new(); + for kernel_image in kernel_image { + // Guess the guest architecture. + let mut cmd = Command::new("file"); + let output = cmd + .arg("--brief") + .arg(&kernel_image) + .output() + .with_context(|| format!("failed to run {cmd:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{cmd:?} failed: {output:?}") + } + let Output { stdout, .. } = output; + + // Now parse the output of the file command, which looks something like + // + // - Linux kernel ARM64 boot executable Image, little-endian, 4K pages + // + // - Linux kernel x86 boot executable bzImage, version 6.1.0-10-cloud-amd64 [..] + + let stdout = String::from_utf8(stdout) + .with_context(|| format!("invalid UTF-8 in {cmd:?} stdout"))?; + let (_, stdout) = stdout + .split_once("Linux kernel") + .ok_or_else(|| anyhow!("failed to parse {cmd:?} stdout: {stdout}"))?; + let (guest_arch, _) = stdout + .split_once("boot executable") + .ok_or_else(|| anyhow!("failed to parse {cmd:?} stdout: {stdout}"))?; + let guest_arch = guest_arch.trim(); + + let (guest_arch, machine, cpu) = match guest_arch { + "ARM64" => ("aarch64", Some("virt"), Some("cortex-a57")), + "x86" => ("x86_64", Some("q35"), Some("qemu64")), + guest_arch => (guest_arch, None, None), + }; + + let target = format!("{guest_arch}-unknown-linux-musl"); + + // Build our init program. The contract is that it will run anything it finds in /bin. + let init = build(Some(&target), |cmd| { + cmd.args(["--package", "init", "--profile", "release"]) + }) + .context("building init program failed")?; + + let init = match &*init { + [(name, init)] => { + if name != "init" { + bail!("expected init program to be named init, found {name}") + } + init + } + init => bail!("expected exactly one init program, found {init:?}"), + }; + + let binaries = binaries(Some(&target))?; + + let tmp_dir = tempfile::tempdir().context("tempdir failed")?; + + let initrd_image = tmp_dir.path().join("qemu-initramfs.img"); + let initrd_image_file = File::create(&initrd_image).with_context(|| { + format!("failed to create {} for writing", initrd_image.display()) + })?; + + let mut gen_init_cpio = Command::new(&gen_init_cpio); + let mut gen_init_cpio_child = gen_init_cpio + .arg("-") + .stdin(Stdio::piped()) + .stdout(initrd_image_file) + .spawn() + .with_context(|| format!("failed to spawn {gen_init_cpio:?}"))?; + let Child { stdin, .. } = &mut gen_init_cpio_child; + let mut stdin = stdin.take().unwrap(); + + use std::os::unix::ffi::OsStrExt as _; + + // Send input into gen_init_cpio which looks something like + // + // file /init path-to-init 0755 0 0 + // dir /bin 0755 0 0 + // file /bin/foo path-to-foo 0755 0 0 + // file /bin/bar path-to-bar 0755 0 0 + + for bytes in [ + "file /init ".as_bytes(), + init.as_os_str().as_bytes(), + " 0755 0 0\n".as_bytes(), + "dir /bin 0755 0 0\n".as_bytes(), + ] { + stdin.write_all(bytes).expect("write"); + } + + for (profile, binaries) in binaries { + for (name, binary) in binaries { + let name = format!("{}-{}", profile, name); + let path = tmp_dir.path().join(&name); + copy(&binary, &path).with_context(|| { + format!("copy({}, {}) failed", binary.display(), path.display()) + })?; + for bytes in [ + "file /bin/".as_bytes(), + name.as_bytes(), + " ".as_bytes(), + path.as_os_str().as_bytes(), + " 0755 0 0\n".as_bytes(), + ] { + stdin.write_all(bytes).expect("write"); + } + } + } + // Must explicitly close to signal EOF. + drop(stdin); + + let output = gen_init_cpio_child + .wait_with_output() + .with_context(|| format!("failed to wait for {gen_init_cpio:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{gen_init_cpio:?} failed: {output:?}") + } + + copy(&initrd_image, "/tmp/initrd.img").context("copy failed")?; + + let mut qemu = Command::new(format!("qemu-system-{guest_arch}")); + if let Some(machine) = machine { + qemu.args(["-machine", machine]); + } + if guest_arch == ARCH { + match OS { + "linux" => match metadata("/dev/kvm") { + Ok(metadata) => { + use std::os::unix::fs::FileTypeExt as _; + if metadata.file_type().is_char_device() { + qemu.args(["-accel", "kvm"]); + } + } + Err(error) => { + if error.kind() != ErrorKind::NotFound { + Err(error).context("failed to check existence of /dev/kvm")?; + } + } + }, + "macos" => { + qemu.args(["-accel", "hvf"]); + } + os => bail!("unsupported OS: {os}"), + } + } else if let Some(cpu) = cpu { + qemu.args(["-cpu", cpu]); + } + let console = OsString::from("ttyS0"); + let kernel_args = std::iter::once(("console", &console)) + .chain(run_args.clone().map(|run_arg| ("init.arg", run_arg))) + .enumerate() + .fold(OsString::new(), |mut acc, (i, (k, v))| { + if i != 0 { + acc.push(" "); + } + acc.push(k); + acc.push("="); + acc.push(v); + acc + }); + qemu.args(["-no-reboot", "-nographic", "-m", "512M", "-smp", "2"]) + .arg("-append") + .arg(kernel_args) + .arg("-kernel") + .arg(&kernel_image) + .arg("-initrd") + .arg(&initrd_image); + if guest_arch == "aarch64" { + match OS { + "linux" => { + let mut cmd = Command::new("locate"); + let output = cmd + .arg("QEMU_EFI.fd") + .output() + .with_context(|| format!("failed to run {cmd:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{qemu:?} failed: {output:?}") + } + let Output { stdout, .. } = output; + let bios = String::from_utf8(stdout) + .with_context(|| format!("failed to parse output of {cmd:?}"))?; + qemu.args(["-bios", bios.trim()]); + } + "macos" => { + let mut cmd = Command::new("brew"); + let output = cmd + .args(["list", "qemu", "-1", "-v"]) + .output() + .with_context(|| format!("failed to run {cmd:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{qemu:?} failed: {output:?}") + } + let Output { stdout, .. } = output; + let output = String::from_utf8(stdout) + .with_context(|| format!("failed to parse output of {cmd:?}"))?; + const NAME: &str = "edk2-aarch64-code.fd"; + let bios = output.lines().find(|line| line.contains(NAME)).ok_or_else( + || anyhow!("failed to find {NAME} in output of {cmd:?}: {output}"), + )?; + qemu.args(["-bios", bios.trim()]); + } + os => bail!("unsupported OS: {os}"), + }; + } + let mut qemu_child = qemu + .stdout(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn {qemu:?}"))?; + let Child { stdout, .. } = &mut qemu_child; + let stdout = stdout.take().unwrap(); + let stdout = BufReader::new(stdout); + + let mut outcome = None; + for line in stdout.lines() { + let line = + line.with_context(|| format!("failed to read line from {qemu:?}"))?; + println!("{}", line); + if let Some(line) = line.strip_prefix("init: ") { + let previous = match line { + "success" => outcome.replace(Ok(())), + "failure" => outcome.replace(Err(())), + line => bail!("unexpected init output: {}", line), + }; + if let Some(previous) = previous { + bail!("multiple exit status: previous={previous:?}, current={line}"); + } + // Try to get QEMU to exit on kernel panic; otherwise it might hang indefinitely. + if line.contains("end Kernel panic") { + qemu_child.kill().context("failed to kill {qemu:?}")?; + } + } + } + + let output = qemu_child + .wait_with_output() + .with_context(|| format!("failed to wait for {qemu:?}"))?; + let Output { status, .. } = &output; + if status.code() != Some(0) { + bail!("{qemu:?} failed: {output:?}") + } + + let outcome = outcome.ok_or(anyhow!("init did not exit"))?; + match outcome { + Ok(()) => {} + Err(()) => { + errors.push(anyhow!("VM binaries failed on {}", kernel_image.display())) + } + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(Errors(errors).into()) + } + } } }