ci: cache downloads

We're seeing 429 from Github trying to download gen_init_cpio, so cache
it using actions cache. Since I'm here add this for kernel images as
well to save time waiting on slow Debian servers.

.github/scripts/download_kernel_images.sh

@@ -29,5 +29,30 @@ for VERSION in "${VERSIONS[@]}"; do
   )"
 done
 
-printf '%s\n' "${FILES[@]}" \
+# TODO(https://github.com/curl/curl/issues/15729): restore --parallel here if and when it properly
-| xargs -t curl -sfSL --create-dirs --output-dir "$OUTPUT_DIR" --parallel --remote-name-all
+# supports ETags.
+# TODO(https://github.com/curl/curl/issues/15730): restore --create-dirs when it works in the
+# presence of `--etag-save`.`
+#
+# Note: `--etag-{compare,save}` are not idempotent until curl 8.9.0 which included
+# https://github.com/curl/curl/commit/85efbb92b8e6679705e122cee45ce76c56414a3e. At the time of
+# writing our CI uses Ubuntu 22.04 which has curl 7.81.0 and the latest available is Ubuntu 24.04
+# which has curl 8.5.0. Since neither has a new enough curl, we don't bother to update, but we
+# should do so when Ubuntu 24.10 or later is available.
+mkdir -p "$OUTPUT_DIR"
+KEEP=()
+for FILE in "${FILES[@]}"; do
+  name=$(basename "$FILE")
+  etag_name="$name.etag"
+  KEEP+=("$name" "$etag_name")
+
+  etag="$OUTPUT_DIR/$etag_name"
+  curl -sfSL --output-dir "$OUTPUT_DIR" --remote-name-all --etag-compare "$etag" --etag-save "$etag" "$FILE"
+done
+
+# Remove any files that were previously downloaded that are no longer needed.
+FIND_ARGS=()
+for FILE in "${KEEP[@]}"; do
+  FIND_ARGS+=("!" "-name" "$FILE")
+done
+find "$OUTPUT_DIR" -type f "${FIND_ARGS[@]}" -exec rm {} +

.github/workflows/ci.yml

@@ -226,7 +226,9 @@ jobs:
 
       - name: Install prerequisites
         if: runner.os == 'macOS'
-        # The xargs shipped on macOS always exits 0 with -P0, so we need GNU findutils.
+        # The curl shipped on macOS doesn't contain
+        # https://github.com/curl/curl/commit/85efbb92b8e6679705e122cee45ce76c56414a3e which is
+        # needed for proper handling of `--etag-{compare,save}`.
         #
         # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar.
         #
@@ -237,9 +239,10 @@ jobs:
           # https://github.com/actions/setup-python/issues/577
           find /usr/local/bin -type l -exec sh -c 'readlink -f "$1" \
           | grep -q ^/Library/Frameworks/Python.framework/Versions/' _ {} \; -exec rm -v {} \;
-          brew install --formula dpkg gnu-tar llvm lynx pkg-config qemu
+          brew install --formula curl dpkg gnu-tar llvm lynx pkg-config qemu
-          echo $(brew --prefix)/opt/gnu-tar/libexec/gnubin >> $GITHUB_PATH
+          echo $(brew --prefix curl)/bin >> $GITHUB_PATH
-          echo $(brew --prefix)/opt/llvm/bin >> $GITHUB_PATH
+          echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH
+          echo $(brew --prefix llvm)/bin >> $GITHUB_PATH
 
       - uses: dtolnay/rust-toolchain@nightly
         with:
@@ -278,6 +281,12 @@ jobs:
         # Do this on all system (not just macOS) to avoid relying on rustc-provided libLLVM.so.
         run: cargo install --force bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features
 
+      - name: Cache test cache
+        uses: actions/cache@v4
+        with:
+          path: test/.tmp
+          key: ${{ runner.arch }}-${{ runner.os }}-test-cache
+
       - name: Download debian kernels
         if: runner.arch == 'ARM64'
         # TODO: enable tests on kernels before 6.0.
@@ -302,7 +311,7 @@ jobs:
         run: |
           set -euxo pipefail
           find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
-            cargo xtask integration-test vm --github-api-token ${{ secrets.GITHUB_TOKEN }}
+            cargo xtask integration-test vm --cache-dir test/.tmp --github-api-token ${{ secrets.GITHUB_TOKEN }}
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

test/README.md

@@ -27,7 +27,7 @@ cargo xtask integration-test local
 ### Virtualized
 
 ```bash
-cargo xtask integration-test vm <KERNEL IMAGE>
+cargo xtask integration-test vm --cache-dir <CACHE_DIR> <KERNEL_IMAGE>...
 ```
 
 ### Writing an integration test

xtask/src/run.rs

@@ -3,7 +3,7 @@ use std::{
     fmt::Write as _,
     fs::{copy, create_dir_all, OpenOptions},
     io::{BufRead as _, BufReader, Write as _},
-    path::{Path, PathBuf},
+    path::PathBuf,
     process::{Child, ChildStdin, Command, Output, Stdio},
     sync::{Arc, Mutex},
     thread,
@@ -25,6 +25,10 @@ enum Environment {
     },
     /// Runs the integration tests in a VM.
     VM {
+        /// The cache directory in which to store intermediate artifacts.
+        #[clap(long)]
+        cache_dir: PathBuf,
+
         /// The Github API token to use if network requests to Github are made.
         ///
         /// This may be required if Github rate limits are exceeded.
@@ -175,6 +179,7 @@ pub fn run(opts: Options) -> Result<()> {
             }
         }
         Environment::VM {
+            cache_dir,
             github_api_token,
             kernel_image,
         } => {
@@ -195,13 +200,13 @@ pub fn run(opts: Options) -> Result<()> {
             // We consume the output of QEMU, looking for the output of our init program. This is
             // the only way to distinguish success from failure. We batch up the errors across all
             // VM images and report to the user. The end.
-            let cache_dir = Path::new("test/.tmp");
+            create_dir_all(&cache_dir).context("failed to create cache dir")?;
-            create_dir_all(cache_dir).context("failed to create cache dir")?;
             let gen_init_cpio = cache_dir.join("gen_init_cpio");
             if !gen_init_cpio
                 .try_exists()
                 .context("failed to check existence of gen_init_cpio")?
             {
+                // TODO(https://github.com/oxidecomputer/third-party-api-clients/issues/96): Use ETag-based caching.
                 let client = octorust::Client::new(
                     String::from("aya-xtask-integration-test-run"),
                     github_api_token.map(octorust::auth::Credentials::Token),