ci: download gen_init_cpio with authentication

The raw endpoint[0] now[1] seems to return HTTP 429 on the first request
and unauthenticated API limits are exceeded after one request (so one of
ubuntu/macos builders fails). Hopefully this works with authentication.

Link: https://raw.githubusercontent.com/torvalds/linux/refs/heads/master/usr/gen_init_cpio.c [0]
Link: https://github.com/orgs/community/discussions/146957 [1]

.github/workflows/ci.yml

@@ -299,7 +299,10 @@ jobs:
         run: cargo xtask integration-test local
 
       - name: Run virtualized integration tests
-        run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm
+        run: |
+          set -euxo pipefail
+          find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \
+            cargo xtask integration-test vm --github-api-token ${{ secrets.GITHUB_TOKEN }}
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions

Cargo.toml

@@ -59,6 +59,7 @@ edition = "2021"
 # them to do that, but in the meantime we need to be careful.
 [workspace.dependencies]
 anyhow = { version = "1", default-features = false }
+base64 = { version = "0.22.1", default-features = false }
 assert_matches = { version = "1.5.0", default-features = false }
 async-io = { version = "2.0", default-features = false }
 bindgen = { version = "0.71", default-features = false }
@@ -80,6 +81,7 @@ netns-rs = { version = "0.1", default-features = false }
 nix = { version = "0.29.0", default-features = false }
 num_enum = { version = "0.7", default-features = false }
 object = { version = "0.36", default-features = false }
+octorust = { version = "0.7.0", default-features = false }
 once_cell = { version = "1.20.1", default-features = false }
 proc-macro2 = { version = "1", default-features = false }
 proc-macro2-diagnostics = { version = "0.10.1", default-features = false }

xtask/Cargo.toml

@@ -11,11 +11,13 @@ edition.workspace = true
 [dependencies]
 anyhow = { workspace = true, features = ["std"] }
 aya-tool = { path = "../aya-tool", version = "0.1.0", default-features = false }
+base64 = { workspace = true, features = ["std"] }
 cargo_metadata = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 dialoguer = { workspace = true }
 diff = { workspace = true }
 indoc = { workspace = true }
+octorust = { workspace = true, features = ["rustls-tls"] }
 proc-macro2 = { workspace = true }
 public-api = { workspace = true }
 quote = { workspace = true }
@@ -23,4 +25,5 @@ rustdoc-json = { workspace = true }
 rustup-toolchain = { workspace = true }
 syn = { workspace = true }
 tempfile = { workspace = true }
+tokio = { workspace = true, features = ["rt"] }
 which = { workspace = true }

xtask/src/run.rs

@@ -10,9 +10,10 @@ use std::{
 };
 
 use anyhow::{anyhow, bail, Context as _, Result};
+use base64::engine::Engine as _;
 use cargo_metadata::{Artifact, CompilerMessage, Message, Target};
 use clap::Parser;
-use xtask::{exec, Errors, AYA_BUILD_INTEGRATION_BPF};
+use xtask::{Errors, AYA_BUILD_INTEGRATION_BPF};
 
 #[derive(Parser)]
 enum Environment {
@@ -24,6 +25,12 @@ enum Environment {
     },
     /// Runs the integration tests in a VM.
     VM {
+        /// The Github API token to use if network requests to Github are made.
+        ///
+        /// This may be required if Github rate limits are exceeded.
+        #[clap(long)]
+        github_api_token: Option<String>,
+
         /// The kernel images to use.
         ///
         /// You can download some images with:
@@ -167,7 +174,10 @@ pub fn run(opts: Options) -> Result<()> {
                 Err(anyhow!("failures:\n{}", failures))
             }
         }
-        Environment::VM { kernel_image } => {
+        Environment::VM {
+            github_api_token,
+            kernel_image,
+        } => {
             // The user has asked us to run the tests on a VM. This is involved; strap in.
             //
             // We need tools to build the initramfs; we use gen_init_cpio from the Linux repository,
@@ -192,37 +202,56 @@ pub fn run(opts: Options) -> Result<()> {
                 .try_exists()
                 .context("failed to check existence of gen_init_cpio")?
             {
-                let mut curl = Command::new("curl");
+                let client = octorust::Client::new(
-                curl.args([
+                    String::from("aya-xtask-integration-test-run"),
-                    "-sfSL",
+                    github_api_token.map(octorust::auth::Credentials::Token),
-                    "https://raw.githubusercontent.com/torvalds/linux/master/usr/gen_init_cpio.c",
+                )?;
-                ]);
+                let octorust::Response {
-                let mut curl_child = curl
+                    status: _,
+                    headers: _,
+                    body: octorust::types::ContentFile { mut content, .. },
+                } = tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .unwrap()
+                    .block_on(client.repos().get_content_file(
+                        "torvalds",
+                        "linux",
+                        "usr/gen_init_cpio.c",
+                        "master",
+                    ))
+                    .context("failed to download gen_init_cpio.c")?;
+                // Github very helpfully wraps their base64 at 10 columns /s.
+                content.retain(|c| !c.is_whitespace());
+                let content = base64::engine::general_purpose::STANDARD
+                    .decode(content)
+                    .context("failed to decode gen_init_cpio.c")?;
+
+                let mut clang = Command::new("clang");
+                clang
+                    .args(["-g", "-O2", "-x", "c", "-", "-o"])
+                    .arg(&gen_init_cpio);
+                let mut child = clang
+                    .stdin(Stdio::piped())
                     .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
                     .spawn()
-                    .with_context(|| format!("failed to spawn {curl:?}"))?;
+                    .with_context(|| format!("failed to spawn {clang:?}"))?;
-                let Child { stdout, .. } = &mut curl_child;
-                let curl_stdout = stdout.take().unwrap();
 
-                let mut clang = Command::new("clang");
+                let Child { stdin, .. } = &mut child;
-                let clang = exec(
+                let mut stdin = stdin.take().unwrap();
-                    clang
+                stdin
-                        .args(["-g", "-O2", "-x", "c", "-", "-o"])
+                    .write_all(&content)
-                        .arg(&gen_init_cpio)
+                    .with_context(|| format!("failed to write to {clang:?} stdin"))?;
-                        .stdin(curl_stdout),
+                std::mem::drop(stdin); // Send EOF.
-                );
+
-
+                let output = child
-                let output = curl_child
                     .wait_with_output()
-                    .with_context(|| format!("failed to wait for {curl:?}"))?;
+                    .with_context(|| format!("failed to wait for {clang:?}"))?;
                 let Output { status, .. } = &output;
                 if status.code() != Some(0) {
-                    bail!("{curl:?} failed: {output:?}")
+                    bail!("{clang:?} failed: {output:?}")
                 }
-
-                // Check the result of clang *after* checking curl; in case the download failed,
-                // only curl's output will be useful.
-                clang?;
             }
 
             let mut errors = Vec::new();