diff --git a/test/integration-test/src/tests/load.rs b/test/integration-test/src/tests/load.rs index 795f396d..5e193dbf 100644 --- a/test/integration-test/src/tests/load.rs +++ b/test/integration-test/src/tests/load.rs @@ -154,22 +154,36 @@ fn unload_xdp() { fn test_loaded_at() { let mut bpf = Bpf::load(crate::TEST).unwrap(); let prog: &mut Xdp = bpf.program_mut("pass").unwrap().try_into().unwrap(); - let t1 = SystemTime::now(); - prog.load().unwrap(); - let t2 = SystemTime::now(); - assert_loaded("pass"); - - let loaded_at = prog.info().unwrap().loaded_at(); - let range = t1..t2; + // SystemTime is not monotonic, which can cause this test to flake. We don't expect the clock + // timestamp to continuously jump around, so we add some retries. If the test is ever correct, + // we know that the value returned by loaded_at() was reasonable relative to SystemTime::now(). + let mut failures = Vec::new(); + for _ in 0..5 { + let t1 = SystemTime::now(); + prog.load().unwrap(); + let t2 = SystemTime::now(); + let loaded_at = prog.info().unwrap().loaded_at(); + prog.unload().unwrap(); + let range = t1..t2; + if range.contains(&loaded_at) { + failures.clear(); + break; + } + failures.push(LoadedAtRange(loaded_at, range)); + } assert!( - range.contains(&loaded_at), - "{range:?}.contains({loaded_at:?})" + failures.is_empty(), + "loaded_at was not in range: {failures:?}", ); - prog.unload().unwrap(); - - assert_unloaded("pass"); + struct LoadedAtRange(SystemTime, std::ops::Range); + impl std::fmt::Debug for LoadedAtRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self(loaded_at, range) = self; + write!(f, "{range:?}.contains({loaded_at:?})") + } + } } #[test] diff --git a/test/integration-test/src/tests/xdp.rs b/test/integration-test/src/tests/xdp.rs index 88423fe9..092f280e 100644 --- a/test/integration-test/src/tests/xdp.rs +++ b/test/integration-test/src/tests/xdp.rs @@ -82,17 +82,20 @@ fn cpumap_chain() { xdp.load().unwrap(); xdp.attach("lo", XdpFlags::default()).unwrap(); - let sock = UdpSocket::bind("127.0.0.1:1777").unwrap(); - sock.set_read_timeout(Some(Duration::from_millis(1))) + const PAYLOAD: &str = "hello cpumap"; + + let sock = UdpSocket::bind("127.0.0.1:0").unwrap(); + let addr = sock.local_addr().unwrap(); + sock.set_read_timeout(Some(Duration::from_secs(60))) .unwrap(); - sock.send_to(b"hello cpumap", "127.0.0.1:1777").unwrap(); + sock.send_to(PAYLOAD.as_bytes(), addr).unwrap(); - // Read back the packet to ensure it wenth through the entire network stack, including our two + // Read back the packet to ensure it went through the entire network stack, including our two // probes. - let mut buf = vec![0u8; 1000]; + let mut buf = [0u8; PAYLOAD.len() + 1]; let n = sock.recv(&mut buf).unwrap(); - assert_eq!(&buf[..n], b"hello cpumap"); + assert_eq!(&buf[..n], PAYLOAD.as_bytes()); assert_eq!(hits.get(&0, 0).unwrap(), 1); assert_eq!(hits.get(&1, 0).unwrap(), 1); } diff --git a/xtask/src/run.rs b/xtask/src/run.rs index a9ba17f4..64114620 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -456,20 +456,30 @@ pub fn run(opts: Options) -> Result<()> { let stderr = stderr.take().unwrap(); let stderr = BufReader::new(stderr); - fn terminate_if_contains_kernel_panic( - line: &str, - stdin: &Arc>, - ) -> anyhow::Result<()> { - if line.contains("end Kernel panic") { - println!("kernel panic detected; terminating QEMU"); - let mut stdin = stdin.lock().unwrap(); - stdin - .write_all(&[0x01, b'x']) - .context("failed to write to stdin")?; - println!("waiting for QEMU to terminate"); - } - Ok(()) - } + const TERMINATE_AFTER_COUNT: &[(&str, usize)] = + &[("end Kernel panic", 0), ("watchdog: BUG: soft lockup", 1)]; + let mut counts = [0; TERMINATE_AFTER_COUNT.len()]; + + let mut terminate_if_kernel_hang = + move |line: &str, stdin: &Arc>| -> anyhow::Result<()> { + if let Some(i) = TERMINATE_AFTER_COUNT + .iter() + .position(|(marker, _)| line.contains(marker)) + { + counts[i] += 1; + + let (marker, max) = TERMINATE_AFTER_COUNT[i]; + if counts[i] > max { + println!("{marker} detected > {max} times; terminating QEMU"); + let mut stdin = stdin.lock().unwrap(); + stdin + .write_all(&[0x01, b'x']) + .context("failed to write to stdin")?; + println!("waiting for QEMU to terminate"); + } + } + Ok(()) + }; let stderr = { let stdin = stdin.clone(); @@ -478,8 +488,7 @@ pub fn run(opts: Options) -> Result<()> { for line in stderr.lines() { let line = line.context("failed to read line from stderr")?; eprintln!("{}", line); - // Try to get QEMU to exit on kernel panic; otherwise it might hang indefinitely. - terminate_if_contains_kernel_panic(&line, &stdin)?; + terminate_if_kernel_hang(&line, &stdin)?; } anyhow::Ok(()) }) @@ -490,8 +499,7 @@ pub fn run(opts: Options) -> Result<()> { for line in stdout.lines() { let line = line.context("failed to read line from stdout")?; println!("{}", line); - // Try to get QEMU to exit on kernel panic; otherwise it might hang indefinitely. - terminate_if_contains_kernel_panic(&line, &stdin)?; + terminate_if_kernel_hang(&line, &stdin)?; // The init program will print "init: success" or "init: failure" to indicate // the outcome of running the binaries it found in /bin. if let Some(line) = line.strip_prefix("init: ") {