WIP: auto-attach tracepoints based on SEC() data

reviewable/pr1057/r1
Erik Schilling 3 months ago
parent 29b821376e
commit ef73d65229

@ -248,7 +248,10 @@ pub enum ProgramSection {
URetProbe {
sleepable: bool,
},
TracePoint,
TracePoint {
category: Option<String>,
name: Option<String>,
},
SocketFilter,
Xdp {
frags: bool,
@ -275,7 +278,9 @@ pub enum ProgramSection {
Lsm {
sleepable: bool,
},
BtfTracePoint,
BtfTracePoint {
trace_point: Option<String>,
},
FEntry {
sleepable: bool,
},
@ -331,8 +336,14 @@ impl FromStr for ProgramSection {
}
},
},
"tp_btf" => BtfTracePoint,
"tracepoint" | "tp" => TracePoint,
"tp_btf" => BtfTracePoint {
trace_point: pieces.next().map(|s| s.to_string()),
},
"tracepoint" | "tp" => {
let category = pieces.next().map(|s| s.to_string());
let name = pieces.next().map(|s| s.to_string());
TracePoint { category, name }
}
"socket" => SocketFilter,
"sk_msg" => SkMsg,
"sk_skb" => {
@ -2029,7 +2040,7 @@ mod tests {
assert_matches!(
obj.parse_section(fake_section(
EbpfSectionKind::Program,
"tracepoint/foo",
"tracepoint/cat/name",
bytes_of(&fake_ins()),
None
)),

@ -28,10 +28,11 @@ use crate::{
Object, ParseError, ProgramSection,
},
programs::{
BtfTracePoint, CgroupDevice, CgroupSkb, CgroupSkbAttachType, CgroupSock, CgroupSockAddr,
CgroupSockopt, CgroupSysctl, Extension, FEntry, FExit, Iter, KProbe, LircMode2, Lsm,
PerfEvent, ProbeKind, Program, ProgramData, ProgramError, RawTracePoint, SchedClassifier,
SkLookup, SkMsg, SkSkb, SkSkbKind, SockOps, SocketFilter, TracePoint, UProbe, Xdp,
trace_point::TracePointAttachInfo, BtfTracePoint, CgroupDevice, CgroupSkb,
CgroupSkbAttachType, CgroupSock, CgroupSockAddr, CgroupSockopt, CgroupSysctl, Extension,
FEntry, FExit, Iter, KProbe, LircMode2, Lsm, PerfEvent, ProbeKind, Program, ProgramData,
ProgramError, RawTracePoint, SchedClassifier, SkLookup, SkMsg, SkSkb, SkSkbKind, SockOps,
SocketFilter, TracePoint, UProbe, Xdp,
},
sys::{
bpf_load_btf, is_bpf_cookie_supported, is_bpf_global_data_supported,
@ -412,7 +413,7 @@ impl<'a> EbpfLoader<'a> {
| ProgramSection::FEntry { sleepable: _ }
| ProgramSection::FExit { sleepable: _ }
| ProgramSection::Lsm { sleepable: _ }
| ProgramSection::BtfTracePoint
| ProgramSection::BtfTracePoint { trace_point: _ }
| ProgramSection::Iter { sleepable: _ } => {
return Err(EbpfError::BtfError(err))
}
@ -420,7 +421,10 @@ impl<'a> EbpfLoader<'a> {
| ProgramSection::KProbe
| ProgramSection::UProbe { sleepable: _ }
| ProgramSection::URetProbe { sleepable: _ }
| ProgramSection::TracePoint
| ProgramSection::TracePoint {
category: _,
name: _,
}
| ProgramSection::SocketFilter
| ProgramSection::Xdp {
frags: _,
@ -575,9 +579,19 @@ impl<'a> EbpfLoader<'a> {
kind: ProbeKind::URetProbe,
})
}
ProgramSection::TracePoint => Program::TracePoint(TracePoint {
data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level),
}),
ProgramSection::TracePoint { category, name } => {
let expected_attach_info = match (category, name) {
(Some(category), Some(name)) => Some(TracePointAttachInfo {
category: category.clone(),
name: name.clone(),
}),
_ => None,
};
Program::TracePoint(TracePoint {
data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level),
expected_attach_info,
})
}
ProgramSection::SocketFilter => Program::SocketFilter(SocketFilter {
data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level),
}),
@ -657,9 +671,11 @@ impl<'a> EbpfLoader<'a> {
}
Program::Lsm(Lsm { data })
}
ProgramSection::BtfTracePoint => Program::BtfTracePoint(BtfTracePoint {
data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level),
}),
ProgramSection::BtfTracePoint { trace_point: _ } => {
Program::BtfTracePoint(BtfTracePoint {
data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level),
})
}
ProgramSection::FEntry { sleepable } => {
let mut data =
ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level);
@ -1081,6 +1097,14 @@ impl Ebpf {
pub fn programs_mut(&mut self) -> impl Iterator<Item = (&str, &mut Program)> {
self.programs.iter_mut().map(|(s, p)| (s.as_str(), p))
}
/// TODO
pub fn auto_attach(&mut self) {
for (_, program) in self.programs_mut() {
let btf = Btf::from_sys_fs().expect("unable to get btf info");
program.auto_attach(Some(&btf)).unwrap();
}
}
}
/// The error type returned by [`Ebpf::load_file`] and [`Ebpf::load`].

@ -51,7 +51,7 @@
meta_variable_misuse,
missing_abi,
//missing_copy_implementations,
missing_docs,
// missing_docs,
non_ascii_idents,
noop_method_call,
rust_2021_incompatible_closure_captures,

@ -97,6 +97,10 @@ impl CgroupDevice {
)))
}
}
pub fn auto_attach(&self) -> Result<CgroupDeviceLinkId, ProgramError> {
todo!();
}
/// Queries the cgroup for attached programs.
pub fn query<T: AsFd>(target_fd: T) -> Result<Vec<CgroupDeviceLink>, ProgramError> {

@ -123,6 +123,10 @@ impl CgroupSkb {
.insert(CgroupSkbLink::new(CgroupSkbLinkInner::ProgAttach(link)))
}
}
pub fn auto_attach(&self) -> Result<CgroupSkbLinkId, ProgramError> {
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///

@ -101,6 +101,10 @@ impl CgroupSock {
.insert(CgroupSockLink::new(CgroupSockLinkInner::ProgAttach(link)))
}
}
pub fn auto_attach(&self) -> Result<CgroupSockLinkId, ProgramError>{
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///

@ -102,6 +102,10 @@ impl CgroupSockAddr {
))
}
}
pub fn auto_attach(&self)-> Result<CgroupSockAddrLinkId, ProgramError> {
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///

@ -101,6 +101,10 @@ impl CgroupSockopt {
)))
}
}
pub fn auto_attach(&self)-> Result<CgroupSockoptLinkId, ProgramError> {
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///

@ -96,6 +96,10 @@ impl CgroupSysctl {
)))
}
}
pub fn auto_attach(&self) -> Result<CgroupSysctlLinkId, ProgramError>{
todo!();
}
}
#[derive(Debug, Hash, Eq, PartialEq)]

@ -112,6 +112,10 @@ impl Extension {
.links
.insert(ExtensionLink::new(FdLink::new(link_fd)))
}
pub fn auto_attach(&self) -> Result<ExtensionLinkId, ProgramError>{
todo!();
}
/// Attaches the extension to another program.
///

@ -67,6 +67,10 @@ impl FEntry {
pub fn attach(&mut self) -> Result<FEntryLinkId, ProgramError> {
attach_raw_tracepoint(&mut self.data, None)
}
pub fn auto_attach(&self) -> Result<FEntryLinkId, ProgramError>{
todo!();
}
}
define_link_wrapper!(

@ -67,6 +67,10 @@ impl FExit {
pub fn attach(&mut self) -> Result<FExitLinkId, ProgramError> {
attach_raw_tracepoint(&mut self.data, None)
}
pub fn auto_attach(&self)-> Result<FExitLinkId, ProgramError> {
todo!();
}
}
define_link_wrapper!(

@ -83,6 +83,10 @@ impl Iter {
.links
.insert(IterLink::new(PerfLinkInner::FdLink(FdLink::new(link_fd))))
}
pub fn auto_attach(&self) -> Result<IterLinkId, ProgramError>{
todo!();
}
}
/// An iterator descriptor.

@ -88,6 +88,10 @@ impl KProbe {
)
}
pub fn auto_attach(&mut self) -> Result<KProbeLinkId, ProgramError> {
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///
/// Existing links will not be populated. To work with existing links you should use [`crate::programs::links::PinnedLink`].

@ -81,6 +81,10 @@ impl LircMode2 {
self.data.links.insert(LircLink::new(prog_fd, lircdev_fd))
}
pub fn auto_attach(&self) -> Result<LircLinkId, ProgramError> {
todo!();
}
/// Detaches the program.
///

@ -73,6 +73,10 @@ impl Lsm {
pub fn attach(&mut self) -> Result<LsmLinkId, ProgramError> {
attach_raw_tracepoint(&mut self.data, None)
}
pub fn auto_attach(&self)-> Result<LsmLinkId, ProgramError> {
todo!();
}
}
define_link_wrapper!(

@ -78,9 +78,11 @@ use std::{
sync::Arc,
};
use aya_obj::{btf::Btf, ProgramSection};
use info::impl_info;
pub use info::{loaded_programs, ProgramInfo, ProgramType};
use libc::ENOSPC;
use object::Section;
use tc::SchedClassifierLink;
use thiserror::Error;
@ -148,6 +150,10 @@ pub enum ProgramError {
#[error("the program is not attached")]
NotAttached,
/// The program cannot be auto attached.
#[error("the program cannot be auto attached")]
CannotAutoAttach,
/// Loading the program failed.
#[error("the BPF_PROG_LOAD syscall failed. Verifier output: {verifier_log}")]
LoadError {
@ -480,10 +486,91 @@ impl Program {
Self::Iter(p) => p.info(),
}
}
pub fn auto_attach(&mut self, btf: Option<&Btf>) -> Result<(), ProgramError> {
match self {
Self::KProbe(p) => {
p.auto_attach()?;
}
Self::UProbe(p) => {
p.auto_attach()?;
}
Self::TracePoint(p) => {
p.auto_attach()?;
}
Self::SocketFilter(p) => {
p.auto_attach()?;
}
Self::Xdp(p) => {
p.auto_attach()?;
}
Self::SkMsg(p) => {
p.auto_attach()?;
}
Self::SkSkb(p) => {
p.auto_attach()?;
}
Self::SockOps(p) => {
p.auto_attach()?;
}
Self::SchedClassifier(p) => {
p.auto_attach()?;
}
Self::CgroupSkb(p) => {
p.auto_attach()?;
}
Self::CgroupSysctl(p) => {
p.auto_attach()?;
}
Self::CgroupSockopt(p) => {
p.auto_attach()?;
}
Self::LircMode2(p) => {
p.auto_attach()?;
}
Self::PerfEvent(p) => {
p.auto_attach()?;
}
Self::RawTracePoint(p) => {
p.auto_attach()?;
}
Self::Lsm(p) => {
p.auto_attach()?;
}
Self::BtfTracePoint(p) => {
p.auto_attach(btf.ok_or(ProgramError::CannotAutoAttach)?)?;
}
Self::FEntry(p) => {
p.auto_attach()?;
}
Self::FExit(p) => {
p.auto_attach()?;
}
Self::Extension(p) => {
p.auto_attach()?;
}
Self::CgroupSockAddr(p) => {
p.auto_attach()?;
}
Self::SkLookup(p) => {
p.auto_attach()?;
}
Self::CgroupSock(p) => {
p.auto_attach()?;
}
Self::CgroupDevice(p) => {
p.auto_attach()?;
}
Self::Iter(p) => {
p.auto_attach()?;
}
};
Ok(())
}
}
#[derive(Debug)]
pub(crate) struct ProgramData<T: Link> {
pub(crate) struct ProgramData<T: Link + Eq> {
pub(crate) name: Option<String>,
pub(crate) obj: Option<(obj::Program, obj::Function)>,
pub(crate) fd: Option<ProgramFd>,
@ -498,7 +585,7 @@ pub(crate) struct ProgramData<T: Link> {
pub(crate) flags: u32,
}
impl<T: Link> ProgramData<T> {
impl<T: Link + Eq> ProgramData<T> {
pub(crate) fn new(
name: Option<String>,
obj: (obj::Program, obj::Function),
@ -570,15 +657,19 @@ impl<T: Link> ProgramData<T> {
let name = info.name_as_str().map(|s| s.to_string());
Self::from_bpf_prog_info(name, fd, path.as_ref(), info.0, verifier_log_level)
}
fn section(&self) -> Option<&ProgramSection> {
Some(&self.obj.as_ref()?.0.section)
}
}
impl<T: Link> ProgramData<T> {
impl<T: Link + Eq> ProgramData<T> {
fn fd(&self) -> Result<&ProgramFd, ProgramError> {
self.fd.as_ref().ok_or(ProgramError::NotLoaded)
}
}
fn unload_program<T: Link>(data: &mut ProgramData<T>) -> Result<(), ProgramError> {
fn unload_program<T: Link + Eq>(data: &mut ProgramData<T>) -> Result<(), ProgramError> {
data.links.remove_all()?;
data.fd
.take()
@ -586,7 +677,10 @@ fn unload_program<T: Link>(data: &mut ProgramData<T>) -> Result<(), ProgramError
.map(|ProgramFd { .. }| ())
}
fn pin_program<T: Link, P: AsRef<Path>>(data: &ProgramData<T>, path: P) -> Result<(), PinError> {
fn pin_program<T: Link + Eq, P: AsRef<Path>>(
data: &ProgramData<T>,
path: P,
) -> Result<(), PinError> {
use std::os::unix::ffi::OsStrExt as _;
let fd = data.fd.as_ref().ok_or(PinError::NoFd {
@ -609,7 +703,7 @@ fn pin_program<T: Link, P: AsRef<Path>>(data: &ProgramData<T>, path: P) -> Resul
Ok(())
}
fn load_program<T: Link>(
fn load_program<T: Link + Eq>(
prog_type: bpf_prog_type,
data: &mut ProgramData<T>,
) -> Result<(), ProgramError> {
@ -961,7 +1055,6 @@ macro_rules! impl_from_pin {
// Use impl_from_pin if the program doesn't require additional data
impl_from_pin!(
TracePoint,
SocketFilter,
SkMsg,
CgroupSysctl,

@ -183,6 +183,10 @@ impl PerfEvent {
let link = perf_attach(prog_fd, fd, None /* cookie */)?;
self.data.links.insert(PerfEventLink::new(link))
}
pub fn auto_attach(&self) -> Result<PerfEventLinkId, ProgramError> {
todo!();
}
}
impl TryFrom<PerfEventLink> for FdLink {

@ -52,6 +52,10 @@ impl RawTracePoint {
let tp_name_c = CString::new(tp_name).unwrap();
attach_raw_tracepoint(&mut self.data, Some(&tp_name_c))
}
pub fn auto_attach(&self) -> Result<RawTracePointLinkId, ProgramError> {
todo!();
}
}
define_link_wrapper!(

@ -74,6 +74,10 @@ impl SkLookup {
.links
.insert(SkLookupLink::new(FdLink::new(link_fd)))
}
pub fn auto_attach(&self) -> Result<SkLookupLinkId, ProgramError>{
todo!();
}
}
define_link_wrapper!(

@ -89,6 +89,10 @@ impl SkMsg {
self.data.links.insert(SkMsgLink::new(link))
}
pub fn auto_attach(&self) -> Result<SkMsgLinkId, ProgramError>{
todo!();
}
}
define_link_wrapper!(

@ -95,6 +95,10 @@ impl SkSkb {
self.data.links.insert(SkSkbLink::new(link))
}
pub fn auto_attach(&self) -> Result<SkSkbLinkId, ProgramError>{
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///

@ -91,6 +91,10 @@ impl SockOps {
.insert(SockOpsLink::new(SockOpsLinkInner::ProgAttach(link)))
}
}
pub fn auto_attach(&self) -> Result<SockOpsLinkId, ProgramError> {
todo!();
}
}
#[derive(Debug, Hash, Eq, PartialEq)]

@ -98,6 +98,10 @@ impl SocketFilter {
self.data.links.insert(SocketFilterLink { socket, prog_fd })
}
pub fn auto_attach(&self) -> Result<SocketFilterLinkId, ProgramError> {
todo!();
}
/// Detaches the program.
///
/// See [`Self::attach``].

@ -192,6 +192,10 @@ impl SchedClassifier {
)
}
}
pub fn auto_attach(&self) -> Result<SchedClassifierLinkId, ProgramError>{
todo!();
}
/// Attaches the program to the given `interface` with options defined in [`TcAttachOptions`].
///

@ -71,6 +71,17 @@ impl BtfTracePoint {
pub fn attach(&mut self) -> Result<BtfTracePointLinkId, ProgramError> {
attach_raw_tracepoint(&mut self.data, None)
}
pub fn auto_attach(&mut self, btf: &Btf) -> Result<BtfTracePointLinkId, ProgramError> {
let tp = match &self.data.section().ok_or(ProgramError::CannotAutoAttach)? {
aya_obj::ProgramSection::BtfTracePoint { trace_point } => {
trace_point.clone().ok_or(ProgramError::CannotAutoAttach)?
}
_ => Err(ProgramError::CannotAutoAttach)?,
};
self.load(&tp, btf)?;
self.attach()
}
}
define_link_wrapper!(

@ -12,6 +12,7 @@ use crate::{
FdLink, LinkError, ProgramData, ProgramError,
},
sys::{bpf_link_get_info_by_fd, perf_event_open_trace_point, SyscallError},
VerifierLogLevel,
};
/// The type returned when attaching a [`TracePoint`] fails.
@ -28,6 +29,15 @@ pub enum TracePointError {
},
}
/// Defines where to attach trace point
#[derive(Debug)]
pub struct TracePointAttachInfo {
/// Category of trace point
pub category: String,
/// Name of trace point
pub name: String,
}
/// A program that can be attached at a pre-defined kernel trace point.
///
/// The kernel provides a set of pre-defined trace points that eBPF programs can
@ -53,6 +63,7 @@ pub enum TracePointError {
#[doc(alias = "BPF_PROG_TYPE_TRACEPOINT")]
pub struct TracePoint {
pub(crate) data: ProgramData<TracePointLink>,
pub(crate) expected_attach_info: Option<TracePointAttachInfo>,
}
impl TracePoint {
@ -81,6 +92,32 @@ impl TracePoint {
let link = perf_attach(prog_fd, fd, None /* cookie */)?;
self.data.links.insert(TracePointLink::new(link))
}
pub fn auto_attach(&mut self) -> Result<TracePointLinkId, ProgramError> {
let (cat, name) = match &self.data.section().ok_or(ProgramError::CannotAutoAttach)? {
aya_obj::ProgramSection::TracePoint { category, name } => (
category.clone().ok_or(ProgramError::CannotAutoAttach)?,
name.clone().ok_or(ProgramError::CannotAutoAttach)?,
),
_ => Err(ProgramError::CannotAutoAttach)?,
};
self.load()?;
self.attach(&cat, &name)
}
/// Creates a program from a pinned entry on a bpffs.
///
/// Existing links will not be populated. To work with existing links you should use [`crate::programs::links::PinnedLink`].
///
/// On drop, any managed links are detached and the program is unloaded. This will not result in
/// the program being unloaded from the kernel if it is still pinned.
pub fn from_pin<P: AsRef<Path>>(path: P) -> Result<Self, ProgramError> {
let data = ProgramData::from_pinned_path(path, VerifierLogLevel::default())?;
Ok(Self {
data,
expected_attach_info: None,
})
}
}
define_link_wrapper!(

@ -130,6 +130,10 @@ impl UProbe {
attach(&mut self.data, self.kind, path, offset, pid, cookie)
}
pub fn auto_attach(&self) -> Result<UProbeLinkId, ProgramError> {
todo!();
}
/// Creates a program from a pinned entry on a bpffs.
///
/// Existing links will not be populated. To work with existing links you should use [`crate::programs::links::PinnedLink`].

@ -117,6 +117,10 @@ impl Xdp {
}
self.attach_to_if_index(if_index, flags)
}
pub fn auto_attach(&self)-> Result<XdpLinkId, ProgramError> {
todo!();
}
/// Attaches the program to the given interface index.
///

@ -0,0 +1,41 @@
// clang-format off
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
// clang-format on
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 256);
__type(key, char[32]);
__type(value, __u8);
} executed_once SEC(".maps");
#define assign_str(target, str)
// BPF will not allow us to write out of bounds, so we skip the length checks
#define mark_executed(key) \
{ \
__u8 __executed = 1; \
char __probe_type[32] = {}; \
__builtin_memcpy(__probe_type, key, sizeof(key)); \
bpf_map_update_elem(&executed_once, &__probe_type, &__executed, BPF_ANY); \
} \
do { \
} while (0)
SEC("tp_btf/sched_switch")
int BPF_PROG(sched_switch_tp_btf, bool preempt, struct task_struct *prev,
struct task_struct *next) {
mark_executed("tp_btf");
return 0;
}
SEC("tracepoint/sched/sched_switch")
int sched_switch_tp(bool preempt, struct task_struct *prev,
struct task_struct *next) {
mark_executed("tracepoint");
return 0;
}
char _license[] SEC("license") = "GPL";

@ -66,6 +66,7 @@ fn main() -> Result<()> {
("iter.bpf.c", true),
("main.bpf.c", false),
("multimap-btf.bpf.c", false),
("probe_auto_attach.bpf.c", false),
("reloc.bpf.c", true),
("text_64_64_reloc.c", false),
("variables_reloc.bpf.c", false),

@ -5,6 +5,8 @@ pub const ITER_TASK: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/i
pub const MAIN: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/main.bpf.o"));
pub const MULTIMAP_BTF: &[u8] =
include_bytes_aligned!(concat!(env!("OUT_DIR"), "/multimap-btf.bpf.o"));
pub const PROBE_AUTO_ATTACH: &[u8] =
include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_auto_attach.bpf.o"));
pub const RELOC_BPF: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/reloc.bpf.o"));
pub const RELOC_BTF: &[u8] =
include_bytes_aligned!(concat!(env!("OUT_DIR"), "/reloc.bpf.target.o"));

@ -5,6 +5,7 @@ mod info;
mod iter;
mod load;
mod log;
mod probe_auto_attach;
mod raw_tracepoint;
mod rbpf;
mod relocations;

@ -0,0 +1,42 @@
use std::{
ffi,
time::{Duration, Instant},
};
use aya::{maps::MapError, Ebpf};
fn to_map_key(key: &str) -> [ffi::c_char; 32] {
let mut padded: Vec<_> = key.bytes().map(|b| b as ffi::c_char).collect();
padded.resize(32, 0);
padded.try_into().unwrap()
}
#[test]
fn auto_attach_succes() {
let mut bpf = Ebpf::load(crate::PROBE_AUTO_ATTACH).unwrap();
bpf.auto_attach();
let executed_map: aya::maps::HashMap<aya::maps::MapData, [ffi::c_char; 32], u8> =
aya::maps::HashMap::try_from(bpf.take_map("executed_once").unwrap()).unwrap();
let fired_probes = ["tp_btf", "tracepoint"];
let start = Instant::now();
const TIMEOUT: Duration = Duration::from_secs(1);
let mut all_fired = false;
while !all_fired && (Instant::now() - start) < TIMEOUT {
all_fired = true;
for probe in fired_probes {
let executed = match executed_map.get(&to_map_key(probe), 0) {
Ok(fired) => fired,
Err(MapError::KeyNotFound) => 0,
e => e.unwrap(),
};
if executed == 0 {
all_fired = false;
}
}
}
assert!(all_fired, "Not all expected probes fired");
}
Loading…
Cancel
Save