Merge pull request #522 from dave-tucker/perf_link

Use bpf_link for perf_attach programs (Kprobe/Uprobe/Tracepoint etc...)

aya/src/bpf.rs

@@ -35,8 +35,8 @@ use crate::{
     sys::{
         bpf_load_btf, bpf_map_freeze, bpf_map_update_elem_ptr, is_btf_datasec_supported,
         is_btf_decl_tag_supported, is_btf_float_supported, is_btf_func_global_supported,
-        is_btf_func_supported, is_btf_supported, is_btf_type_tag_supported, is_prog_name_supported,
-        retry_with_verifier_logs,
+        is_btf_func_supported, is_btf_supported, is_btf_type_tag_supported, is_perf_link_supported,
+        is_prog_name_supported, retry_with_verifier_logs,
     },
     util::{bytes_of, possible_cpus, VerifierLog, POSSIBLE_CPUS},
 };
@@ -65,50 +65,39 @@ unsafe impl<T: Pod, const N: usize> Pod for [T; N] {}
 
 pub use aya_obj::maps::{bpf_map_def, PinningType};
 
+lazy_static! {
+    pub(crate) static ref FEATURES: Features = Features::new();
+}
+
 // Features implements BPF and BTF feature detection
 #[derive(Default, Debug)]
 pub(crate) struct Features {
     pub bpf_name: bool,
+    pub bpf_perf_link: bool,
     pub btf: Option<BtfFeatures>,
 }
 
 impl Features {
-    fn probe_features(&mut self) {
-        self.bpf_name = is_prog_name_supported();
-        debug!("[FEAT PROBE] BPF program name support: {}", self.bpf_name);
-
-        self.btf = if is_btf_supported() {
-            Some(BtfFeatures::default())
+    fn new() -> Self {
+        let btf = if is_btf_supported() {
+            Some(BtfFeatures {
+                btf_func: is_btf_func_supported(),
+                btf_func_global: is_btf_func_global_supported(),
+                btf_datasec: is_btf_datasec_supported(),
+                btf_float: is_btf_float_supported(),
+                btf_decl_tag: is_btf_decl_tag_supported(),
+                btf_type_tag: is_btf_type_tag_supported(),
+            })
         } else {
             None
         };
-        debug!("[FEAT PROBE] BTF support: {}", self.btf.is_some());
-
-        if let Some(ref mut btf) = self.btf {
-            btf.btf_func = is_btf_func_supported();
-            debug!("[FEAT PROBE] BTF func support: {}", btf.btf_func);
-
-            btf.btf_func_global = is_btf_func_global_supported();
-            debug!(
-                "[FEAT PROBE] BTF global func support: {}",
-                btf.btf_func_global
-            );
-
-            btf.btf_datasec = is_btf_datasec_supported();
-            debug!(
-                "[FEAT PROBE] BTF var and datasec support: {}",
-                btf.btf_datasec
-            );
-
-            btf.btf_float = is_btf_float_supported();
-            debug!("[FEAT PROBE] BTF float support: {}", btf.btf_float);
-
-            btf.btf_decl_tag = is_btf_decl_tag_supported();
-            debug!("[FEAT PROBE] BTF decl_tag support: {}", btf.btf_decl_tag);
-
-            btf.btf_type_tag = is_btf_type_tag_supported();
-            debug!("[FEAT PROBE] BTF type_tag support: {}", btf.btf_type_tag);
-        }
+        let f = Features {
+            bpf_name: is_prog_name_supported(),
+            bpf_perf_link: is_perf_link_supported(),
+            btf,
+        };
+        debug!("BPF Feature Detection: {:#?}", f);
+        f
     }
 }
 
@@ -139,7 +128,6 @@ pub struct BpfLoader<'a> {
     map_pin_path: Option<PathBuf>,
     globals: HashMap<&'a str, &'a [u8]>,
     max_entries: HashMap<&'a str, u32>,
-    features: Features,
     extensions: HashSet<&'a str>,
     verifier_log_level: VerifierLogLevel,
 }
@@ -169,14 +157,11 @@ impl Default for VerifierLogLevel {
 impl<'a> BpfLoader<'a> {
     /// Creates a new loader instance.
     pub fn new() -> BpfLoader<'a> {
-        let mut features = Features::default();
-        features.probe_features();
         BpfLoader {
             btf: Btf::from_sys_fs().ok().map(Cow::Owned),
             map_pin_path: None,
             globals: HashMap::new(),
             max_entries: HashMap::new(),
-            features,
             extensions: HashSet::new(),
             verifier_log_level: VerifierLogLevel::default(),
         }
@@ -360,8 +345,8 @@ impl<'a> BpfLoader<'a> {
         let mut obj = Object::parse(data)?;
         obj.patch_map_data(self.globals.clone())?;
 
-        let btf_fd = if let Some(ref btf) = self.features.btf {
-            if let Some(btf) = obj.fixup_and_sanitize_btf(btf)? {
+        let btf_fd = if let Some(features) = &FEATURES.btf {
+            if let Some(btf) = obj.fixup_and_sanitize_btf(features)? {
                 // load btf to the kernel
                 Some(load_btf(btf.to_bytes())?)
             } else {
@@ -449,7 +434,7 @@ impl<'a> BpfLoader<'a> {
             .programs
             .drain()
             .map(|(name, obj)| {
-                let prog_name = if self.features.bpf_name {
+                let prog_name = if FEATURES.bpf_name {
                     Some(name.clone())
                 } else {
                     None

aya/src/programs/kprobe.rs

@@ -6,7 +6,7 @@ use crate::{
     generated::bpf_prog_type::BPF_PROG_TYPE_KPROBE,
     programs::{
         define_link_wrapper, load_program,
-        perf_attach::{PerfLink, PerfLinkId},
+        perf_attach::{PerfLinkIdInner, PerfLinkInner},
         probe::{attach, ProbeKind},
         ProgramData, ProgramError,
     },
@@ -101,8 +101,8 @@ define_link_wrapper!(
     KProbeLink,
     /// The type returned by [KProbe::attach]. Can be passed to [KProbe::detach].
     KProbeLinkId,
-    PerfLink,
-    PerfLinkId
+    PerfLinkInner,
+    PerfLinkIdInner
 );
 
 /// The type returned when attaching a [`KProbe`] fails.

aya/src/programs/perf_attach.rs

@@ -3,11 +3,42 @@ use libc::close;
 use std::os::unix::io::RawFd;
 
 use crate::{
-    programs::{probe::detach_debug_fs, Link, ProbeKind, ProgramData, ProgramError},
-    sys::perf_event_ioctl,
-    PERF_EVENT_IOC_DISABLE, PERF_EVENT_IOC_ENABLE, PERF_EVENT_IOC_SET_BPF,
+    generated::bpf_attach_type::BPF_PERF_EVENT,
+    programs::{probe::detach_debug_fs, FdLink, Link, ProbeKind, ProgramError},
+    sys::{bpf_link_create, perf_event_ioctl},
+    FEATURES, PERF_EVENT_IOC_DISABLE, PERF_EVENT_IOC_ENABLE, PERF_EVENT_IOC_SET_BPF,
 };
 
+#[derive(Debug, Hash, Eq, PartialEq)]
+pub(crate) enum PerfLinkIdInner {
+    FdLinkId(<FdLink as Link>::Id),
+    PerfLinkId(<PerfLink as Link>::Id),
+}
+
+#[derive(Debug)]
+pub(crate) enum PerfLinkInner {
+    FdLink(FdLink),
+    PerfLink(PerfLink),
+}
+
+impl Link for PerfLinkInner {
+    type Id = PerfLinkIdInner;
+
+    fn id(&self) -> Self::Id {
+        match self {
+            PerfLinkInner::FdLink(link) => PerfLinkIdInner::FdLinkId(link.id()),
+            PerfLinkInner::PerfLink(link) => PerfLinkIdInner::PerfLinkId(link.id()),
+        }
+    }
+
+    fn detach(self) -> Result<(), ProgramError> {
+        match self {
+            PerfLinkInner::FdLink(link) => link.detach(),
+            PerfLinkInner::PerfLink(link) => link.detach(),
+        }
+    }
+}
+
 /// The identifer of a PerfLink.
 #[derive(Debug, Hash, Eq, PartialEq)]
 pub struct PerfLinkId(RawFd);
@@ -41,29 +72,36 @@ impl Link for PerfLink {
     }
 }
 
-pub(crate) fn perf_attach<T: Link + From<PerfLink>>(
-    data: &mut ProgramData<T>,
-    fd: RawFd,
-) -> Result<T::Id, ProgramError> {
-    perf_attach_either(data, fd, None, None)
+pub(crate) fn perf_attach(prog_fd: RawFd, fd: RawFd) -> Result<PerfLinkInner, ProgramError> {
+    if FEATURES.bpf_perf_link {
+        let link_fd =
+            bpf_link_create(prog_fd, fd, BPF_PERF_EVENT, None, 0).map_err(|(_, io_error)| {
+                ProgramError::SyscallError {
+                    call: "bpf_link_create".to_owned(),
+                    io_error,
+                }
+            })? as RawFd;
+        Ok(PerfLinkInner::FdLink(FdLink::new(link_fd)))
+    } else {
+        perf_attach_either(prog_fd, fd, None, None)
+    }
 }
 
-pub(crate) fn perf_attach_debugfs<T: Link + From<PerfLink>>(
-    data: &mut ProgramData<T>,
+pub(crate) fn perf_attach_debugfs(
+    prog_fd: RawFd,
     fd: RawFd,
     probe_kind: ProbeKind,
     event_alias: String,
-) -> Result<T::Id, ProgramError> {
-    perf_attach_either(data, fd, Some(probe_kind), Some(event_alias))
+) -> Result<PerfLinkInner, ProgramError> {
+    perf_attach_either(prog_fd, fd, Some(probe_kind), Some(event_alias))
 }
 
-fn perf_attach_either<T: Link + From<PerfLink>>(
-    data: &mut ProgramData<T>,
+fn perf_attach_either(
+    prog_fd: RawFd,
     fd: RawFd,
     probe_kind: Option<ProbeKind>,
     event_alias: Option<String>,
-) -> Result<T::Id, ProgramError> {
-    let prog_fd = data.fd_or_err()?;
+) -> Result<PerfLinkInner, ProgramError> {
     perf_event_ioctl(fd, PERF_EVENT_IOC_SET_BPF, prog_fd).map_err(|(_, io_error)| {
         ProgramError::SyscallError {
             call: "PERF_EVENT_IOC_SET_BPF".to_owned(),
@@ -77,12 +115,9 @@ fn perf_attach_either<T: Link + From<PerfLink>>(
         }
     })?;
 
-    data.links.insert(
-        PerfLink {
+    Ok(PerfLinkInner::PerfLink(PerfLink {
         perf_fd: fd,
         probe_kind,
         event_alias,
-        }
-        .into(),
-    )
+    }))
 }

aya/src/programs/perf_event.rs

@@ -13,8 +13,9 @@ use crate::{
         },
     },
     programs::{
+        links::define_link_wrapper,
         load_program, perf_attach,
-        perf_attach::{PerfLink, PerfLinkId},
+        perf_attach::{PerfLinkIdInner, PerfLinkInner},
         ProgramData, ProgramError,
     },
     sys::perf_event_open,
@@ -119,7 +120,7 @@ pub enum PerfEventScope {
 #[derive(Debug)]
 #[doc(alias = "BPF_PROG_TYPE_PERF_EVENT")]
 pub struct PerfEvent {
-    pub(crate) data: ProgramData<PerfLink>,
+    pub(crate) data: ProgramData<PerfEventLink>,
 }
 
 impl PerfEvent {
@@ -141,7 +142,7 @@ impl PerfEvent {
         config: u64,
         scope: PerfEventScope,
         sample_policy: SamplePolicy,
-    ) -> Result<PerfLinkId, ProgramError> {
+    ) -> Result<PerfEventLinkId, ProgramError> {
         let (sample_period, sample_frequency) = match sample_policy {
             SamplePolicy::Period(period) => (period, None),
             SamplePolicy::Frequency(frequency) => (0, Some(frequency)),
@@ -168,13 +169,14 @@ impl PerfEvent {
             io_error,
         })? as i32;
 
-        perf_attach(&mut self.data, fd)
+        let link = perf_attach(self.data.fd_or_err()?, fd)?;
+        self.data.links.insert(PerfEventLink::new(link))
     }
 
     /// Detaches the program.
     ///
     /// See [PerfEvent::attach].
-    pub fn detach(&mut self, link_id: PerfLinkId) -> Result<(), ProgramError> {
+    pub fn detach(&mut self, link_id: PerfEventLinkId) -> Result<(), ProgramError> {
         self.data.links.remove(link_id)
     }
 
@@ -182,7 +184,16 @@ impl PerfEvent {
     ///
     /// The link will be detached on `Drop` and the caller is now responsible
     /// for managing its lifetime.
-    pub fn take_link(&mut self, link_id: PerfLinkId) -> Result<PerfLink, ProgramError> {
+    pub fn take_link(&mut self, link_id: PerfEventLinkId) -> Result<PerfEventLink, ProgramError> {
         self.data.take_link(link_id)
     }
 }
+
+define_link_wrapper!(
+    /// The link used by [PerfEvent] programs.
+    PerfEventLink,
+    /// The type returned by [PerfEvent::attach]. Can be passed to [PerfEvent::detach].
+    PerfEventLinkId,
+    PerfLinkInner,
+    PerfLinkIdInner
+);

aya/src/programs/probe.rs

@@ -8,7 +8,7 @@ use std::{
 
 use crate::{
     programs::{
-        kprobe::KProbeError, perf_attach, perf_attach::PerfLink, perf_attach_debugfs,
+        kprobe::KProbeError, perf_attach, perf_attach::PerfLinkInner, perf_attach_debugfs,
         trace_point::read_sys_fs_trace_point_id, uprobe::UProbeError, utils::find_tracefs_path,
         Link, ProgramData, ProgramError,
     },
@@ -37,7 +37,7 @@ impl ProbeKind {
     }
 }
 
-pub(crate) fn attach<T: Link + From<PerfLink>>(
+pub(crate) fn attach<T: Link + From<PerfLinkInner>>(
     program_data: &mut ProgramData<T>,
     kind: ProbeKind,
     fn_name: &str,
@@ -49,13 +49,18 @@ pub(crate) fn attach<T: Link + From<PerfLink>>(
     let k_ver = kernel_version().unwrap();
     if k_ver < (4, 17, 0) {
         let (fd, event_alias) = create_as_trace_point(kind, fn_name, offset, pid)?;
-
-        return perf_attach_debugfs(program_data, fd, kind, event_alias);
+        let link = T::from(perf_attach_debugfs(
+            program_data.fd_or_err()?,
+            fd,
+            kind,
+            event_alias,
+        )?);
+        return program_data.links.insert(link);
     };
 
     let fd = create_as_probe(kind, fn_name, offset, pid)?;
-
-    perf_attach(program_data, fd)
+    let link = T::from(perf_attach(program_data.fd_or_err()?, fd)?);
+    program_data.links.insert(link)
 }
 
 pub(crate) fn detach_debug_fs(kind: ProbeKind, event_alias: &str) -> Result<(), ProgramError> {

aya/src/programs/trace_point.rs

@@ -6,7 +6,7 @@ use crate::{
     generated::bpf_prog_type::BPF_PROG_TYPE_TRACEPOINT,
     programs::{
         define_link_wrapper, load_program,
-        perf_attach::{perf_attach, PerfLink, PerfLinkId},
+        perf_attach::{perf_attach, PerfLinkIdInner, PerfLinkInner},
         utils::find_tracefs_path,
         ProgramData, ProgramError,
     },
@@ -87,7 +87,8 @@ impl TracePoint {
             }
         })? as i32;
 
-        perf_attach(&mut self.data, fd)
+        let link = perf_attach(self.data.fd_or_err()?, fd)?;
+        self.data.links.insert(TracePointLink::new(link))
     }
 
     /// Detaches from a trace point.
@@ -111,8 +112,8 @@ define_link_wrapper!(
     TracePointLink,
     /// The type returned by [TracePoint::attach]. Can be passed to [TracePoint::detach].
     TracePointLinkId,
-    PerfLink,
-    PerfLinkId
+    PerfLinkInner,
+    PerfLinkIdInner
 );
 
 pub(crate) fn read_sys_fs_trace_point_id(

aya/src/programs/uprobe.rs

@@ -17,7 +17,7 @@ use crate::{
     generated::bpf_prog_type::BPF_PROG_TYPE_KPROBE,
     programs::{
         define_link_wrapper, load_program,
-        perf_attach::{PerfLink, PerfLinkId},
+        perf_attach::{PerfLinkIdInner, PerfLinkInner},
         probe::{attach, ProbeKind},
         ProgramData, ProgramError,
     },
@@ -154,8 +154,8 @@ define_link_wrapper!(
     UProbeLink,
     /// The type returned by [UProbe::attach]. Can be passed to [UProbe::detach].
     UProbeLinkId,
-    PerfLink,
-    PerfLinkId
+    PerfLinkInner,
+    PerfLinkIdInner
 );
 
 /// The type returned when attaching an [`UProbe`] fails.

aya/src/sys/bpf.rs

@@ -599,6 +599,37 @@ pub(crate) fn is_prog_name_supported() -> bool {
     }
 }
 
+pub(crate) fn is_perf_link_supported() -> bool {
+    let mut attr = unsafe { mem::zeroed::<bpf_attr>() };
+    let u = unsafe { &mut attr.__bindgen_anon_3 };
+
+    let prog: &[u8] = &[
+        0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r0 = 0
+        0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit
+    ];
+
+    let gpl = b"GPL\0";
+    u.license = gpl.as_ptr() as u64;
+
+    let insns = copy_instructions(prog).unwrap();
+    u.insn_cnt = insns.len() as u32;
+    u.insns = insns.as_ptr() as u64;
+    u.prog_type = bpf_prog_type::BPF_PROG_TYPE_TRACEPOINT as u32;
+
+    if let Ok(fd) = sys_bpf(bpf_cmd::BPF_PROG_LOAD, &attr) {
+        if let Err((code, _)) =
+            // Uses an invalid target FD so we get EBADF if supported.
+            bpf_link_create(fd as i32, -1, bpf_attach_type::BPF_PERF_EVENT, None, 0)
+        {
+            // Returns EINVAL if unsupported. EBADF if supported.
+            let res = code == (-libc::EBADF).into();
+            unsafe { libc::close(fd as i32) };
+            return res;
+        }
+    }
+    false
+}
+
 pub(crate) fn is_btf_supported() -> bool {
     let mut btf = Btf::new();
     let name_offset = btf.add_string("int".to_string());