feat(aya-obj): Support loading programs with ksyms

By default, BTF of extern functions has `extern` linkage. However,
verifier is not accepting functions with other linkages than `global`
and `static`.

Fixup of sizes and offsets of ksyms can't be retrieved from ELF.
Instead, the size of an unsigned integer is used for all types which
were found in the ksyms section.

Based on similar fixups which are done in libbpf.[0]

[0] https://github.com/torvalds/linux/commit/5bd022ec01f06

Fixes #1245

aya-obj/src/btf/btf.rs

@@ -15,12 +15,12 @@ use crate::{
     Object,
     btf::{
         Array, BtfEnum, BtfKind, BtfMember, BtfType, Const, Enum, FuncInfo, FuncLinkage, Int,
-        IntEncoding, LineInfo, Struct, Typedef, Union, VarLinkage,
+        IntEncoding, LineInfo, Struct, Typedef, Union, Var, VarLinkage,
         info::{FuncSecInfo, LineSecInfo},
         relocation::Relocation,
     },
     generated::{btf_ext_header, btf_header},
-    util::{HashMap, bytes_of},
+    util::{HashMap, HashSet, bytes_of},
 };
 
 pub(crate) const MAX_RESOLVE_DEPTH: usize = 32;
@@ -365,6 +365,15 @@ impl Btf {
             // read() reads POD values from ELF, which is sound, but the values can still contain
             // internally inconsistent values (like out of bound offsets and such).
             let ty = unsafe { BtfType::read(data, endianness)? };
+            if let BtfType::DataSec(ref datasec) = ty {
+                for entry in &datasec.entries {
+                    let btf_ty = types.type_by_id(entry.btf_type)?;
+                    if let BtfType::Func(_) = btf_ty {
+                        types.has_ksym_funcs = true;
+                    }
+                    types.ksym_types.insert(entry.btf_type);
+                }
+            }
             data = &data[ty.type_info_size()..];
             types.push(ty);
         }
@@ -484,7 +493,7 @@ impl Btf {
         symbol_offsets: &HashMap<String, u64>,
         features: &BtfFeatures,
     ) -> Result<(), BtfError> {
-        // ENUM64 placeholder type needs to be added before we take ownership of
+        // Placeholder types need to be added before we take ownership of
         // self.types to ensure that the offsets in the BtfHeader are correct.
         let placeholder_name = self.add_string("enum64_placeholder");
         let enum64_placeholder_id = (!features.btf_enum64
@@ -497,6 +506,22 @@ impl Btf {
                 0,
             )))
         });
+        let ksym_func_placeholder_id = self.types.has_ksym_funcs.then(|| {
+            let int_btf_id = self
+                .types()
+                .enumerate()
+                .find(|(_, t)| {
+                    if let BtfType::Int(int) = t {
+                        int.size == 4 && int.encoding() == IntEncoding::Signed
+                    } else {
+                        false
+                    }
+                })
+                .map(|(i, _)| i as u32)
+                .expect("no `int` type in BTF");
+            let name = self.add_string("ksym_func_placeholder");
+            self.add_type(BtfType::Var(Var::new(name, int_btf_id, VarLinkage::Global)))
+        });
         let mut types = mem::take(&mut self.types);
         for i in 0..types.types.len() {
             let t = &mut types.types[i];
@@ -566,6 +591,45 @@ impl Btf {
                     // There are some cases when the compiler does indeed populate the size.
                     if d.size > 0 {
                         debug!("{kind} {name}: size fixup not required");
+                    } else if name == ".ksyms" {
+                        // Size of `.ksyms` can't be retrieved from ELF. We know that each entry
+                        // has a size of a signed 32-bit integer.
+                        let size = d.entries.len() * mem::size_of::<i32>();
+                        debug!("{kind} {name}: fixup size to {size}");
+                        d.size = size as u32;
+
+                        let mut entries = mem::take(&mut d.entries);
+                        let mut fixed_section = d.clone();
+
+                        for (i, e) in entries.iter_mut().enumerate() {
+                            let size = mem::size_of::<i32>();
+                            let offset = i * size;
+                            e.offset = offset as u32;
+                            e.size = size as u32;
+                            match types.type_by_id(e.btf_type)? {
+                                BtfType::Func(func) => {
+                                    e.btf_type = ksym_func_placeholder_id
+                                        .expect("ksym_func_placeholder_id must be set");
+
+                                    let func_name = self.string_at(func.name_offset)?;
+
+                                    debug!(
+                                        "{kind} {name}: FUNC {func_name}: fixup offset {offset}"
+                                    );
+                                }
+                                BtfType::Var(var) => {
+                                    let var_name = self.string_at(var.name_offset)?;
+
+                                    debug!("{kind} {name}: VAR {var_name}: fixup offset {offset}");
+                                }
+                                _ => unreachable!("`.ksyms` members have to be either FUNC or VAR"),
+                            }
+                        }
+                        fixed_section.entries = entries;
+
+                        // Must reborrow here because we borrow `types` immutably above.
+                        let t = &mut types.types[i];
+                        *t = BtfType::DataSec(fixed_section);
                     } else {
                         // We need to get the size of the section from the ELF file.
                         // Fortunately, we cached these when parsing it initially
@@ -641,6 +705,9 @@ impl Btf {
                 // Sanitize FUNC.
                 BtfType::Func(ty) => {
                     let name = self.string_at(ty.name_offset)?;
+                    if types.ksym_types.contains(&(i as u32)) {
+                        ty.set_linkage(FuncLinkage::Global);
+                    }
                     // Sanitize FUNC.
                     if !features.btf_func {
                         debug!("{kind}: not supported. replacing with TYPEDEF");
@@ -1010,12 +1077,16 @@ impl<'a> Iterator for SecInfoIter<'a> {
 #[derive(Debug, Clone)]
 pub(crate) struct BtfTypes {
     pub(crate) types: Vec<BtfType>,
+    pub(crate) ksym_types: HashSet<u32>,
+    pub(crate) has_ksym_funcs: bool,
 }
 
 impl Default for BtfTypes {
     fn default() -> Self {
         Self {
             types: vec![BtfType::Unknown],
+            ksym_types: HashSet::new(),
+            has_ksym_funcs: false,
         }
     }
 }

test/integration-test/bpf/ksym.bpf.c

@@ -0,0 +1,30 @@
+// clang-format off
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_tracing.h>
+// clang-format on
+
+char _license[] SEC("license") = "GPL";
+
+struct {
+  __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+  __uint(map_flags, BPF_F_NO_PREALLOC);
+  __type(key, int);
+  __type(value, __u32);
+} task_storage SEC(".maps");
+
+extern void bpf_rcu_read_lock(void) __ksym;
+extern void bpf_rcu_read_unlock(void) __ksym;
+
+SEC("tp_btf/sys_enter")
+int BPF_PROG(sys_enter, struct pt_regs *regs, long id) {
+  __u32 value = 1;
+  struct task_struct *task = bpf_get_current_task_btf();
+  bpf_rcu_read_lock();
+  struct task_struct *group_leader = task->group_leader;
+  bpf_task_storage_get(&task_storage, group_leader, &value,
+                       BPF_LOCAL_STORAGE_GET_F_CREATE);
+  bpf_rcu_read_unlock();
+  return 0;
+}

test/integration-test/build.rs

@@ -64,6 +64,7 @@ fn main() -> Result<()> {
     const C_BPF: &[(&str, bool)] = &[
         ("ext.bpf.c", false),
         ("iter.bpf.c", true),
+        ("ksym.bpf.c", true),
         ("main.bpf.c", false),
         ("multimap-btf.bpf.c", false),
         ("reloc.bpf.c", true),

test/integration-test/src/lib.rs

@@ -2,6 +2,7 @@ use aya::include_bytes_aligned;
 
 pub const EXT: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/ext.bpf.o"));
 pub const ITER_TASK: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/iter.bpf.o"));
+pub const KSYM: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/ksym.bpf.o"));
 pub const MAIN: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/main.bpf.o"));
 pub const MULTIMAP_BTF: &[u8] =
     include_bytes_aligned!(concat!(env!("OUT_DIR"), "/multimap-btf.bpf.o"));

test/integration-test/src/tests.rs

@@ -3,6 +3,7 @@ mod btf_relocations;
 mod elf;
 mod info;
 mod iter;
+mod ksym;
 mod load;
 mod log;
 mod raw_tracepoint;

test/integration-test/src/tests/ksym.rs

@@ -0,0 +1,12 @@
+use aya::{Btf, Ebpf, programs::BtfTracePoint};
+use test_log::test;
+
+#[test]
+fn test_ksym() {
+    let mut ebpf = Ebpf::load(crate::KSYM).unwrap();
+
+    let prog: &mut BtfTracePoint = ebpf.program_mut("sys_enter").unwrap().try_into().unwrap();
+    let btf = Btf::from_sys_fs().unwrap();
+    prog.load("sys_enter", &btf).unwrap();
+    prog.attach().unwrap();
+}