diff --git a/scripts/gen-bindings b/scripts/gen-bindings index 6cece633..09703be2 100755 --- a/scripts/gen-bindings +++ b/scripts/gen-bindings @@ -24,6 +24,39 @@ BPF_TYPES="\ BPF_VARS="\ BPF_PSEUDO_.* + BPF_ALU \ + BPF_ALU64 \ + BPF_LDX \ + BPF_ST \ + BPF_STX \ + BPF_LD \ + BPF_K \ + BPF_DW \ + BPF_W \ + BPF_H \ + BPF_B + " + +BTF_TYPES="\ + btf_header \ + btf_ext_header \ + btf_ext_info \ + btf_ext_info_sec \ + bpf_core_relo \ + bpf_core_relo_kind \ + btf_type \ + btf_enum \ + btf_array \ + btf_member \ + btf_param \ + btf_var \ + btf_var_secinfo + + " + +BTF_VARS="\ + BTF_KIND_.* + BTF_INT_.* " PERF_TYPES="\ @@ -52,6 +85,28 @@ bindgen $LIBBPF_DIR/include/uapi/linux/bpf.h \ done) \ > $OUTPUT_DIR/bpf_bindings.rs +bindgen $LIBBPF_DIR/include/uapi/linux/btf.h \ + --no-layout-tests \ + --default-enum-style moduleconsts \ + $(for ty in $BTF_TYPES; do + echo --whitelist-type "$ty" + done) \ + $(for var in $BTF_VARS; do + echo --whitelist-var "$var" + done) \ + > $OUTPUT_DIR/btf_bindings.rs + +bindgen $LIBBPF_DIR/src/libbpf_internal.h \ + --no-layout-tests \ + --default-enum-style moduleconsts \ + $(for ty in $BTF_TYPES; do + echo --whitelist-type "$ty" + done) \ + $(for var in $BTF_VARS; do + echo --whitelist-var "$var" + done) \ + > $OUTPUT_DIR/btf_internal_bindings.rs + bindgen include/perf_wrapper.h \ --no-layout-tests \ --default-enum-style moduleconsts \ diff --git a/src/bpf.rs b/src/bpf.rs index e06a98e6..e73c377e 100644 --- a/src/bpf.rs +++ b/src/bpf.rs @@ -9,7 +9,7 @@ use thiserror::Error; use crate::{ generated::bpf_insn, maps::{Map, MapError}, - obj::{relocate, Object, ParseError, RelocationError}, + obj::{Object, ParseError, RelocationError}, programs::{KProbe, Program, ProgramData, ProgramError, SocketFilter, TracePoint, UProbe, Xdp}, syscalls::bpf_map_update_elem_ptr, }; @@ -68,7 +68,7 @@ impl Bpf { maps.push(map); } - relocate(&mut obj, maps.as_slice())?; + obj.relocate(maps.as_slice())?; let programs = obj .programs diff --git a/src/generated/bpf_bindings.rs b/src/generated/bpf_bindings.rs index d21bed5a..f790700e 100644 --- a/src/generated/bpf_bindings.rs +++ b/src/generated/bpf_bindings.rs @@ -81,6 +81,17 @@ where } } } +pub const BPF_LD: u32 = 0; +pub const BPF_LDX: u32 = 1; +pub const BPF_ST: u32 = 2; +pub const BPF_STX: u32 = 3; +pub const BPF_ALU: u32 = 4; +pub const BPF_W: u32 = 0; +pub const BPF_H: u32 = 8; +pub const BPF_B: u32 = 16; +pub const BPF_K: u32 = 0; +pub const BPF_ALU64: u32 = 7; +pub const BPF_DW: u32 = 24; pub const BPF_PSEUDO_MAP_FD: u32 = 1; pub const BPF_PSEUDO_MAP_VALUE: u32 = 2; pub const BPF_PSEUDO_BTF_ID: u32 = 3; diff --git a/src/generated/btf_bindings.rs b/src/generated/btf_bindings.rs new file mode 100644 index 00000000..cf251db7 --- /dev/null +++ b/src/generated/btf_bindings.rs @@ -0,0 +1,90 @@ +/* automatically generated by rust-bindgen 0.55.1 */ + +pub const BTF_KIND_UNKN: u32 = 0; +pub const BTF_KIND_INT: u32 = 1; +pub const BTF_KIND_PTR: u32 = 2; +pub const BTF_KIND_ARRAY: u32 = 3; +pub const BTF_KIND_STRUCT: u32 = 4; +pub const BTF_KIND_UNION: u32 = 5; +pub const BTF_KIND_ENUM: u32 = 6; +pub const BTF_KIND_FWD: u32 = 7; +pub const BTF_KIND_TYPEDEF: u32 = 8; +pub const BTF_KIND_VOLATILE: u32 = 9; +pub const BTF_KIND_CONST: u32 = 10; +pub const BTF_KIND_RESTRICT: u32 = 11; +pub const BTF_KIND_FUNC: u32 = 12; +pub const BTF_KIND_FUNC_PROTO: u32 = 13; +pub const BTF_KIND_VAR: u32 = 14; +pub const BTF_KIND_DATASEC: u32 = 15; +pub const BTF_KIND_MAX: u32 = 15; +pub const BTF_INT_SIGNED: u32 = 1; +pub const BTF_INT_CHAR: u32 = 2; +pub const BTF_INT_BOOL: u32 = 4; +pub type __u8 = ::std::os::raw::c_uchar; +pub type __u16 = ::std::os::raw::c_ushort; +pub type __s32 = ::std::os::raw::c_int; +pub type __u32 = ::std::os::raw::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_header { + pub magic: __u16, + pub version: __u8, + pub flags: __u8, + pub hdr_len: __u32, + pub type_off: __u32, + pub type_len: __u32, + pub str_off: __u32, + pub str_len: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct btf_type { + pub name_off: __u32, + pub info: __u32, + pub __bindgen_anon_1: btf_type__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union btf_type__bindgen_ty_1 { + pub size: __u32, + pub type_: __u32, + _bindgen_union_align: u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_enum { + pub name_off: __u32, + pub val: __s32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_array { + pub type_: __u32, + pub index_type: __u32, + pub nelems: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_member { + pub name_off: __u32, + pub type_: __u32, + pub offset: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_param { + pub name_off: __u32, + pub type_: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var { + pub linkage: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var_secinfo { + pub type_: __u32, + pub offset: __u32, + pub size: __u32, +} diff --git a/src/generated/btf_internal_bindings.rs b/src/generated/btf_internal_bindings.rs new file mode 100644 index 00000000..08dda04a --- /dev/null +++ b/src/generated/btf_internal_bindings.rs @@ -0,0 +1,86 @@ +/* automatically generated by rust-bindgen 0.55.1 */ + +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField(::std::marker::PhantomData, [T; 0]); +impl __IncompleteArrayField { + #[inline] + pub const fn new() -> Self { + __IncompleteArrayField(::std::marker::PhantomData, []) + } + #[inline] + pub fn as_ptr(&self) -> *const T { + self as *const _ as *const T + } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::std::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} +impl ::std::fmt::Debug for __IncompleteArrayField { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__IncompleteArrayField") + } +} +pub type __u8 = ::std::os::raw::c_uchar; +pub type __u16 = ::std::os::raw::c_ushort; +pub type __u32 = ::std::os::raw::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_ext_info { + pub info: *mut ::std::os::raw::c_void, + pub rec_size: __u32, + pub len: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_ext_header { + pub magic: __u16, + pub version: __u8, + pub flags: __u8, + pub hdr_len: __u32, + pub func_info_off: __u32, + pub func_info_len: __u32, + pub line_info_off: __u32, + pub line_info_len: __u32, + pub core_relo_off: __u32, + pub core_relo_len: __u32, +} +#[repr(C)] +#[derive(Debug)] +pub struct btf_ext_info_sec { + pub sec_name_off: __u32, + pub num_info: __u32, + pub data: __IncompleteArrayField<__u8>, +} +pub mod bpf_core_relo_kind { + pub type Type = ::std::os::raw::c_uint; + pub const BPF_FIELD_BYTE_OFFSET: Type = 0; + pub const BPF_FIELD_BYTE_SIZE: Type = 1; + pub const BPF_FIELD_EXISTS: Type = 2; + pub const BPF_FIELD_SIGNED: Type = 3; + pub const BPF_FIELD_LSHIFT_U64: Type = 4; + pub const BPF_FIELD_RSHIFT_U64: Type = 5; + pub const BPF_TYPE_ID_LOCAL: Type = 6; + pub const BPF_TYPE_ID_TARGET: Type = 7; + pub const BPF_TYPE_EXISTS: Type = 8; + pub const BPF_TYPE_SIZE: Type = 9; + pub const BPF_ENUMVAL_EXISTS: Type = 10; + pub const BPF_ENUMVAL_VALUE: Type = 11; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_core_relo { + pub insn_off: __u32, + pub type_id: __u32, + pub access_str_off: __u32, + pub kind: bpf_core_relo_kind::Type, +} diff --git a/src/generated/mod.rs b/src/generated/mod.rs index 4fa2ab83..3030c9be 100644 --- a/src/generated/mod.rs +++ b/src/generated/mod.rs @@ -3,7 +3,11 @@ // FIXME: generate for x86_64 and aarch64 mod bpf_bindings; +mod btf_bindings; +mod btf_internal_bindings; mod perf_bindings; pub use bpf_bindings::*; +pub use btf_bindings::*; +pub use btf_internal_bindings::*; pub use perf_bindings::*; diff --git a/src/obj/btf/btf.rs b/src/obj/btf/btf.rs new file mode 100644 index 00000000..44df8e49 --- /dev/null +++ b/src/obj/btf/btf.rs @@ -0,0 +1,381 @@ +use std::{ + borrow::Cow, + convert::TryInto, + ffi::{c_void, CStr}, + mem, ptr, +}; + +use thiserror::Error; + +use crate::generated::{btf_ext_header, btf_header}; + +use super::{BtfType, Relocation}; + +unsafe impl object::pod::Pod for btf_header {} +unsafe impl object::pod::Pod for btf_ext_header {} + +pub(crate) const MAX_RESOLVE_DEPTH: u8 = 32; +pub(crate) const MAX_SPEC_LEN: usize = 64; + +#[derive(Error, Debug, Clone, Eq, PartialEq)] +pub enum BtfError { + #[error("error parsing BTF header")] + InvalidHeader, + + #[error("invalid type info segment")] + InvalidTypeInfo, + + #[error("invalid relocation info segment")] + InvalidRelocationInfo, + + #[error("invalid BTF type kind `{kind}`")] + InvalidTypeKind { kind: u32 }, + + #[error("invalid BTF relocation kind `{kind}`")] + InvalidRelocationKind { kind: u32 }, + + #[error("invalid BTF string offset: {offset}")] + InvalidStringOffset { offset: usize }, + + #[error("invalid BTF info, offset: {offset} len: {len} section_len: {section_len}")] + InvalidInfo { + offset: usize, + len: usize, + section_len: usize, + }, + + #[error("invalid BTF line info, offset: {offset} len: {len} section_len: {section_len}")] + InvalidLineInfo { + offset: usize, + len: usize, + section_len: usize, + }, + + #[error("Unknown BTF type id `{type_id}`")] + UnknownBtfType { type_id: u32 }, + + #[error("Unexpected BTF type id `{type_id}`")] + UnexpectedBtfType { type_id: u32 }, + + #[error("maximum depth reached resolving BTF type")] + MaximumTypeDepthReached { type_id: u32 }, +} + +#[derive(Clone, Debug)] +pub struct Btf { + header: btf_header, + strings: Vec, + types: Vec, +} + +impl Btf { + pub(crate) fn parse(data: &[u8]) -> Result { + if data.len() < mem::size_of::() { + return Err(BtfError::InvalidHeader); + } + + // safety: btf_header is POD so read_unaligned is safe + let header = unsafe { ptr::read_unaligned(data.as_ptr() as *const btf_header) }; + + let str_off = header.hdr_len as usize + header.str_off as usize; + let str_len = header.str_len as usize; + if str_off + str_len > data.len() { + return Err(BtfError::InvalidHeader); + } + + let strings = data[str_off..str_off + str_len].to_vec(); + let types = Btf::read_type_info(&header, data)?; + + Ok(Btf { + header, + strings, + types, + }) + } + + fn read_type_info(header: &btf_header, data: &[u8]) -> Result, BtfError> { + let hdr_len = header.hdr_len as usize; + let type_off = header.type_off as usize; + let type_len = header.type_len as usize; + let base = hdr_len + type_off; + if base + type_len > data.len() { + return Err(BtfError::InvalidTypeInfo); + } + + let mut data = &data[base..base + type_len]; + let mut types = vec![BtfType::Unknown]; + while !data.is_empty() { + // Safety: + // read() reads POD values from ELF, which is sound, but the values can still contain + // internally inconsistent values (like out of bound offsets and such). + let ty = unsafe { BtfType::read(data)? }; + data = &data[ty.type_info_size()..]; + types.push(ty); + } + + Ok(types) + } + + pub(crate) fn string_at(&self, offset: u32) -> Result, BtfError> { + let btf_header { + hdr_len, + mut str_off, + str_len, + .. + } = self.header; + str_off += hdr_len; + if offset >= str_off + str_len { + return Err(BtfError::InvalidStringOffset { + offset: offset as usize, + }); + } + + let offset = offset as usize; + let nul = self.strings[offset..] + .iter() + .position(|c| *c == 0u8) + .ok_or(BtfError::InvalidStringOffset { offset })?; + + let s = CStr::from_bytes_with_nul(&self.strings[offset..=offset + nul]) + .map_err(|_| BtfError::InvalidStringOffset { offset })?; + + Ok(s.to_string_lossy()) + } + + pub(crate) fn type_by_id(&self, type_id: u32) -> Result<&BtfType, BtfError> { + self.types + .get(type_id as usize) + .ok_or(BtfError::UnknownBtfType { type_id }) + } + + pub(crate) fn types(&self) -> impl Iterator { + self.types.iter() + } + + pub(crate) fn resolve_type(&self, root_type_id: u32) -> Result { + let mut type_id = root_type_id; + for _ in 0..MAX_RESOLVE_DEPTH { + let ty = self.type_by_id(type_id)?; + + use BtfType::*; + match ty { + Volatile(ty) | Const(ty) | Restrict(ty) | Typedef(ty) => { + // Safety: union + type_id = unsafe { ty.__bindgen_anon_1.type_ }; + continue; + } + _ => return Ok(type_id), + } + } + + Err(BtfError::MaximumTypeDepthReached { + type_id: root_type_id, + }) + } + + pub(crate) fn type_name(&self, ty: &BtfType) -> Result>, BtfError> { + ty.name_offset() + .map(|off| Ok(self.string_at(off)?)) + .transpose() + } + + pub(crate) fn err_type_name(&self, ty: &BtfType) -> Option { + ty.name_offset() + .and_then(|off| self.string_at(off).ok().map(String::from)) + } + + pub(crate) fn type_size(&self, root_type_id: u32) -> Result { + let mut type_id = root_type_id; + let mut n_elems = 1; + for _ in 0..MAX_RESOLVE_DEPTH { + let ty = self.type_by_id(type_id)?; + + use BtfType::*; + let size = match ty { + Int(ty, _) | Struct(ty, _) | Union(ty, _) | Enum(ty, _) | DataSec(ty, _) => { + // Safety: union + unsafe { ty.__bindgen_anon_1.size as usize } + } + Ptr(_) => mem::size_of::<*const c_void>(), // FIXME + Typedef(ty) | Volatile(ty) | Const(ty) | Restrict(ty) | Var(ty, _) => { + // Safety: union + type_id = unsafe { ty.__bindgen_anon_1.type_ }; + continue; + } + Array(_, array) => { + n_elems *= array.nelems as usize; + type_id = array.type_; + continue; + } + Unknown | Fwd(_) | Func(_) | FuncProto(_, _) => { + return Err(BtfError::UnexpectedBtfType { type_id }) + } + }; + + return Ok(size * n_elems); + } + + Err(BtfError::MaximumTypeDepthReached { + type_id: root_type_id, + }) + } +} + +#[derive(Debug, Clone)] +pub struct BtfExt { + data: Vec, + relocations: Vec<(u32, Vec)>, + header: btf_ext_header, + func_info_rec_size: usize, + line_info_rec_size: usize, + core_relo_rec_size: usize, +} + +impl BtfExt { + pub(crate) fn parse(data: &[u8]) -> Result { + // Safety: btf_ext_header is POD so read_unaligned is safe + let header = unsafe { + ptr::read_unaligned::(data.as_ptr() as *const btf_ext_header) + }; + + let rec_size = |offset, len| { + let offset = mem::size_of::() + offset as usize; + let len = len as usize; + // check that there's at least enough space for the `rec_size` field + if (len > 0 && len < 4) || offset + len > data.len() { + return Err(BtfError::InvalidInfo { + offset, + len, + section_len: data.len(), + }); + } + Ok(if len > 0 { + /* FIXME: endianness */ + u32::from_ne_bytes(data[offset..offset + 4].try_into().unwrap()) as usize + } else { + 0 + }) + }; + + let btf_ext_header { + func_info_off, + func_info_len, + line_info_off, + line_info_len, + core_relo_off, + core_relo_len, + .. + } = header; + + let mut ext = BtfExt { + header, + relocations: Vec::new(), + func_info_rec_size: rec_size(func_info_off, func_info_len)?, + line_info_rec_size: rec_size(line_info_off, line_info_len)?, + core_relo_rec_size: rec_size(core_relo_off, core_relo_len)?, + data: data.to_vec(), + }; + + let rec_size = ext.core_relo_rec_size; + ext.relocations.extend( + SecInfoIter::new(ext.core_relo_data(), ext.core_relo_rec_size) + .map(move |sec| { + let relos = sec + .data + .chunks(rec_size) + .enumerate() + .map(|(n, rec)| unsafe { Relocation::parse(rec, n) }) + .collect::, _>>()?; + Ok((sec.sec_name_off, relos)) + }) + .collect::, _>>()?, + ); + + Ok(ext) + } + + fn info_data(&self, offset: u32, len: u32) -> &[u8] { + let offset = (self.header.hdr_len + offset) as usize; + let data = &self.data[offset..offset + len as usize]; + if len > 0 { + // skip `rec_size` + &data[4..] + } else { + data + } + } + + fn func_info_data(&self) -> &[u8] { + self.info_data(self.header.func_info_off, self.header.func_info_len) + } + + fn line_info_data(&self) -> &[u8] { + self.info_data(self.header.line_info_off, self.header.line_info_len) + } + + fn core_relo_data(&self) -> &[u8] { + self.info_data(self.header.core_relo_off, self.header.core_relo_len) + } + + pub(crate) fn func_info(&self) -> SecInfoIter<'_> { + SecInfoIter::new(self.func_info_data(), self.func_info_rec_size) + } + + pub(crate) fn line_info(&self) -> SecInfoIter<'_> { + SecInfoIter::new(self.line_info_data(), self.line_info_rec_size) + } + + pub(crate) fn relocations(&self) -> impl Iterator)> { + self.relocations.iter() + } +} + +pub(crate) struct SecInfoIter<'a> { + data: &'a [u8], + offset: usize, + rec_size: usize, +} + +impl<'a> SecInfoIter<'a> { + fn new(data: &'a [u8], rec_size: usize) -> Self { + Self { + data, + rec_size, + offset: 0, + } + } +} + +impl<'a> Iterator for SecInfoIter<'a> { + type Item = SecInfo<'a>; + + fn next(&mut self) -> Option { + let data = self.data; + if self.offset + 8 >= data.len() { + return None; + } + + // FIXME: endianness + let sec_name_off = + u32::from_ne_bytes(data[self.offset..self.offset + 4].try_into().unwrap()); + self.offset += 4; + let num_info = u32::from_ne_bytes(data[self.offset..self.offset + 4].try_into().unwrap()); + self.offset += 4; + + let data = &data[self.offset..self.offset + (self.rec_size * num_info as usize)]; + self.offset += self.rec_size * num_info as usize; + + Some(SecInfo { + sec_name_off, + num_info, + data, + }) + } +} + +#[derive(Debug)] +pub(crate) struct SecInfo<'a> { + sec_name_off: u32, + num_info: u32, + data: &'a [u8], +} diff --git a/src/obj/btf/mod.rs b/src/obj/btf/mod.rs new file mode 100644 index 00000000..02c1266b --- /dev/null +++ b/src/obj/btf/mod.rs @@ -0,0 +1,7 @@ +mod btf; +mod relocation; +mod types; + +pub use btf::*; +pub(crate) use relocation::*; +pub(crate) use types::*; diff --git a/src/obj/btf/relocation.rs b/src/obj/btf/relocation.rs new file mode 100644 index 00000000..ef7ac22a --- /dev/null +++ b/src/obj/btf/relocation.rs @@ -0,0 +1,991 @@ +use std::{ + collections::HashMap, + convert::{TryFrom, TryInto}, + fs, io, mem, ptr, +}; + +use thiserror::Error; + +use crate::{ + generated::{ + bpf_core_relo, bpf_core_relo_kind::*, bpf_insn, BPF_ALU, BPF_ALU64, BPF_B, BPF_DW, BPF_H, + BPF_K, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BPF_W, BTF_INT_SIGNED, + }, + obj::{ + btf::{ + fields_are_compatible, member_bit_field_size, member_bit_offset, types_are_compatible, + BtfType, MAX_SPEC_LEN, + }, + Btf, BtfError, Object, Program, + }, +}; + +#[derive(Error, Debug)] +pub enum BtfRelocationError { + #[error("{error}")] + BtfError { + #[from] + error: BtfError, + }, + + #[error("{error}")] + IOError { + #[from] + error: io::Error, + }, + + #[error("section `{name}` not found")] + SectionNotFound { name: String }, + + #[error("invalid BTF relocation access string {access_str}")] + InvalidAccessString { access_str: String }, + + #[error("invalid instruction index #{index} referenced by relocation #{relocation_number} in section `{section_name}`")] + InvalidInstructionIndex { + index: usize, + num_instructions: usize, + section_name: String, + relocation_number: usize, + }, + + #[error("error relocating {type_name}, multiple candidate target types found with different memory layouts: {candidates:?}")] + ConflictingCandidates { + type_name: String, + candidates: Vec, + }, + + #[error("maximum nesting level reached evaluating candidate type `{}`", err_type_name(.type_name))] + MaximumNestingLevelReached { type_name: Option }, + + #[error("invalid access string `{spec}` for type `{}`: {error}", err_type_name(.type_name))] + InvalidAccessIndex { + type_name: Option, + spec: String, + index: usize, + max_index: usize, + error: String, + }, + + #[error( + "relocation #{relocation_number} of kind `{relocation_kind}` not valid for type `{type_kind}`: {error}" + )] + InvalidRelocationKindForType { + relocation_number: usize, + relocation_kind: String, + type_kind: String, + error: String, + }, + + #[error( + "instruction #{index} referenced by relocation #{relocation_number} is invalid: {error}" + )] + InvalidInstruction { + relocation_number: usize, + index: usize, + error: String, + }, +} + +fn err_type_name(name: &Option) -> String { + name.clone().unwrap_or_else(|| "[unknown name]".to_string()) +} + +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +enum RelocationKind { + FieldByteOffset = BPF_FIELD_BYTE_OFFSET, + FieldByteSize = BPF_FIELD_BYTE_SIZE, + FieldExists = BPF_FIELD_EXISTS, + FieldSigned = BPF_FIELD_SIGNED, + FieldLShift64 = BPF_FIELD_LSHIFT_U64, + FieldRShift64 = BPF_FIELD_RSHIFT_U64, + TypeIdLocal = BPF_TYPE_ID_LOCAL, + TypeIdTarget = BPF_TYPE_ID_TARGET, + TypeExists = BPF_TYPE_EXISTS, + TypeSize = BPF_TYPE_SIZE, + EnumVariantExists = BPF_ENUMVAL_EXISTS, + EnumVariantValue = BPF_ENUMVAL_VALUE, +} + +impl TryFrom for RelocationKind { + type Error = BtfError; + + fn try_from(v: u32) -> Result { + use RelocationKind::*; + + Ok(match v { + BPF_FIELD_BYTE_OFFSET => FieldByteOffset, + BPF_FIELD_BYTE_SIZE => FieldByteSize, + BPF_FIELD_EXISTS => FieldExists, + BPF_FIELD_SIGNED => FieldSigned, + BPF_FIELD_LSHIFT_U64 => FieldLShift64, + BPF_FIELD_RSHIFT_U64 => FieldRShift64, + BPF_TYPE_ID_LOCAL => TypeIdLocal, + BPF_TYPE_ID_TARGET => TypeIdTarget, + BPF_TYPE_EXISTS => TypeExists, + BPF_TYPE_SIZE => TypeSize, + BPF_ENUMVAL_EXISTS => EnumVariantExists, + BPF_ENUMVAL_VALUE => EnumVariantValue, + kind => return Err(BtfError::InvalidRelocationKind { kind }), + }) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct Relocation { + kind: RelocationKind, + ins_offset: usize, + type_id: u32, + access_str_offset: u32, + number: usize, +} + +impl Relocation { + #[allow(unused_unsafe)] + pub(crate) unsafe fn parse(data: &[u8], number: usize) -> Result { + if mem::size_of::() > data.len() { + return Err(BtfError::InvalidRelocationInfo); + } + + let rel = unsafe { ptr::read_unaligned::(data.as_ptr() as *const _) }; + + Ok(Relocation { + kind: rel.kind.try_into()?, + ins_offset: rel.insn_off as usize, + type_id: rel.type_id, + access_str_offset: rel.access_str_off, + number, + }) + } +} + +impl Object { + pub fn relocate_btf(&mut self) -> Result<(), BtfRelocationError> { + let (local_btf, btf_ext) = match (&self.btf, &self.btf_ext) { + (Some(btf), Some(btf_ext)) => (btf, btf_ext), + _ => return Ok(()), + }; + + let target_btf = fs::read("/sys/kernel/btf/vmlinux")?; + let target_btf = Btf::parse(&target_btf)?; + + let mut candidates_cache = HashMap::>::new(); + + for (sec_name_off, relos) in btf_ext.relocations() { + let section_name = local_btf.string_at(*sec_name_off)?; + + // FIXME + let parts = section_name.split("/").collect::>(); + if parts.len() < 2 { + continue; + } + let section_name = parts[1]; + let program = self.programs.get_mut(section_name).ok_or_else(|| { + BtfRelocationError::SectionNotFound { + name: section_name.to_string(), + } + })?; + + for rel in relos { + let instructions = &mut program.instructions; + let ins_index = rel.ins_offset as usize / std::mem::size_of::(); + if ins_index >= instructions.len() { + return Err(BtfRelocationError::InvalidInstructionIndex { + index: ins_index, + num_instructions: instructions.len(), + section_name: section_name.to_string(), + relocation_number: rel.number, + }); + } + + let local_ty = local_btf.type_by_id(rel.type_id)?; + let local_name = &*local_btf.type_name(local_ty)?.unwrap(); + let access_str = &*local_btf.string_at(rel.access_str_offset)?; + let local_spec = AccessSpec::new(local_btf, rel.type_id, access_str, *rel)?; + + let mut matches = match rel.kind { + RelocationKind::TypeIdLocal => Vec::new(), // we don't need to look at target types to relocate this value + _ => { + let candidates = match candidates_cache.get(&rel.type_id) { + Some(cands) => cands, + None => { + candidates_cache.insert( + rel.type_id, + find_candidates(local_ty, local_name, &target_btf)?, + ); + candidates_cache.get(&rel.type_id).unwrap() + } + }; + + let mut matches = Vec::new(); + for candidate in candidates { + if let Some(candidate_spec) = match_candidate(&local_spec, candidate)? { + let comp_rel = ComputedRelocation::new( + rel, + &local_spec, + Some(&candidate_spec), + )?; + matches.push((candidate.name.clone(), candidate_spec, comp_rel)); + } + } + + matches + } + }; + + let comp_rel = if !matches.is_empty() { + let mut matches = matches.drain(..); + let (_, target_spec, target_comp_rel) = matches.next().unwrap(); + + // if there's more than one candidate, make sure that they all resolve to the + // same value, else the relocation is ambiguous and can't be applied + let conflicts = matches + .filter_map(|(cand_name, cand_spec, cand_comp_rel)| { + if cand_spec.bit_offset != target_spec.bit_offset + || cand_comp_rel.target.value != target_comp_rel.target.value + { + Some(cand_name.clone()) + } else { + None + } + }) + .collect::>(); + if !conflicts.is_empty() { + return Err(BtfRelocationError::ConflictingCandidates { + type_name: local_name.to_string(), + candidates: conflicts, + }); + } + target_comp_rel + } else { + // there are no candidate matches and therefore no target_spec. This might mean + // that matching failed, or that the relocation can be applied looking at local + // types only + ComputedRelocation::new(rel, &local_spec, None)? + }; + + comp_rel.apply(program, rel, section_name, local_btf, &target_btf)?; + } + } + + Ok(()) + } +} + +fn find_candidates<'target>( + local_ty: &BtfType, + local_name: &str, + target_btf: &'target Btf, +) -> Result>, BtfError> { + let flavorless_name = |name: &str| name.splitn(2, "___").next().unwrap().to_string(); + + let mut candidates = Vec::new(); + let local_name = flavorless_name(local_name); + for (type_id, ty) in target_btf.types().enumerate() { + if local_ty.kind()? != ty.kind()? { + continue; + } + let name = &*target_btf.type_name(ty)?.unwrap(); + if local_name != flavorless_name(name) { + continue; + } + + candidates.push(Candidate { + name: name.to_owned(), + btf: &target_btf, + ty, + type_id: type_id as u32, + }); + } + + Ok(candidates) +} + +fn match_candidate<'target>( + local_spec: &AccessSpec, + candidate: &'target Candidate, +) -> Result>, BtfRelocationError> { + let mut target_spec = AccessSpec { + btf: candidate.btf, + root_type_id: candidate.type_id, + relocation: local_spec.relocation, + parts: Vec::new(), + accessors: Vec::new(), + bit_offset: 0, + }; + + match local_spec.relocation.kind { + RelocationKind::TypeIdLocal + | RelocationKind::TypeIdTarget + | RelocationKind::TypeExists + | RelocationKind::TypeSize => { + if types_are_compatible( + local_spec.btf, + local_spec.root_type_id, + candidate.btf, + candidate.type_id, + )? { + return Ok(Some(target_spec)); + } else { + return Ok(None); + } + } + RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => todo!(), + RelocationKind::FieldByteOffset + | RelocationKind::FieldByteSize + | RelocationKind::FieldExists + | RelocationKind::FieldSigned + | RelocationKind::FieldLShift64 + | RelocationKind::FieldRShift64 => { + let mut target_id = candidate.type_id; + for accessor in &local_spec.accessors { + target_id = candidate.btf.resolve_type(target_id)?; + + if accessor.name.is_some() { + if let Some(next_id) = match_member( + local_spec.btf, + &local_spec, + accessor, + candidate.btf, + target_id, + &mut target_spec, + )? { + target_id = next_id; + } else { + return Ok(None); + } + } else { + // array access + + if target_spec.parts.len() == MAX_SPEC_LEN { + return Err(BtfRelocationError::MaximumNestingLevelReached { + type_name: Some(candidate.name.clone()), + }); + } + + target_spec.parts.push(accessor.index); + target_spec.accessors.push(Accessor { + index: accessor.index, + type_id: target_id, + name: None, + }); + target_spec.bit_offset += + accessor.index * candidate.btf.type_size(target_id)? * 8; + } + } + } + }; + + Ok(Some(target_spec)) +} + +fn match_member<'local, 'target>( + local_btf: &Btf, + local_spec: &AccessSpec<'local>, + local_accessor: &Accessor, + target_btf: &'target Btf, + target_id: u32, + target_spec: &mut AccessSpec<'target>, +) -> Result, BtfRelocationError> { + let local_ty = local_btf.type_by_id(local_accessor.type_id)?; + let local_member = match local_ty { + BtfType::Struct(_, members) | BtfType::Union(_, members) => { + // this won't panic, bounds are checked when local_spec is built in AccessSpec::new + members[local_accessor.index] + } + _ => panic!("bug! this should only be called for structs and unions"), + }; + + let local_name = &*local_btf.string_at(local_member.name_off)?; + let target_id = target_btf.resolve_type(target_id)?; + let target_ty = target_btf.type_by_id(target_id)?; + + let target_members = match target_ty { + BtfType::Struct(ty, members) | BtfType::Union(ty, members) => members, + // not a fields type, no match + _ => return Ok(None), + }; + + for (index, target_member) in target_members.iter().enumerate() { + if target_spec.parts.len() == MAX_SPEC_LEN { + let root_ty = target_spec.btf.type_by_id(target_spec.root_type_id)?; + return Err(BtfRelocationError::MaximumNestingLevelReached { + type_name: target_spec.btf.err_type_name(root_ty), + }); + } + + let bit_offset = member_bit_offset(target_ty.info().unwrap(), target_member); + let target_name = &*target_btf.string_at(target_member.name_off)?; + + if target_name == "" { + let ret = match_member( + local_btf, + local_spec, + local_accessor, + target_btf, + target_member.type_, + target_spec, + )?; + if ret.is_some() { + target_spec.bit_offset += bit_offset; + target_spec.parts.push(index); + return Ok(ret); + } + } else if local_name == target_name { + if fields_are_compatible( + local_spec.btf, + local_member.type_, + target_btf, + target_member.type_, + )? { + target_spec.bit_offset += bit_offset; + target_spec.parts.push(index); + target_spec.accessors.push(Accessor { + type_id: target_id, + index, + name: Some(target_name.to_owned()), + }); + return Ok(Some(target_member.type_)); + } else { + return Ok(None); + } + } + } + + Ok(None) +} + +#[derive(Debug)] +struct AccessSpec<'a> { + btf: &'a Btf, + root_type_id: u32, + parts: Vec, + accessors: Vec, + relocation: Relocation, + bit_offset: usize, +} + +impl<'a> AccessSpec<'a> { + fn new( + btf: &'a Btf, + root_type_id: u32, + spec: &str, + relocation: Relocation, + ) -> Result, BtfRelocationError> { + let parts = spec + .split(":") + .map(|s| s.parse::()) + .collect::, _>>() + .map_err(|_| BtfRelocationError::InvalidAccessString { + access_str: spec.to_string(), + })?; + + let mut type_id = btf.resolve_type(root_type_id)?; + let ty = btf.type_by_id(type_id)?; + + let spec = match relocation.kind { + RelocationKind::TypeIdLocal + | RelocationKind::TypeIdTarget + | RelocationKind::TypeExists + | RelocationKind::TypeSize => { + if parts != [0] { + return Err(BtfRelocationError::InvalidAccessString { + access_str: spec.to_string(), + }); + } + AccessSpec { + btf, + root_type_id, + relocation, + parts, + accessors: Vec::new(), + bit_offset: 0, + } + } + RelocationKind::EnumVariantExists | RelocationKind::EnumVariantValue => match ty { + BtfType::Enum(_, members) => { + if parts.len() != 1 { + return Err(BtfRelocationError::InvalidAccessString { + access_str: spec.to_string(), + }); + } + let index = parts[0]; + if index >= members.len() { + return Err(BtfRelocationError::InvalidAccessIndex { + type_name: btf.err_type_name(ty), + spec: spec.to_string(), + index: index, + max_index: members.len(), + error: "tried to access nonexistant enum variant".to_string(), + }); + } + let accessors = vec![Accessor { + type_id, + index, + name: btf.type_name(ty)?.map(String::from), + }]; + + AccessSpec { + btf, + root_type_id, + relocation, + parts, + accessors, + bit_offset: 0, + } + } + _ => { + return Err(BtfRelocationError::InvalidRelocationKindForType { + relocation_number: relocation.number, + relocation_kind: format!("{:?}", relocation.kind), + type_kind: format!("{:?}", ty.kind()?.unwrap()), + error: "enum relocation on non-enum type".to_string(), + }) + } + }, + + RelocationKind::FieldByteOffset + | RelocationKind::FieldByteSize + | RelocationKind::FieldExists + | RelocationKind::FieldSigned + | RelocationKind::FieldLShift64 + | RelocationKind::FieldRShift64 => { + let mut accessors = vec![Accessor { + type_id, + index: parts[0], + name: None, + }]; + let mut bit_offset = accessors[0].index as usize * btf.type_size(type_id)?; + for index in parts.iter().skip(1).cloned() { + type_id = btf.resolve_type(type_id)?; + let ty = btf.type_by_id(type_id)?; + + use BtfType::*; + match ty { + Struct(t, members) | Union(t, members) => { + if index >= members.len() { + return Err(BtfRelocationError::InvalidAccessIndex { + type_name: btf.err_type_name(ty), + spec: spec.to_string(), + index: index, + max_index: members.len(), + error: "out of bounds struct or union access".to_string(), + }); + } + + let member = members[index]; + bit_offset += member_bit_offset(t.info, &member); + + if member.name_off != 0 { + accessors.push(Accessor { + type_id, + index, + name: Some(btf.string_at(member.name_off)?.to_string()), + }); + } + + type_id = member.type_; + } + + Array(_, array) => { + type_id = btf.resolve_type(array.type_)?; + let var_len = array.nelems == 0 && { + // an array is potentially variable length if it's the last field + // of the parent struct and has 0 elements + let parent = accessors.last().unwrap(); + let parent_ty = btf.type_by_id(parent.type_id)?; + match parent_ty { + Struct(_, members) => index == members.len() - 1, + _ => false, + } + }; + if !var_len && index >= array.nelems as usize { + return Err(BtfRelocationError::InvalidAccessIndex { + type_name: btf.err_type_name(ty), + spec: spec.to_string(), + index, + max_index: array.nelems as usize, + error: "array index out of bounds".to_string(), + }); + } + accessors.push(Accessor { + type_id, + index, + name: None, + }); + let size = btf.type_size(type_id)?; + bit_offset += index * size * 8; + } + rel_kind => { + return Err(BtfRelocationError::InvalidRelocationKindForType { + relocation_number: relocation.number, + relocation_kind: format!("{:?}", rel_kind), + type_kind: format!("{:?}", ty.kind()), + error: "field relocation on a type that doesn't have fields" + .to_string(), + }); + } + }; + } + + AccessSpec { + btf, + root_type_id, + relocation, + parts, + accessors, + bit_offset, + } + } + }; + + Ok(spec) + } +} + +#[derive(Debug)] +struct Accessor { + type_id: u32, + index: usize, + name: Option, +} + +#[derive(Debug)] +struct Candidate<'a> { + name: String, + btf: &'a Btf, + ty: &'a BtfType, + type_id: u32, +} + +#[derive(Debug)] +struct ComputedRelocation { + local: ComputedRelocationValue, + target: ComputedRelocationValue, +} + +#[derive(Debug)] +struct ComputedRelocationValue { + value: u32, + size: u32, + type_id: Option, +} + +impl ComputedRelocation { + fn new( + rel: &Relocation, + local_spec: &AccessSpec, + target_spec: Option<&AccessSpec>, + ) -> Result { + use RelocationKind::*; + let ret = match rel.kind { + FieldByteOffset | FieldByteSize | FieldExists | FieldSigned | FieldLShift64 + | FieldRShift64 => ComputedRelocation { + local: Self::compute_field_relocation(rel, Some(local_spec))?, + target: Self::compute_field_relocation(rel, target_spec)?, + }, + TypeIdLocal | TypeIdTarget | TypeExists | TypeSize => ComputedRelocation { + local: Self::compute_type_relocation(rel, local_spec, target_spec)?, + target: Self::compute_type_relocation(rel, local_spec, target_spec)?, + }, + EnumVariantExists | EnumVariantValue => ComputedRelocation { + local: Self::compute_enum_relocation(rel, Some(local_spec))?, + target: Self::compute_enum_relocation(rel, target_spec)?, + }, + }; + + Ok(ret) + } + + fn apply( + &self, + program: &mut Program, + rel: &Relocation, + section_name: &str, + local_btf: &Btf, + target_btf: &Btf, + ) -> Result<(), BtfRelocationError> { + let instructions = &mut program.instructions; + let num_instructions = instructions.len(); + let ins_index = rel.ins_offset as usize / std::mem::size_of::(); + let mut ins = + instructions + .get_mut(ins_index) + .ok_or(BtfRelocationError::InvalidInstructionIndex { + index: rel.ins_offset as usize, + num_instructions, + section_name: section_name.to_string(), + relocation_number: rel.number, + })?; + + let class = (ins.code & 0x07) as u32; + + let target_value = self.target.value; + + match class { + BPF_ALU | BPF_ALU64 => { + let src_reg = ins.src_reg(); + if src_reg != BPF_K as u8 { + return Err(BtfRelocationError::InvalidInstruction { + relocation_number: rel.number, + index: ins_index, + error: format!("invalid src_reg={:x} expected {:x}", src_reg, BPF_K), + }); + } + + ins.imm = target_value as i32; + } + BPF_LDX | BPF_ST | BPF_STX => { + if target_value > std::i16::MAX as u32 { + return Err(BtfRelocationError::InvalidInstruction { + relocation_number: rel.number, + index: ins_index, + error: format!("value `{}` overflows 16 bits offset field", target_value), + }); + } + + ins.off = target_value as i16; + + if self.local.size != self.target.size { + let local_ty = local_btf.type_by_id(self.local.type_id.unwrap())?; + let target_ty = target_btf.type_by_id(self.target.type_id.unwrap())?; + let unsigned = |info: u32| ((info >> 24) & 0x0F) & BTF_INT_SIGNED == 0; + use BtfType::*; + match (local_ty, target_ty) { + (Ptr(_), Ptr(_)) => {} + (Int(_, local_info), Int(_, target_info)) + if unsigned(*local_info) && unsigned(*target_info) => {} + _ => { + return Err(BtfRelocationError::InvalidInstruction { + relocation_number: rel.number, + index: ins_index, + error: format!( + "original type {} has size {} but target type {} has size {}", + err_type_name(&local_btf.err_type_name(local_ty)), + self.local.size, + err_type_name(&target_btf.err_type_name(target_ty)), + self.target.size, + ), + }) + } + } + + let size = match self.target.size { + 8 => BPF_DW, + 4 => BPF_W, + 2 => BPF_H, + 1 => BPF_B, + size => { + return Err(BtfRelocationError::InvalidInstruction { + relocation_number: rel.number, + index: ins_index, + error: format!("invalid target size {}", size), + }) + } + } as u8; + ins.code = ins.code & 0xE0 | size | ins.code & 0x07; + } + } + BPF_LD => { + ins.imm = target_value as i32; + let mut next_ins = instructions.get_mut(ins_index + 1).ok_or( + BtfRelocationError::InvalidInstructionIndex { + index: ins_index + 1, + num_instructions, + section_name: section_name.to_string(), + relocation_number: rel.number, + }, + )?; + + next_ins.imm = 0; + } + class => { + return Err(BtfRelocationError::InvalidInstruction { + relocation_number: rel.number, + index: ins_index, + error: format!("invalid instruction class {:x}", class), + }) + } + }; + + Ok(()) + } + + fn compute_enum_relocation( + rel: &Relocation, + spec: Option<&AccessSpec>, + ) -> Result { + use RelocationKind::*; + let value = match rel.kind { + EnumVariantExists => spec.is_some() as u32, + EnumVariantValue => { + let spec = spec.unwrap(); + let accessor = &spec.accessors[0]; + match spec.btf.type_by_id(accessor.type_id)? { + BtfType::Enum(_, variants) => variants[accessor.index].val as u32, + _ => panic!("should not be reached"), + } + } + // this function is only called for enum relocations + _ => panic!("should not be reached"), + }; + + Ok(ComputedRelocationValue { + value, + size: 0, + type_id: None, + }) + } + + fn compute_field_relocation( + rel: &Relocation, + spec: Option<&AccessSpec>, + ) -> Result { + use RelocationKind::*; + + if let FieldExists = rel.kind { + // this is the bpf_preserve_field_info(member_access, FIELD_EXISTENCE) case. If we + // managed to build a spec, it means the field exists. + return Ok(ComputedRelocationValue { + value: spec.is_some() as u32, + size: 0, + type_id: None, + }); + } + + let spec = spec.unwrap(); + let accessor = spec.accessors.last().unwrap(); + if accessor.name.is_none() { + // the last accessor is unnamed, meaning that this is an array access + return match rel.kind { + FieldByteOffset => Ok(ComputedRelocationValue { + value: (spec.bit_offset / 8) as u32, + size: spec.btf.type_size(accessor.type_id)? as u32, + type_id: Some(accessor.type_id), + }), + FieldByteSize => Ok(ComputedRelocationValue { + value: spec.btf.type_size(accessor.type_id)? as u32, + size: 0, + type_id: Some(accessor.type_id), + }), + rel_kind => { + let ty = spec.btf.type_by_id(accessor.type_id)?; + return Err(BtfRelocationError::InvalidRelocationKindForType { + relocation_number: rel.number, + relocation_kind: format!("{:?}", rel_kind), + type_kind: format!("{:?}", ty.kind()), + error: "invalid relocation kind for array type".to_string(), + }); + } + }; + } + + let ty = spec.btf.type_by_id(accessor.type_id)?; + let (ll_ty, member) = match ty { + BtfType::Struct(ty, members) | BtfType::Union(ty, members) => { + (ty, members[accessor.index]) + } + _ => { + return Err(BtfRelocationError::InvalidRelocationKindForType { + relocation_number: rel.number, + relocation_kind: format!("{:?}", rel.kind), + type_kind: format!("{:?}", ty.kind()), + error: "field relocation on a type that doesn't have fields".to_string(), + }); + } + }; + + let bit_off = spec.bit_offset as u32; + let member_type_id = spec.btf.resolve_type(member.type_)?; + let member_ty = spec.btf.type_by_id(member_type_id)?; + let ll_member_ty = member_ty.btf_type().unwrap(); + + let mut byte_size; + let mut byte_off; + let mut bit_size = member_bit_field_size(ll_ty, &member) as u32; + let is_bitfield = bit_size > 0; + if is_bitfield { + // find out the smallest int size to load the bitfield + byte_size = unsafe { ll_member_ty.__bindgen_anon_1.size }; + byte_off = bit_off / 8 / byte_size * byte_size; + while bit_off + bit_size - byte_off * 8 > byte_size * 8 { + if byte_size >= 8 { + // the bitfield is larger than 8 bytes!? + return Err(BtfError::InvalidTypeInfo)?; + } + byte_size *= 2; + byte_off = bit_off / 8 / byte_size * byte_size; + } + } else { + byte_size = spec.btf.type_size(member_type_id)? as u32; + bit_size = byte_size * 8; + byte_off = spec.bit_offset as u32 / 8; + } + + let mut value = ComputedRelocationValue { + value: 0, + size: 0, + type_id: None, + }; + match rel.kind { + FieldByteOffset => { + value.value = byte_off; + if !is_bitfield { + value.size = byte_size; + value.type_id = Some(member_type_id); + } + } + FieldByteSize => { + value.value = byte_size; + } + FieldSigned => match member_ty { + BtfType::Enum(_, _) => value.value = 1, + BtfType::Int(_, i) => value.value = ((i >> 24) & 0x0F) & BTF_INT_SIGNED, + _ => (), + }, + #[cfg(target_endian = "little")] + FieldLShift64 => { + value.value = 64 - (bit_off + bit_size - byte_off * 8); + } + #[cfg(target_endian = "big")] + FieldLShift64 => { + value.value = (8 - byte_size) * 8 + (bit_off - byte_off * 8); + } + FieldRShift64 => { + value.value = 64 - bit_size; + } + FieldExists // this is handled at the start of the function + | _ => panic!("bug! this should not be reached"), + } + + Ok(value) + } + + fn compute_type_relocation( + rel: &Relocation, + local_spec: &AccessSpec, + target_spec: Option<&AccessSpec>, + ) -> Result { + use RelocationKind::*; + let value = match rel.kind { + TypeIdLocal => local_spec.root_type_id, + _ => match target_spec { + Some(target_spec) => match rel.kind { + TypeIdTarget => target_spec.root_type_id, + TypeExists => 1, + TypeSize => target_spec.btf.type_size(target_spec.root_type_id)? as u32, + _ => panic!("bug! this should not be reached"), + }, + // FIXME in the case of TypeIdTarget and TypeSize this should probably fail the + // relocation... + None => 0, + }, + }; + + Ok(ComputedRelocationValue { + value, + size: 0, + type_id: None, + }) + } +} diff --git a/src/obj/btf/types.rs b/src/obj/btf/types.rs new file mode 100644 index 00000000..2b587d42 --- /dev/null +++ b/src/obj/btf/types.rs @@ -0,0 +1,388 @@ +use std::{ + convert::{TryFrom, TryInto}, + mem, ptr, +}; + +use crate::{ + generated::{ + btf_array, btf_enum, btf_member, btf_param, btf_type, btf_type__bindgen_ty_1, btf_var, + btf_var_secinfo, BTF_KIND_ARRAY, BTF_KIND_CONST, BTF_KIND_DATASEC, BTF_KIND_ENUM, + BTF_KIND_FUNC, BTF_KIND_FUNC_PROTO, BTF_KIND_FWD, BTF_KIND_INT, BTF_KIND_PTR, + BTF_KIND_RESTRICT, BTF_KIND_STRUCT, BTF_KIND_TYPEDEF, BTF_KIND_UNION, BTF_KIND_UNKN, + BTF_KIND_VAR, BTF_KIND_VOLATILE, + }, + obj::btf::{Btf, BtfError, MAX_RESOLVE_DEPTH}, +}; + +unsafe impl object::pod::Pod for btf_type {} + +#[derive(Clone, Debug)] +pub(crate) enum BtfType { + Unknown, + Fwd(btf_type), + Const(btf_type), + Volatile(btf_type), + Restrict(btf_type), + Ptr(btf_type), + Typedef(btf_type), + Func(btf_type), + Int(btf_type, u32), + Enum(btf_type, Vec), + Array(btf_type, btf_array), + Struct(btf_type, Vec), + Union(btf_type, Vec), + FuncProto(btf_type, Vec), + Var(btf_type, btf_var), + DataSec(btf_type, Vec), +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[repr(u32)] +pub(crate) enum BtfKind { + Unknown = BTF_KIND_UNKN, + Int = BTF_KIND_INT, + Ptr = BTF_KIND_PTR, + Array = BTF_KIND_ARRAY, + Struct = BTF_KIND_STRUCT, + Union = BTF_KIND_UNION, + Enum = BTF_KIND_ENUM, + Fwd = BTF_KIND_FWD, + Typedef = BTF_KIND_TYPEDEF, + Volatile = BTF_KIND_VOLATILE, + Const = BTF_KIND_CONST, + Restrict = BTF_KIND_RESTRICT, + Func = BTF_KIND_FUNC, + FuncProto = BTF_KIND_FUNC_PROTO, + Var = BTF_KIND_VAR, + DataSec = BTF_KIND_DATASEC, +} + +impl TryFrom for BtfKind { + type Error = BtfError; + + fn try_from(v: u32) -> Result { + use BtfKind::*; + Ok(match v { + BTF_KIND_UNKN => Unknown, + BTF_KIND_INT => Int, + BTF_KIND_PTR => Ptr, + BTF_KIND_ARRAY => Array, + BTF_KIND_STRUCT => Struct, + BTF_KIND_UNION => Union, + BTF_KIND_ENUM => Enum, + BTF_KIND_FWD => Fwd, + BTF_KIND_TYPEDEF => Typedef, + BTF_KIND_VOLATILE => Volatile, + BTF_KIND_CONST => Const, + BTF_KIND_RESTRICT => Restrict, + BTF_KIND_FUNC => Func, + BTF_KIND_FUNC_PROTO => FuncProto, + BTF_KIND_VAR => Var, + BTF_KIND_DATASEC => DataSec, + kind => return Err(BtfError::InvalidTypeKind { kind }), + }) + } +} + +unsafe fn read(data: &[u8]) -> Result { + if mem::size_of::() > data.len() { + return Err(BtfError::InvalidTypeInfo); + } + + Ok(ptr::read_unaligned::(data.as_ptr() as *const T)) +} + +unsafe fn read_array(data: &[u8], len: usize) -> Result, BtfError> { + if mem::size_of::() * len > data.len() { + return Err(BtfError::InvalidTypeInfo); + } + + Ok((0..len) + .map(|i| { + ptr::read_unaligned::((data.as_ptr() as usize + i * mem::size_of::()) as *const T) + }) + .collect::>()) +} + +impl BtfType { + #[allow(unused_unsafe)] + pub(crate) unsafe fn read(data: &[u8]) -> Result { + let ty = unsafe { read::(data)? }; + let data = &data[mem::size_of::()..]; + + let vlen = type_vlen(&ty) as usize; + use BtfType::*; + Ok(match type_kind(&ty)? { + BtfKind::Unknown => Unknown, + BtfKind::Fwd => Fwd(ty), + BtfKind::Const => Const(ty), + BtfKind::Volatile => Volatile(ty), + BtfKind::Restrict => Restrict(ty), + BtfKind::Ptr => Ptr(ty), + BtfKind::Typedef => Typedef(ty), + BtfKind::Func => Func(ty), + BtfKind::Int => { + // FIXME: endianness + if mem::size_of::() > data.len() { + return Err(BtfError::InvalidTypeInfo); + } + Int( + ty, + u32::from_ne_bytes(data[..mem::size_of::()].try_into().unwrap()), + ) + } + BtfKind::Enum => Enum(ty, unsafe { read_array(data, vlen)? }), + BtfKind::Array => Array(ty, unsafe { read(data)? }), + BtfKind::Struct => Struct(ty, unsafe { read_array(data, vlen)? }), + BtfKind::Union => Union(ty, unsafe { read_array(data, vlen)? }), + BtfKind::FuncProto => FuncProto(ty, unsafe { read_array(data, vlen)? }), + BtfKind::Var => Var(ty, unsafe { read(data)? }), + BtfKind::DataSec => DataSec(ty, unsafe { read_array(data, vlen)? }), + }) + } + + pub(crate) fn type_info_size(&self) -> usize { + let ty_size = mem::size_of::(); + + use BtfType::*; + match self { + Unknown => 0, + Fwd(_) | Const(_) | Volatile(_) | Restrict(_) | Ptr(_) | Typedef(_) | Func(_) => { + ty_size + } + Int(_, _) => ty_size + mem::size_of::(), + Enum(ty, _) => ty_size + type_vlen(ty) * mem::size_of::(), + Array(_, _) => ty_size + mem::size_of::(), + Struct(ty, _) => ty_size + type_vlen(ty) * mem::size_of::(), + Union(ty, _) => ty_size + type_vlen(ty) * mem::size_of::(), + FuncProto(ty, _) => ty_size + type_vlen(ty) * mem::size_of::(), + Var(_, _) => ty_size + mem::size_of::(), + DataSec(ty, _) => ty_size + type_vlen(ty) * mem::size_of::(), + } + } + + pub(crate) fn btf_type(&self) -> Option<&btf_type> { + use BtfType::*; + Some(match self { + Unknown => return None, + Fwd(ty) => ty, + Const(ty) => ty, + Volatile(ty) => ty, + Restrict(ty) => ty, + Ptr(ty) => ty, + Typedef(ty) => ty, + Func(ty) => ty, + Int(ty, _) => ty, + Enum(ty, _) => ty, + Array(ty, _) => ty, + Struct(ty, _) => ty, + Union(ty, _) => ty, + FuncProto(ty, _) => ty, + Var(ty, _) => ty, + DataSec(ty, _) => ty, + }) + } + + pub(crate) fn info(&self) -> Option { + self.btf_type().map(|ty| ty.info) + } + + pub(crate) fn name_offset(&self) -> Option { + self.btf_type().map(|ty| ty.name_off) + } + + pub(crate) fn kind(&self) -> Result, BtfError> { + self.btf_type().map(type_kind).transpose() + } + + pub(crate) fn is_composite(&self) -> bool { + match self { + BtfType::Struct(_, _) | BtfType::Union(_, _) => true, + _ => false, + } + } +} + +fn type_kind(ty: &btf_type) -> Result { + ((ty.info >> 24) & 0x0F).try_into() +} + +fn type_vlen(ty: &btf_type) -> usize { + (ty.info & 0xFFFF) as usize +} + +pub(crate) fn member_bit_offset(info: u32, member: &btf_member) -> usize { + let k_flag = info >> 31 == 1; + let bit_offset = if k_flag { + member.offset & 0xFFFFFF + } else { + member.offset + }; + + bit_offset as usize +} + +pub(crate) fn member_bit_field_size(ty: &btf_type, member: &btf_member) -> usize { + let k_flag = (ty.info >> 31) == 1; + let size = if k_flag { member.offset >> 24 } else { 0 }; + + size as usize +} + +pub(crate) fn types_are_compatible( + local_btf: &Btf, + root_local_id: u32, + target_btf: &Btf, + root_target_id: u32, +) -> Result { + let mut local_id = root_local_id; + let mut target_id = root_target_id; + let local_ty = local_btf.type_by_id(local_id)?; + let target_ty = target_btf.type_by_id(target_id)?; + + if local_ty.kind()? != target_ty.kind()? { + return Ok(false); + } + + for _ in 0..MAX_RESOLVE_DEPTH { + local_id = local_btf.resolve_type(local_id)?; + target_id = target_btf.resolve_type(target_id)?; + let local_ty = local_btf.type_by_id(local_id)?; + let target_ty = target_btf.type_by_id(target_id)?; + + if local_ty.kind()? != target_ty.kind()? { + return Ok(false); + } + + use BtfType::*; + match local_ty { + Unknown | Struct(_, _) | Union(_, _) | Enum(_, _) | Fwd(_) => return Ok(true), + Int(_, local_off) => { + if let Int(_, target_off) = target_ty { + return Ok(*local_off == 0 && *target_off == 0); + } + } + Ptr(l_ty) => { + if let Ptr(t_ty) = target_ty { + // Safety: union + unsafe { + local_id = l_ty.__bindgen_anon_1.type_; + target_id = t_ty.__bindgen_anon_1.type_; + } + continue; + } + } + Array(l_ty, _) => { + if let Array(t_ty, _) = target_ty { + // Safety: union + unsafe { + local_id = l_ty.__bindgen_anon_1.type_; + target_id = t_ty.__bindgen_anon_1.type_; + } + continue; + } + } + FuncProto(l_ty, l_params) => { + if let FuncProto(t_ty, t_params) = target_ty { + if l_params.len() != t_params.len() { + return Ok(false); + } + + for (l_param, t_param) in l_params.iter().zip(t_params.iter()) { + let local_id = local_btf.resolve_type(l_param.type_)?; + let target_id = target_btf.resolve_type(t_param.type_)?; + if !types_are_compatible(local_btf, local_id, target_btf, target_id)? { + return Ok(false); + } + } + + // Safety: union + unsafe { + local_id = l_ty.__bindgen_anon_1.type_; + target_id = t_ty.__bindgen_anon_1.type_; + } + continue; + } + } + _ => panic!("this shouldn't be reached"), + } + } + + Err(BtfError::MaximumTypeDepthReached { type_id: local_id }) +} + +pub(crate) fn fields_are_compatible( + local_btf: &Btf, + mut local_id: u32, + target_btf: &Btf, + mut target_id: u32, +) -> Result { + for _ in 0..MAX_RESOLVE_DEPTH { + local_id = local_btf.resolve_type(local_id)?; + target_id = target_btf.resolve_type(target_id)?; + let local_ty = local_btf.type_by_id(local_id)?; + let target_ty = target_btf.type_by_id(target_id)?; + + if local_ty.is_composite() && target_ty.is_composite() { + return Ok(true); + } + + if local_ty.kind()? != target_ty.kind()? { + return Ok(false); + } + + use BtfType::*; + match local_ty { + Fwd(_) | Enum(_, _) => { + let flavorless_name = + |name: &str| name.splitn(2, "___").next().unwrap().to_string(); + + let local_name = flavorless_name(&*local_btf.type_name(local_ty)?.unwrap()); + let target_name = flavorless_name(&*target_btf.type_name(target_ty)?.unwrap()); + + return Ok(local_name == target_name); + } + Int(_, local_off) => { + let local_off = (local_off >> 16) & 0xFF; + if let Int(_, target_off) = target_ty { + let target_off = (target_off >> 16) & 0xFF; + return Ok(local_off == 0 && target_off == 0); + } + } + Ptr(_) => return Ok(true), + Array(l_ty, _) => { + if let Array(t_ty, _) = target_ty { + // Safety: union + unsafe { + local_id = l_ty.__bindgen_anon_1.type_; + target_id = t_ty.__bindgen_anon_1.type_; + } + continue; + } + } + _ => panic!("this shouldn't be reached"), + } + } + + Err(BtfError::MaximumTypeDepthReached { type_id: local_id }) +} + +impl std::fmt::Debug for btf_type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("btf_type") + .field("name_off", &self.name_off) + .field("info", &self.info) + .field("__bindgen_anon_1", &self.__bindgen_anon_1) + .finish() + } +} + +impl std::fmt::Debug for btf_type__bindgen_ty_1 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Safety: union + f.debug_struct("btf_type__bindgen_ty_1") + .field("size", unsafe { &self.size }) + .field("type_", unsafe { &self.type_ }) + .finish() + } +} diff --git a/src/obj/mod.rs b/src/obj/mod.rs index 6f859413..edc2beaf 100644 --- a/src/obj/mod.rs +++ b/src/obj/mod.rs @@ -1,3 +1,4 @@ +mod btf; mod relocation; use object::{ @@ -14,7 +15,8 @@ use std::{ }; use thiserror::Error; -pub use self::relocation::{relocate, RelocationError}; +use btf::{Btf, BtfError, BtfExt}; +pub use relocation::*; use crate::{ bpf_map_def, @@ -24,11 +26,13 @@ use crate::{ const KERNEL_VERSION_ANY: u32 = 0xFFFF_FFFE; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct Object { pub(crate) endianness: Endianness, pub license: CString, pub kernel_version: KernelVersion, + pub btf: Option, + pub btf_ext: Option, pub(crate) maps: HashMap, pub(crate) programs: HashMap, pub(crate) relocations: HashMap>, @@ -100,6 +104,8 @@ impl Object { endianness: endianness.into(), license, kernel_version, + btf: None, + btf_ext: None, maps: HashMap::new(), programs: HashMap::new(), relocations: HashMap::new(), @@ -135,6 +141,9 @@ pub enum ParseError { source: object::read::Error, }, + #[error("Error parsing BTF: {0}")] + BTF(#[from] BtfError), + #[error("no license specified")] MissingLicense, @@ -193,7 +202,7 @@ impl<'data, 'file, 's> TryFrom<&'s Section<'data, 'file>> for BPFSection<'s> { relocations: section .relocations() .map(|(offset, r)| { - Ok(Relocation { + Ok::<_, ParseError>(Relocation { kind: r.kind(), target: r.target(), addend: r.addend(), @@ -321,6 +330,17 @@ fn parse_program(bpf: &Object, section: &BPFSection, ty: &str) -> Result Result<(), BtfError> { + obj.btf = Some(Btf::parse(section.data)?); + + Ok(()) +} + +fn parse_btf_ext(obj: &mut Object, section: &BPFSection) -> Result<(), BtfError> { + obj.btf_ext = Some(BtfExt::parse(section.data)?); + Ok(()) +} + fn parse_section(bpf: &mut Object, section: BPFSection) -> Result<(), ParseError> { let parts = section.name.split("/").collect::>(); @@ -329,6 +349,8 @@ fn parse_section(bpf: &mut Object, section: BPFSection) -> Result<(), ParseError bpf.maps .insert(name.to_string(), parse_map(§ion, name)?); } + &[".BTF"] => parse_btf(bpf, §ion)?, + &[".BTF.ext"] => parse_btf_ext(bpf, §ion)?, &["maps", name] => { bpf.maps .insert(name.to_string(), parse_map(§ion, name)?); diff --git a/src/obj/relocation.rs b/src/obj/relocation.rs index f1148806..1d225df3 100644 --- a/src/obj/relocation.rs +++ b/src/obj/relocation.rs @@ -1,15 +1,15 @@ -use std::collections::HashMap; +use std::{collections::HashMap, io}; use object::{RelocationKind, RelocationTarget, SectionIndex}; use thiserror::Error; -use super::Object; use crate::{ generated::{bpf_insn, BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_VALUE}, maps::Map, + obj::{btf::BtfRelocationError, Object}, }; -#[derive(Debug, Clone, Error)] +#[derive(Debug, Error)] pub enum RelocationError { #[error("unknown symbol, index `{index}`")] UnknownSymbol { index: usize }, @@ -19,7 +19,7 @@ pub enum RelocationError { #[error("section `{section_index}` not found, referenced by symbol `{}`", .symbol_name.clone().unwrap_or_else(|| .symbol_index.to_string()))] - RelocationSectionNotFound { + SectionNotFound { section_index: usize, symbol_index: usize, symbol_name: Option, @@ -28,8 +28,24 @@ pub enum RelocationError { #[error("the map `{name}` at section `{section_index}` has not been created")] MapNotCreated { section_index: usize, name: String }, - #[error("invalid relocation offset `{offset}`")] - InvalidRelocationOffset { offset: u64 }, + #[error("invalid instruction index `{index}` referenced by relocation #{relocation_number}")] + InvalidInstructionIndex { + index: usize, + num_instructions: usize, + relocation_number: usize, + }, + + #[error("BTF error: {error}")] + BtfRelocationError { + #[from] + error: BtfRelocationError, + }, + + #[error("IO error: {io_error}")] + IO { + #[from] + io_error: io::Error, + }, } #[derive(Debug, Copy, Clone)] @@ -47,62 +63,73 @@ pub(crate) struct Symbol { pub(crate) address: u64, } -pub fn relocate(obj: &mut Object, maps: &[Map]) -> Result<(), RelocationError> { - let maps_by_section = maps - .iter() - .map(|map| (map.obj.section_index, map)) - .collect::>(); - - for program in obj.programs.values_mut() { - if let Some(relocations) = obj.relocations.get(&program.section_index) { - for rel in relocations { - match rel.target { - RelocationTarget::Symbol(index) => { - let sym = obj - .symbol_table - .get(&index) - .ok_or(RelocationError::UnknownSymbol { index: index.0 })?; - - let section_index = sym - .section_index - .ok_or(RelocationError::UnknownSymbolSection { index: index.0 })?; - - let map = maps_by_section.get(§ion_index.0).ok_or( - RelocationError::RelocationSectionNotFound { - symbol_index: index.0, - symbol_name: sym.name.clone(), +impl Object { + pub fn relocate(&mut self, maps: &[Map]) -> Result<(), RelocationError> { + self.relocate_maps(maps)?; + self.relocate_btf()?; + + Ok(()) + } + + pub fn relocate_maps(&mut self, maps: &[Map]) -> Result<(), RelocationError> { + let maps_by_section = maps + .iter() + .map(|map| (map.obj.section_index, map)) + .collect::>(); + + for program in self.programs.values_mut() { + if let Some(relocations) = self.relocations.get(&program.section_index) { + for (rel_n, rel) in relocations.iter().enumerate() { + match rel.target { + RelocationTarget::Symbol(index) => { + let sym = self + .symbol_table + .get(&index) + .ok_or(RelocationError::UnknownSymbol { index: index.0 })?; + + let section_index = sym + .section_index + .ok_or(RelocationError::UnknownSymbolSection { index: index.0 })?; + + let map = maps_by_section.get(§ion_index.0).ok_or( + RelocationError::SectionNotFound { + symbol_index: index.0, + symbol_name: sym.name.clone(), + section_index: section_index.0, + }, + )?; + + let map_fd = map.fd.ok_or_else(|| RelocationError::MapNotCreated { + name: map.obj.name.clone(), section_index: section_index.0, - }, - )?; - - let map_fd = map.fd.ok_or_else(|| RelocationError::MapNotCreated { - name: map.obj.name.clone(), - section_index: section_index.0, - })?; - - let instructions = &mut program.instructions; - let ins_index = - (rel.offset / std::mem::size_of::() as u64) as usize; - if ins_index >= instructions.len() { - return Err(RelocationError::InvalidRelocationOffset { - offset: rel.offset, - }); - } - if !map.obj.data.is_empty() { - instructions[ins_index].set_src_reg(BPF_PSEUDO_MAP_VALUE as u8); - instructions[ins_index + 1].imm = - instructions[ins_index].imm + sym.address as i32; - } else { - instructions[ins_index].set_src_reg(BPF_PSEUDO_MAP_FD as u8); + })?; + + let instructions = &mut program.instructions; + let ins_index = + (rel.offset / std::mem::size_of::() as u64) as usize; + if ins_index >= instructions.len() { + return Err(RelocationError::InvalidInstructionIndex { + index: ins_index, + num_instructions: instructions.len(), + relocation_number: rel_n, + }); + } + if !map.obj.data.is_empty() { + instructions[ins_index].set_src_reg(BPF_PSEUDO_MAP_VALUE as u8); + instructions[ins_index + 1].imm = + instructions[ins_index].imm + sym.address as i32; + } else { + instructions[ins_index].set_src_reg(BPF_PSEUDO_MAP_FD as u8); + } + instructions[ins_index].imm = map_fd; } - instructions[ins_index].imm = map_fd; + RelocationTarget::Section(_index) => {} + RelocationTarget::Absolute => todo!(), } - RelocationTarget::Section(_index) => {} - RelocationTarget::Absolute => todo!(), } } } - } - Ok(()) + Ok(()) + } }