diff --git a/.gitignore b/.gitignore index 1afdc5c..3585f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ target/ -images/ \ No newline at end of file +images/ diff --git a/Makefile b/Makefile index 4895b23..074ab78 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ WORKSPACE := /home/rocker USER_NAME := rocker -PASSWORD = asd123 +GROUP_NAME := $(USER_NAME) +UID = 7788 +GID = 7788 +PASSWORD = quant_u2Oh0Go9J76aQb3h7Ybgapw@yanguangshaonian build: cargo build --release @@ -8,14 +11,14 @@ build: # 创建WORKSPACE 对应的文件夹 install: @# 创建一个普通用户 - -@echo "Creating user $(USER_NAME)" - -@sudo useradd -m -s /bin/bash $(USER_NAME) + -@sudo useradd -m -s /bin/bash $(USER_NAME) -u $(UID) + @# 将用户添加到用户组 + -@sudo usermod -g $(GID) -aG $(GROUP_NAME) $(USER_NAME) @# 设置密码 - -@echo "Setting password for $(USER_NAME)" -@echo "$(USER_NAME):$(PASSWORD)" | sudo chpasswd - @# 授权sudo - -@echo "Authorizing $(USER_NAME) to use sudo" - -@sudo sh -c "echo '$(USER_NAME) ALL=(ALL) ALL' >> /etc/sudoers" +# @# 授权sudo +# -@echo "Authorizing $(USER_NAME) to use sudo" +# -@sudo sh -c "echo '$(USER_NAME) ALL=(ALL) ALL' >> /etc/sudoers" mkdir -p $(WORKSPACE)/images mkdir -p $(WORKSPACE)/volumes @@ -24,12 +27,23 @@ install: cp images/* $(WORKSPACE)/images/ chown -R rocker:rocker $(WORKSPACE) + -mkdir /sys/fs/cgroup/rocker_1 + -mkdir /sys/fs/cgroup/rocker_2 + -mkdir /sys/fs/cgroup/rocker_3 + clean: + -./target/debug/rocker --rm all -rocker --rm all -rm -rf $(WORKSPACE)/* -rm -rf /usr/bin/rocker + -rmdir /sys/fs/cgroup/rocker_1/* + -rmdir /sys/fs/cgroup/rocker_2/* + -rmdir /sys/fs/cgroup/rocker_3/* + -rmdir /sys/fs/cgroup/rocker_* + -@echo "Cleaning up" -@sudo userdel -r $(USER_NAME) + -@sudo groupdel $(GROUP_NAME) -@sudo sed -i "/^$(USER_NAME)/d" /etc/sudoers diff --git a/src/cgroup.rs b/src/cgroup.rs index e69de29..0ed3d31 100644 --- a/src/cgroup.rs +++ b/src/cgroup.rs @@ -0,0 +1,92 @@ +use std::fmt::{Display, Formatter}; +use nix::sys::signal::kill; +use nix::unistd::Pid; +use nix::sys::signal::Signal; +use std::path::Path; +use crate::{create_dir, get_container_info, ContainerInfo}; +use crate::error::Result; + +static CGROUP_PATH: &str = "/sys/fs/cgroup/rocker"; + +pub enum CgroupLevel { + One, + Two, + Three, +} + +impl From<&str> for CgroupLevel { + fn from(value: &str) -> Self { + match value { + "1" => CgroupLevel::One, + "2" => CgroupLevel::Two, + "3" => CgroupLevel::Three, + _ => todo!("invalid cgroup level"), + } + } +} + +impl Display for CgroupLevel { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::One => write!(f, "1"), + Self::Two => write!(f, "2"), + Self::Three => write!(f, "3"), + } + } +} + +impl Default for CgroupLevel { + fn default() -> Self { + CgroupLevel::One + } +} + +pub struct Cgroup { + pub procs: Vec +} + +impl From<&ContainerInfo> for Cgroup { + fn from(container_inro: &ContainerInfo) -> Self { + // let container_inro = get_container_info(container_id).unwrap(); + let l = CgroupLevel::from(container_inro.cgroup_level.as_str()); + let cgroup_path = format!("{CGROUP_PATH}_{l}/{}", container_inro.id); + let mut v = vec![]; + for pid in std::fs::read_to_string(format!("{cgroup_path}/cgroup.procs")).unwrap_or_default().lines() { + let pid = pid.parse::().unwrap(); + v.push(pid); + } + Self { + procs: v + } + } +} + +impl Cgroup { + pub fn create(container_id: &str, pid: i32, l: CgroupLevel) -> Result<()> { + let cgroup_path = format!("{CGROUP_PATH}_{l}/{container_id}"); + create_dir(&cgroup_path, false)?; + // 打开 cgroup.procs, 写入pid + std::fs::write(format!("{cgroup_path}/cgroup.procs"), pid.to_string())?; + Ok(()) + } + + pub fn remove(container_id: &str) { + let container_inro = get_container_info(container_id).unwrap(); + let _self = Self::from(&container_inro); + for pid in _self.procs { + let _ = kill(Pid::from_raw(pid), Signal::SIGKILL); + let pid_path = Path::new("/proc").join(pid.to_string()); + (0..100).any(|i|{ + std::thread::sleep(std::time::Duration::from_millis(i)); + pid_path.exists() == false + }); + } + let l = container_inro.cgroup_level; + let container_group_path = format!("{CGROUP_PATH}_{l}/{container_id}"); + let _ = std::fs::remove_dir(&container_group_path); + println!("删除cgroup {container_group_path}") + } +} + + + diff --git a/src/main.rs b/src/main.rs index 2465807..bdb8195 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,3 @@ -use std::fs::{remove_dir_all, OpenOptions}; -use std::os::unix::fs::OpenOptionsExt; use std::{io, fs, fmt, os, path, process, time}; use fmt::Display; use os::fd::{AsFd, AsRawFd}; @@ -8,8 +6,8 @@ use path::{Path, PathBuf}; use std::sync::OnceLock; use nix::sched::{clone, CloneCb, CloneFlags, setns}; -use nix::sys::{signal::{kill, Signal}, wait::{waitpid, WaitPidFlag}}; -use nix::unistd::{dup2, getpgid, pivot_root, setgid, setgroups, sethostname, setuid, Gid, Pid, Uid, User}; +use nix::sys::wait::{waitpid, WaitPidFlag}; +use nix::unistd::{dup2, pivot_root, setgid, setgroups, sethostname, setuid, Gid, Pid, Uid, User}; use nix::mount::{mount, MntFlags, MsFlags, umount2}; use uuid; use toml; @@ -20,6 +18,8 @@ use error::{Result, RockerError}; mod error; use network::{create_network, remove_network}; mod network; +use cgroup::{Cgroup, CgroupLevel}; +mod cgroup; static WORKSPACE: &str = "/home/rocker"; static USER_NAME: &str = "rocker"; @@ -73,6 +73,9 @@ struct RockerArgs { // --psa #[arg(long)] psa: bool, + // --root + #[arg(long)] + root: bool, // rm "container_id_1, container_id_2, container_id_3" #[arg(long)] @@ -86,6 +89,8 @@ struct RockerArgs { #[arg(long)] restart: Option, + #[arg(long)] + cgroup: Option, } macro_rules! rocker_println { @@ -125,11 +130,11 @@ fn extend_image(image_name: &String) -> Result { if out.status.success() { println!("解压缩完毕: {image_name:?}"); // 把解压缩后的文件全部给 rocker:rocker - let _ = process::Command::new("chown") - .arg("-R") - .arg("rocker:rocker") - .arg(volume_path_str) - .output()?; + // let _ = process::Command::new("chown") + // .arg("-R") + // .arg("rocker:rocker") + // .arg(volume_path_str) + // .output()?; Ok(volume_path) } else { // 删除 volume_path @@ -164,7 +169,13 @@ fn init_container_overlay>(volume_path: P, upper_path: P, merged_ } fn init_container_dev>(container_merged_path: P) -> Result<()> { - let dev_vec = [("urandom", 0), ("random", 0), ("null", 0), ("zero", 0), ("shm", 1)]; + let dev_vec = [ + ("urandom", 0), + ("random", 0), + ("null", 0), + ("zero", 0), + // ("shm", 1), + ]; for (dev, tp) in dev_vec { let host_dev_path = format!("/dev/{dev}"); @@ -310,7 +321,7 @@ fn init_container_user(uid: Uid, gid: Gid) -> Result<()>{ Ok(()) } -fn create_dir>(path: P, is_any:bool) -> Result<()> { +pub fn create_dir>(path: P, is_any:bool) -> Result<()> { fs::create_dir_all(&path)?; if is_any { fs::set_permissions(&path, PermissionsExt::from_mode(0o777))?; @@ -348,7 +359,7 @@ fn create_pause(container_root_pause_path: &Path) -> Result<()> { Ok(()) } -fn start(container_info: &ContainerInfo, cb: CloneCb, clong_flags: CloneFlags, container_merged_pause_path: &PathBuf) -> Result{ +fn start(container_info: &ContainerInfo, cb: CloneCb, clong_flags: CloneFlags, container_merged_pause_path: &PathBuf) -> Result<()>{ match unsafe {clone(cb, STACK.as_mut_slice(), clong_flags, None)} { Ok(child_pid) => { { // 执行成功就保存吧~ @@ -359,18 +370,18 @@ fn start(container_info: &ContainerInfo, cb: CloneCb, clong_flags: CloneFlags, c // exec之前的步骤 create_network(&container_info.id, child_pid.as_raw()); + Cgroup::create(&container_info.id, child_pid.as_raw(), CgroupLevel::from(container_info.cgroup_level.as_str()))?; + // 删除 pause标志文件, 解开阻塞, 使其执行exec - for _ in 0..500 { - std::thread::sleep(std::time::Duration::from_millis(10)); - if std::fs::remove_file(container_merged_pause_path).is_ok() { - break; - } - } + (0..100).any(|i| { + std::thread::sleep(std::time::Duration::from_millis(i)); + std::fs::remove_file(container_merged_pause_path).is_ok() + }); // 检查是否执行exec了 let main_exe = std::env::current_exe()?; - for _ in 0..500 { - std::thread::sleep(std::time::Duration::from_millis(10)); + for i in 0..100 { + std::thread::sleep(std::time::Duration::from_millis(i)); if let Ok(true) = check_container_is_running(&child_pid, &main_exe) { break; } @@ -388,7 +399,7 @@ fn start(container_info: &ContainerInfo, cb: CloneCb, clong_flags: CloneFlags, c } } } - Ok(child_pid.as_raw()) + Ok(()) } Err(e) => { Err(RockerError::OtherError(format!("clone err: {e}"))) @@ -396,7 +407,7 @@ fn start(container_info: &ContainerInfo, cb: CloneCb, clong_flags: CloneFlags, c } } -fn run_container(container_info: &ContainerInfo, is_exec_cmd: Option<&String>) -> Result { +fn run_container(container_info: &ContainerInfo, is_exec_cmd: Option<&String>) -> Result<()> { // 禁止同时wait和log if container_info.wait && container_info.log { return Err(RockerError::OtherError("--wait/--log 禁止同时使用".to_string())); @@ -425,7 +436,9 @@ fn run_container(container_info: &ContainerInfo, is_exec_cmd: Option<&String>) - let _cb = || { init_exec_ns(container_info.pid).unwrap(); let env_vec = get_env_vec(&Default::default()).unwrap(); - init_container_user(rocker_uid, rocker_gid).unwrap(); + if container_info.root == false { + init_container_user(rocker_uid, rocker_gid).unwrap(); + } init_container_env(env_vec).unwrap(); create_pause(container_root_pause_path).unwrap(); @@ -477,8 +490,10 @@ fn run_container(container_info: &ContainerInfo, is_exec_cmd: Option<&String>) - } // 设置用户 - init_container_user(rocker_uid, rocker_gid).unwrap(); - + if container_info.root == false { + init_container_user(rocker_uid, rocker_gid).unwrap(); + } + // 暂停等待外部主进程设置网络设备 create_pause(&container_root_pause_path).unwrap(); while container_root_pause_path.exists() { @@ -514,7 +529,6 @@ impl Display for ContainerStatus { Self::STOP => write!(f, "🔴"), } } - } @@ -530,6 +544,9 @@ struct ContainerInfo { env: String, // a=1,b=2,c=3 或者 env文件路径 log: bool, wait: bool, + root: bool, + + cgroup_level: String, // one/two/three } impl ContainerInfo { @@ -551,39 +568,25 @@ impl Display for ContainerInfo { } } -fn save_container_info(args: &RockerArgs, container_id: &String, pid: i32) -> Result<()> { - let container_info_path = Path::new(WORKSPACE).join("containers").join(container_id).join(INFO_FILE); - let container_info = ContainerInfo { - id: container_id.clone(), - pid: pid, - run: args.run.as_ref().unwrap().clone(), - image: args.image.as_ref().unwrap().clone(), - volume: args.volume.clone().unwrap_or("".to_string()), - env: args.env.clone().unwrap_or("".to_string()), - status: ContainerStatus::READY, - log: args.log, - wait: args.wait, - }; - let toml_str = toml::to_string(&container_info)?; - fs::write(container_info_path, toml_str)?; - Ok(()) -} - fn get_container_info(container_id: &str) -> Result { let container_work_path = Path::new(WORKSPACE).join("containers").join(container_id); let container_info_path = container_work_path.join(INFO_FILE); let info_str = fs::read_to_string(container_info_path)?; let mut container_info: ContainerInfo = toml::from_str(&info_str)?; + // todo 判断当前进程组 是否还有进程 + let cgroup = Cgroup::from(&container_info); + // 判断是否正在运行, 进入proc/pid/判断当前进程网络中是否有下面的设备 - let dev_path = Path::new("/proc").join(container_info.pid.to_string()).join("net").join("dev"); - let is_running = if let Ok(dev_text) = fs::read_to_string(dev_path){ - let slave_veth_name = format!("ro_{container_id}_3"); - dev_text.lines().any(|l|l.starts_with(&slave_veth_name)) - } else { - false - }; - if is_running { + // let dev_path = Path::new("/proc").join(container_info.pid.to_string()).join("net").join("dev"); + // let is_running = if let Ok(dev_text) = fs::read_to_string(dev_path){ + // let slave_veth_name = format!("ro_{container_id}_3"); + // dev_text.lines().any(|l|l.starts_with(&slave_veth_name)) + // } else { + // false + // }; + + if cgroup.procs.len() > 0 { container_info.status = ContainerStatus::RUNNING; } else { container_info.status = ContainerStatus::STOP; @@ -624,17 +627,15 @@ fn stop_container(containers_id: &str, is_remove: bool) -> Result<()> { return Ok(()) } for container_id in containers_id.split(" ") { + // 容器网络删除 + remove_network(container_id); + let container_work_path = containers_path.join(container_id); let container_merged_path = container_work_path.join("merged"); if let Ok(container_info) = get_container_info(container_id) { - // 正在运行中的需要 kill - if container_info.status == ContainerStatus::RUNNING { - let _ = kill(Pid::from_raw(-container_info.pid), Signal::SIGKILL); - let pid_path = Path::new("/proc").join(container_info.pid.to_string()); - while pid_path.exists() { - std::thread::sleep(std::time::Duration::from_millis(10)); - } - } + // 删除进程组 + Cgroup::remove(container_id); + // 卸载自定义挂载点 if container_info.volume != "" { container_info.volume @@ -650,14 +651,16 @@ fn stop_container(containers_id: &str, is_remove: bool) -> Result<()> { .for_each(|s| { match umount2(s.as_str(), MntFlags::MNT_DETACH) { Ok(_) => println!("卸载自定卷{s}"), - Err(e) => println!("卸载卷{s}失败: {e:?}"), + Err(e) => println!("卸载自定卷{s}失败: {e:?}"), } }); } // 卸载dev文件夹中的设备 let container_dev_path = container_merged_path.join("dev"); - container_dev_path.read_dir()?.filter_map(|d|d.ok()) - .for_each(|p|{let _ = umount2(p.path().to_str().unwrap(), MntFlags::MNT_DETACH);}); + let _ = container_dev_path.read_dir().map(|dev| { + dev.filter_map(|d|d.ok()) + .for_each(|p|{let _ = umount2(p.path().to_str().unwrap(), MntFlags::MNT_DETACH);}); + }); // 卸载overlayfs match umount2(container_merged_path.to_str().unwrap(), MntFlags::MNT_DETACH) { @@ -676,14 +679,14 @@ fn stop_container(containers_id: &str, is_remove: bool) -> Result<()> { } else { println!("容器不存在: {container_id}, 强制删除"); // 需要强制删除一下目录 - match fs::remove_dir_all(&container_work_path) { - Ok(_) => println!("强制删除 {container_work_path:?} 成功"), - Err(e) => println!("强制删除失败: {e:?}"), + if container_work_path.exists() { + match fs::remove_dir_all(&container_work_path) { + Ok(_) => println!("强制删除 {container_work_path:?} 成功"), + Err(e) => println!("强制删除失败: {e:?}"), + } } } - // 容器网络删除 - remove_network(container_id); } Ok(()) } @@ -708,7 +711,9 @@ fn main() -> Result<()>{ env: args.env.clone().unwrap_or_default(), status: ContainerStatus::READY, log: args.log, - wait: args.wait + wait: args.wait, + cgroup_level: args.cgroup.clone().unwrap_or(CgroupLevel::default().to_string()), + root: args.root } } (None, None, Some(_container_id)) => { diff --git a/src/network.rs b/src/network.rs index 7fd96e7..b29f49d 100644 --- a/src/network.rs +++ b/src/network.rs @@ -147,6 +147,7 @@ pub fn remove_network(uuid_name: &str) { } // 回写到文件中 write_network_info(all_network); + println!("删除网络设备: {uuid_name}"); } diff --git a/test/data b/test/data index 34eb56b..8b13789 100755 Binary files a/test/data and b/test/data differ diff --git a/test/test.py b/test/test.py index c35b2d9..49121e2 100755 --- a/test/test.py +++ b/test/test.py @@ -1,25 +1,67 @@ # import urllib.request + # from datetime import datetime -# import time -# import mmap -# import os -# print(os.getpid()) -# print(os.getpgid(os.getpid())) +# +import time +import mmap +import os +import threading +import ctypes + +def read_memory(address): + # 创建一个足够大的缓冲区来读取内存 + buffer_size = 8 # 假设我们读取的是一个64位整数 + buffer = ctypes.create_string_buffer(buffer_size) + + # 使用ctypes的cast函数将地址转换为ctypes的指针类型 + ptr = ctypes.cast(address, ctypes.POINTER(ctypes.c_char * buffer_size)) + + # 读取内存到buffer中 + ptr.contents[:buffer_size] + + # 解析buffer中的数据 + data = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_int8)).contents.value + return data + + + +print(os.getpid()) +print(os.getpgid(os.getpid())) + + +def t(): + time.sleep(100) + cnt = 0 + while True: + cnt += 1 + + +t = threading.Thread(target=t) +t.start() + + +def dealy(): + time.sleep(100000) +# time.sleep(20) +print("start") + +with open("/tmp/test/data", "r+b") as f: + mm = mmap.mmap(f.fileno(), 0, mmap.PROT_READ | mmap.MAP_PRIVATE) + for i in range(400): + for _ in range(1024 * 1024): + idx = _ * i + a = mm[idx] + mm[idx] = idx % 230 + + # memory_block = bytearray(1024*1024) + # lis.append(memory_block) + time.sleep(1) + print(i, idx, mm[idx], a) + print("ok") -# with open("./data", "r+b") as f: -# mm = mmap.mmap(f.fileno(), 0) -# time.sleep(1000000) -# # lis = [] -# # while True: -# # lis.clear() -# # for _ in range(1024 * 1024 * 400): -# # if len(lis) < 1024 * 1024 * 50: -# # lis.append(_) -# # print("Ok") -