From 92b33f4c100cc6e156c1016484f432b640e19adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=B3=E5=85=89=E5=B0=91=E5=B9=B4?= <849317537@qq.com> Date: Sat, 10 Aug 2024 02:00:16 +0000 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AE=B9=E5=99=A8=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E5=AD=98=E6=B4=BB=E7=9A=84=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.rs | 92 ++++++++++++++++++++++--------------- src/network.rs | 121 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 145 insertions(+), 68 deletions(-) diff --git a/src/main.rs b/src/main.rs index e239102..5f32d71 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ use path::{Path, PathBuf}; use nix::sched::{clone, CloneCb, CloneFlags, setns}; use nix::sys::{signal::{kill, Signal}, wait::{waitpid, WaitPidFlag}}; -use nix::unistd::{pause, dup2, pivot_root, setgid, setgroups, sethostname, setuid, Gid, Pid, Uid, User}; +use nix::unistd::{dup2, pivot_root, setgid, setgroups, sethostname, setuid, Gid, Pid, Uid, User}; use nix::mount::{mount, MntFlags, MsFlags, umount2}; use uuid; use toml; @@ -16,6 +16,8 @@ use clap::Parser; use error::{Result, RockerError}; mod error; +use network::create_network; +mod network; static WORKSPACE: &str = "/home/rocker"; static USER_NAME: &str = "rocker"; @@ -318,13 +320,28 @@ fn init_exec_ns(pid: i32) -> Result<()>{ Ok(()) } -fn start(is_wait: bool, cb: CloneCb, clong_flags: CloneFlags) -> Result{ +fn create_pause() -> Result<()> { + fs::OpenOptions::new() + .write(true) + .create(true) + .read(true) + .open("/pause")?; + Ok(()) +} + +fn check_pause>(pause_path: P) -> bool { + // 判断该文件还在不在 + pause_path.as_ref().exists() +} + +fn start(is_wait: bool, cb: CloneCb, clong_flags: CloneFlags, container_id: &String, container_merged_pause_path: &PathBuf) -> Result{ match unsafe {clone(cb, STACK.as_mut_slice(), clong_flags, None)} { - Ok(child_pid) => { + Ok(child_pid) => { println!("clone ok: {child_pid:?}"); // exec之前的步骤 - - + create_network(container_id, child_pid.as_raw()); + // 删除 container_merged_pause_path 解开阻塞 + std::fs::remove_file(container_merged_pause_path)?; // 检查是否执行exec了 let mut cnt = 0; @@ -368,18 +385,28 @@ fn run_container(_container_id: &String, cmd: &String, args: &RockerArgs, volume let clone_flags; + let container_work_path = Path::new(WORKSPACE).join("containers").join(&_container_id); + let container_upper_path = container_work_path.join("upper"); + let container_merged_path = container_work_path.join("merged"); + let container_merged_pause_path = container_work_path.join("merged").join("pause"); + let rocker_user_info = User::from_name(USER_NAME)?.ok_or(RockerError::OtherError(format!("没找到 用户: {USER_NAME}")))?; let rocker_uid = rocker_user_info.uid; let rocker_gid = rocker_user_info.gid; let _cb = if is_exec { - let _cb = move || { + let _cb = || { + dbg!(1); let container_info = get_container_info(_container_id).unwrap(); init_exec_ns(container_info.pid).unwrap(); init_container_env(None).unwrap(); init_container_user(rocker_uid, rocker_gid).unwrap(); + dbg!(2); + create_pause().unwrap(); + while container_merged_pause_path.exists() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } - // pause(); let cmd_vec = cmd.split(" ").collect::>(); let err = process::Command::new(cmd_vec[0]) .args(&cmd_vec[1..]) @@ -392,14 +419,11 @@ fn run_container(_container_id: &String, cmd: &String, args: &RockerArgs, volume Box::new(_cb) as CloneCb } else { // 初始化容器工作目录 - let container_work_path = Path::new(WORKSPACE).join("containers").join(&_container_id); - let container_upper_path = container_work_path.join("upper"); - let container_merged_path = container_work_path.join("merged"); create_dir(&container_work_path, true)?; create_dir(&container_upper_path, true)?; create_dir(&container_merged_path, true)?; - let _cb = move || { + let _cb = || { init_container_lock(&container_work_path).unwrap(); init_container_overlay(volume_path, &container_upper_path, &container_merged_path).unwrap(); if let Some(custom_volume) = &args.volume { @@ -412,7 +436,11 @@ fn run_container(_container_id: &String, cmd: &String, args: &RockerArgs, volume init_container_log(args.log).unwrap(); init_container_user(rocker_uid, rocker_gid).unwrap(); - // pause(); + create_pause().unwrap(); + while container_merged_pause_path.exists() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let cmd_vec = cmd.split(" ").collect::>(); let err = process::Command::new(cmd_vec[0]) .args(&cmd_vec[1..]) @@ -424,7 +452,7 @@ fn run_container(_container_id: &String, cmd: &String, args: &RockerArgs, volume Box::new(_cb) as CloneCb }; - start(args.wait, _cb, clone_flags) + start(args.wait, _cb, clone_flags, _container_id, &container_merged_pause_path) } #[derive(Deserialize, Serialize, Debug, PartialEq)] @@ -490,12 +518,11 @@ fn get_container_info(container_id: &str) -> Result { let info_str = fs::read_to_string(container_info_path)?; let mut container_info: ContainerInfo = toml::from_str(&info_str)?; - // 判断是否正在运行, 首先得到该容器进程对应的所有的fd - let proc_fd_path = Path::new("/proc").join(container_info.pid.to_string()).join("fd"); - let is_running = if let Ok(fd_dir) = fs::read_dir(proc_fd_path) { - fd_dir.filter_map(|p|p.ok()) - .filter_map(|f| fs::read_link(f.path()).ok()) - .any(|p|p == lock_path) + // 判断是否正在运行, 进入proc/pid/判断当前进程网络中是否有下面的设备 + let dev_path = Path::new("/proc").join(container_info.pid.to_string()).join("net").join("dev"); + let is_running = if let Ok(dev_text) = fs::read_to_string(dev_path){ + let slave_veth_name = format!("ro_{container_id}_3"); + dev_text.lines().any(|l|l.starts_with(&slave_veth_name)) } else { false }; @@ -542,10 +569,9 @@ fn stop_container(containers_id: &str, is_remove: bool) -> Result<()> { let container_work_path = Path::new(WORKSPACE).join("containers").join(container_id); let container_merged_path = container_work_path.join("merged"); if let Ok(container_info) = get_container_info(container_id) { - println!("container_merged_path: {container_merged_path:?}"); // 正在运行中的需要 kill if container_info.status == ContainerStatus::RUNNING { - let _ = kill(Pid::from_raw(container_info.pid), Signal::SIGTERM); + let _ = kill(Pid::from_raw(container_info.pid), Signal::SIGKILL); let pid_path = Path::new("/proc").join(container_info.pid.to_string()); while pid_path.exists() { std::thread::sleep(std::time::Duration::from_millis(10)); @@ -605,16 +631,21 @@ fn main() -> Result<()>{ let cmd; match (&args.run, &args.image, &args.restart) { (Some(_cmd), Some(image_name), None) => { + dbg!(100); volume_path = extend_image(image_name)?; + dbg!(101); container_id = uuid::Uuid::new_v4().to_string()[0..8].to_string(); + dbg!(102); cmd = _cmd.clone(); } (None, None, Some(_container_id)) => { + dbg!(211); let container_info = get_container_info(_container_id)?; + dbg!(210); volume_path = extend_image(&container_info.image)?; container_id = _container_id.clone(); cmd = container_info.run; - + dbg!(200); args.run = Some(cmd.clone()); args.image = Some(container_info.image); args.log = container_info.log; @@ -624,12 +655,14 @@ fn main() -> Result<()>{ if container_info.env.len() > 0 { args.env = Some(container_info.env); } + dbg!(201); stop_container(&container_id, false)?; } _ => { unreachable!() } } + dbg!(3); match run_container(&container_id, &cmd, &args, &volume_path, false) { Ok(child_pid) => { save_container_info(&args, &container_id, child_pid)?; @@ -642,21 +675,6 @@ fn main() -> Result<()>{ } } } - // if let (Some(cmd), Some(image_name)) = (&args.run, &args.image) { - // // run - // let volume_path = extend_image(image_name)?; - // let container_id = uuid::Uuid::new_v4().to_string()[0..8].to_string(); - // match run_container(&container_id, &cmd, &args, &volume_path, false) { - // Ok(child_pid) => { - // save_container_info(&args, &container_id, child_pid)?; - // } - // Err(e) => { - // // clone 之后的错误 这里已经无法捕获了 - // println!("run_container失败: {e}"); - // stop_container(&container_id, true)?; - // } - // } - // } else if args.ps || args.psa { // --ps show_containers(args.psa)? diff --git a/src/network.rs b/src/network.rs index 35e4983..144a802 100644 --- a/src/network.rs +++ b/src/network.rs @@ -1,4 +1,4 @@ -use std::{fs::OpenOptions, io::Read}; +use std::{fmt::{Display, Formatter}, fs::OpenOptions, io::{Read, Write}}; use rand::Rng; static NETWORK_FILE: &str = "/home/rocker/network"; @@ -7,7 +7,7 @@ fn get_gateway_addr(addr: &str) -> String { let prefix = &addr[..addr.len() - 1]; format!("{}0", prefix) } -fn create_bridge_dev(bridge_name: &str) { +fn create_bridge_dev(bridge_name: &str) -> bool { // brctl addbr br0 let args = ["addbr", bridge_name]; let out = std::process::Command::new("brctl") @@ -17,10 +17,12 @@ fn create_bridge_dev(bridge_name: &str) { if !out.status.success() { println!("create_bridge_dev: {args:?}"); println!("{:?}", out); + return false } + true } -fn set_bridge_ip(bridge_name: &str, addr: &str){ +fn set_bridge_ip(bridge_name: &str, addr: &str) -> bool { // 设置容器内从ip: ip addr add 192.168.124.1/24 dev bridge_name let args = ["addr", "add", &format!("{addr}/24"), "dev", bridge_name]; let out = std::process::Command::new("ip") @@ -30,10 +32,12 @@ fn set_bridge_ip(bridge_name: &str, addr: &str){ if !out.status.success() { println!("set_bridge_ip: {args:?}"); println!("{:?}", out); + return false; } + true } -fn set_slave_ip(slave_veth_name: &str, addr: &str, pid: &str) { +fn set_slave_ip(slave_veth_name: &str, addr: &str, pid: &str) -> bool { // nsenter -t 10050 -n -- ip addr add / dev veth1 let args = ["-t", pid, "-n", "--", "ip", "addr", "add", &format!("{addr}/24"), "dev", slave_veth_name]; let out = std::process::Command::new("nsenter") @@ -43,10 +47,12 @@ fn set_slave_ip(slave_veth_name: &str, addr: &str, pid: &str) { if !out.status.success() { println!("set_slave_ip: {args:?}"); println!("{:?}", out); + return false; } + true } -fn set_snat(gateway_addr: &str, bridge_name: &str) { +fn set_snat(gateway_addr: &str, bridge_name: &str) -> bool { // sudo iptables -t nat -A POSTROUTING -s 172.18.0.0/24 ! -o br0 -j MASQUERADE let args = ["-t", "nat", "-A", "POSTROUTING", "-s", &format!("{gateway_addr}/24"), "!", "-o", bridge_name, "-j", "MASQUERADE"]; let out = std::process::Command::new("iptables") @@ -56,10 +62,12 @@ fn set_snat(gateway_addr: &str, bridge_name: &str) { if !out.status.success() { println!("set_snat: {args:?}"); println!("{:?}", out); + return false; } + true } -fn set_iptables_forward() { +fn set_iptables_forward() -> bool { // sudo sysctl net.ipv4.conf.all.forwarding=1 let args = ["net.ipv4.conf.all.forwarding=1"]; let out = std::process::Command::new("sysctl") @@ -69,6 +77,7 @@ fn set_iptables_forward() { if !out.status.success() { println!("set_iptables_forward: {args:?}"); println!("{:?}", out); + return false; } // sudo iptables -t filter -P FORWARD ACCEPT let args = ["-t", "filter", "-P", "FORWARD", "ACCEPT"]; @@ -79,11 +88,13 @@ fn set_iptables_forward() { if !out.status.success() { println!("set_iptables_forward: {args:?}"); println!("{:?}", out); + return false; } + true } -fn set_slave_route(slave_veth_name: &str, bridge_addr: &str, pid: &str) { +fn set_slave_route(slave_veth_name: &str, bridge_addr: &str, pid: &str) -> bool { // sudo nsenter -t 1851 -n -- ip route add default via 172.18.0.1 dev veth3 let args = ["-t", pid, "-n", "--", "ip", "route", "add", "default", "via", bridge_addr, "dev", slave_veth_name]; let out = std::process::Command::new("nsenter") @@ -93,10 +104,12 @@ fn set_slave_route(slave_veth_name: &str, bridge_addr: &str, pid: &str) { if !out.status.success() { println!("set_slave_route: {args:?}"); println!("{:?}", out); + return false; } + true } -fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_name: &str) { +fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_name: &str) -> bool { // sudo ip link set bridge_name up let args = ["link", "set", bridge_name, "up"]; let out = std::process::Command::new("ip") @@ -106,6 +119,7 @@ fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_n if !out.status.success() { println!("set_up: {args:?}"); println!("{:?}", out); + return false; } // sudo ip link set master_veth_name up @@ -117,6 +131,7 @@ fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_n if !out.status.success() { println!("set_up: {args:?}"); println!("{:?}", out); + return false; } // nsenter -t 1970 -n -- ip link set dev slave_veth_name up @@ -128,6 +143,7 @@ fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_n if !out.status.success() { println!("set_up: {args:?}"); println!("{:?}", out); + return false; } // ip link set lo up @@ -139,7 +155,9 @@ fn set_net_up(bridge_name: &str, master_veth_name: &str, pid: &str, slave_veth_n if !out.status.success() { println!("set_up: {args:?}"); println!("{:?}", out); + return false; } + true } fn set_up(dev_name: &str) { @@ -211,7 +229,7 @@ fn del_bridge(bridge_name: &str) { } -fn create_veth_pair(master_veth_name: &str, slave_veth_name: &str) { +fn create_veth_pair(master_veth_name: &str, slave_veth_name: &str) -> bool { // sudo ip link add master_veth_name type veth peer name slave_veth_name let args = ["link", "add", master_veth_name, "type", "veth", "peer", "name", slave_veth_name]; let out = std::process::Command::new("ip") @@ -221,7 +239,9 @@ fn create_veth_pair(master_veth_name: &str, slave_veth_name: &str) { if !out.status.success() { println!("create_veth_pair: {args:?}"); println!("{:?}", out); + return false; } + return true; } @@ -236,6 +256,12 @@ struct NetWrok { slave_veth_name: String, // ro_uuid[:8]_3 // ip 就是 192.168.124.3, 插在容器中 } + +impl Display for NetWrok { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{},{},{},{},{},{}", self.gateway_addr, self.bridge_addr, self.slave_addr, self.bridge_name, self.master_veth_name, self.slave_veth_name) + } +} fn get_all_network() -> Vec { // 打开 NETWORK_FILE 所有已经配置的信息, 如果没有则创建 let mut f = OpenOptions::new() @@ -246,10 +272,24 @@ fn get_all_network() -> Vec { .unwrap(); let mut text = String::new(); f.read_to_string(&mut text).unwrap(); - vec![] + + let mut all_network = vec![]; + for line in text.lines() { + let network_info_vec = line.split(",").collect::>(); + all_network.push(NetWrok { + gateway_addr: network_info_vec[0].to_string(), + bridge_addr: network_info_vec[1].to_string(), + slave_addr: network_info_vec[2].to_string(), + bridge_name: network_info_vec[3].to_string(), + master_veth_name: network_info_vec[4].to_string(), + slave_veth_name: network_info_vec[5].to_string(), + }) + + } + all_network } -fn master_join_bridge(bridge_name: &str, master_veth_name: &str) { +fn master_join_bridge(bridge_name: &str, master_veth_name: &str) -> bool { // 将主端链接到桥 //sudo brctl addif bridge_name master_veth_name let args = ["addif", bridge_name, master_veth_name]; @@ -260,10 +300,23 @@ fn master_join_bridge(bridge_name: &str, master_veth_name: &str) { if !out.status.success() { println!("master_join_bridge: {args:?}"); println!("{:?}", out); + return false; + } + true +} + +fn write_network_info(all_network: Vec) { + let mut f = OpenOptions::new() + .write(true) + .create(true) + .open(NETWORK_FILE) + .unwrap(); + for n in all_network { + writeln!(f, "{}", n).unwrap(); } } -fn slave_join_bridge(slave_veth_name: &str, pid: &str) { +fn slave_join_bridge(slave_veth_name: &str, pid: &str) -> bool { // 将从端 链接到容器 // sudo ip link set slave_veth_name netns 1234 let args = ["link", "set", slave_veth_name, "netns", pid]; @@ -274,10 +327,12 @@ fn slave_join_bridge(slave_veth_name: &str, pid: &str) { if !out.status.success() { println!("slave_join_bridge: {args:?}"); println!("{:?}", out); + return false; } + true } -fn create_network(uuid_name: &str, pid: i32) { +pub fn create_network(uuid_name: &str, pid: i32) -> bool { // 创建一个随机地址段的 没分配过的ip let mut all_network = get_all_network(); let mut rg = rand::thread_rng(); @@ -305,34 +360,40 @@ fn create_network(uuid_name: &str, pid: i32) { } }; + println!("{network}"); all_network.push(network.clone()); // 写入到文件中 - let line = format!("{},{},{},{},{},{}", network.gateway_addr, network.bridge_addr, network.slave_addr, network.bridge_name, network.master_veth_name, network.slave_veth_name); - println!("{line:?}"); + write_network_info(all_network); + // 系统中创建桥 - create_bridge_dev(&network.bridge_name); + if create_bridge_dev(&network.bridge_name) && // 给桥分配ip - set_bridge_ip(&network.bridge_name, &network.bridge_addr); + set_bridge_ip(&network.bridge_name, &network.bridge_addr) && // 创建veth pair - create_veth_pair(&network.master_veth_name, &network.slave_veth_name); + create_veth_pair(&network.master_veth_name, &network.slave_veth_name) && // 宿主机 主pair 连接桥 - master_join_bridge(&network.bridge_name, &network.master_veth_name); + master_join_bridge(&network.bridge_name, &network.master_veth_name) && // 把从 pair 添加到容器内 - slave_join_bridge(&network.slave_veth_name, pid.to_string().as_str()); + slave_join_bridge(&network.slave_veth_name, pid.to_string().as_str()) && // 给容器内 pair 分配ip - set_slave_ip(&network.slave_veth_name, &network.slave_addr, pid.to_string().as_str()); + set_slave_ip(&network.slave_veth_name, &network.slave_addr, pid.to_string().as_str()) && // 激活 上线 - set_net_up(&network.bridge_name, &network.master_veth_name, pid.to_string().as_str(), &&network.slave_veth_name); + set_net_up(&network.bridge_name, &network.master_veth_name, pid.to_string().as_str(), &&network.slave_veth_name) && // 访问公网设置 // 配置路由 - set_slave_route(&network.slave_veth_name, &network.bridge_addr, pid.to_string().as_str()); + set_slave_route(&network.slave_veth_name, &network.bridge_addr, pid.to_string().as_str()) && // 设置转发规则 - set_iptables_forward(); + set_iptables_forward() && // 设置snat - set_snat(&network.gateway_addr, &network.bridge_name); - - // 如果出错, 清理资源 + set_snat(&network.gateway_addr, &network.bridge_name) { + println!("create_network success"); + true + } else { + println!("create_network failed"); + remove_network(&uuid_name); + false + } } @@ -347,14 +408,12 @@ fn remove_network(uuid_name: &str,) { all_network.retain(|n|n.bridge_name != bridge_name); // todo 回写到文件中 - + } fn main(){ - // get_all_network(); - let container_id = uuid::Uuid::new_v4().to_string()[0..8].to_string(); - create_network(&container_id, 15992); + create_network(&container_id, 17543); return; let addr = "192.168.124.1";