diff --git a/pypy.env b/pypy.env deleted file mode 100644 index c128c8f..0000000 --- a/pypy.env +++ /dev/null @@ -1,4 +0,0 @@ -TERM=xterm -VIRTUAL_ENV=/python_env -VIRTUAL_ENV_PROMPT=(python_env) -PATH=/python_env/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ No newline at end of file diff --git a/quant.env b/quant.env new file mode 100644 index 0000000..04e0e8e --- /dev/null +++ b/quant.env @@ -0,0 +1,6 @@ +TERM=xterm +HOME=/home/rocker +VIRTUAL_ENV=/home/rocker/pypy_env +VIRTUAL_ENV_PROMPT=(pypy_env) + +PATH=/home/rocker/go/go/bin:/home/rocker/pypy_env/bin:/home/rocker/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin \ No newline at end of file diff --git a/readme.md b/readme.md index 043e1ce..bbfbe8b 100644 --- a/readme.md +++ b/readme.md @@ -1,14 +1,19 @@ - - ## 基础镜像构建 ```bash +# 更新 +$ apt update +$ apt install sudo +$ apt install vim +$ apt install curl +$ apt install build-essential # 安装 jupyter所需要的基础软件 $ apt install ca-certificates # 创建一个rocker用户和他的home $ useradd -m -s /bin/bash rocker -u 7788 -$ passwd rocker quant_u2Oh0Go9J76aQb3h7Ybgapw@yanguangshaonian +# 创建密码 quant_u2Oh0Go9J76aQb3h7Ybgapw@yanguangshaonian +$ passwd rocker # 创建一个rocker的用户组并把上面的用户添加到组中 $ usermod -g 7788 -aG rocker rocker @@ -26,25 +31,58 @@ $ vi /etc/group $ vi /etc/passwd # 创建 /log/logs 文件 -$ mkdir /log -$ touch /log/logs +$ mkdir /logs && touch /logs/log && chmod 777 /logs && chmod 777 /logs/log + +# 切换用户 +$ su rocker && cd ~ + +# rocker用户 安装rust +$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +# rocker用户 安装pypy +# 官网下载上传 + +# rocker用户 安装go +# 官网下载上传 + +# rocker用户 安装g++ +# 上面已经有了 -# 删除history历史命令 -$ history -c -# 清空.viminfo +# 清空root和rocker用户的 .viminfo $ echo > ~/.viminfo +# 清空root和rocker用户的 history历史命令 +$ history -c && echo > ~/.bash_history + + +# docker 导出 +$ docker export CONTAINER > base_ubuntu_image.tar + +# 解压后的操作, 解压后检查 +$ tar -xvf base_ubuntu_image.tar +$ rm -rf base_ubuntu_image.tar + +# vi 编辑 etc/resolv.conf, 添加 nameserver 8.8.8.8 +$ echo "nameserver 8.8.8.8" > etc/resolv.conf + +# vi 编辑 etc/hosts 添加 127.0.0.1 rocker +$ echo "127.0.0.1 rocker" > etc/hosts + +# 删除多余文件 +$ rm -rf /.dockerenv -# vi 编辑 /etc/resolv.conf, 添加 nameserver 8.8.8.8 -$ vi /etc/resolv.conf +# 删除需要编译的语言 测试用生成的可执行程序(比如 rust/go/cpp) +$ rm -rf rust_project/target/rust_project +$ rm -rf cpp_project/main +$ rm -rf go_project/go_project -# vi 编辑 /etc/hosts 添加 127.0.0.1 rocker -$ vi /etc/hosts +# 在入镜像的主目录压缩, 再次压缩 即可使用 +tar -czf ../base_ubuntu_image.tar.gz . -# 上传 pypy 解压之后根据文件夹 放到 /usr 对应的目录 -# 到根目录, 使用 python -m venv python_env 创建虚拟环境 +# copy到images +$ cp base_ubuntu_image.tar.gz /home/rocker/images/base_ubuntu_image diff --git a/test/service/client.py b/test/service/client.py index 55fc202..3cb0e3e 100644 --- a/test/service/client.py +++ b/test/service/client.py @@ -1,23 +1,60 @@ -code = """ +python_code = """ import time import os import http.client - +import random +import ctypes +size_in_bytes = 100 * 1024 * 1024 +fixed_size_memory = (ctypes.c_ubyte * size_in_bytes)() +for i in range(100 * 1024): + fixed_size_memory[i * 1024] = 0 print("hello world") print(os.environ) print(os.listdir()) +memory_list = [] +# for _ in range(100_0000): +# memory_list.append(random.randint(0, 1000_0000)) +exit() conn = http.client.HTTPConnection("www.baidu.com") conn.request("GET", "/") print(conn.getresponse().read().decode('utf-8')) """ +cpp_code = """ +#include +using namespace std; + +int main(){ + cout << "hello world~" << endl; +} +""" + +go_code = """ +package main +import ( + "fmt" +) +func main(){ + fmt.Println1("hello world~") +} +""" + +rust_code = """ +fn main(){ + println!("hello world~"); +} +""" + if __name__ == "__main__": import requests - url = "http://code.quant.cm" - resp = requests.post(url, json={"code": code}) + url = "http://127.0.0.1:8011" + # resp = requests.post(url, json={"code": python_code, "language": "python"}) + # resp = requests.post(url, json={"code": cpp_code, "language": "cpp"}) + # resp = requests.post(url, json={"code": go_code, "language": "go"}) + resp = requests.post(url, json={"code": rust_code, "language": "rust"}) print(resp.json()["msg"]) diff --git a/test/service/handler.py b/test/service/handler.py new file mode 100644 index 0000000..d3a451c --- /dev/null +++ b/test/service/handler.py @@ -0,0 +1,92 @@ +import os +import sys +import subprocess +import traceback + +TARGET_LANGUAGE = set(["python", "cpp", "go", "node", "rust"]) +PROJECT_PATH = "/home/rocker/project" + +def go_handler(): + source_path = f"{PROJECT_PATH}/go_project/main.go" + subprocess.run(["cp", "/tmp/code", source_path], capture_output=True) + main_path = f"{PROJECT_PATH}/go_project/main" + ret = subprocess.run(["go", "build", "-o", main_path], cwd=f"{PROJECT_PATH}/go_project", capture_output=True) + std_err = ret.stderr.decode("utf-8") + if len(std_err) > 0: + return std_err, "" + return "", main_path + +def cpp_handler(): + source_path = f"{PROJECT_PATH}/cpp_project/main.cpp" + subprocess.run(["cp", "/tmp/code", source_path], capture_output=True) + main_path = f"{PROJECT_PATH}/cpp_project/main" + ret = subprocess.run(["g++", source_path, "-o", main_path], capture_output=True) + std_err = ret.stderr.decode("utf-8") + if len(std_err) > 0: + return std_err, "" + return "", main_path + +def node_handler(): + pass + +def rust_handler(): + source_path = f"{PROJECT_PATH}/rust_project/src/main.rs" + subprocess.run(["cp", "/tmp/code", source_path], capture_output=True) + ret = subprocess.run(["cargo", "build"], cwd=f"{PROJECT_PATH}/rust_project", capture_output=True) + std_err = ret.stderr.decode("utf-8") + if len(std_err) > 0 and ret.returncode != 0: + return std_err, "" + main_path = f"{PROJECT_PATH}/rust_project/target/debug/main" + subprocess.run(["mv", f"{PROJECT_PATH}/rust_project/target/debug/rust_project", main_path], capture_output=True) + return "", main_path + +def python_handler(): + main_path = f"{PROJECT_PATH}/python_project/main.py" + subprocess.run(["cp", "/tmp/code", main_path], capture_output=True) + return "", main_path + + +def main(): + language = sys.argv[1] + if language not in TARGET_LANGUAGE: + print(f"不支持的编程语言: {language}") + return + + # 编译和执行 + cmd = ["/usr/bin/time", "-f", "'%M %U %S %e'"] + err = "" + if language == "python": + err, main_path = python_handler() + cmd.extend(["python", "-u", main_path]) + elif language == "cpp": + err, main_path = cpp_handler() + cmd.extend([main_path]) + elif language == "go": + err, main_path = go_handler() + cmd.extend([main_path]) + elif language == "node": + pass + elif language == "rust": + err, main_path = rust_handler() + cmd.extend([main_path]) + else: + ... + + # 阻塞执行 + if err != "": + print(err) + else: + ret = subprocess.run(cmd, capture_output=True) + out_lines = f"{ret.stdout[:1_0000].decode('utf-8')} \n {ret.stderr[:1_0000].decode('utf-8')}".rstrip().split("\n") + try: + end_line_lis = out_lines[-1].replace("'", "").strip().split(" ") + end_line = f"MaxRSS: {int(end_line_lis[0])//1024}mb, User: {end_line_lis[1]}s, Sys: {end_line_lis[2]}s, Elapsed: {end_line_lis[3]}s" + out_lines[-1] = end_line + except Exception as e: + print(traceback.format_exc()) + out = "\n".join(out_lines) + print(out) + + +if __name__ == "__main__": + main() diff --git a/test/service/server.py b/test/service/server.py index 9c8cf33..ffa26d5 100644 --- a/test/service/server.py +++ b/test/service/server.py @@ -3,50 +3,65 @@ import random import os import subprocess import time +import traceback app = Flask(__name__) -USER_NAME = "ubuntu" +USER_NAME = "yanguangshaonian" ROCKER_PATH = f"/home/{USER_NAME}/rocker/target/debug/rocker" +HANDLER_FILE = "handler.py" +HANDLER_CODE = "" +with open(f"./{HANDLER_FILE}") as f: + HANDLER_CODE = f.read() @app.route("/", methods=["POST"]) def main(): code = request.json.get("code", "") - + language = request.json.get("language", "") _id = str(random.randint(8000_0000, 9000_0000)) + resp = {"msg": "", code: "", "id": _id} + + if len(code) > 1_0000: + resp["msg"] = "代码过长" + return resp + user_path = f"/tmp/{_id}" os.mkdir(user_path) - with open(f"{user_path}/main.py", "w") as f: + with open(f"{user_path}/code", "w") as f: f.write(code) + with open(f"{user_path}/{HANDLER_FILE}", "w") as f: + f.write(HANDLER_CODE) try: _ = subprocess.run(["sudo", ROCKER_PATH, "--id", _id, - "--image", "ubuntu_pypy_numpy_pandas_user", - "--run", "python -u /tmp/main.py", - "--env", f"/home/{USER_NAME}/rocker/pypy.env", + "--image", "base_ubuntu_image", + "--run", f"python3 -u /tmp/handler.py {language}", + "--env", f"/home/{USER_NAME}/rocker/quant.env", "--volume", f"{user_path}:/tmp", "--log"]) except Exception as e: + resp["msg"] += f"{traceback.format_exc()}\n" print(e) - time.sleep(1) - + time.sleep(3) + # 读取文件 - out = "" try: log_path = f"/home/rocker/containers/{_id}/upper/logs/log" with open(log_path) as f: - out = f.read() + log_str = f"{f.read()}\n" + resp["msg"] += log_str except Exception as e: - print(e) + resp["msg"] += f"{traceback.format_exc()}\n" + resp["msg"] = resp["msg"][:10000].rstrip() subprocess.run(["sudo", ROCKER_PATH, "--rm", _id]) subprocess.run(["sudo", "rm", "-rf", user_path]) - return {"msg": out, code: code, "id": _id} + return resp if __name__ == "__main__": app.run(host="0.0.0.0", port=8011, threaded=True) \ No newline at end of file