更新依赖版本,限制最大并发

This commit is contained in:
MeiK 2021-02-08 16:24:53 +08:00
parent ccbf906d11
commit 5d74e89804
96 changed files with 136 additions and 28 deletions

View File

@ -5,14 +5,15 @@ authors = ["MeiK <meik2333@gmail.com>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
tonic = "0.3" tonic = "0.4"
prost = "0.6" prost = "0.7"
tokio = { version = "0.2", features = ["macros", "sync", "stream", "time", "fs"] } tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1", features = ["net"] }
async-stream = "0.3"
futures = { version = "0.3" } futures = { version = "0.3" }
futures-core = "0.3" futures-core = "0.3"
futures-util = { version = "0.3", default-features = false } futures-util = { version = "0.3", default-features = false }
async-stream = "0.2" tower = "0.4"
tower = "0.3"
tempfile = "3" tempfile = "3"
libc = "0.2" libc = "0.2"
log = "0.4.0" log = "0.4.0"
@ -22,8 +23,8 @@ nix = "0.19.0"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.8" serde_yaml = "0.8"
lazy_static = "1.4.0" lazy_static = "1.4.0"
zip = "0.5" num_cpus = "1.0"
[build-dependencies] [build-dependencies]
tonic-build ={ version = "0.3" } tonic-build ={ version = "0.3" }
cc = "1.0" cc = "1.0"

View File

@ -1,5 +1,4 @@
fn main() -> Result<(), Box<dyn std::error::Error>> { fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic_build::compile_protos("proto/river.proto")?; tonic_build::compile_protos("proto/river.proto")?;
// cc::Build::new().file("src/memory.c").compile("memory");
Ok(()) Ok(())
} }

View File

@ -10,7 +10,7 @@ appenders:
encoder: encoder:
pattern: "{d(%Y-%m-%d %H:%M:%S)} [{l}] {m}{n}" pattern: "{d(%Y-%m-%d %H:%M:%S)} [{l}] {m}{n}"
root: root:
level: debug level: info
appenders: appenders:
- file - file
- console - console

1
runtime/data/1000/10.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/10.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/11.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/11.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/12.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/12.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/13.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/13.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/14.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/14.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/15.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/15.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/16.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/16.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/17.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/17.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/18.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/18.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/19.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/19.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/20.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/20.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/21.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/21.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/22.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/22.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/23.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/23.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/24.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/24.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/25.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/25.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/26.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/26.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/27.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/27.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/28.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/28.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/29.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/29.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/3.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/3.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/30.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/30.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/31.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/31.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/32.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/32.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/33.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/33.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/34.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/34.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/35.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/35.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/36.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/36.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/37.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/37.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/38.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/38.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/39.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/39.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/4.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/4.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/40.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/40.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/41.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/41.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/42.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/42.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/43.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/43.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/44.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/44.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/45.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/45.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/46.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/46.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/47.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/47.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/5.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/5.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/6.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/6.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/7.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/7.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/8.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/8.out Normal file
View File

@ -0,0 +1 @@
3

1
runtime/data/1000/9.in Normal file
View File

@ -0,0 +1 @@
1 2

1
runtime/data/1000/9.out Normal file
View File

@ -0,0 +1 @@
3

View File

@ -1,7 +1,10 @@
use lazy_static::lazy_static; use lazy_static::lazy_static;
use num_cpus;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::fs; use std::fs;
use std::sync::Arc;
use tokio::sync::Semaphore;
pub static STDIN_FILENAME: &str = "stdin.txt"; pub static STDIN_FILENAME: &str = "stdin.txt";
pub static STDOUT_FILENAME: &str = "stdout.txt"; pub static STDOUT_FILENAME: &str = "stdout.txt";
@ -14,6 +17,12 @@ lazy_static! {
debug!("{:?}", cfg); debug!("{:?}", cfg);
cfg cfg
}; };
pub static ref CPU_SEMAPHORE: Arc<Semaphore> = {
let num = num_cpus::get();
info!("cpus = {}", num);
// 设置最大并发量与 CPU 核数相同,以防止因资源不足而产生系统错误
Arc::new(Semaphore::new(num))
};
} }
#[derive(Debug, PartialEq, Serialize, Deserialize)] #[derive(Debug, PartialEq, Serialize, Deserialize)]

View File

@ -1,4 +1,4 @@
use crate::config::{CONFIG, STDERR_FILENAME, STDIN_FILENAME, STDOUT_FILENAME}; use crate::config::{CONFIG, STDERR_FILENAME, STDIN_FILENAME, STDOUT_FILENAME, CPU_SEMAPHORE};
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::process::{Process, Runner}; use crate::process::{Process, Runner};
use crate::result::{ use crate::result::{
@ -23,8 +23,11 @@ pub async fn compile(language: &str, code: &str, path: &Path) -> Result<JudgeRes
1024 * 1024, 1024 * 1024,
path.to_path_buf(), path.to_path_buf(),
); );
let semaphore = CPU_SEMAPHORE.clone();
let permit = semaphore.acquire().await;
let result = Runner::from(process)?; let result = Runner::from(process)?;
let status = result.await?; let status = result.await?;
drop(permit);
let mem_used = if status.memory_used < status.cgroup_memory_used { let mem_used = if status.memory_used < status.cgroup_memory_used {
status.memory_used status.memory_used
} else { } else {
@ -67,8 +70,12 @@ pub async fn judge(
memory_limit, memory_limit,
path.to_path_buf(), path.to_path_buf(),
); );
// 信号量控制并发
let semaphore = CPU_SEMAPHORE.clone();
let permit = semaphore.acquire().await;
let result = Runner::from(process)?; let result = Runner::from(process)?;
let status = result.await?; let status = result.await?;
drop(permit);
// 为了更好的体验,此处内存用量取 getrusage 报告与 cgroup 报告中较小的一个 // 为了更好的体验,此处内存用量取 getrusage 报告与 cgroup 报告中较小的一个
let mem_used = if status.memory_used < status.cgroup_memory_used { let mem_used = if status.memory_used < status.cgroup_memory_used {
status.memory_used status.memory_used

View File

@ -9,7 +9,7 @@ use river::judge_request::Data;
use river::river_server::{River, RiverServer}; use river::river_server::{River, RiverServer};
use river::{ use river::{
Empty, JudgeRequest, JudgeResponse, JudgeResultEnum, LanguageConfigResponse, LanguageItem, Empty, JudgeRequest, JudgeResponse, JudgeResultEnum, LanguageConfigResponse, LanguageItem,
LsRequest, LsResponse, LsCase LsCase, LsRequest, LsResponse,
}; };
use std::path::Path; use std::path::Path;
use std::pin::Pin; use std::pin::Pin;
@ -47,6 +47,7 @@ impl River for RiverService {
let mut stream = request.into_inner(); let mut stream = request.into_inner();
let output = async_stream::try_stream! { let output = async_stream::try_stream! {
info!("new judge request");
let pwd = match tempdir_in(&config::CONFIG.judge_dir) { let pwd = match tempdir_in(&config::CONFIG.judge_dir) {
Ok(val) => val, Ok(val) => val,
Err(e) => { Err(e) => {
@ -60,7 +61,6 @@ impl River for RiverService {
let mut language = String::from(""); let mut language = String::from("");
while let Some(req) = stream.next().await { while let Some(req) = stream.next().await {
yield result::pending(); yield result::pending();
// TODO: 限制并发数量
yield result::running(); yield result::running();
let req = req?; let req = req?;
let result = match &req.data { let result = match &req.data {
@ -96,7 +96,6 @@ impl River for RiverService {
} }
}, },
None => Err(error::Error::CustomError(String::from("unrecognized request types"))), None => Err(error::Error::CustomError(String::from("unrecognized request types"))),
_ => Err(error::Error::CustomError(String::from("unrecognized request types"))),
}; };
let res = match result { let res = match result {
Ok(res) => res, Ok(res) => res,
@ -129,29 +128,32 @@ impl River for RiverService {
} }
async fn ls(&self, request: Request<LsRequest>) -> Result<Response<LsResponse>, Status> { async fn ls(&self, request: Request<LsRequest>) -> Result<Response<LsResponse>, Status> {
// TODO: 将获取文件的接口剥离出来,归入评测文件管理系统中
// TODO: 最终将会删除这个接口
// TODO: 目前有安全隐患,可以获取到任意目录文件
let pid = request.into_inner().pid; let pid = request.into_inner().pid;
let mut response = LsResponse { cases: vec![] }; let mut response = LsResponse { cases: vec![] };
let directory_stream = match read_dir(Path::new("runtime/data").join(pid.to_string())).await { let mut directory_stream =
Ok(val) => val, match read_dir(Path::new("runtime/data").join(pid.to_string())).await {
Err(_) => return Ok(Response::new(response)), Ok(val) => val,
}; Err(_) => return Ok(Response::new(response)),
let files: Vec<_> = directory_stream };
.filter_map(|file| async move { let mut files: Vec<String> = vec![];
Some(file.unwrap().file_name().into_string().unwrap()) while let Ok(Some(entry)) = directory_stream.next_entry().await {
}) let file = entry.file_name().into_string().unwrap();
.collect() files.push(file);
.await; }
// let files: Vec<_> = directory_stream
// .filter_map(|file| async move {
// Some(file.unwrap().file_name().into_string().unwrap())
// })
// .collect()
// .await;
let mut iter = 1; let mut iter = 1;
loop { loop {
let in_file = format!("{}.in", iter); let in_file = format!("{}.in", iter);
let out_file = format!("{}.out", iter); let out_file = format!("{}.out", iter);
if files.contains(&in_file) && files.contains((&out_file)) { if files.contains(&in_file) && files.contains((&out_file)) {
response.cases.push(LsCase{ response.cases.push(LsCase {
r#in: in_file, r#in: in_file,
out: out_file out: out_file,
}); });
iter += 1; iter += 1;
} else { } else {