diff --git a/Cargo.lock b/Cargo.lock index d071865..dd48e21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -346,6 +346,37 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb474a9c3219a8254ead020421ecf1b90427f29b55f6aae9a2471fa62c126ef" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "deranged" version = "0.3.11" @@ -770,6 +801,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -981,6 +1021,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.10.6" @@ -1158,6 +1218,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "sysinfo" +version = "0.31.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "tap" version = "1.0.1" @@ -1535,6 +1609,22 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -1544,6 +1634,65 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -1661,6 +1810,7 @@ dependencies = [ "prost-types", "serde", "serde_json", + "sysinfo", "tokio", "toml", "tonic", diff --git a/protos/protocol.proto b/protos/protocol.proto index bbc2bae..459e094 100644 --- a/protos/protocol.proto +++ b/protos/protocol.proto @@ -15,7 +15,7 @@ message DataResponse { string node_id = 1; string uuid = 2; string hashmap_id = 3; - string length = 4; + uint32 length = 4; bytes data = 5; } diff --git a/yotei-nodes/Cargo.toml b/yotei-nodes/Cargo.toml index 08ce724..3b4ad46 100644 --- a/yotei-nodes/Cargo.toml +++ b/yotei-nodes/Cargo.toml @@ -27,6 +27,7 @@ tonic-reflection = "0.12.2" prost = "0.13.2" prost-types = "0.13.2" colorful = "0.3.2" +sysinfo = "0.31.4" [build-dependencies] tonic-build = "0.12.2" diff --git a/yotei-nodes/src/head.rs b/yotei-nodes/src/head.rs index 8257236..c28466c 100644 --- a/yotei-nodes/src/head.rs +++ b/yotei-nodes/src/head.rs @@ -7,6 +7,8 @@ use clap::{Parser, Subcommand}; use log::{debug, info}; use sched::AliveCheckRequest; use tonic::transport::Server; + +mod head_util; // Import generated protobuf mod sched { diff --git a/yotei-nodes/src/head_util/mod.rs b/yotei-nodes/src/head_util/mod.rs new file mode 100644 index 0000000..e8ae652 --- /dev/null +++ b/yotei-nodes/src/head_util/mod.rs @@ -0,0 +1 @@ +pub mod queue; diff --git a/yotei-nodes/src/head_util/queue.rs b/yotei-nodes/src/head_util/queue.rs new file mode 100644 index 0000000..e69de29 diff --git a/yotei-nodes/src/util/grpc.rs b/yotei-nodes/src/util/grpc.rs index 2707831..60efb98 100644 --- a/yotei-nodes/src/util/grpc.rs +++ b/yotei-nodes/src/util/grpc.rs @@ -9,7 +9,6 @@ pub mod sched { tonic::include_proto!("sched"); } - #[cfg(feature = "head")] use crate::util::state::STATE; @@ -30,7 +29,7 @@ impl data_server::Data for DataService { node_id: input.node_id.clone(), hashmap_id: input.hashmap_id.clone(), uuid: input.uuid.clone(), - length: data.len().to_string(), + length: data.len().try_into().unwrap(), data, }; debug!("{:?}", &res); diff --git a/yotei-nodes/src/util/state.rs b/yotei-nodes/src/util/state.rs index 948e841..9a913b1 100644 --- a/yotei-nodes/src/util/state.rs +++ b/yotei-nodes/src/util/state.rs @@ -1,10 +1,12 @@ -use std::{collections::HashMap, sync::RwLock }; - +use std::{ + collections::{HashMap, VecDeque}, + sync::RwLock, +}; lazy_static::lazy_static! { pub static ref STATE: RwLock = RwLock::new(State { nodes: vec![], - tasks: HashMap::new(), + tasks: HashMap::new(), }); } @@ -16,11 +18,11 @@ pub struct Node { pub struct Task { pub id: uuid::Uuid, - pub data: Vec, + pub data: Box<[u8]>, } pub struct JobQueue { - pub jobs: Vec, + pub jobs: VecDeque, pub size: u128, } diff --git a/yotei-nodes/src/worker.rs b/yotei-nodes/src/worker.rs index 8a822b4..844fce8 100644 --- a/yotei-nodes/src/worker.rs +++ b/yotei-nodes/src/worker.rs @@ -3,6 +3,7 @@ use std::{ sync::{atomic::AtomicBool, Arc}, }; mod util; +mod worker_util; use crate::util::config::CONFIG; diff --git a/yotei-nodes/src/worker_util/mod.rs b/yotei-nodes/src/worker_util/mod.rs new file mode 100644 index 0000000..8e3228e --- /dev/null +++ b/yotei-nodes/src/worker_util/mod.rs @@ -0,0 +1 @@ +pub mod specs; diff --git a/yotei-nodes/src/worker_util/specs.rs b/yotei-nodes/src/worker_util/specs.rs new file mode 100644 index 0000000..ece4fa0 --- /dev/null +++ b/yotei-nodes/src/worker_util/specs.rs @@ -0,0 +1,24 @@ +use crate::util::state::NodeSpecs; +pub fn collect_specs() -> NodeSpecs { + let mut sys = sysinfo::System::new_all(); + NodeSpecs { + threads: sys.cpus().len().try_into().unwrap(), + mem: sys.total_memory(), + net_speed: 13.0, + } +} + +fn network() { + let networks = sysinfo::Networks::new_with_refreshed_list(); + for (interface_name, data) in &networks { + println!( + "{interface_name}: {} B (down) / {} B (up)", + data.total_received(), + data.total_transmitted(), + ); + // If you want the amount of data received/transmitted since last call + // to `Networks::refresh`, use `received`/`transmitted`. + } + +} +