Compare commits

...

4 commits

Author SHA1 Message Date
xqtc bc84e9164f idk
Some checks failed
/ build (push) Failing after 1m51s
/ clippy (push) Failing after 1m40s
2024-09-12 00:01:31 +02:00
xqtc a634b174b9 bump version 2024-09-12 00:01:31 +02:00
xqtc 79efbf8907 Add WIP notice to README 2024-09-12 00:01:31 +02:00
xqtc fef0c39da3 Cargo publish stuff 2024-09-12 00:01:26 +02:00
31 changed files with 70 additions and 8366 deletions

2
Cargo.lock generated
View file

@ -1646,7 +1646,7 @@ dependencies = [
[[package]]
name = "yotei"
version = "0.1.0"
version = "0.1.1"
dependencies = [
"anyhow",
"bincode",

11
LICENSE Normal file
View file

@ -0,0 +1,11 @@
Copyright (c) 2024 xqtc161.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View file

@ -9,6 +9,12 @@ fmt:
run *ARGS:
cargo run {{ARGS}}
head-node *ARGS:
cargo run --bin head-node
worker-node *ARGS:
cargo run --bin worker-node
# Run 'cargo watch' to run the project (auto-recompiles)
watch *ARGS:
cargo watch -x "run -- {{ARGS}}"

View file

@ -1,3 +0,0 @@
{
"vimMode": true
}

View file

@ -1,4 +0,0 @@
{
"theme": "obsidian",
"cssTheme": "Catppuccin"
}

View file

@ -1,30 +0,0 @@
{
"file-explorer": true,
"global-search": true,
"switcher": true,
"graph": true,
"backlink": true,
"canvas": true,
"outgoing-link": true,
"tag-pane": true,
"properties": false,
"page-preview": true,
"daily-notes": true,
"templates": true,
"note-composer": true,
"command-palette": true,
"slash-command": false,
"editor-status": true,
"bookmarks": true,
"markdown-importer": false,
"zk-prefixer": false,
"random-note": false,
"outline": true,
"word-count": true,
"slides": false,
"audio-recorder": false,
"workspaces": false,
"file-recovery": true,
"publish": false,
"sync": false
}

View file

@ -1,20 +0,0 @@
[
"file-explorer",
"global-search",
"switcher",
"graph",
"backlink",
"canvas",
"outgoing-link",
"tag-pane",
"page-preview",
"daily-notes",
"templates",
"note-composer",
"command-palette",
"editor-status",
"bookmarks",
"outline",
"word-count",
"file-recovery"
]

View file

@ -1,7 +0,0 @@
{
"name": "Catppuccin",
"version": "0.4.24",
"minAppVersion": "1.0.0",
"author": "Marshall Beckrich",
"authorUrl": "https://github.com/catppuccin/obsidian"
}

File diff suppressed because one or more lines are too long

View file

@ -1,155 +0,0 @@
{
"main": {
"id": "ba6656b2e09ced8b",
"type": "split",
"children": [
{
"id": "a5f8ef865112d597",
"type": "tabs",
"children": [
{
"id": "394ad00be71f8b2e",
"type": "leaf",
"state": {
"type": "markdown",
"state": {
"file": "UwUsched/Protocol.md",
"mode": "source",
"source": false
}
}
}
]
}
],
"direction": "vertical"
},
"left": {
"id": "351ee0d58b83aaeb",
"type": "split",
"children": [
{
"id": "5e4f44266924bf54",
"type": "tabs",
"children": [
{
"id": "9f2faf4f1827783f",
"type": "leaf",
"state": {
"type": "file-explorer",
"state": {
"sortOrder": "alphabetical"
}
}
},
{
"id": "24d16549c3e9f26c",
"type": "leaf",
"state": {
"type": "search",
"state": {
"query": "",
"matchingCase": false,
"explainSearch": false,
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical"
}
}
},
{
"id": "78556ee640811968",
"type": "leaf",
"state": {
"type": "bookmarks",
"state": {}
}
}
]
}
],
"direction": "horizontal",
"width": 300
},
"right": {
"id": "41231040187f060c",
"type": "split",
"children": [
{
"id": "3bb571b4d8a6b510",
"type": "tabs",
"children": [
{
"id": "f123b7a34597da70",
"type": "leaf",
"state": {
"type": "backlink",
"state": {
"file": "UwUsched/Protocol.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
"showSearch": false,
"searchQuery": "",
"backlinkCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "d3b5730a2ea06ff4",
"type": "leaf",
"state": {
"type": "outgoing-link",
"state": {
"file": "UwUsched/Protocol.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "53131ea4b1beb2c9",
"type": "leaf",
"state": {
"type": "tag",
"state": {
"sortOrder": "frequency",
"useHierarchy": true
}
}
},
{
"id": "f3e3169cbae74ce5",
"type": "leaf",
"state": {
"type": "outline",
"state": {
"file": "UwUsched/Protocol.md"
}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"left-ribbon": {
"hiddenItems": {
"switcher:Open quick switcher": false,
"graph:Open graph view": false,
"canvas:Create new canvas": false,
"daily-notes:Open today's daily note": false,
"templates:Insert template": false,
"command-palette:Open command palette": false
}
},
"active": "394ad00be71f8b2e",
"lastOpenFiles": [
"UwUsched/UwUSched.md",
"UwUsched/Protocol.md",
"UwUsched/Cluster Architecture.md"
]
}

View file

@ -1,4 +0,0 @@
{
"vimMode": true,
"alwaysUpdateLinks": true
}

View file

@ -1,6 +0,0 @@
{
"cssTheme": "Catppuccin",
"theme": "obsidian",
"monospaceFontFamily": "Source Code Pro",
"textFontFamily": "Source Code Pro"
}

View file

@ -1,30 +0,0 @@
{
"file-explorer": true,
"global-search": true,
"switcher": true,
"graph": true,
"backlink": true,
"canvas": true,
"outgoing-link": true,
"tag-pane": true,
"properties": false,
"page-preview": true,
"daily-notes": true,
"templates": true,
"note-composer": true,
"command-palette": true,
"slash-command": false,
"editor-status": true,
"bookmarks": true,
"markdown-importer": false,
"zk-prefixer": false,
"random-note": false,
"outline": true,
"word-count": true,
"slides": false,
"audio-recorder": false,
"workspaces": false,
"file-recovery": true,
"publish": false,
"sync": false
}

View file

@ -1,20 +0,0 @@
[
"file-explorer",
"global-search",
"switcher",
"graph",
"backlink",
"canvas",
"outgoing-link",
"tag-pane",
"page-preview",
"daily-notes",
"templates",
"note-composer",
"command-palette",
"editor-status",
"bookmarks",
"outline",
"word-count",
"file-recovery"
]

View file

@ -1,22 +0,0 @@
{
"collapse-filter": true,
"search": "",
"showTags": false,
"showAttachments": false,
"hideUnresolved": false,
"showOrphans": true,
"collapse-color-groups": true,
"colorGroups": [],
"collapse-display": true,
"showArrow": false,
"textFadeMultiplier": 0,
"nodeSizeMultiplier": 1,
"lineSizeMultiplier": 1,
"collapse-forces": true,
"centerStrength": 0.518713248970312,
"repelStrength": 10,
"linkStrength": 1,
"linkDistance": 250,
"scale": 1,
"close": true
}

View file

@ -1,7 +0,0 @@
{
"name": "Catppuccin",
"version": "0.4.24",
"minAppVersion": "1.0.0",
"author": "Marshall Beckrich",
"authorUrl": "https://github.com/catppuccin/obsidian"
}

File diff suppressed because one or more lines are too long

View file

@ -1,159 +0,0 @@
{
"main": {
"id": "6226bb31998275a0",
"type": "split",
"children": [
{
"id": "a0eecd0f2f82e63c",
"type": "tabs",
"children": [
{
"id": "0f16c2e047ea76e2",
"type": "leaf",
"state": {
"type": "markdown",
"state": {
"file": "Protocol.md",
"mode": "source",
"source": false
}
}
}
]
}
],
"direction": "vertical"
},
"left": {
"id": "404cda8cf3aaef46",
"type": "split",
"children": [
{
"id": "56fd14939d4e1db3",
"type": "tabs",
"children": [
{
"id": "7ec07b2cf2f9abff",
"type": "leaf",
"state": {
"type": "file-explorer",
"state": {
"sortOrder": "alphabetical"
}
}
},
{
"id": "443ca75514af4821",
"type": "leaf",
"state": {
"type": "search",
"state": {
"query": "",
"matchingCase": false,
"explainSearch": false,
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical"
}
}
},
{
"id": "394fb16cdcdeec29",
"type": "leaf",
"state": {
"type": "bookmarks",
"state": {}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"right": {
"id": "c5deb5eb9a13ad1e",
"type": "split",
"children": [
{
"id": "6eddb59da5c4428c",
"type": "tabs",
"children": [
{
"id": "1a540ce1673cf6bb",
"type": "leaf",
"state": {
"type": "backlink",
"state": {
"file": "Protocol.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
"showSearch": false,
"searchQuery": "",
"backlinkCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "94cc7396d731b430",
"type": "leaf",
"state": {
"type": "outgoing-link",
"state": {
"file": "Protocol.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "76e3b53e26acc5df",
"type": "leaf",
"state": {
"type": "tag",
"state": {
"sortOrder": "frequency",
"useHierarchy": true
}
}
},
{
"id": "02804ac32c45e084",
"type": "leaf",
"state": {
"type": "outline",
"state": {
"file": "Protocol.md"
}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"left-ribbon": {
"hiddenItems": {
"switcher:Open quick switcher": false,
"graph:Open graph view": false,
"canvas:Create new canvas": false,
"daily-notes:Open today's daily note": false,
"templates:Insert template": false,
"command-palette:Open command palette": false
}
},
"active": "0f16c2e047ea76e2",
"lastOpenFiles": [
"UwUSched.md",
"Cluster Architecture.md",
"active-active.md",
"active-passive.png",
"active-active.png",
"create a link.md"
]
}

View file

@ -1,7 +0,0 @@
# Active-Active
![[active-active.png]]
The active-active architecture offers in high-availability, scalability and no single-point-of-failure.
-> Several Head-Nodes
# Active-Passive
![[active-passive.png]]Contrary to [[Cluster Architecture#Active-Active |active-active]] clustering, the active-passive method uses only one head-node and no load-balancer, with one additional head-node running as a failover.

View file

@ -1,123 +0,0 @@
- UUID-identifier with data
- Data binary- or hex-encoded
- Client <-> Host negotiating what to do => Task queue synchronization
=> Two seperate ways of communicating: One for neogtioating task queue stuff and one for data (Linked with UUID)
## Conn-Status
### Login
*Request:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"specs": {
"threads":<threads:u64>,
"clock":<clock:f64>,
"mem":<mem:u64>,
"net_speed_kib":<net_speed:f64>
}
}
```
*Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"status":<ACCEPTED|REJECTED>,
}
```
### Alive Check
*Request/Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>
}
```
### Logout
*Request:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
}
```
*Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"status":<ACK|OK|ERR>
}
```
>If the re-scheduling fails trigger the worker-node to give all HashMap-data to the head-node
## Pull
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>,
}
```
## Response
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"status":<OK|HASH_ERR|TIMEOUT_ERR>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>
}
```
## Push
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>,
"length_decomp": <length_decomp>,
"length_comp": <length_comp>,
"comp_format":<comp_format>,
"attributes":{go wild :3}
"data":<data>
}
```
> **Possible Traffic-Sequences**
> `PULL -> PUSH -> RESPONSE`
> `PUSH -> RESPONSE`
----
## JSON-Field Descriptions
**Data-Related**
`uuid`: sha-256 of decompressed data
`hashmap_identifier`: identifier of associated HashMap
`length_comp`: length of compressed data
`length_decomp`: length of decompressed data
`comp_format`: compression algorithm used
`data`: compressed data
`data_type`: either `permutation` or `state` (complete tree)
`attributes`: Implementation specific *EXTRAWURST*
**Control Flow**
`protocol_id`: `data_push | data_pull | data_response | hash_push | hash_pull | hash_response`
`node_id`:unique identifier of nodes. (Hostnames)
`status`: error codes indicating success of transmission

View file

@ -1,3 +0,0 @@
UwUsched is an [[Cluster Architecture#Active-Passive| active-passive]] Cluster.
The protocol is defined [[Protocol |here]]

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

View file

@ -2,7 +2,10 @@
authors = ["xqtc"]
description = "Distributed computing with abstract data"
name = "yotei"
version = "0.1.0"
license = "BSD-3-Clause"
readme = "./README.md"
repository = "https://git.heroin.trade/xqtc/yotei"
version = "0.1.1"
edition = "2021"
[dependencies]
@ -33,5 +36,5 @@ name = "head-node"
path = "src/head.rs"
[[bin]]
name = "compute-node"
path = "src/compute.rs"
name = "worker-node"
path = "src/worker.rs"

View file

@ -1,6 +1,8 @@
# UwUsched
# 🚧THIS IS WIP AND NOT WORKING RN🚧
# YOTEI
UwUsched is an active-active, high-availability clustering soloution.
YOTEI is an active-active, high-availability clustering soloution.
## Basic Architecture

View file

@ -2,10 +2,16 @@ use std::{env, path::PathBuf};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
std::env::set_var("CARGO_MANIFEST_DIR", std::env::var("PWD").unwrap());
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
let proto_dir = manifest_dir.join("./protos/");
let proto_dir_file = manifest_dir.join("./protos/protocol.proto");
tonic_build::configure()
.file_descriptor_set_path(out_dir.join("protocol_descriptor.bin"))
.compile(&["../protos/protocol.proto"], &["../protos"])?;
tonic_build::compile_protos("../protos/protocol.proto")?;
// .compile(&["../protos/protocol.proto"], &["../protos"])?;
.compile(&[proto_dir_file.clone()], &[proto_dir])?;
tonic_build::compile_protos(proto_dir_file)?;
Ok(())
}

View file

@ -30,8 +30,9 @@ impl data_server::Data for DataService {
}
}
async fn data_client() {
todo!()
async fn data_client(node_id: String, hashmap_id: String, uuid: String) -> tonic::Request<DataRequest>{
let req = DataRequest {node_id, uuid, hashmap_id };
tonic::Request::new(req)
}
#[derive(Debug, Default)]
@ -64,10 +65,6 @@ impl auth_server::Auth for AuthService {
}
}
async fn login(node_id: String) -> tonic::Request<LoginRequest> {
let req = LoginRequest { node_id };
tonic::Request::new(req)
}
// async fn logout(node_id: String) -> tonic::Request<LoginRequest> {}

View file

@ -72,14 +72,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
pretty_env_logger::init();
use colorful::Color;
use colorful::Colorful;
println!(" {}", "+--------------+".color(Color::LightBlue));
use colorful::HSL;
println!(" {}", "+-------------+".color(Color::LightBlue));
println!(" {}", "| 予 定 |".color(Color::LightPink1));
println!(
" {}",
"+--------------+".color(colorful::RGB::new(252, 252, 252))
"+-------------+".color(colorful::RGB::new(252, 252, 252))
);
println!(" {}", "YOITEI SCHEDULER".color(Color::LightPink1));
println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " HEAD NODE \n".color(Color::LightBlue));
let cli = Cli::parse();
match &cli.command {

View file

@ -5,27 +5,38 @@ mod config;
use crate::config::CONFIG;
use log::{debug, info};
use sched::LoginRequest;
use sched::{LoginRequest, LogoutRequest};
pub mod sched {
tonic::include_proto!("sched");
}
fn greet() {
use colorful::Color;
use colorful::Colorful;
println!(" {}", "+-------------+".color(Color::LightBlue));
println!(" {}", "| 予 定 |".color(Color::LightPink1));
println!(
" {}",
"+-------------+".color(colorful::RGB::new(252, 252, 252))
);
println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " WORKER NODE\n".color(Color::LightBlue));
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::env::set_var("RUST_LOG", CONFIG.log.level.clone());
pretty_env_logger::init();
use colorful::Color;
use colorful::Colorful;
println!(" {}", "+--------------+".color(Color::LightBlue));
println!(" {}", "| 予定 |".color(Color::LightPink1));
println!(
" {}",
"+--------------+".color(colorful::RGB::new(252, 252, 252))
);
println!(" {}", "YOITEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " COMPUTE NODE\n".color(Color::LightBlue));
greet();
// println!(" {}", "+-------------+".color(Color::LightBlue));
// println!(" {}", "| 予 定 |".color(Color::LightPink1));
// println!(
// " {}",
// "+-------------+".color(colorful::RGB::new(252, 252, 252))
// );
// println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
// println!(" {}", " WORKER NODE\n".color(Color::LightBlue));
let server_addr = "http://[::1]:50051";
let addr: SocketAddr = "[::1]:50053".parse()?;
let data = sched::data_client::DataClient::connect(server_addr).await?;
@ -39,6 +50,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}))
.await?
);
debug!("{:#?}", auth.logout(tonic::Request::new(LogoutRequest {
node_id: "compute-1".to_string(),
})).await?);
// Await server at end
alive_check_server.await?;
Ok(())
@ -47,11 +61,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
async fn alive_check_serve(addr: SocketAddr) -> Result<(), Box<dyn std::error::Error>> {
use grpc::AliveCheckService;
use sched::alive_check_server::AliveCheckServer;
use tokio::sync::mpsc;
use tokio::task;
use tonic::transport::Server;
let (tx, mut rx) = mpsc::channel::<Server>(100);
task::spawn(async move {
info!(
@ -66,8 +78,5 @@ async fn alive_check_serve(addr: SocketAddr) -> Result<(), Box<dyn std::error::E
.unwrap();
});
while let Some(msg) = rx.recv().await {
println!("REC: {:?}", msg);
}
Ok(())
}