xqtc/yotei
1
0
Fork 0
Code Issues 2 Pull requests Actions Packages Projects Releases Wiki Activity

Cargo publish stuff

Browse source
This commit is contained in:
xqtc 2024-09-11 20:55:16 +02:00
parent 4ea68a5439
commit fef0c39da3
28 changed files with 53 additions and 8352 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
LICENSE Normal file
View file

@ -0,0 +1,11 @@
Copyright (c) 2024 xqtc161.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3
notes/.obsidian/app.json vendored
View file

@ -1,3 +0,0 @@
{
"vimMode": true
}

4
notes/.obsidian/appearance.json vendored
View file

@ -1,4 +0,0 @@
{
"theme": "obsidian",
"cssTheme": "Catppuccin"
}

30
notes/.obsidian/core-plugins-migration.json vendored
View file

@ -1,30 +0,0 @@
{
"file-explorer": true,
"global-search": true,
"switcher": true,
"graph": true,
"backlink": true,
"canvas": true,
"outgoing-link": true,
"tag-pane": true,
"properties": false,
"page-preview": true,
"daily-notes": true,
"templates": true,
"note-composer": true,
"command-palette": true,
"slash-command": false,
"editor-status": true,
"bookmarks": true,
"markdown-importer": false,
"zk-prefixer": false,
"random-note": false,
"outline": true,
"word-count": true,
"slides": false,
"audio-recorder": false,
"workspaces": false,
"file-recovery": true,
"publish": false,
"sync": false
}

20
notes/.obsidian/core-plugins.json vendored
View file

@ -1,20 +0,0 @@
[
"file-explorer",
"global-search",
"switcher",
"graph",
"backlink",
"canvas",
"outgoing-link",
"tag-pane",
"page-preview",
"daily-notes",
"templates",
"note-composer",
"command-palette",
"editor-status",
"bookmarks",
"outline",
"word-count",
"file-recovery"
]

7
notes/.obsidian/themes/Catppuccin/manifest.json vendored
View file

@ -1,7 +0,0 @@
{
"name": "Catppuccin",
"version": "0.4.24",
"minAppVersion": "1.0.0",
"author": "Marshall Beckrich",
"authorUrl": "https://github.com/catppuccin/obsidian"
}

3865
notes/.obsidian/themes/Catppuccin/theme.css vendored
View file

File diff suppressed because one or more lines are too long

155
notes/.obsidian/workspace.json vendored
View file

@ -1,155 +0,0 @@
{
"main": {
"id": "ba6656b2e09ced8b",
"type": "split",
"children": [
{
"id": "a5f8ef865112d597",
"type": "tabs",
"children": [
{
"id": "394ad00be71f8b2e",
"type": "leaf",
"state": {
"type": "markdown",
"state": {
"file": "UwUsched/Protocol.md",
"mode": "source",
"source": false
}
}
}
]
}
],
"direction": "vertical"
},
"left": {
"id": "351ee0d58b83aaeb",
"type": "split",
"children": [
{
"id": "5e4f44266924bf54",
"type": "tabs",
"children": [
{
"id": "9f2faf4f1827783f",
"type": "leaf",
"state": {
"type": "file-explorer",
"state": {
"sortOrder": "alphabetical"
}
}
},
{
"id": "24d16549c3e9f26c",
"type": "leaf",
"state": {
"type": "search",
"state": {
"query": "",
"matchingCase": false,
"explainSearch": false,
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical"
}
}
},
{
"id": "78556ee640811968",
"type": "leaf",
"state": {
"type": "bookmarks",
"state": {}
}
}
]
}
],
"direction": "horizontal",
"width": 300
},
"right": {
"id": "41231040187f060c",
"type": "split",
"children": [
{
"id": "3bb571b4d8a6b510",
"type": "tabs",
"children": [
{
"id": "f123b7a34597da70",
"type": "leaf",
"state": {
"type": "backlink",
"state": {
"file": "UwUsched/Protocol.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
"showSearch": false,
"searchQuery": "",
"backlinkCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "d3b5730a2ea06ff4",
"type": "leaf",
"state": {
"type": "outgoing-link",
"state": {
"file": "UwUsched/Protocol.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "53131ea4b1beb2c9",
"type": "leaf",
"state": {
"type": "tag",
"state": {
"sortOrder": "frequency",
"useHierarchy": true
}
}
},
{
"id": "f3e3169cbae74ce5",
"type": "leaf",
"state": {
"type": "outline",
"state": {
"file": "UwUsched/Protocol.md"
}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"left-ribbon": {
"hiddenItems": {
"switcher:Open quick switcher": false,
"graph:Open graph view": false,
"canvas:Create new canvas": false,
"daily-notes:Open today's daily note": false,
"templates:Insert template": false,
"command-palette:Open command palette": false
}
},
"active": "394ad00be71f8b2e",
"lastOpenFiles": [
"UwUsched/UwUSched.md",
"UwUsched/Protocol.md",
"UwUsched/Cluster Architecture.md"
]
}

4
notes/UwUsched/.obsidian/app.json vendored
View file

@ -1,4 +0,0 @@
{
"vimMode": true,
"alwaysUpdateLinks": true
}

6
notes/UwUsched/.obsidian/appearance.json vendored
View file

@ -1,6 +0,0 @@
{
"cssTheme": "Catppuccin",
"theme": "obsidian",
"monospaceFontFamily": "Source Code Pro",
"textFontFamily": "Source Code Pro"
}

30
notes/UwUsched/.obsidian/core-plugins-migration.json vendored
View file

@ -1,30 +0,0 @@
{
"file-explorer": true,
"global-search": true,
"switcher": true,
"graph": true,
"backlink": true,
"canvas": true,
"outgoing-link": true,
"tag-pane": true,
"properties": false,
"page-preview": true,
"daily-notes": true,
"templates": true,
"note-composer": true,
"command-palette": true,
"slash-command": false,
"editor-status": true,
"bookmarks": true,
"markdown-importer": false,
"zk-prefixer": false,
"random-note": false,
"outline": true,
"word-count": true,
"slides": false,
"audio-recorder": false,
"workspaces": false,
"file-recovery": true,
"publish": false,
"sync": false
}

20
notes/UwUsched/.obsidian/core-plugins.json vendored
View file

@ -1,20 +0,0 @@
[
"file-explorer",
"global-search",
"switcher",
"graph",
"backlink",
"canvas",
"outgoing-link",
"tag-pane",
"page-preview",
"daily-notes",
"templates",
"note-composer",
"command-palette",
"editor-status",
"bookmarks",
"outline",
"word-count",
"file-recovery"
]

22
notes/UwUsched/.obsidian/graph.json vendored
View file

@ -1,22 +0,0 @@
{
"collapse-filter": true,
"search": "",
"showTags": false,
"showAttachments": false,
"hideUnresolved": false,
"showOrphans": true,
"collapse-color-groups": true,
"colorGroups": [],
"collapse-display": true,
"showArrow": false,
"textFadeMultiplier": 0,
"nodeSizeMultiplier": 1,
"lineSizeMultiplier": 1,
"collapse-forces": true,
"centerStrength": 0.518713248970312,
"repelStrength": 10,
"linkStrength": 1,
"linkDistance": 250,
"scale": 1,
"close": true
}

7
notes/UwUsched/.obsidian/themes/Catppuccin/manifest.json vendored
View file

@ -1,7 +0,0 @@
{
"name": "Catppuccin",
"version": "0.4.24",
"minAppVersion": "1.0.0",
"author": "Marshall Beckrich",
"authorUrl": "https://github.com/catppuccin/obsidian"
}

3865
notes/UwUsched/.obsidian/themes/Catppuccin/theme.css vendored
View file

File diff suppressed because one or more lines are too long

159
notes/UwUsched/.obsidian/workspace.json vendored
View file

@ -1,159 +0,0 @@
{
"main": {
"id": "6226bb31998275a0",
"type": "split",
"children": [
{
"id": "a0eecd0f2f82e63c",
"type": "tabs",
"children": [
{
"id": "0f16c2e047ea76e2",
"type": "leaf",
"state": {
"type": "markdown",
"state": {
"file": "Protocol.md",
"mode": "source",
"source": false
}
}
}
]
}
],
"direction": "vertical"
},
"left": {
"id": "404cda8cf3aaef46",
"type": "split",
"children": [
{
"id": "56fd14939d4e1db3",
"type": "tabs",
"children": [
{
"id": "7ec07b2cf2f9abff",
"type": "leaf",
"state": {
"type": "file-explorer",
"state": {
"sortOrder": "alphabetical"
}
}
},
{
"id": "443ca75514af4821",
"type": "leaf",
"state": {
"type": "search",
"state": {
"query": "",
"matchingCase": false,
"explainSearch": false,
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical"
}
}
},
{
"id": "394fb16cdcdeec29",
"type": "leaf",
"state": {
"type": "bookmarks",
"state": {}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"right": {
"id": "c5deb5eb9a13ad1e",
"type": "split",
"children": [
{
"id": "6eddb59da5c4428c",
"type": "tabs",
"children": [
{
"id": "1a540ce1673cf6bb",
"type": "leaf",
"state": {
"type": "backlink",
"state": {
"file": "Protocol.md",
"collapseAll": false,
"extraContext": false,
"sortOrder": "alphabetical",
"showSearch": false,
"searchQuery": "",
"backlinkCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "94cc7396d731b430",
"type": "leaf",
"state": {
"type": "outgoing-link",
"state": {
"file": "Protocol.md",
"linksCollapsed": false,
"unlinkedCollapsed": true
}
}
},
{
"id": "76e3b53e26acc5df",
"type": "leaf",
"state": {
"type": "tag",
"state": {
"sortOrder": "frequency",
"useHierarchy": true
}
}
},
{
"id": "02804ac32c45e084",
"type": "leaf",
"state": {
"type": "outline",
"state": {
"file": "Protocol.md"
}
}
}
]
}
],
"direction": "horizontal",
"width": 300,
"collapsed": true
},
"left-ribbon": {
"hiddenItems": {
"switcher:Open quick switcher": false,
"graph:Open graph view": false,
"canvas:Create new canvas": false,
"daily-notes:Open today's daily note": false,
"templates:Insert template": false,
"command-palette:Open command palette": false
}
},
"active": "0f16c2e047ea76e2",
"lastOpenFiles": [
"UwUSched.md",
"Cluster Architecture.md",
"active-active.md",
"active-passive.png",
"active-active.png",
"create a link.md"
]
}

0
notes/UwUsched/.trash/active-active.md
View file

0
notes/UwUsched/.trash/create a link.md
View file

7
notes/UwUsched/Cluster Architecture.md
View file

@ -1,7 +0,0 @@
# Active-Active
![[active-active.png]]
The active-active architecture offers in high-availability, scalability and no single-point-of-failure.
-> Several Head-Nodes
# Active-Passive
![[active-passive.png]]Contrary to [[Cluster Architecture#Active-Active |active-active]] clustering, the active-passive method uses only one head-node and no load-balancer, with one additional head-node running as a failover.

123
notes/UwUsched/Protocol.md
View file

@ -1,123 +0,0 @@
- UUID-identifier with data
- Data binary- or hex-encoded
- Client <-> Host negotiating what to do => Task queue synchronization
=> Two seperate ways of communicating: One for neogtioating task queue stuff and one for data (Linked with UUID)
## Conn-Status
### Login
*Request:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"specs": {
"threads":<threads:u64>,
"clock":<clock:f64>,
"mem":<mem:u64>,
"net_speed_kib":<net_speed:f64>
}
}
```
*Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"status":<ACCEPTED|REJECTED>,
}
```
### Alive Check
*Request/Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>
}
```
### Logout
*Request:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
}
```
*Response:*
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"transaction_uuid":<transaction_uuid>,
"status":<ACK|OK|ERR>
}
```
>If the re-scheduling fails trigger the worker-node to give all HashMap-data to the head-node
## Pull
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>,
}
```
## Response
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"status":<OK|HASH_ERR|TIMEOUT_ERR>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>
}
```
## Push
```json
{
"protocol_id":<protocol_id>,
"node_id":<node_id>,
"uuid":<uuid>,
"hashmap_id":<hashmap_id>,
"length_decomp": <length_decomp>,
"length_comp": <length_comp>,
"comp_format":<comp_format>,
"attributes":{go wild :3}
"data":<data>
}
```
> **Possible Traffic-Sequences**
> `PULL -> PUSH -> RESPONSE`
> `PUSH -> RESPONSE`
----
## JSON-Field Descriptions
**Data-Related**
`uuid`: sha-256 of decompressed data
`hashmap_identifier`: identifier of associated HashMap
`length_comp`: length of compressed data
`length_decomp`: length of decompressed data
`comp_format`: compression algorithm used
`data`: compressed data
`data_type`: either `permutation` or `state` (complete tree)
`attributes`: Implementation specific *EXTRAWURST*
**Control Flow**
`protocol_id`: `data_push | data_pull | data_response | hash_push | hash_pull | hash_response`
`node_id`:unique identifier of nodes. (Hostnames)
`status`: error codes indicating success of transmission

3
notes/UwUsched/UwUSched.md
View file

@ -1,3 +0,0 @@
UwUsched is an [[Cluster Architecture#Active-Passive| active-passive]] Cluster.
The protocol is defined [[Protocol |here]]

BIN
notes/UwUsched/active-active.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 51 KiB

BIN
notes/UwUsched/active-passive.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 41 KiB

7
yotei-nodes/Cargo.toml
View file

@ -2,6 +2,9 @@
authors = ["xqtc"] authors = ["xqtc"]
description = "Distributed computing with abstract data" description = "Distributed computing with abstract data"
name = "yotei" name = "yotei"
license = "BSD-3-Clause"
readme = "./README.md"
repository = "https://git.heroin.trade/xqtc/yotei"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
@ -33,5 +36,5 @@ name = "head-node"
path = "src/head.rs" path = "src/head.rs"
[[bin]] [[bin]]
name = "compute-node" name = "worker-node"
path = "src/compute.rs" path = "src/worker.rs"

4
README.md → yotei-nodes/README.md
View file

@ -1,6 +1,6 @@
# UwUsched # YOTEI
UwUsched is an active-active, high-availability clustering soloution. YOTEI is an active-active, high-availability clustering soloution.
## Basic Architecture ## Basic Architecture

10
yotei-nodes/build.rs
View file

@ -2,10 +2,16 @@ use std::{env, path::PathBuf};
fn main() -> Result<(), Box<dyn std::error::Error>> { fn main() -> Result<(), Box<dyn std::error::Error>> {
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
std::env::set_var("CARGO_MANIFEST_DIR", std::env::var("PWD").unwrap());
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
let proto_dir = manifest_dir.join("./protos/");
let proto_dir_file = manifest_dir.join("./protos/protocol.proto");
tonic_build::configure() tonic_build::configure()
.file_descriptor_set_path(out_dir.join("protocol_descriptor.bin")) .file_descriptor_set_path(out_dir.join("protocol_descriptor.bin"))
.compile(&["../protos/protocol.proto"], &["../protos"])?; // .compile(&["../protos/protocol.proto"], &["../protos"])?;
tonic_build::compile_protos("../protos/protocol.proto")?; .compile(&[proto_dir_file.clone()], &[proto_dir])?;
tonic_build::compile_protos(proto_dir_file)?;
Ok(()) Ok(())
} }

8
yotei-nodes/src/head.rs
View file

@ -72,14 +72,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
pretty_env_logger::init(); pretty_env_logger::init();
use colorful::Color; use colorful::Color;
use colorful::Colorful; use colorful::Colorful;
use colorful::HSL;
println!(" {}", "+--------------+".color(Color::LightBlue)); println!(" {}", "+-------------+".color(Color::LightBlue));
println!(" {}", "| 予 定 |".color(Color::LightPink1)); println!(" {}", "| 予 定 |".color(Color::LightPink1));
println!( println!(
" {}", " {}",
"+--------------+".color(colorful::RGB::new(252, 252, 252)) "+-------------+".color(colorful::RGB::new(252, 252, 252))
); );
println!(" {}", "YOITEI SCHEDULER".color(Color::LightPink1)); println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " HEAD NODE \n".color(Color::LightBlue)); println!(" {}", " HEAD NODE \n".color(Color::LightBlue));
let cli = Cli::parse(); let cli = Cli::parse();
match &cli.command { match &cli.command {

33
yotei-nodes/src/compute.rs → yotei-nodes/src/worker.rs
View file

@ -11,21 +11,32 @@ pub mod sched {
tonic::include_proto!("sched"); tonic::include_proto!("sched");
} }
fn greet() {
use colorful::Color;
use colorful::Colorful;
println!(" {}", "+-------------+".color(Color::LightBlue));
println!(" {}", "| 予 定 |".color(Color::LightPink1));
println!(
" {}",
"+-------------+".color(colorful::RGB::new(252, 252, 252))
);
println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " WORKER NODE\n".color(Color::LightBlue));
}
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> { async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::env::set_var("RUST_LOG", CONFIG.log.level.clone()); std::env::set_var("RUST_LOG", CONFIG.log.level.clone());
pretty_env_logger::init(); pretty_env_logger::init();
use colorful::Color; greet();
use colorful::Colorful; // println!(" {}", "+-------------+".color(Color::LightBlue));
// println!(" {}", "| 予 定 |".color(Color::LightPink1));
println!(" {}", "+--------------+".color(Color::LightBlue)); // println!(
println!(" {}", "| 予定 |".color(Color::LightPink1)); // " {}",
println!( // "+-------------+".color(colorful::RGB::new(252, 252, 252))
" {}", // );
"+--------------+".color(colorful::RGB::new(252, 252, 252)) // println!(" {}", "YOTEI SCHEDULER".color(Color::LightPink1));
); // println!(" {}", " WORKER NODE\n".color(Color::LightBlue));
println!(" {}", "YOITEI SCHEDULER".color(Color::LightPink1));
println!(" {}", " COMPUTE NODE\n".color(Color::LightBlue));
let server_addr = "http://[::1]:50051"; let server_addr = "http://[::1]:50051";
let addr: SocketAddr = "[::1]:50053".parse()?; let addr: SocketAddr = "[::1]:50053".parse()?;
let data = sched::data_client::DataClient::connect(server_addr).await?; let data = sched::data_client::DataClient::connect(server_addr).await?;