diff --git a/README.md b/README.md index a58a88c..4bfe404 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Project or files: ├── __misc │   ├── better_panic │   ├── bytes +│   ├── erasure-coding │   ├── fancy-regex │   ├── human_panic │   ├── lazy_static @@ -143,6 +144,10 @@ Project or files: ├── __search │   ├── simsearch │   └── tantivy +├── __security +│   ├── seckey +│   ├── secmem-proc +│   └── sensitive ├── __serialization │   ├── bytes │   ├── cbor @@ -183,6 +188,7 @@ Project or files: ├── __translate │   └── retranslate ├── __wasm +│   ├── connect.ssh │   ├── deno_rust_wasm_import_functions │   ├── deno_rust_wasm_js_sandbox │   ├── deno_rust_wasm_qr_decode @@ -232,7 +238,7 @@ Project or files: ├── vec.rs └── while.rs -202 directories, 33 files +207 directories, 36 files ``` diff --git a/__misc/erasure-coding/Cargo.lock b/__misc/erasure-coding/Cargo.lock new file mode 100644 index 0000000..08dd784 --- /dev/null +++ b/__misc/erasure-coding/Cargo.lock @@ -0,0 +1,145 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "erasure-coding" +version = "0.1.0" +dependencies = [ + "reed-solomon-erasure", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "libm" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da83a57f3f5ba3680950aa3cbc806fc297bc0b289d42e8942ed528ace71b8145" + +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reed-solomon-erasure" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fe31452b684b8b33f65f8730c8b8812c3f5a0bb8a096934717edb1ac488641" +dependencies = [ + "libm", + "parking_lot", + "smallvec", + "spin", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "smallvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" + +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/__misc/erasure-coding/Cargo.toml b/__misc/erasure-coding/Cargo.toml new file mode 100644 index 0000000..a45d68a --- /dev/null +++ b/__misc/erasure-coding/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "erasure-coding" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reed-solomon-erasure = "5.0.3" diff --git a/__misc/erasure-coding/README.md b/__misc/erasure-coding/README.md new file mode 100644 index 0000000..ed59de8 --- /dev/null +++ b/__misc/erasure-coding/README.md @@ -0,0 +1,5 @@ +EC libraries: +* https://github.com/cberner/erasure-coding +* https://github.com/rust-rse/reed-solomon-erasure +* https://crates.io/crates/liberasurecode + diff --git a/__misc/erasure-coding/src/main.rs b/__misc/erasure-coding/src/main.rs new file mode 100644 index 0000000..62eaa7d --- /dev/null +++ b/__misc/erasure-coding/src/main.rs @@ -0,0 +1,38 @@ +#[macro_use(shards)] +extern crate reed_solomon_erasure; + +use reed_solomon_erasure::galois_8::ReedSolomon; +// or use the following for Galois 2^16 backend +// use reed_solomon_erasure::galois_16::ReedSolomon; + +fn main () { + let r = ReedSolomon::new(3, 2).unwrap(); // 3 data shards, 2 parity shards + + let mut master_copy = shards!( + [0, 1, 2, 3], + [4, 5, 6, 7], + [8, 9, 10, 11], + [0, 0, 0, 0], // last 2 rows are parity shards + [0, 0, 0, 0] + ); + + // Construct the parity shards + r.encode(&mut master_copy).unwrap(); + + // Make a copy and transform it into option shards arrangement + // for feeding into reconstruct_shards + let mut shards: Vec<_> = master_copy.iter().cloned().map(Some).collect(); + + // We can remove up to 2 shards, which may be data or parity shards + shards[0] = None; + shards[4] = None; + + // Try to reconstruct missing shards + r.reconstruct(&mut shards).unwrap(); + + // Convert back to normal shard arrangement + let result: Vec<_> = shards.into_iter().filter_map(|x| x).collect(); + + assert!(r.verify(&result).unwrap()); + assert_eq!(master_copy, result); +}