feat: add __network/html-crawl-parse, __wasm/wasmtime-serde-demo

__wasm/wasmtime-serde-demo/crates/guest/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "wasmtime_serde_guest"
+version = "0.1.0"
+edition = "2021"
+license = "Unlicense OR MIT"
+authors = ["Heráclito <heraclitoqsaldanha@gmail.com>"]
+description = "Simple library for serializing complex types to the wasmtime runtime using serde"
+repository = "https://github.com/Heraclito-Q-Saldanha/wasmtime_serde"
+
+[dependencies]
+serde = "1.0.163"
+bincode = "1.3.3"
+wasmtime_serde_guest_macro = "0.1.0"

__wasm/wasmtime-serde-demo/crates/guest/src/lib.rs

@@ -0,0 +1,60 @@
+//! Simple library for serializing complex types to the wasmtime runtime using serde
+
+pub use bincode::{deserialize, serialize};
+pub use wasmtime_serde_guest_macro::*;
+
+#[inline]
+#[no_mangle]
+pub extern "C" fn alloc(len: u32) -> *mut u8 {
+	let mut buf = Vec::with_capacity(len as _);
+	let ptr = buf.as_mut_ptr();
+	std::mem::forget(buf);
+	return ptr;
+}
+
+#[inline]
+#[no_mangle]
+pub unsafe extern "C" fn dealloc(value: u64) {
+	let (ptr, len) = from_bitwise(value);
+	let ptr = std::mem::transmute::<usize, *mut u8>(ptr as _);
+	let buffer = Vec::from_raw_parts(ptr, len as _, len as _);
+	std::mem::drop(buffer);
+}
+
+pub fn write_msg<T: serde::ser::Serialize>(value: &T) -> u64 {
+	let mut buffer = bincode::serialize(value).unwrap();
+	let len = buffer.len();
+	let ptr = buffer.as_mut_ptr();
+	std::mem::forget(buffer);
+	into_bitwise(ptr as _, len as _)
+}
+
+pub unsafe fn read_msg<T: serde::de::DeserializeOwned>(value: u64) -> T {
+	let (ptr, len) = from_bitwise(value);
+	let ptr = std::mem::transmute::<usize, *mut u8>(ptr as _);
+	let buffer = Vec::from_raw_parts(ptr, len as _, len as _);
+	bincode::deserialize(&buffer).unwrap()
+}
+
+#[inline(always)]
+const fn from_bitwise(value: u64) -> (u32, u32) {
+	((value << 32 >> 32) as u32, (value >> 32) as u32)
+}
+
+#[inline(always)]
+const fn into_bitwise(a: u32, b: u32) -> u64 {
+	(a as u64) | (b as u64) << 32
+}
+
+#[cfg(test)]
+mod test {
+	use crate::*;
+
+	#[test]
+	fn bitwise() {
+		const DATA: (u32, u32) = (10, 20);
+		const INTO: u64 = into_bitwise(DATA.0, DATA.1);
+		const FROM: (u32, u32) = from_bitwise(INTO);
+		assert_eq!(DATA, FROM)
+	}
+}

__wasm/wasmtime-serde-demo/crates/guest_macro/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "wasmtime_serde_guest_macro"
+version = "0.1.0"
+edition = "2021"
+license = "Unlicense OR MIT"
+authors = ["Heráclito <heraclitoqsaldanha@gmail.com>"]
+description = "Simple library for serializing complex types to the wasmtime runtime using serde"
+repository = "https://github.com/Heraclito-Q-Saldanha/wasmtime_serde"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = {version = "2.0.18", features = ["full"] }
+quote = "1.0.28"

__wasm/wasmtime-serde-demo/crates/guest_macro/src/lib.rs

@@ -0,0 +1,89 @@
+//! Simple library for serializing complex types to the wasmtime runtime using serde
+
+use quote::quote;
+
+#[proc_macro_attribute]
+pub fn export_fn(_attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let data = syn::parse_macro_input!(item as syn::ItemFn);
+	let name = &data.sig.ident;
+	let extern_name = quote::format_ident!("_wasm_guest_{}", name);
+	let gen = {
+		let mut argument_types = quote!();
+		let mut call = quote!();
+		for (i, arg) in data.sig.inputs.iter().enumerate() {
+			let i = syn::Index::from(i);
+			call = quote!(#call message.#i,);
+			if let syn::FnArg::Typed(t) = arg {
+				let ty = &t.ty;
+				argument_types = quote!(#argument_types #ty,);
+			} else {
+				panic!();
+			}
+		}
+		argument_types = quote! { (#argument_types) };
+		quote! {
+			#[no_mangle]
+			pub unsafe extern "C" fn #extern_name(value: u64) -> u64 {
+				let message:#argument_types = wasmtime_serde_guest::read_msg(value);
+				wasmtime_serde_guest::write_msg(&#name(#call))
+			}
+		}
+	};
+	quote!(#gen #data).into()
+}
+
+struct FnImports {
+	functions: Vec<syn::Signature>,
+}
+
+impl syn::parse::Parse for FnImports {
+	fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
+		let mut functions = vec![];
+		while let Ok(f) = input.parse::<syn::Signature>() {
+			functions.push(f);
+			input.parse::<syn::Token![;]>()?;
+		}
+		Ok(FnImports { functions })
+	}
+}
+
+#[proc_macro]
+pub fn import_fn(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let mut remote_fns = quote!();
+	let mut local_fns = quote!();
+	let data = syn::parse_macro_input!(input as FnImports);
+	for f in data.functions.iter().cloned() {
+		let remote_name = quote::format_ident!("_wasm_host_{}", f.ident);
+		let mut inputs = quote!();
+		for item in &f.inputs {
+			if let syn::FnArg::Typed(syn::PatType { pat: p, .. }) = item {
+				if let syn::Pat::Ident(i) = p.as_ref() {
+					inputs = quote!(#inputs #i,);
+				} else {
+					panic!()
+				}
+			} else {
+				panic!()
+			}
+		}
+		inputs = quote!((#inputs));
+		local_fns = quote!(
+			#local_fns
+			#f {
+				let ptr = wasmtime_serde_guest::write_msg(&#inputs);
+				unsafe{wasmtime_serde_guest::read_msg(#remote_name(ptr))}
+			}
+		);
+		remote_fns = quote!(
+			#remote_fns
+			fn #remote_name(ptr: u64) -> u64;
+		);
+	}
+	quote! {
+		#local_fns
+		extern "C" {
+			#remote_fns
+		}
+	}
+	.into()
+}

__wasm/wasmtime-serde-demo/crates/host/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "wasmtime_serde_host"
+version = "0.1.3"
+edition = "2021"
+license = "Unlicense OR MIT"
+authors = ["Heráclito <heraclitoqsaldanha@gmail.com>"]
+description = "Simple library for serializing complex types to the wasmtime runtime using serde"
+repository = "https://github.com/Heraclito-Q-Saldanha/wasmtime_serde"
+
+[dependencies]
+wasmtime_serde_host_macro = "0.1.0"
+wasmtime = {version = "9.0.2", default-features = false, features = ["cranelift"]}
+serde = "1.0.163"
+bincode = "1.3.3"
+anyhow = "1.0.71"

__wasm/wasmtime-serde-demo/crates/host/src/func.rs

@@ -0,0 +1,29 @@
+use crate::*;
+use std::{cell::RefCell, rc::Rc};
+
+pub struct Func<P: serde::ser::Serialize, R: serde::de::DeserializeOwned> {
+	pub(crate) wasm_fn: wasmtime::TypedFunc<u64, u64>,
+	pub(crate) store: Rc<RefCell<wasmtime::Store<Option<RuntimeCaller>>>>,
+	pub(crate) par: std::marker::PhantomData<P>,
+	pub(crate) rtn: std::marker::PhantomData<R>,
+}
+
+impl<P: serde::ser::Serialize, R: serde::de::DeserializeOwned> Func<P, R> {
+	/// a more ergonomic version of the check_call function, which panic if it fails, using an analogy to an array, if checked_call were array.get(i), call would be array\[i\]
+	pub fn call(&self, value: &P) -> R {
+		self.checked_call(value).unwrap()
+	}
+	/// fail if the function in the guest panic and does not return
+	pub fn checked_call(&self, value: &P) -> anyhow::Result<R> {
+		let RuntimeCaller { memory, alloc_fn, .. } = self.store.borrow().data().unwrap();
+		let buffer = serialize(value)?;
+		let len = buffer.len() as _;
+		let ptr = alloc_fn.call(&mut *self.store.borrow_mut(), len)?;
+		memory.write(&mut *self.store.borrow_mut(), ptr as _, &buffer)?;
+		let ptr = self.wasm_fn.call(&mut *self.store.borrow_mut(), into_bitwise(ptr, len))?;
+		let (ptr, len) = from_bitwise(ptr);
+		let mut buffer = vec![0u8; len as _];
+		memory.read(&*self.store.borrow(), ptr as _, &mut buffer)?;
+		Ok(deserialize(&buffer)?)
+	}
+}

__wasm/wasmtime-serde-demo/crates/host/src/lib.rs

@@ -0,0 +1,32 @@
+//! Simple library for serializing complex types to the wasmtime runtime using serde
+
+mod func;
+mod runtime;
+
+pub use bincode::{deserialize, serialize};
+pub use func::*;
+pub use runtime::*;
+pub use wasmtime_serde_host_macro::*;
+
+#[inline(always)]
+const fn from_bitwise(value: u64) -> (u32, u32) {
+	((value << 32 >> 32) as u32, (value >> 32) as u32)
+}
+
+#[inline(always)]
+const fn into_bitwise(a: u32, b: u32) -> u64 {
+	(a as u64) | (b as u64) << 32
+}
+
+#[cfg(test)]
+mod test {
+	use crate::*;
+
+	#[test]
+	fn bitwise() {
+		const DATA: (u32, u32) = (10, 20);
+		const INTO: u64 = into_bitwise(DATA.0, DATA.1);
+		const FROM: (u32, u32) = from_bitwise(INTO);
+		assert_eq!(DATA, FROM)
+	}
+}

__wasm/wasmtime-serde-demo/crates/host/src/runtime.rs

@@ -0,0 +1,68 @@
+use crate::*;
+use std::{cell::RefCell, rc::Rc};
+
+pub struct Runtime {
+	instance: wasmtime::Instance,
+	store: Rc<RefCell<wasmtime::Store<Option<RuntimeCaller>>>>,
+}
+
+#[derive(Clone, Copy)]
+pub(crate) struct RuntimeCaller {
+	pub(crate) memory: wasmtime::Memory,
+	pub(crate) alloc_fn: wasmtime::TypedFunc<u32, u32>,
+	pub(crate) dealloc_fn: wasmtime::TypedFunc<u64, ()>,
+}
+
+impl Runtime {
+	pub fn from_file(file: impl AsRef<std::path::Path>, imports: &'static [(&'static str, fn(&[u8]) -> Vec<u8>)]) -> anyhow::Result<Self> {
+		Self::new(&std::fs::read(&file)?, imports)
+	}
+	pub fn new(bytes: impl AsRef<[u8]>, imports: &'static [(&'static str, fn(&[u8]) -> Vec<u8>)]) -> anyhow::Result<Self> {
+		let engine = wasmtime::Engine::default();
+		let module = wasmtime::Module::new(&engine, bytes)?;
+		let mut store = wasmtime::Store::new(&engine, None);
+		let mut linker = wasmtime::Linker::new(&engine);
+		for (name, callback) in imports {
+			linker.func_wrap("env", name, |mut caller: wasmtime::Caller<Option<RuntimeCaller>>, ptr: u64| -> u64 {
+				let RuntimeCaller {
+					memory,
+					alloc_fn,
+					dealloc_fn,
+				} = caller.data().unwrap();
+				let (ptr, len) = from_bitwise(ptr);
+				let mut buffer = vec![0u8; len as _];
+				memory.read(&caller, ptr as _, &mut buffer).unwrap();
+				dealloc_fn.call(&mut caller, into_bitwise(ptr, len)).unwrap();
+				let buffer = (callback)(&buffer);
+				let ptr = alloc_fn.call(&mut caller, buffer.len() as _).unwrap();
+				memory.write(&mut caller, ptr as _, &buffer).unwrap();
+				into_bitwise(ptr, buffer.len() as _)
+			})?;
+		}
+		let instance = linker.instantiate(&mut store, &module)?;
+		let memory = instance.get_memory(&mut store, "memory").unwrap();
+		let alloc_fn = instance.get_typed_func(&mut store, "alloc")?;
+		let dealloc_fn = instance.get_typed_func(&mut store, "dealloc")?;
+		*store.data_mut() = Some(RuntimeCaller {
+			memory,
+			alloc_fn,
+			dealloc_fn,
+		});
+		Ok(Self {
+			instance,
+			store: Rc::new(RefCell::new(store)),
+		})
+	}
+
+	pub fn get_func<P: serde::ser::Serialize, R: serde::de::DeserializeOwned>(&self, name: &str) -> anyhow::Result<Func<P, R>> {
+		let wasm_fn = self
+			.instance
+			.get_typed_func::<u64, u64>(&mut *self.store.borrow_mut(), &format!("_wasm_guest_{name}"))?;
+		Ok(Func {
+			wasm_fn,
+			store: self.store.clone(),
+			par: std::marker::PhantomData::<P>,
+			rtn: std::marker::PhantomData::<R>,
+		})
+	}
+}

__wasm/wasmtime-serde-demo/crates/host/tests/runtime.rs

@@ -0,0 +1,32 @@
+#[cfg(test)]
+mod test {
+	use wasmtime_serde_host::*;
+	const GUEST_DATA: &[u8] = include_bytes!("guest.wasm");
+
+	#[test]
+	fn load_runtime() {
+		assert!(Runtime::new(GUEST_DATA, &[]).is_ok())
+	}
+
+	#[test]
+	fn get_func() {
+		let runtime = Runtime::new(GUEST_DATA, &[]).unwrap();
+		assert!(runtime.get_func::<(i32, i32), i32>("add").is_ok())
+	}
+
+	#[test]
+	fn call() {
+		let runtime = Runtime::new(GUEST_DATA, &[]).unwrap();
+		let add_fn = runtime.get_func::<(i32, i32), i32>("add").unwrap();
+		let result = add_fn.call(&(10, 10));
+		assert_eq!(result, 20)
+	}
+
+	#[test]
+	fn checked_call() {
+		let runtime = Runtime::new(GUEST_DATA, &[]).unwrap();
+		let panic_fn = runtime.get_func::<(), ()>("panic").unwrap();
+		let result = panic_fn.checked_call(&());
+		assert!(result.is_err())
+	}
+}

__wasm/wasmtime-serde-demo/crates/host_macro/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "wasmtime_serde_host_macro"
+version = "0.1.0"
+edition = "2021"
+license = "Unlicense OR MIT"
+authors = ["Heráclito <heraclitoqsaldanha@gmail.com>"]
+description = "Simple library for serializing complex types to the wasmtime runtime using serde"
+repository = "https://github.com/Heraclito-Q-Saldanha/wasmtime_serde"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = {version = "2.0.18", features = ["full"] }
+proc-macro2 = "1.0.59"
+quote = "1.0.28"

__wasm/wasmtime-serde-demo/crates/host_macro/src/lib.rs

@@ -0,0 +1,70 @@
+//! Simple library for serializing complex types to the wasmtime runtime using serde
+
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+use syn::{parse_macro_input, ItemFn};
+
+struct FnHost {
+	functions: Vec<syn::Ident>,
+}
+
+impl syn::parse::Parse for FnHost {
+	fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
+		let mut functions = vec![];
+		let mut end = false;
+		while let Ok(f) = input.parse::<syn::Ident>() {
+			if end {
+				panic!("comma")
+			}
+			functions.push(f);
+			end = input.parse::<syn::Token![,]>().is_err();
+		}
+		Ok(Self { functions })
+	}
+}
+
+#[proc_macro]
+pub fn host_funcs(input: TokenStream) -> TokenStream {
+	let input = parse_macro_input!(input as FnHost);
+	let mut list = quote!();
+	let len = input.functions.len();
+	for name in input.functions {
+		let name = format_ident!("_wasm_host_{}", name);
+		let str_name: proc_macro2::TokenStream = format!(r#""{name}""#).parse().unwrap();
+		list = quote!(#list  (#str_name, #name),);
+	}
+	quote!({
+		const HOST_FUNC:[(&str, fn(&[u8]) -> Vec<u8>);#len] = [#list];
+		&HOST_FUNC
+	})
+	.into()
+}
+
+#[proc_macro_attribute]
+pub fn export_fn(_attr: TokenStream, item: TokenStream) -> TokenStream {
+	let data = parse_macro_input!(item as ItemFn);
+	let name = &data.sig.ident;
+	let extern_name = format_ident!("_wasm_host_{}", name);
+	let gen = {
+		let mut argument_types = quote!();
+		let mut call = quote!();
+		for (i, arg) in data.sig.inputs.iter().enumerate() {
+			let i = syn::Index::from(i);
+			call = quote!(#call message.#i,);
+			if let syn::FnArg::Typed(t) = arg {
+				let ty = &t.ty;
+				argument_types = quote!(#argument_types #ty,);
+			} else {
+				panic!();
+			}
+		}
+		argument_types = quote! { (#argument_types) };
+		quote! {
+			fn #extern_name(value: &[u8]) -> Vec<u8> {
+				let message:#argument_types = wasmtime_serde_host::deserialize(value).unwrap();
+				wasmtime_serde_host::serialize(&#name(#call)).unwrap()
+			}
+		}
+	};
+	quote!(#gen #data).into()
+}