commit 92a14e865bdeb54d2e9665a66d7a61156ac76986
parent 50cda03e571a8313bc28e7850f456ec116b9c523
Author: minerva-jupiter <ryouturn@gmail.com>
Date: Sun, 19 Oct 2025 11:14:24 +0900
make dict .zst
Diffstat:
10 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/app/components/1_chat.tsx b/app/components/1_chat.tsx
@@ -118,7 +118,7 @@ function ChatPage() {
const [messages, setMessages] = useState<Message[]>(initialMessages);
const [ dict, setDict ] = useState<Uint8Array|undefined>(undefined);
const messagesEndRef = useRef<HTMLDivElement>(null);
- const dictPath = '/system.dic';
+ const dictPath = '/system.dic.zst';
useEffect(() => {
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
diff --git a/package.json b/package.json
@@ -5,6 +5,7 @@
"scripts": {
"dev": "next dev --turbopack",
"build": "next build --turbopack",
+ "wasmbuild": "cd ./rust-wasm/ && wasm-pack build --target web",
"start": "next start",
"lint": "eslint"
},
diff --git a/public/system.dic.zst b/public/system.dic.zst
Binary files differ.
diff --git a/rust-wasm/Cargo.lock b/rust-wasm/Cargo.lock
@@ -126,6 +126,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7"
dependencies = [
"find-msvc-tools",
+ "jobserver",
+ "libc",
"shlex",
]
@@ -324,6 +326,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
+[[package]]
name = "js-sys"
version = "0.3.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -397,6 +409,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
+[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -526,6 +544,7 @@ dependencies = [
"js-sys",
"vibrato",
"wasm-bindgen",
+ "zstd",
]
[[package]]
@@ -971,3 +990,31 @@ dependencies = [
"quote",
"syn",
]
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
diff --git a/rust-wasm/Cargo.toml b/rust-wasm/Cargo.toml
@@ -12,3 +12,4 @@ vibrato = "0.5.2"
wasm-bindgen = "0.2.104"
getrandom_v02 = { package = "getrandom", version = "0.2", features = ["js"] }
getrandom_v03 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
+zstd = "0.13.3"
diff --git a/rust-wasm/pkg/.gitignore b/rust-wasm/pkg/.gitignore
@@ -1 +1 @@
-
+*+
\ No newline at end of file
diff --git a/rust-wasm/pkg/rust_wasm.d.ts b/rust-wasm/pkg/rust_wasm.d.ts
@@ -7,6 +7,14 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly chat: (a: number, b: number, c: number, d: number) => [number, number, number, number];
+ readonly rust_zstd_wasm_shim_qsort: (a: number, b: number, c: number, d: number) => void;
+ readonly rust_zstd_wasm_shim_malloc: (a: number) => number;
+ readonly rust_zstd_wasm_shim_memcmp: (a: number, b: number, c: number) => number;
+ readonly rust_zstd_wasm_shim_calloc: (a: number, b: number) => number;
+ readonly rust_zstd_wasm_shim_free: (a: number) => void;
+ readonly rust_zstd_wasm_shim_memcpy: (a: number, b: number, c: number) => number;
+ readonly rust_zstd_wasm_shim_memmove: (a: number, b: number, c: number) => number;
+ readonly rust_zstd_wasm_shim_memset: (a: number, b: number, c: number) => number;
readonly __wbindgen_export_0: WebAssembly.Table;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
diff --git a/rust-wasm/pkg/rust_wasm_bg.wasm b/rust-wasm/pkg/rust_wasm_bg.wasm
Binary files differ.
diff --git a/rust-wasm/pkg/rust_wasm_bg.wasm.d.ts b/rust-wasm/pkg/rust_wasm_bg.wasm.d.ts
@@ -2,6 +2,14 @@
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const chat: (a: number, b: number, c: number, d: number) => [number, number, number, number];
+export const rust_zstd_wasm_shim_qsort: (a: number, b: number, c: number, d: number) => void;
+export const rust_zstd_wasm_shim_malloc: (a: number) => number;
+export const rust_zstd_wasm_shim_memcmp: (a: number, b: number, c: number) => number;
+export const rust_zstd_wasm_shim_calloc: (a: number, b: number) => number;
+export const rust_zstd_wasm_shim_free: (a: number) => void;
+export const rust_zstd_wasm_shim_memcpy: (a: number, b: number, c: number) => number;
+export const rust_zstd_wasm_shim_memmove: (a: number, b: number, c: number) => number;
+export const rust_zstd_wasm_shim_memset: (a: number, b: number, c: number) => number;
export const __wbindgen_export_0: WebAssembly.Table;
export const __wbindgen_malloc: (a: number, b: number) => number;
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
diff --git a/rust-wasm/src/lib.rs b/rust-wasm/src/lib.rs
@@ -4,7 +4,8 @@ use wasm_bindgen::prelude::*;
#[wasm_bindgen]
pub fn chat(dict_data: &[u8], input: &str) -> Result<String, JsValue> {
- let reader = Cursor::new(dict_data);
+ let encoded = Cursor::new(dict_data);
+ let reader = zstd::Decoder::new(encoded).unwrap();
let dict = Dictionary::read(reader)
.map_err(|_|{JsValue::from(js_sys::Error::new("Dictionary road Error"))})?;
let tokenizer = Tokenizer::new(dict);
@@ -13,6 +14,6 @@ pub fn chat(dict_data: &[u8], input: &str) -> Result<String, JsValue> {
worker.reset_sentence(input);
worker.tokenize();
- let ans : String= "そんなことを言ってるんじゃない。".to_string();
+ let ans : String= "馬鹿なことを言ってないで。".to_string();
Ok(ans)
}
