From fd3f5cc5654ffe5f2140e703bfb481525ca3a04b Mon Sep 17 00:00:00 2001 From: quantumaikr Date: Fri, 10 Apr 2026 20:54:32 +0900 Subject: [PATCH 1/2] fix: n-gram loop detection + generation regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of ~530-token text collapse: small model + T=0 greedy enters repetition loop → KV quant error compounds through softmax → collapse. NOT a code bug — FP32 also degenerates, just slower. Fix: - Add 4-gram loop detection (stop when same 4-gram repeats 3+ times) - Increase rep_window 32→64, recent_tokens buffer 64→128 - Add bench/generation_regression_test.sh (4 tests): 1. T=0 500-token coherence check (no garbage output) 2. Loop detection fires on repetitive generation 3. No false positives at T=0.7 4. PPL within 15% of FP32 Why this wasn't caught before: - PPL tests are teacher-forced (no error accumulation) - Generation tests were ≤100 tokens (collapse at ~530) - No T=0 stress test existed Co-Authored-By: Claude Opus 4.6 (1M context) --- bench/generation_regression_test.sh | 139 ++++++++++++++++++++++++++++ src/engine/tq_generate.c | 47 ++++++++-- 2 files changed, 179 insertions(+), 7 deletions(-) create mode 100755 bench/generation_regression_test.sh diff --git a/bench/generation_regression_test.sh b/bench/generation_regression_test.sh new file mode 100755 index 0000000..73703d1 --- /dev/null +++ b/bench/generation_regression_test.sh @@ -0,0 +1,139 @@ +#!/bin/bash +# quant.cpp — Generation Regression Test +# +# Detects autoregressive generation collapse that PPL tests miss. +# Tests: T=0 greedy 500-token generation → verify no garbage output. +# +# The key insight: PPL (teacher-forced) is near-identical for FP32 and +# turbo_kv_4b at all context lengths. But autoregressive generation +# can collapse at ~500 tokens when T=0 repetition compounds KV quant error. +# +# This test catches that class of bugs by checking: +# 1. Loop detection triggers (prevents garbage, so verify it fires) +# 2. Output before loop detection is coherent (no random Unicode) +# 3. PPL sanity check at multiple context lengths +# +# Usage: +# bash bench/generation_regression_test.sh [model.gguf] +# +# Requires: built quant binary in build/ + +set -e + +MODEL="${1:-models/Llama-3.2-1B-Instruct-Q8_0.gguf}" +TQ_RUN="./build/quant" +THREADS=4 +PASS=0 +FAIL=0 + +if [ ! -f "$TQ_RUN" ]; then + echo "Error: $TQ_RUN not found. Build first." + exit 1 +fi +if [ ! -f "$MODEL" ]; then + echo "SKIP: Model not found: $MODEL" + exit 0 +fi + +echo "============================================" +echo " Generation Regression Test" +echo " Model: $MODEL" +echo "============================================" +echo "" + +check() { + local desc="$1" result="$2" + if [ "$result" = "PASS" ]; then + echo " [PASS] $desc" + PASS=$((PASS + 1)) + else + echo " [FAIL] $desc" + FAIL=$((FAIL + 1)) + fi +} + +# Test 1: T=0 generation should NOT produce garbage at 500 tokens +echo "[Test 1] T=0 500-token generation — no garbage output" +OUTPUT=$($TQ_RUN "$MODEL" -p "Explain the theory of relativity in detail" \ + -n 500 -T 0.0 -j $THREADS -k turbo_kv_4b --chat 2>/dev/null) + +# Check for garbage patterns: random Unicode, excessive non-ASCII +# Garbage typically has lots of CJK/Arabic/Thai mixed with Latin +GARBAGE_CHARS=$(echo "$OUTPUT" | tr -cd '\200-\377' | wc -c | tr -d ' ') +TOTAL_CHARS=$(echo "$OUTPUT" | wc -c | tr -d ' ') +if [ "$TOTAL_CHARS" -gt 0 ]; then + GARBAGE_RATIO=$((GARBAGE_CHARS * 100 / TOTAL_CHARS)) +else + GARBAGE_RATIO=100 +fi +if [ "$GARBAGE_RATIO" -lt 30 ]; then + check "turbo_kv_4b output coherence (${GARBAGE_RATIO}% non-ASCII)" "PASS" +else + check "turbo_kv_4b output coherence (${GARBAGE_RATIO}% non-ASCII, threshold 30%)" "FAIL" +fi + +# Test 2: Loop detection should fire for T=0 repetitive prompt +echo "" +echo "[Test 2] Loop detection fires on repetitive T=0 generation" +LOOP_OUTPUT=$($TQ_RUN "$MODEL" -p "what is your name?" \ + -n 1000 -T 0.0 -j $THREADS -k turbo_kv_4b 2>&1) + +if echo "$LOOP_OUTPUT" | grep -q "repetition loop detected"; then + LOOP_TOKENS=$(echo "$LOOP_OUTPUT" | grep "repetition loop" | grep -o "after [0-9]* tokens" | grep -o "[0-9]*") + check "loop detected at ${LOOP_TOKENS} tokens (before 500)" "PASS" +else + TOTAL_TOK=$(echo "$LOOP_OUTPUT" | grep "tok/s" | grep -o "^[0-9]*") + if [ "${TOTAL_TOK:-1000}" -lt 500 ]; then + check "EOS hit at ${TOTAL_TOK} tokens (no loop needed)" "PASS" + else + check "no loop detection in 1000 tokens" "FAIL" + fi +fi + +# Test 3: Non-repetitive generation should NOT trigger loop detection +echo "" +echo "[Test 3] Non-repetitive generation (T=0.7) — no false positives" +NORMAL_OUTPUT=$($TQ_RUN "$MODEL" -p "Tell me a creative story" \ + -n 200 -T 0.7 -j $THREADS -k turbo_kv_4b --chat 2>&1) + +if echo "$NORMAL_OUTPUT" | grep -q "repetition loop detected"; then + check "no false loop detection at T=0.7" "FAIL" +else + check "no false loop detection at T=0.7" "PASS" +fi + +# Test 4: FP32 vs turbo_kv_4b PPL sanity (if ppl data exists) +PPL_FILE="bench/data/ppl_test_1k.txt" +if [ -f "$PPL_FILE" ]; then + echo "" + echo "[Test 4] PPL sanity: turbo_kv_4b within 15% of FP32" + FP32_PPL=$($TQ_RUN "$MODEL" --ppl "$PPL_FILE" -k fp32 -j $THREADS 2>&1 \ + | grep "PPL_CSV" | cut -d, -f3) + Q4_PPL=$($TQ_RUN "$MODEL" --ppl "$PPL_FILE" -k turbo_kv_4b -j $THREADS 2>&1 \ + | grep "PPL_CSV" | cut -d, -f3) + + if [ -n "$FP32_PPL" ] && [ -n "$Q4_PPL" ]; then + # Compare using integer math (multiply by 1000) + FP32_INT=$(echo "$FP32_PPL" | awk '{printf "%d", $1 * 1000}') + Q4_INT=$(echo "$Q4_PPL" | awk '{printf "%d", $1 * 1000}') + THRESHOLD=$((FP32_INT * 115 / 100)) # 15% margin + if [ "$Q4_INT" -le "$THRESHOLD" ]; then + DELTA=$(echo "$FP32_PPL $Q4_PPL" | awk '{printf "%.1f", ($2/$1 - 1)*100}') + check "PPL delta: ${DELTA}% (within 15%)" "PASS" + else + DELTA=$(echo "$FP32_PPL $Q4_PPL" | awk '{printf "%.1f", ($2/$1 - 1)*100}') + check "PPL delta: ${DELTA}% (exceeds 15%)" "FAIL" + fi + else + check "PPL comparison (could not parse results)" "FAIL" + fi +fi + +echo "" +echo "============================================" +echo " Results: ${PASS} passed, ${FAIL} failed" +echo "============================================" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi diff --git a/src/engine/tq_generate.c b/src/engine/tq_generate.c index fb964e7..4180472 100644 --- a/src/engine/tq_generate.c +++ b/src/engine/tq_generate.c @@ -254,13 +254,22 @@ int tq_generate(tq_model_t* model, tq_tokenizer_t* tokenizer, int vocab_size = model->config.vocab_size; float rep_penalty = config->rep_penalty; int rep_window = config->rep_window; - if (rep_window > 64) rep_window = 64; - int recent_tokens[64]; + if (rep_window > 128) rep_window = 128; + int recent_tokens[128]; int recent_count = 0; + /* N-gram loop detection: track recent 4-grams to detect infinite loops. + * Small models with T=0 greedy decoding enter repetition loops where + * the same ~30-token pattern repeats endlessly. KV quantization error + * compounds through these repetitions, eventually collapsing output + * into garbage. Detecting loops early prevents wasted compute. */ + uint32_t ngram_hashes[64]; + int ngram_hash_count = 0; + int loop_detected = 0; + /* Seed recent tokens with tail of prompt for better penalty coverage */ for (int i = (n_prompt > rep_window ? n_prompt - rep_window : 0); i < n_prompt; i++) { - recent_tokens[recent_count % 64] = prompt_tokens[i]; + recent_tokens[recent_count % 128] = prompt_tokens[i]; recent_count++; } @@ -268,8 +277,8 @@ int tq_generate(tq_model_t* model, tq_tokenizer_t* tokenizer, if (rep_penalty > 1.0f) { int window = recent_count < rep_window ? recent_count : rep_window; for (int r = 0; r < window; r++) { - int idx = (recent_count - 1 - r) % 64; - if (idx < 0) idx += 64; + int idx = (recent_count - 1 - r) % 128; + if (idx < 0) idx += 128; int tok = recent_tokens[idx]; if (tok >= 0 && tok < vocab_size) { if (state->logits[tok] > 0) @@ -288,7 +297,7 @@ int tq_generate(tq_model_t* model, tq_tokenizer_t* tokenizer, &rng_state); /* Record first sampled token */ - recent_tokens[recent_count % 64] = next_token; + recent_tokens[recent_count % 128] = next_token; recent_count++; int generated = 0; @@ -483,8 +492,32 @@ int tq_generate(tq_model_t* model, tq_tokenizer_t* tokenizer, &rng_state); /* Record sampled token for repetition penalty */ - recent_tokens[recent_count % 64] = next_token; + recent_tokens[recent_count % 128] = next_token; recent_count++; + + /* N-gram loop detection: hash recent 4-gram and check for repeats */ + if (recent_count >= 4) { + uint32_t h = 0; + for (int r = 0; r < 4; r++) { + int gi = (recent_count - 4 + r) % 128; + h = h * 31 + (uint32_t)recent_tokens[gi]; + } + int matches = 0; + int ring_len = ngram_hash_count < 64 ? ngram_hash_count : 64; + for (int r = 0; r < ring_len; r++) { + if (ngram_hashes[r] == h) matches++; + } + ngram_hashes[ngram_hash_count % 64] = h; + ngram_hash_count++; + if (matches >= 3) { + loop_detected = 1; + break; + } + } + } + + if (loop_detected) { + fprintf(stderr, "[generate] repetition loop detected after %d tokens, stopping\n", generated); } /* Null-terminate output */ From 1f110d342d1f239e7991fb5f2161779a5fc9b276 Mon Sep 17 00:00:00 2001 From: quantumaikr Date: Fri, 10 Apr 2026 20:54:46 +0900 Subject: [PATCH 2/2] feat(wasm): Llama 3.2 1B Instruct default + skip Q4 reconversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes for WASM demo reliability and speed: 1. Model: switch from Qwen3.5-0.8B (base, gated, Qwen arch issues) to Llama 3.2 1B Instruct (verified working, good quality, public HuggingFace URL, proper Instruct tuning for chat). 2. Speed: add -DTQ_NO_Q4=1 to WASM build. Skips the load-time Q4 reconversion (GGUF Q4_K_M → FP32 → internal Q4) which was expensive and redundant for already-quantized models. Uses GGUF on-the-fly dequant instead. Saves several seconds of model init and reduces peak memory usage. Added compile-time #ifdef TQ_NO_Q4 guard in quant.h so it works in WASM (no getenv). Native builds are unaffected. Co-Authored-By: Claude Opus 4.6 (1M context) --- quant.h | 7 ++++++- wasm/build.sh | 1 + wasm/index.html | 21 ++++----------------- wasm/quant.wasm | Bin 292932 -> 292901 bytes 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/quant.h b/quant.h index 720521c..8cf1383 100644 --- a/quant.h +++ b/quant.h @@ -12179,8 +12179,13 @@ tq_model_t* tq_load_gguf(const char* path) { } const size_t MAX_FP32_BYTES = (size_t)16 * 1024 * 1024 * 1024ULL; /* 16 GB */ - /* TQ_NO_Q4=1 disables Q4 recompression → use direct GGUF dequant for better quality */ + /* TQ_NO_Q4=1 disables Q4 recompression → use direct GGUF dequant for better quality. + * Can be set via environment variable or compile-time define (useful for WASM). */ +#ifdef TQ_NO_Q4 + if (1) { +#else if (getenv("TQ_NO_Q4")) { +#endif fprintf(stderr, "tq_load_gguf: TQ_NO_Q4 set — skipping Q4 conversion, using GGUF on-the-fly dequant\n"); goto skip_q4_conversion; } diff --git a/wasm/build.sh b/wasm/build.sh index a611472..df74c62 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -40,6 +40,7 @@ emcc "$SCRIPT_DIR/quant_wasm.c" \ -lm \ -DNDEBUG \ -D__EMSCRIPTEN__ \ + -DTQ_NO_Q4=1 \ -Wno-gnu-zero-variadic-macro-arguments \ -Wno-dollar-in-identifier-extension diff --git a/wasm/index.html b/wasm/index.html index a70c6a5..6f4a663 100644 --- a/wasm/index.html +++ b/wasm/index.html @@ -174,16 +174,11 @@

Run an LLM in your browser

No install. No API key. No server.

-
@@ -223,17 +218,9 @@

Run an LLM in your browser

let activeModelId = null; const MODELS = { - 'qwen3.5-0.8b': { - url: 'https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q4_K_M.gguf', - name: 'Qwen3.5 0.8B', - size: 508, - cacheKey: 'qwen3.5-0.8b-q4km', - chatTemplate: (t) => `<|im_start|>user\n${t}<|im_end|>\n<|im_start|>assistant\n`, - cardId: 'card-qwen', metaId: 'meta-qwen', - }, 'llama-3.2-1b': { url: 'https://huggingface.co/hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF/resolve/main/llama-3.2-1b-instruct-q4_k_m.gguf', - name: 'Llama 3.2 1B', + name: 'Llama 3.2 1B Instruct', size: 770, cacheKey: 'llama-3.2-1b-q4km', chatTemplate: (t) => `<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n${t}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`, diff --git a/wasm/quant.wasm b/wasm/quant.wasm index 1159b15f49ce16bf71d7d540d4f8e00cb8838d0e..dc5a3ffbec4be0bb449c33205b6a09b6fea4195f 100755 GIT binary patch delta 12464 zcmbt44O~>k_V>)(Wp~+KSb;@86z;D0S(v6Nnkd(vir<>P^1aNoEDbfIkbG)_Woc!l zU21xjWs3QQ3Zh;sD=JJ)D^x-=%SG`k!H*(RllY&Ry9>C&`~QCL>5rY6^EKz3nK^T2 z?i62YvHDVr{6YdtwXNhjZ5w<N-skPA`NzcA}Mm3TOuj^zj{OqliQh!FN0dF zCCl%?e(k6{GOqeJQ0PE8MIkCy(-i1IBn2Fx?m@ym+91^o&s3NKGL=Pe>R6Qb4gB(HXek9~*4ic46n-A+T8!{1L$mFE>7a&s`7nI~v z`}LqBLSe2-GYY?CexWzYPSs}ZH_*j zxQoo(iQF_3R_r#LUp3nR9BJEBm1JO-ziF`{=V6a#4js%JcZY7^MP2A9?0J;)B?t0z zb4=(H?BJ|(QZkpw2b~X!La>4!vEZz+TWv~^Z-_l)Fw*02D?|yJ<`LvL6Vzi`R@5!` zS#7H1mXP?iG!I3Wz4JTo0~vBf#SS)g?eEY(t0#7{0xQ`ovhknHn@FkztM)YY@pSdb z`tR49p5&V3|L&VI*_u5}aa^x9bViBQ)mpExSalHM~QXvDQ42if&WL@K`>J5GEWc+@dZ<6J>7ILacEAs(j*P@yc1O7|!CYr9P`^ z3XTBH@ZhkB3G2b}#;K%;fxm(kTxF=Hstj^S)TWkbmtC$PJ|}QEK8woFyTas@F3e2dkD^iqaLOb?7PmRgr+IR@8?m@b@zuNx5(aV4qykz|}N zT5X8_SVSu>2yGVORV*-4A6^&6Y--Y4a}(CyXILsVwp4f~!(;M=rL(d7Vz3(#R?%ak zM-iGO-lO(St8bd@jRy2KE4sm_irH|cKk^NZJ1Dv!+$hscUEC4>RU2dYd_mp~IF6uL zL>Feo7?WZxPPrqPJ5J2o{e|}{)ik84j~e9c=aC%*H;@YLfrQymr5#CF3>Afw+Agx; zdOp|XURYbGcHL@*@3fkvfv~GExmPiTcMD1Kwud0UaL0<-;PUQ*+OZP;`U=8Z+JKcq zn!)6*OlbylwOO8VtD584xT?8+?77YHOv-JJrzrRKShOVVR!Q2DlZ2~LQWV85f?wvX zlDE*Y`W}G8+T;&BP^K;VAP&CQ3f8!+5rU4YrP_CET{#pa?r!^eXs_+-(xHM(0 z1!Z+y%ViB(I|yaHy0%$a|8^Qti z_Eve(6vseIc{EZRk=Lb}xuWGYH-^9FH8<{qHa5re!N%rz{%n@#pPQQFnUou$eYvSe zGn$8NzC{=wtNnd*G#Wp1GdF$_*5QQqet!El<)46m&X{r1?F5fmRk-saFy~X0phR3= z=ksIWq!v`55br12q#Z6Tx!`VOUs}+qdG_uN>^~K7_A3Rg8rb`2Q?^8+i1}OogNU&W zBKB>Gi9gv$Q&mbu6~`-WaiWuq+VPDg|Gkh)o?h7LzDBmmhHbK8TSmR)2AQ(zyIlCZ&#~H8?%0i6xnp->9nNUSx3vuCuC)3Y6@?~Br#0(# zH=NRL*&YKG+Q98?ZaI@z@BF0xofofB>!fs#Zy0c^Hf$7wa9YdR?rwFef!=0VGFD7+ z9Hk2FK%l_(K7n?_c84Lb4;p0V%Eaqj8CB=XjMv+s+S$6Hw%wp?g_I^vxn5B27nEo8 z*0^jY?r5dCck~stfBM>odAl(3Loc;@t8kOXZ820kV0%ED@X39M_~?@~8A=P=eo;ak zJOAKYEYEF&RXpSDT0~6y|KJz$h!0k{=3rFlp17!Pf?{;X!=0vu6 z`_5PpGr9CU#QFtFicQTI!ZCs3DUVvohRuN(ZDCm}a;ZjJS(dA3^J_-BBP0}y7e0^orW4sM!(9)@XJik~#U@Y%zoDY>*nD?3_6 zs+VZHew@UM4#ytA>|^n}b%u*ig-6knLieBj2;^y#YLz>4UdHnp?y+he z{-f!GJd%U(v_JoX80X-D2$0n3D&F$oAczWgYIT*ivo;PV`H!`&Q}Zrh0xmd^Mo3a5 zF*5aH+!9J{1LE|eG%qscF9k(T`YS5j5EsM6iZis$7OHtq#}{t+Ypyk`BoC%57!f4J zio3f92R;2($q(eI7s;##rEx-nFgXB55f+3g1?Lk)GM5I8qzS0$W^q|kP!saZf$}U| z9to6Zy;+_V+=Mt=pg21hN4dCUpS40iM-%cwnvfUjm&Ybq!PUFo>K0A#hc&?;UeBL@ z1HY>Y{>UcyTh{Z}A-@eF5$s$9%tHMia6weCIJJnoA-2E;kxnc-8Qw@sMysaQBh2(=n}g#O0hYk)M9ot9^!)$WgL9=FseXD zDAf}cxFO~{nvVHwk{jAb@}M7N1bs|3jQDxIhP~^C)-E315yzvuS@lPER^o=ZR36?D z#lyQ<^@n!@)=I$4hA4}h?-dpv)_B}U79RJ_sz2_t9v(0PKcaZxH>>`@&v51#ksnb! z@|#tEf!sl4D=m#?pn$-`c0yfX4zI$I%*_rg_iGs z3;ZbOu_!RuV|5U`y}@nkeHJ?!;*hrYXqbz|_R+8!i}81ZhQ+13VI3B4je$F`C>aAY zTs|>FabE+QP3%Z&k$(4B=!G`uOU6Ne3skW0CPL?arB$Fh6)VOa=g3%oCnc#yjw0>> zjKpUkN>R7+dn|P^xo#8rkTKJDus)OEwDs&RN$uh6qyk&;Ahc$w528t{*uNfx2Tea0 z;Z>7Xur-`5eGu$=-Gh)sphQo47-9%4WMig4FF4I!o&sOM8hzAM&>XOnRm_56c1()I z1s&h0S)WWu#q^Cmp9wsTV|kg-t+~8{)nr27CgxOq8mCCZF#l{w z)xYk8Z&>%ixbCfD-T(2_jq6^SE!O>z*>I-$%oOkccxKvz+p%ztVbA(G&}?4Hew_pV zb)uv{JQp^S;EFn)+_){Om$jOYVbNQM4&Z}bq4%8+vnf>Q`U0phh40)83YqQ>Qd%fM zij(gXMf#>i&=p8+Usk>t9=E*HS1~y|s)T)>N+R`{FTesjhPunI!(`0X^?R66e*7lP z;q4x80q#Pj`q#N|jkkAwV8Bz>!fU*JXdT?!30CO0l)^O=mhFy0&(N@<>9zNeqUj)h z3IAm?&%v~&;FZn5AN{3i`qRz8A3EPOy|)?o{gsg3gg&~r5`v%!1iMlNQ*iZ7yZ}9E zQ}}h4n^vIRuT8^OH3L_EYnpy}Gw_&dn9-Cz|E`9aO~KK?IF|G~R5pb~BYUzLH6o>;4B zRVsN5qf}Ubk`C|a3;UB@Cit2SNhLF}`B5sd@#*u`V6u&b`c73If_IE2j215rBdZ{f zbskQ7SyuK{2u{}|Rz94xWX}&LF|d-oKb$0CaTE(b!JP&{*AXNdF+)d?o_D;C!#r$BiOmW%!P59821^-hHPthJPxsh)-1ecnh~%3+7vr`lWtxEJ*{? zkEijZs;RxA*;DtB`>j__;lacii=lJ$VB%uG-h+x&uv_jW_rp2;g?mYe6W(KQP9c5a zBmM9cGK}IBN|;7ggZbEgNp0usq?Ol3veVPaFzd75NNOh$g_qq+BJ`0nNPocW?zP8A zCM;#vnWP&Q$uo%uffHvESO2BYkXlKW6i02SC$_eYC#H6cLyA$I>IxsC5l~U{Zw40W z`b_d9^euUY;I4(KMU476ic=3Xw7=l(gAMI3dHYa(8ylti)O6BW#t~<;$Y@xrpPNO( zWLV3t|BDQTHp}Gb9ZM{?cN?chBAu z9_ihfL%I2}2GZ;%e#{Uw1aVj|XBX#^ILD2kn0cgUs~bRED&DVv!E}Pnm`D1j{3i+@ zgRhs9z9!`g$#!BBYOss*$SqcsBDC|eHcyigVO~t2@KZ<{epFLX>E4o;J^M82XhJwf z?1HDs(D3EwB)7>%viQ9We)U3gm$T6Mqzif|Wj^T+%k^3F$wQ`8P%0dzmyHFf>-2^9aXc`K|TtcA(p1?Wo z)kiKQzXd}!yX#ewj5BM&t7Hu<)jKXFizsG61sYk3xsl%gb<)#8>h|h$SCT3COL0*ib3kUTzC&_FG+M4Xd?}U2WQ=}L=AhSq@ZEB?u;{j9=w`+~$VEBS9zCxyxx>8ni1!Ku37JZdG0tZ<3Rq~+qIz~4h&O!y_8`nr2Jj>#*VbER9 zdS4??!wRyV?LyUqjt zG5z;iQUkD?Ro0PhP_}I){e;4HeWFYwFhe<^XPRggi9T{3Jl&M8^OSB`o~~JdMU;?s(c7i+_)&?eJ7uG@eFA6z>K;-7=@ULll=1iAt-* z`Y+=te)HPN@+SzkuP4yXNPl$#XVaAj=!XEm=|?BhWZ8QDYr}&tvSl-9gx>vOnoq3x zc;@Fr@-lNzqv6}9(kKc$_2_A6k@cez1NS;-XvxZ_(^MR&PBUmc?h!*~&^Wi4&{mbf zK)&ez{6bQ{mvE0^&;~zJ4s_UJ^tWfw0s`f1_~UdArt*6pr-R@WbEea_@D1ycPLqJ? zGt=q6_!>X=B)zNMu4M5+km6>I7r)8Qb^|;{{c_ia`XT-949Z8GO_)U!@HGGIEV|Ss zsKq6$pcS=4tV1UK6Q@SPQ}kPaH}v0T(*Xd_vtDy(HXbTInnQ7?{*J}Zqh)ZG)y$(K zU_Be~G(7^_S=@YjAK+#(pMFmMIL4B*&?~Fh=`1=0s@aHaIv#S^hHUz_@@XH{;p~T- zJL>EZaT?Uu`lzj({RC(;dt?EfANE5ZH5k8j+7KqY@jgM_uh%S~r3BvA_bsGTDeTs} zFQ$*lr0!cicM1I%$X`F|lU|{xY;aN^^cGDEhC+QyE`39W=k&qr=rI`+qIR3;Y61mp ze?Fa$+v3my9HiAOr+~WAob3g)4UW~(0(uv=;S5l;{Ps-wejkP%_Lj8Ly&WWqm|wUwS5WC$4Gru91u zzXtM!EQ&Y!>ZLmIego36F5w1@ZIy4HT%w3@LfGbQw0|26pi-?39}!R^&SAmh9L1AW zkcux6sCt$~Y^R+dV!)`maNlPqYM+c0sQ4egZivOr_1~PZwj;j zNzd^g8hYx(KcdtMKeEwZ(s{U7e)}c8&As>)qT(OlUwCE+(=zaRm4pcbHu!U&5n3bao(Fk|1*gkY&`9v6N%f zmE{-ucs-nVkG*se?6v^*>49Xo3bNn;vZ;Y&9R!&*fNXM3R)B2@g3b~^H!)D0BFM}E zWa9(LS_!hC0J3q7$k>>lXm~Hb6UH{e2q#1c0djzVQT1f(y`LKSa9BOu=R><-CjsoK zfn*jz1_5M$3nY^TnG`^ll4J4vP`~qM+~D9a%R5dx#*|Gj^n@)@!f+GiZv^<5@;Q1IE8cNs{FYSeWz3-*1pp1R$ zrBTt}BTtJZcm*Lo+H#THqlr=}yU0ZzYCz*t2ae_*OOy_>_;N%aXh7pz0FLG^NR&$0 z$a0MRhuOR3xbv2=o#no+eAeN&Z%*9Z*Hsml#k=yMeO=*16 zgs^JS3EEnC&}WX{gHeL)d-f@p!aeB2`#k6r=tJxRM{^JQ&^{01Kw~EdSlmhB!5$~+ zr5G{y@EZ2!<<3$n8ci-fb~l89BrzRA1ExbpAQ`%)iDV zKQKUy3M9i}MKw(UM)S#ggEYG!3yc=2Ik-RGK!?)+#nAw9e6rs_CJQox&te3%Yl>L){K zk z!Lm^l`rga*U<5vJV9SH#gse;AcxfDx?dbP3<7*Bk^v-b11Z^r37-w2cX`E>ZlDhR} zqGlI~4|{o7UHZxwqK=>D)lU2zozL+Gfz6h40@_OI^A_8;xp1jp#&fPi`9ECj7y820 z>EpIt>^J&8y`NcL7p-4bX0o zz%cxakkrz>tjig2x?{vz%(P7@}n``Fs4azflQ-+(P72C*?Q{1^XOCAAQNfSrHFenGLp zRlFG~saLk_V>)(Wp~+KSb+rr#k*Jh4$~|J73EqfnxAFmdzooj5^72z`P2m8qojYD z*0sjIE%Dn-1ie~Xlv-w1RAl}Q6BHFw4E)w4|7Ygz0jB+<$djeppY zBBiSm=c0)UWvYA}yfVqgQOoATz$t@}5;9dH$PVb0wofhRQQPx{tL$C^HTu_%SD{oN>+BoN z`%rNRJCfY9ppuTnrizg()eD@dIOtcMx0p1F>JFCbR^cuQy#WD0H){Z)ZM4 zRz8T_Gz(Uo4u@Yg$K5#7jx>#AVwdOj*pO3jK(~et=8bzp*YVuW{NPN|0}gRWcaqak>?v1Wi^2IeLP2TF-9T#@OmQs`VpI!C%)Ou|g~fUoUw{Nj(qujx+_(9E?wZ0hBwM1-2d?ydT8}4)`vgwMrx9LD4NBdvS4MP=6^m7y zsnE%qHABT^Dke^>Ts^_lAz4rgOOe}&vTz!mL1rO4xE*{<$UTj1LJz>0V!7jdH#tP*q;-rwHP)f`i%M0*as}Z6zQt-mdMQ&C?tw9EWz<%$8LCwD(qS$U zyl}b#J-I8C=o?y&iZLBYk#CN7`tax|y<4;#gKI^&3n#fTd?CpgU$j~m{fUTHToBqU z!mC(dq&~dn#%yTPT5A*5K4e-dHMUfYOr~n_g{7+4Z(^_;5ms@;M2{jgOT0(@+g9H; z*&7Y$ZANs%m?~CtF#VBlaNLig3&M@E+|dmJ z1oMFtep^=fuyUS;TX~g~>^~D6A?Z$(tX8ZG~Hx%>+-)Zm1hA;jgbCyrtj$!H{M!`5&Y-gSpZy z&zR-S@qDfgi+O#AQSEHmTic zh6+6nf6Mj0t8PQF(^jd<@j#|xiE}0TxOji9f%nrIj_TF;i*h`=vf75SI<4li2CW{1 zvKFmwR@Rt$SwA2{rTz#09@SlIe6mv3a9M+~S*}l7!}WS|4L5hk8o#-jJ^{j9J#?)N z*;}qP**m29kQIT35N^jrAGTb7dTlB!)<0Q$JFL=c)<(hNLfg7YSk1zcoCTMDN0ZRF0WG?W8j<~RG^TW z9r}c=9zD6>USwZf(7Ac`+Z)&q6>#>;1+5y``|6W6MWTqgoBoZ6(G4QL*%Z^}Tq8|2 zDHT;5tF%K$Cz*9)8%w^wkV~FY*!iJGw#lY#vT0jpz2pX&vgYPyPxqtD?vw>M2(sDT z9PG~O>rSsNe8D%cdJ!Mk4~zJ~?#4P?(hqNr3g<&<_cJOAO^`0?_AOqxpm*641LyUD zTiSNHw7!1u69;t7Tcur>GF0C*;8tzgC??^ezGREH)rAInhiS=Z;o>+;=k*@~1$OWW zbeguiOo4sTAS+j1?4_LdBSLo%Vzx6R=Rg< zKT-R;ul);e7e;O?r}mm^+@x_^4Ap;dJfe@=@sI@}?hID`8I36XamT$fR2H`TqJ+5i zUE+H!PjZ8`;zF`}KCzTt!h?eT@>iqbbN%R76Rp?21g}f0EF8Wk9guC?*IS?wn;Y5Y zd~?Evn2DvQAl5HXQXE>b5RRD?&waFOZ0IbA(R25=A~~1z5BA6NV$c5WQ8|||iQR?0 z;vXu52^h~`_UP_k`u5s`VEi+|=1B-sSOHLBxQp9pqK9I>mf|NZF8tRoqbSK)sP8{m zO>!6NyMLX)i;iWFU>cHFwimU%?@%~+8?<#c()QX#(Cl0XRB--p9a8fy;Q;~95t@Ws zEx9L_qZXJjoGHHjnlRbR^^pgn!HZ3_TcBkVy+ptGTbyU9iD@Y?K_=#Hz0=`1&oUFE z2#n3dFOg@fiAfZg5EHXqzj8RW+3_Az z{@;)HH|5`3p``GwDlH5*l0#Q}hLm8Y+>MV^Hl9h#3%!5#C$L?gP^a|CUx)`c+-$Y$ z_>X1`QY9DR`G4^lG0(vR5g=)~qNU;_hzcLn+%x+2x;XUqU+Y?@Za;&WxSvKyQY0}m z^#U=Z!75cfFkQdT~yimV9Ho*>_-t|_uXo5ei3I6bU{*5>A zdz#>nY=S?kp8q=XizyWPM=SQ5rYiu=?o|oUJje1x|Dv!U8$lAgrEq(-K8i zLerTZ3~xb+ajOGfrY*KUsba*#fD!A(Gi*mvWPBF_?-HnDQ(Hj1dCFmj!@}^xfi}U2GEYDU7*(JX zR2m5i+z|6!O~-sT!3!NCdC(6sgFfaQX8c@miM{KE)*c?+5yzvuRr5!8R^o-YR36?D z#lyQ*^M`j6)=I#vrYM`2?-e#4)_B}U79RJlnm_Kdo+_AuA5lE;TQz^+XXrU*J@H!H|Fa-XN5{-(Xa15Y;eKi~&#G=gz=!VddBcKSP3cdq= zrc-SS3|8$fg4Z~>Z56QCQ4oi;y+^@pEVhh-6k#9SuongRx)?46s2p`+huh?O$09np?4B+;NYH<(E^EX67j3LcmOX z2BH*g7r)Tb^2jxZ$cIc%-^2P&faCVs-ICVR-B|-R?@?&YQXfT=*0X;+3XfV&6yu$f zUQiUy7C#D3mp3EM)ghf+YBZEt~{jz(!+a8t5+A$0}#QP$wou;--#o z)U0n7q+(!MN`2V%=YQ_QrLu%oOW>$V@oVd}fLlfjl$q$?eFUW!kfL7Brieva_?`zfP2l z$7jQbB>4Pwp6YmQS`BM87sF!BbsPXb+2@UZb73Zh^M)}GDlOsrz6OO%@dha^lpw{; z_lXi?!+hut-_&xxU~_BEJ@*}CxnGs{nw!YtnI z`6l2lRB7zZhpW83d!-3aSq-o9_D^fz!Ol4KT}t7q1Mz#lu+G<|t9@P|)BMicrN<)R=BiHQvmh)dbt6xQz#ijZKeC6w+ibZ; zp28>QJ*pO5*4VxdQ5(jsmF9(y&B-D4RQVCu*S}RM7Xd-ehHuzemjC3h@_&QXJjhn;-@6{}`l9wZOLS>weANr)RamZg(OKiFyfGKmbO z=tA+6$qKL@+b3!5-JSJ{x=40>3K?o&@U5hE7E$<>2T6o6Vk#K`nBBel6v={R%s!3u zz#@4XQ4u(P8u1KR_D@nL$&%u#3sqz5+Nv>i_qn7P&8=)Y6Q z2UfDwS>z?CVWqRkbdvKD8#tTX#>-i=i5J0d&n9a3ntzgXocNp<3Ew??OL(OBVh-iy z#~Mg?TKF+T&=AC7y_Wqko5Z;!^RV1x%*1Z0Z~`Am!gt_!xY>obxp) zS4egehfst4F^6=qqZFZC4Qu-x86H-H2^7AGq~q(Ff=c(6YS_P?Bb_V=$B13<9Jwca zs(_rWfJNi?F^{{5 z?{w?Hhg0$BY945Tr-k+BSghxsVM;EEh+QJ^UV;uBnvOy-mr&^4Pot057$b7YxnNk# z?p#EY(X-|)BCBAT(P=T6PcaKB(8*%Vjf?@WkzOvcXsmK@}n*5(h%15m`A8^~iW-=mEc4<1@Mo3nx3u8Q|czGI1ZFs$&GNZ-qipetiP zZXmhh9YUmgWo-OMWQgzV@X<%ae7@nN{>K!a?I;%iS;olQNX~R`SuPTM6)om>KqNCA z3su6we)$NA#KB&1gk-g=s^;4%*XCfdw!fElS!}Pdcmco%2*dy_W+_KWTe>)h=!cHC zV=o;gu{_!xB{LytN3t6q42^cj$S&xJ%pw_fXxD@o6;MT#FS}EDVRJu#Noz|?8d{tn zLvf(Sogm||*l~h%$KsC@eH$M0`KGdvc_-8GEornRh22Kohc@fkD`*Zjj7^dEeU|MCpIv;B9; z;$@KHW)0fU$?o=dt7g5ndqcg{R8_#Iu_nxZ{*Ool&|_~E_Z+2+)-zjh|{3G)mLlf?k_-F z+30z6ZrG8&S};C(IuIs%@jgNO*{GaHO9|u~+j40d-@Mg4`jkxazBgW7KtBO8zswl% zGCk&i)5aZd()3{1ZY;~EugmbV(Q6GYlQAKR*g#hh_?UgXk!SOh2rDWG@4W#-(3eshiW*@RwH#qQrkRdkn`o9GMR zFr$|y4Z;{>FQUCW=jQMWQ#3$wCn@6dgukxSp?vOhFfL-)b7*-vg#WRrMf4;}pT3oP zy8FTx#oK)?Cx@p58f0Ky!p#`l8sA8{2odcB_dA>E0Mw>@GxeenvW0dIK_TM93ODg$ zKxDnQ&@&X7Sr>yYQcT6TTKt=dK&2d2?DHb}nopgM|6g_3f7W8iHd70u&&QNvPRsi5 zq;qiV+rE?DYFYX+(U!8Boir8?w$5GjiUal;K_#dNmT&(+d-GlIm!Iere)|7mKe`sK zgroyB4T}#B(2EFt>PYT8R&>B|$EA?ax zJi=$z!#M*267k+IL2{`SuLqIZ2h9v5yIqhu1IVTal64bgjsUVLOS1ji_~hRr=z;_2 z(gMYG6lC@QvWbCY@q)}2KsLS+89P@-!;}2hjctSxgV0I{2nrA|rk;%TKh$UpN7uuB z)j+Aemc`(Ezfcfn-iWMgquEmy8J*Lz|$30J^&Z#mRz93Lr~qM8*^V+)`7X#|k51kef<*d&Unv{J4@xplV zZX*}-w^gnMIwlfvwS7b5pN*CRar2m=DkA0C7VD#i3guSzts<4J1P+ zL9)Pzc2^@ZcJFZ-9^ntX7PQoDya1hVK z)lff(a8M8jx&0vQUZHgrm;#6|BU&LGzpC)mg(AS3C19;hSY%xkrwTEFaV9XT&K*S% z>M@~h5XzFO=-WvRT2bC#7$O?9;$M`g;eL?NihuQ?MsW6!lXSV4e^-J=p)sO{#-I^q zl_;ouE&K5#P4knXbUztN^^>8rTK3TIG}U{yUYe+x(#)!mc9!k>9i{omP@10%rTNKF z+F3U26dma0pRJg>iJB?RtO{x8SkWmu&4Up9at?oe1^6j!<`YK^%AiU_-NUqo zW!GXZRKwm9g<;gvA0zPF2=-EtjFwqx3pFiaUr&+;!TYR5 zn%u$u>o*8>O493S`U zx8MkgL2OJ6|HXfHN!y7)h(KEQ4T=q}