feat(korean_pos): add Ko-HellaSwag benchmarks, loss reweighting, and resilience evaluation report

data/korean_pos_mc/get_dataset.sh

@@ -28,16 +28,21 @@ PY
   fi
 fi
 
-python3 ../template/utils/korean/extract_multicontext_streams.py input.txt . --use-pos --metadata-json '' --metadata-yaml ''
+if [[ ! -f char/train.bin || ! -f pos/train.bin ]]; then
+  python3 ../template/utils/korean/extract_multicontext_streams.py input.txt . --use-pos --metadata-json '' --metadata-yaml ''
+
+  lanes=(script choseong jungseong jongseong jung_base1 jung_base2 jung_has_w jung_has_y jung_has_i jong_base1 jong_base2 jong_base3 choseong_tense choseong_aspirated choseong_nasal_liquid choseong_place jung_height jung_backness jung_round jong_complex has_batchim syllable_index_mod codepoint_mod pos char)
+  for lane in "${lanes[@]}"; do
+    (
+      cd "$lane"
+      python3 ../../template/prepare.py -t input.txt --method char -s -S "$lane"
+      prepared_dir="char_${lane}"
+      cp "${prepared_dir}/meta.pkl" meta.pkl
+      cp "${prepared_dir}/train.bin" train.bin
+      cp "${prepared_dir}/val.bin" val.bin
+    )
+  done
+else
+  echo "Dataset streams already prepared in $script_dir. Skipping extraction."
+fi
 
-lanes=(script choseong jungseong jongseong jung_base1 jung_base2 jung_has_w jung_has_y jung_has_i jong_base1 jong_base2 jong_base3 choseong_tense choseong_aspirated choseong_nasal_liquid choseong_place jung_height jung_backness jung_round jong_complex has_batchim syllable_index_mod codepoint_mod pos char)
-for lane in "${lanes[@]}"; do
-  (
-    cd "$lane"
-    python3 ../../template/prepare.py -t input.txt --method char -s -S "$lane"
-    prepared_dir="char_${lane}"
-    cp "${prepared_dir}/meta.pkl" meta.pkl
-    cp "${prepared_dir}/train.bin" train.bin
-    cp "${prepared_dir}/val.bin" val.bin
-  )
-done

demos/korean_pos_ko_hellaswag_demo.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=========================================================================="
+echo " Korean POS Factorized Tokenizer vs Baseline: Ko-HellaSwag Experiment "
+echo "=========================================================================="
+
+MAX_ITERS="${MAX_ITERS:-3000}"
+EVAL_ITERS="${EVAL_ITERS:-20}"
+MAX_EXAMPLES="${MAX_EXAMPLES:-100}"
+STRUCTURAL_LOSS_WEIGHT="${STRUCTURAL_LOSS_WEIGHT:-0.05}"
+DROPOUT="${DROPOUT:-0.1}"
+DEVICE="${DEVICE:-$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')")}"
+
+# 1. Prepare/verify dataset streams using HangulPosFactorizedTokenizer
+echo "[Step 1/5] Preparing dataset streams..."
+bash data/korean_pos_mc/get_dataset.sh
+
+lanes=(
+  korean_pos_mc/script
+  korean_pos_mc/choseong
+  korean_pos_mc/jungseong
+  korean_pos_mc/jongseong
+  korean_pos_mc/jung_base1
+  korean_pos_mc/jung_base2
+  korean_pos_mc/jung_has_w
+  korean_pos_mc/jung_has_y
+  korean_pos_mc/jung_has_i
+  korean_pos_mc/jong_base1
+  korean_pos_mc/jong_base2
+  korean_pos_mc/jong_base3
+  korean_pos_mc/choseong_tense
+  korean_pos_mc/choseong_aspirated
+  korean_pos_mc/choseong_nasal_liquid
+  korean_pos_mc/choseong_place
+  korean_pos_mc/jung_height
+  korean_pos_mc/jung_backness
+  korean_pos_mc/jung_round
+  korean_pos_mc/jong_complex
+  korean_pos_mc/has_batchim
+  korean_pos_mc/syllable_index_mod
+  korean_pos_mc/codepoint_mod
+  korean_pos_mc/pos
+  korean_pos_mc/char
+)
+
+# 2. Train Multicontext HangulPosFactorizedTokenizer model
+echo ""
+echo "[Step 2/5] Training Multicontext model with HangulPosFactorizedTokenizer (25 lanes)..."
+echo "Params: max_iters=$MAX_ITERS, structural_loss_weight=$STRUCTURAL_LOSS_WEIGHT, dropout=$DROPOUT"
+python3 train.py \
+  --dataset korean_pos_mc/char \
+  --training_mode multicontext \
+  --multicontext \
+  --multicontext_datasets "${lanes[@]}" \
+  --structural_loss_weight "$STRUCTURAL_LOSS_WEIGHT" \
+  --max_iters "$MAX_ITERS" \
+  --eval_iters "$EVAL_ITERS" \
+  --always_save_checkpoint \
+  --dropout "$DROPOUT" \
+  --device "$DEVICE" \
+  --out_dir ./out_mc_korean_pos
+
+# 3. Train Baseline single-context model
+echo ""
+echo "[Step 3/5] Training Baseline single-context model (character level)..."
+echo "Params: max_iters=$MAX_ITERS, dropout=$DROPOUT"
+python3 train.py \
+  --dataset korean_pos_mc/char \
+  --max_iters "$MAX_ITERS" \
+  --eval_iters "$EVAL_ITERS" \
+  --always_save_checkpoint \
+  --dropout "$DROPOUT" \
+  --device "$DEVICE" \
+  --out_dir ./out_baseline_korean_pos
+
+# 4. Evaluate both models on Ko-HellaSwag
+echo ""
+echo "[Step 4/5] Running Ko-HellaSwag evaluation across all normalization modes..."
+
+eval_mc_cmd=(python3 benchmarks/run_ko_hellaswag.py --out_dir ./out_mc_korean_pos --device "$DEVICE" --eval_all_norms --output_json ./out_mc_korean_pos/ko_hellaswag_metrics.json)
+if [ -n "${MAX_EXAMPLES:-}" ]; then
+  eval_mc_cmd+=(--max_examples "$MAX_EXAMPLES")
+fi
+
+eval_base_cmd=(python3 benchmarks/run_ko_hellaswag.py --out_dir ./out_baseline_korean_pos --device "$DEVICE" --eval_all_norms --output_json ./out_baseline_korean_pos/ko_hellaswag_metrics.json)
+if [ -n "${MAX_EXAMPLES:-}" ]; then
+  eval_base_cmd+=(--max_examples "$MAX_EXAMPLES")
+fi
+
+echo "Evaluating Multicontext (HangulPosFactorizedTokenizer) model..."
+"${eval_mc_cmd[@]}"
+
+echo "Evaluating Baseline (Single Context) model..."
+"${eval_base_cmd[@]}"
+
+# 5. Display comparison report
+echo ""
+echo "=========================================================================="
+echo " FINAL EXPERIMENT COMPARISON SUMMARY "
+echo "=========================================================================="
+python3 -c "
+import json
+from pathlib import Path
+
+mc_path = Path('./out_mc_korean_pos/ko_hellaswag_metrics.json')
+base_path = Path('./out_baseline_korean_pos/ko_hellaswag_metrics.json')
+
+if mc_path.exists() and base_path.exists():
+    mc = json.loads(mc_path.read_text())
+    base = json.loads(base_path.read_text())
+
+    print(f'Benchmark Dataset : {mc.get(\"dataset_name\")}')
+    print(f'Total Evaluated   : {mc.get(\"total\")} examples')
+    print('=' * 80)
+    print(f'{\"Normalization Mode\":<20} | {\"Baseline Acc\":<15} | {\"Multicontext Acc\":<18} | {\"Delta\":<10}')
+    print('-' * 80)
+
+    mc_accs = mc.get('accuracies', {})
+    base_accs = base.get('accuracies', {})
+
+    if not mc_accs and 'accuracy' in mc:
+        norm_type = mc.get('norm_type', 'length')
+        mc_accs = {norm_type: mc.get('accuracy', 0.0)}
+        base_accs = {norm_type: base.get('accuracy', 0.0)}
+
+    modes = ['length', 'prior_length', 'unigram_length', 'none', 'prior', 'unigram']
+    for mode in modes:
+        if mode in mc_accs and mode in base_accs:
+            b_acc = base_accs[mode]
+            m_acc = mc_accs[mode]
+            diff = m_acc - b_acc
+            sign = '+' if diff >= 0 else ''
+            print(f'{mode:<20} | {b_acc:.4f}          | {m_acc:.4f}             | {sign}{diff:.4f}')
+    print('=' * 80)
+else:
+    print('Error: Metric JSON output files not found.')
+"
+echo "=========================================================================="

demos/korean_pos_phonetic_slang_demo.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=========================================================================="
+echo " Korean POS Factorized Tokenizer vs Baseline: Experiment 1 Robustness "
+echo " Phonetic Slang & Typo Resilience Evaluation on 3k Checkpoints           "
+echo "=========================================================================="
+
+MAX_EXAMPLES="${MAX_EXAMPLES:-100}"
+CORRUPTION_RATE="${CORRUPTION_RATE:-0.8}"
+DEVICE="${DEVICE:-$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')")}"
+
+MC_DIR="./out_mc_korean_pos"
+BASE_DIR="./out_baseline_korean_pos"
+
+if [[ ! -d "$MC_DIR" || ! -d "$BASE_DIR" ]]; then
+  echo "Error: Checkpoint directories $MC_DIR or $BASE_DIR do not exist."
+  echo "Please ensure 3k checkpoints are present."
+  exit 1
+fi
+
+python3 benchmarks/run_phonetic_slang_eval.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --max_examples "$MAX_EXAMPLES" \
+  --corruption_rate "$CORRUPTION_RATE" \
+  --device "$DEVICE" \
+  --output_json ./phonetic_slang_resilience_results.json
+
+echo "=========================================================================="

demos/korean_pos_vocab_tail_demo.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=========================================================================="
+echo " Korean POS Factorized Tokenizer vs Baseline: Vocabulary Tail Test       "
+echo " Zero-Shot Perplexity across 10 Hangul Syllable Frequency Deciles        "
+echo "=========================================================================="
+
+MAX_EXAMPLES="${MAX_EXAMPLES:-500}"
+DEVICE="${DEVICE:-$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')")}"
+
+MC_DIR="./out_mc_korean_pos"
+BASE_DIR="./out_baseline_korean_pos"
+TRAIN_CORPUS="data/korean_pos_mc/input.txt"
+
+if [[ ! -d "$MC_DIR" || ! -d "$BASE_DIR" ]]; then
+  echo "Error: Checkpoint directories $MC_DIR or $BASE_DIR do not exist."
+  exit 1
+fi
+
+python3 benchmarks/run_vocab_tail_perplexity.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --train_corpus "$TRAIN_CORPUS" \
+  --dataset_name "KETI-AIR/kor_hellaswag" \
+  --split "validation" \
+  --max_examples "$MAX_EXAMPLES" \
+  --device "$DEVICE" \
+  --output_json ./vocab_tail_perplexity_results.json
+
+echo "=========================================================================="

demos/nsmc_phonetic_slang_demo.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=========================================================================="
+echo " Korean POS Factorized Tokenizer vs Baseline: Experiment 1 Robustness "
+echo " Naver Sentiment Movie Corpus (NSMC) Phonetic Slang Resilience Eval      "
+echo "=========================================================================="
+
+MAX_EXAMPLES="${MAX_EXAMPLES:-100}"
+CORRUPTION_RATE="${CORRUPTION_RATE:-0.8}"
+DEVICE="${DEVICE:-$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')")}"
+
+MC_DIR="./out_mc_korean_pos"
+BASE_DIR="./out_baseline_korean_pos"
+
+if [[ ! -d "$MC_DIR" || ! -d "$BASE_DIR" ]]; then
+  echo "Error: Checkpoint directories $MC_DIR or $BASE_DIR do not exist."
+  echo "Please ensure 3k checkpoints are present."
+  exit 1
+fi
+
+python3 benchmarks/run_phonetic_slang_eval.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --dataset_name "Blpeng/nsmc" \
+  --split "test" \
+  --max_examples "$MAX_EXAMPLES" \
+  --corruption_rate "$CORRUPTION_RATE" \
+  --device "$DEVICE" \
+  --output_json ./nsmc_phonetic_slang_resilience_results.json
+
+echo "=========================================================================="

demos/run_full_evals.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=========================================================================="
+echo " Starting Full Evaluation Runs for Phonetic Slang & Typo Resilience    "
+echo "=========================================================================="
+
+DEVICE="${DEVICE:-$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')")}"
+MC_DIR="./out_mc_korean_pos"
+BASE_DIR="./out_baseline_korean_pos"
+CORRUPTION_RATE="${CORRUPTION_RATE:-0.8}"
+
+echo "\n[Run 1/3] Ko-HellaSwag FULL Validation Split (10,042 examples)..."
+python3 benchmarks/run_phonetic_slang_eval.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --dataset_name "KETI-AIR/kor_hellaswag" \
+  --split "validation" \
+  --max_examples 0 \
+  --corruption_rate "$CORRUPTION_RATE" \
+  --device "$DEVICE" \
+  --output_json ./full_ko_hellaswag_val_results.json
+
+echo "\n[Run 2/3] Ko-HellaSwag FULL Test Split (10,003 examples)..."
+python3 benchmarks/run_phonetic_slang_eval.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --dataset_name "KETI-AIR/kor_hellaswag" \
+  --split "test" \
+  --max_examples 0 \
+  --corruption_rate "$CORRUPTION_RATE" \
+  --device "$DEVICE" \
+  --output_json ./full_ko_hellaswag_test_results.json
+
+echo "\n[Run 3/3] NSMC FULL Test Split (50,000 examples)..."
+python3 benchmarks/run_phonetic_slang_eval.py \
+  --mc_dir "$MC_DIR" \
+  --base_dir "$BASE_DIR" \
+  --dataset_name "Blpeng/nsmc" \
+  --split "test" \
+  --max_examples 0 \
+  --corruption_rate "$CORRUPTION_RATE" \
+  --device "$DEVICE" \
+  --output_json ./full_nsmc_test_results.json
+
+echo "=========================================================================="
+echo " All Full Evaluation Runs Completed!                                     "
+echo "=========================================================================="

full_ko_hellaswag_val_results.json

@@ -0,0 +1,23 @@
+{
+  "dataset": "KETI-AIR/kor_hellaswag",
+  "split": "validation",
+  "max_examples": 0,
+  "corruption_rate": 0.8,
+  "baseline": {
+    "clean_accuracy": 0.25851424019119695,
+    "corrupted_accuracy": 0.24387572196773552,
+    "accuracy_drop": 0.014638518223461433,
+    "clean_logprob": -3.116889135833044,
+    "corrupted_logprob": -4.701323254682752,
+    "logprob_degradation": 1.5844341188497078
+  },
+  "multicontext": {
+    "clean_accuracy": 0.2548297151961761,
+    "corrupted_accuracy": 0.24287990440151364,
+    "accuracy_drop": 0.011949810794662452,
+    "clean_logprob": -3.279402141838975,
+    "corrupted_logprob": -4.653083825643806,
+    "logprob_degradation": 1.3736816838048314
+  },
+  "robustness_gain_acc_delta": 0.002688707428798981
+}

full_nsmc_test_results.json

@@ -0,0 +1,23 @@
+{
+  "dataset": "Blpeng/nsmc",
+  "split": "test",
+  "max_examples": 0,
+  "corruption_rate": 0.8,
+  "baseline": {
+    "clean_accuracy": 0.51238,
+    "corrupted_accuracy": 0.4987,
+    "accuracy_drop": 0.01367999999999997,
+    "clean_logprob": -3.356859786963463,
+    "corrupted_logprob": -5.213276321935654,
+    "logprob_degradation": 1.8564165349721908
+  },
+  "multicontext": {
+    "clean_accuracy": 0.49694,
+    "corrupted_accuracy": 0.4962,
+    "accuracy_drop": 0.0007400000000000184,
+    "clean_logprob": -3.8454243441438676,
+    "corrupted_logprob": -5.345579153676033,
+    "logprob_degradation": 1.500154809532165
+  },
+  "robustness_gain_acc_delta": 0.012939999999999952
+}