NNSegmentation/example/run.sh at master · SUTDNLP/NNSegmentation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#!/bin/bash
#absolute path point to NNSegmentation directory
workspace=`pwd`
tooldir=$workspace/../../NNSegmentation
corpus=pku
uni_emb=./embeddings/char.vec
bi_emb=./embeddings/bichar.vec
tri_emb=./embeddings/trichar.vec

outputdir=$corpus.sample

mkdir -p $outputdir
rm $outputdir/* -rf

function extract
{
    #extracting your features here
    echo "[self implementation]"
}

function runSparse
{
    cmd=$1
    echo $cmd
    mkdir $workspace/$outputdir/$cmd -p
    ln -s $workspace/$corpus/$corpus.sample.train.feats $workspace/$outputdir/$cmd/$corpus.train.feats
    ln -s $workspace/$corpus/$corpus.sample.dev.feats $workspace/$outputdir/$cmd/$corpus.dev.feats
    ln -s $workspace/$corpus/$corpus.test.feats $workspace/$outputdir/$cmd/$corpus.test.feats
    cp $tooldir/$cmd $workspace/$outputdir/$cmd/
    train_file=$workspace/$outputdir/$cmd/$corpus.train.feats
    dev_file=$workspace/$outputdir/$cmd/$corpus.dev.feats
    test_file=$workspace/$outputdir/$cmd/$corpus.test.feats
    nohup $workspace/$outputdir/$cmd/$cmd -l -train $train_file -dev $dev_file  -test $test_file -option ./options/option.sparse -model $workspace/$outputdir/$cmd/$cmd.model >$workspace/$outputdir/$cmd.log 2>&1 &
}
function runLSTM
{
    cmd=$1
    option=$2
    echo $cmd, $option
    mkdir $workspace/$outputdir/$cmd -p
    ln -s $workspace/$corpus/$corpus.sample.train.feats $workspace/$outputdir/$cmd/$corpus.train.feats
    ln -s $workspace/$corpus/$corpus.sample.dev.feats $workspace/$outputdir/$cmd/$corpus.dev.feats
    ln -s $workspace/$corpus/$corpus.test.feats $workspace/$outputdir/$cmd/$corpus.test.feats
    cp $tooldir/$cmd $workspace/$outputdir/$cmd/
    train_file=$workspace/$outputdir/$cmd/$corpus.train.feats
    dev_file=$workspace/$outputdir/$cmd/$corpus.dev.feats
    test_file=$workspace/$outputdir/$cmd/$corpus.test.feats
    #character bigram embedding and character trigram embedding should use a comma to separate
    nohup $workspace/$outputdir/$cmd/$cmd -l -train $train_file \
        -dev $dev_file \
        -test $test_file \
        -option $option \
        -model $workspace/$outputdir/$cmd/$cmd.model \
        -word $uni_emb \
        -tag $bi_emb,$tri_emb\
    >$workspace/$outputdir/$cmd.log 2>&1 &
}

echo "Step 1: Extracting Features..."
extract $corpus
echo "Step 2: Running SparseCRFMMLabeler..."
runSparse SparseCRFMMLabeler
echo "Step 3: Running LSTMCRFMMLabeler SparseLSTMCRFMMLabeler..."
cmds="LSTMCRFMMLabeler SparseLSTMCRFMMLabeler"
runLSTM LSTMCRFMMLabeler ./options/option.lstm
runLSTM SparseLSTMCRFMMLabeler ./options/option.sparse+lstm
echo "Successfully run!"