[egs,scripts] Updates for smooth chain2 training (#4290)

Author: nshmyrev

egs/mini_librispeech/s5/local/chain2/tuning/run_tdnn_1a.sh

@@ -148,11 +148,7 @@ if [ $stage -le 12 ]; then
   # This will be a two-level tree (with the smaller number of leaves specified
   # by the '--num-clusters' option); this is needed by the adaptation framework
   # search below for 'tree.map'
-   if [ -f $tree_dir/final.mdl ]; then
-     echo "$0: $tree_dir/final.mdl already exists, refusing to overwrite it."
-     exit 1;
-  fi
-   steps/nnet3/chain/build_tree.sh \
+  steps/nnet3/chain/build_tree.sh \
      --frame-subsampling-factor ${frame_subsampling_factor} \
      --context-opts "--context-width=2 --central-position=1" \
      --cmd "$train_cmd" 3500 ${lores_train_data_dir} \
@@ -200,17 +196,18 @@ if [ $stage -le 14 ]; then
   output-layer name=output-default-xent input=prefinal-xent dim=$num_leaves learning-rate-factor=$learning_rate_factor max-change=1.5
 EOF
   steps/nnet3/xconfig_to_configs.py --xconfig-file $dir/configs/default.xconfig --config-dir $dir/configs/
-  if [ $dir/init/default_trans.mdl ]; then # checking this because it may have been copied in a previous run of the same script
-      copy-transition-model $tree_dir/final.mdl $dir/init/default_trans.mdl  || exit 1 &
-  else
-      echo "Keeping the old $dir/init/default_trans.mdl as it already exists."
-  fi
 fi
 wait;
 
 init_info=$dir/init/info.txt
 if [ $stage -le 15 ]; then
 
+  if [ $dir/init/default_trans.mdl ]; then # checking this because it may have been copied in a previous run of the same script
+      copy-transition-model $tree_dir/final.mdl $dir/init/default_trans.mdl  || exit 1 &
+  else
+      echo "Keeping the old $dir/init/default_trans.mdl as it already exists."
+  fi
+
   if [ ! -f $dir/configs/ref.raw ]; then
       echo "Expected $dir/configs/ref.raw to exist"
       exit

egs/wsj/s5/steps/nnet3/chain2/compute_preconditioning_matrix.sh

@@ -76,7 +76,7 @@ if [ $stage -le 2 ]; then
         $ldafolder/lda.mat $ldafolder/lda_stats || exit 1
 
     rm $ldafolder/lda_stats
-    ln -rs $ldafolder/lda.mat $ldafolder/configs/lda.mat
+    ln -sf $ldafolder/lda.mat $ldafolder/configs/lda.mat
 fi
 
 echo "$0: Finished computing LDA transform"

egs/wsj/s5/steps/nnet3/chain2/get_raw_egs.sh

@@ -130,8 +130,7 @@ latdir=$3
 dir=$4
 
 tree=$chaindir/${lang}.tree
-trans_mdl=$chaindir/init/${lang}.mdl  # contains the transition model and a nnet, but
-                                   # we won't be making use of the nnet part.
+trans_mdl=$chaindir/init/${lang}_trans.mdl
 normalization_fst=$chaindir/den_fsts/${lang}.normalization.fst
 den_fst=$chaindir/den_fsts/${lang}.den.fst
 
@@ -142,13 +141,6 @@ for f in $data/feats.scp $latdir/lat.1.gz $latdir/final.mdl \
          $tree $normalization_fst $den_fst $extra_files; do
   [ ! -f $f ] && echo "$0: no such file $f" && exit 1;
 done
-if [ ! -f $trans_mdl ]; then
-    trans_mdl=$chaindir/init/${lang}_trans.mdl
-    if [ ! -f $trans_mdl ]; then
-        echo "$0: cannot find transition model in $chaindir/init/${lang}_trans.mdl or $trans_mdl"
-        exit 1
-    fi
-fi
 
 nj=$(cat $latdir/num_jobs) || exit 1
 if [ -f $latdir/per_utt ]; then
@@ -227,7 +219,7 @@ else
 fi
 
 feats="scp:$sdata/JOB/feats.scp"
-if [ ! -z $cmvn_opts ]; then
+if [ ! -z "$cmvn_opts" ]; then
     if [ ! -f $data/cmvn.scp ]; then
         echo "Cannot find $data/cmvn.scp. But cmvn_opts=$cmvn_opts"
         exit 1

egs/wsj/s5/steps/nnet3/chain2/internal/get_best_model.sh

@@ -15,7 +15,7 @@ output=output
 if [ -f path.sh ]; then . ./path.sh; fi
 . parse_options.sh || exit 1;
 
-if [ $# -le 1 ]; then
+if [ $# -lt 1 ]; then
     echo "Usage: $0: [options] <model-1-log> <model-2-log> .... <model-N-log>"
     echo "where <model-n> is one of the n models to choose from."
     echo ""

egs/wsj/s5/steps/nnet3/chain2/train.sh

@@ -191,7 +191,12 @@ while [ $x -lt $num_iters ]; do
            "nnet3-copy --learning-rate=$lrate $dir/${x}.raw - |" $den_fst_dir \
            "ark:nnet3-chain-copy-egs $egs_opts scp:$egs_dir/${name}_subset.scp ark:- | nnet3-chain-merge-egs $multilingual_eg_opts --minibatch-size=1:64 ark:- ark:-|" \
            $dir/${next_x}_${name}.mdl || touch $dir/.error_diagnostic &
+
+       # Make sure we do not run more than $num_jobs_final at once
+       [ $num_jobs_final -eq 1 ] && wait
+
     done
+    wait
   fi
 
   if [ $x -gt 0 ]; then