// file: hmm_train.cc
//
// this program does trace projection decoding
//

// isip include files
//
#include "hmm_train.h"
#include "hmm_train_constants.h"

static int_4* trans_states = (int_4*)NULL;
static float_4* trans_scores = (float_4*)NULL;
static int_4 trans_states_size = 0;

// main program
//
int main(int_4 argc, char_1** argv) {

  // variables to hold commandline parameters
  //
  char_1* params_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* monophones_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* transitions_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* states_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* new_trans_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* new_states_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* batch_stats_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* acc_list_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* models_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* phones_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* lexicon_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* input_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* output_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* mfcc_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* state_occ_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* mlf_file = new char_1[ISIP_MAX_STRING_LENGTH];
  char_1* sph_file = new char_1[ISIP_MAX_STRING_LENGTH];
  logical_1 state_occ_mode = HT_OPT_OFF;
  int_4 num_levels = HT_NUM_LEVELS;
  int_4 ph_size = HT_TRIPHONE_SIZE;
  int_4 num_nbest = (int_4)0;
  int_4 context_mode;
  logical_1 output_mode = HT_BINARY_MODE;
  logical_1 train_mode = HT_NORMAL;
  
  // default recognition mode is via a lattice
  //
  logical_1 mlf_mode = HT_WORD_TRANS;

  // parameters to define size
  //
  int_4 num_monophones = (int_4)0;
  int_4 num_features = (int_4)0;  
  int_4 num_trans = (int_4)0;
  int_4 num_states = (int_4)0;
  int_4 num_models = (int_4)0;
  int_4 num_phones = (int_4)0;  
  int_4 num_words = (int_4)0;

  // for the TRAIN
  //
  int_4 file_count = (int_4)0;
  int_4 num_ph = (int_4)0;
  int_4* mod_map = (int_4*)NULL;
  int_4** st_map = (int_4**)NULL;
  int_4* trans_map = (int_4*)NULL;
  int_4* phn_ind = new int_4[HT_MAX_NUM_PHONES];
  int_4* phns = (int_4*)NULL;
  int_4 num_warn = (int_4)0;
  logical_1 op_out = ISIP_FALSE;
  FILE* fmfc = (FILE*)NULL;
  int_4 sp_phn_ind = (int_4)0;
  
  // count of trace generation
  //
  int_4* num_traces_total = new int_4[num_levels];
  int_4* num_traces_gen = new int_4[num_levels];
  int_4* num_traces_del = new int_4[num_levels];
  int_4* total_gen = new int_4[num_levels];
  int_4* total_del = new int_4[num_levels];

  // pruning thresholds and parameters
  //
  int_4 align_mode = HT_DEFAULT_ALIGN_MODE;
  int_4 input_mode = HT_DEFAULT_INPUT_MODE;
  int_4 num_mapmi = (int_4)0;
  int_4 mapmi_limit = (int_4)0;
  float_8 mapmi_thresh = HT_DEFAULT_SCORE;
  float_8* max_score = new float_8[num_levels];
  float_8* beam_width = new float_8[num_levels];
  float_8* beam_thresh = new float_8[num_levels];
  float_8  var_floor = HT_DEFAULT_VAR_FLOOR;

  // phone list and utterance for phone level alignment
  //
  int_4* phn_list = (int_4*)NULL;
  char_1* utterance = (char_1*)NULL;
  char_1* phn_str = (char_1*)NULL;

  // define the lex tree
  //
  Train_Lex_tree* traintree = (Train_Lex_tree*)NULL;
  
  // initialize the pruning parameters
  //
  for (int_4 i = 0; i < num_levels; i++) {
    max_score[i] = HT_DEFAULT_SCORE;
    beam_thresh[i] = HT_DEFAULT_SCORE;
    beam_width[i] = (float_8)0;
  }

  // misc parameters
  //
  int_4 current_frame = (int_4)0;
  int_4 num_steps = (int_4)0;
  
  // read and decipher the commandline
  //
  int_4 context_flag;
  read_cmdline_cc(argc, argv, params_file, context_flag);
  
  // file pointer
  //
  FILE* fp = (FILE*)NULL;
  
  // open the params file
  //
  fp = fopen((char*)params_file, "r");
  if (fp == (FILE*)NULL) {
    fprintf(stdout, "Cannot open file %s\n", params_file);
    exit(ISIP_PROTO_ERROR);
  }

  // context can be specified from the command line also
  //
  if (context_flag == 1){
    context_mode = 3;
  }

  // read parameters
  //
  read_params_cc(fp, monophones_file, transitions_file, states_file,
		 new_trans_file, new_states_file, batch_stats_file, 
		 acc_list_file, models_file, phones_file, lexicon_file,
		 align_mode, input_mode, num_levels, input_file, mlf_file,
		 output_file, ph_size, num_nbest, beam_width, mapmi_limit,
		 mlf_mode, context_mode, var_floor, op_out, output_mode,
		 train_mode, sph_file, state_occ_file, state_occ_mode);

  // create a list for n-best word traces
  //
  int_4 num_hyps = (int_4)0;
  Train_Trace** n_best_array = new Train_Trace*[num_nbest];

  // clean up
  //
  fclose(fp);  
  delete [] params_file;

  // create a memory manager and set it in the trace linked lists
  //
  Train_Memory_manager* manager =
    new Train_Memory_manager(HT_DEFAULT_BLOCK_SIZE, HT_NODE_BLOCK_SIZE);
  manager->set_trace_grow_size_cc(HT_TRACE_BLOCK_SIZE);
  Train_Link_list::set_manager_cc(manager);

  // read the monophones list
  //
  char_1** monophones = read_monophones_cc(num_monophones, monophones_file);
  delete [] monophones_file;
  
  // read the transitions data
  //
  int_4* trans_size = (int_4*)NULL;
  float_4*** transitions = read_trans_cc(num_trans, trans_size,
					 transitions_file);
  delete [] transitions_file;

  // read the list of phones that need special handling wrt context
  //
  int_4* sph_index;
  int_4 num_sph =0;
  sph_index = read_sph_cc(num_sph, monophones, num_monophones, sph_file);
  delete [] sph_file;

  // read the states data
  //
  Train_State** states = read_states_cc(num_states, num_features, states_file);
  delete [] states_file;

  // read the special phone list file;
  // 

  
  // if training mode involves combining accumulators
  //
  if (train_mode == HT_COMBINE) {

    // setup the state counts
    //
    int_4** st_count = new int_4*[num_states];
    for (int_4 i = 0; i < num_states; i++) {
      st_count[i] = new int_4[states[1]->get_num_mixtures_cc()];
    }

    // open the accumulator list file
    //
    FILE* fp_acc_list = fopen((char*)acc_list_file, "r");

    // combine the accumulators
    //
    combine_acc_cc(transitions, states, trans_size, num_states, 
		   states[1]->get_num_mixtures_cc(), num_features,
		   num_trans, var_floor, fp_acc_list, st_count);
    
    // close the file
    //
    fclose(fp_acc_list);
    
    // print the updated states and transitions
    //
    print_state_cc(new_states_file, states, num_states, num_features,
		   output_mode);
    print_trans_cc(new_trans_file, num_trans, trans_size, transitions);

    // determine if we need to output the occupancy
    //
    if (state_occ_mode == HT_OPT_ON) {

      // open the occupancy file
      //
      FILE* fp_occ_file = fopen((char*)state_occ_file, "w");
      if (fp_occ_file == (FILE*)NULL) {
	fprintf(stdout, "Error: unable to open state occupancy file %s\n",
		state_occ_file);
	exit (ISIP_PROTO_ERROR);
      }

      // call the print method
      //
      print_occ_cc(fp_occ_file, st_count, num_states,
		   states[1]->get_num_mixtures_cc());

      fclose(fp_occ_file);
    }
    
    for (int_4 i = 0; i < num_states; i++) {
      delete [] st_count[i];
    }
    delete [] st_count;
    st_count = (int_4**)NULL;

    // exit gracefully
    //
    exit (ISIP_NO_ERROR);
  }
  
  // read the base HMM models, for TRAIN
  //
  Train_Model** models = read_new_models_cc(num_models, models_file, states,
				      transitions, st_map, trans_map);
  delete [] models_file;
  
  // read the phone models data for TRAIN
  //
  int_4* phone_map = (int_4*)NULL;
  Train_Phone** phones = read_new_phones_cc(num_monophones, ph_size, models,
				      phone_map, num_phones, phones_file,
				      mod_map, context_mode, sp_phn_ind);
  delete [] phones_file;
  
  // read the lexicon
  //
  Train_Hash_table* word_table = read_lexicon_cc(num_words, num_monophones,
					   monophones, lexicon_file);
  delete [] lexicon_file;

  // for the TRAIN
  //
  int_4 num_mix = states[1]->get_num_mixtures_cc();
  float_8*** train_mean = new float_8**[num_states];
  for (int_4 i = 0; i < num_states; i++) {
    train_mean[i] = new float_8*[num_mix];
    for (int_4 j = 0; j < num_mix; j++) {
      train_mean[i][j] = new float_8[num_features];
    }
  }

  float_8*** train_covar = new float_8**[num_states];
  for (int_4 i = 0; i < num_states; i++) {
    train_covar[i] = new float_8*[num_mix];
    for (int_4 j = 0; j < num_mix; j++) {
      train_covar[i][j] = new float_8[num_features];
    }
  }

  int_4** count = new int_4*[num_states];
  for (int_4 i = 0; i < num_states; i++) {
    count[i] = new int_4[num_mix];
  }

  int_4*** trans_count = new int_4**[num_trans];
  for (int_4 i = 0; i < num_trans; i++) {
    trans_count[i] = new int_4*[trans_size[i]];
    for (int_4 j = 0; j < trans_size[i]; j++) {
      trans_count[i][j] = new int_4[trans_size[i]];
    }
  }

  // initialize the arrays for TRAIN
  //
  for (int_4 i = 0; i < num_states; i++) {
    for (int_4 j = 0; j < num_mix; j++) {
      for (int_4 k = 0; k < num_features; k++) {
	train_mean[i][j][k] = (float_8)0.0;
      }
    }
  }

  for (int_4 i = 0; i < num_states; i++) {
    for (int_4 j = 0; j < num_mix; j++) {
      for (int_4 k = 0; k < num_features; k++) {
	train_covar[i][j][k] = (float_8)0.0;
      }
    }
  }

  for (int_4 i = 0; i < num_states; i++) {
    for (int_4 j = 0; j < num_mix; j++) {
      count[i][j] = (int_4)0;
    }
  }
  
  for (int_4 i = 0; i < num_trans; i++) {
    for (int_4 j = 0; j < trans_size[i]; j++) {
      for (int_4 k = 0; k < trans_size[i]; k++) {
	trans_count[i][j][k] = (int_4)0;
      }
    }
  }
  
  // need to find the score to skip over the sp phone
  //
  float_8 sp_score = (float_8)0;
  int_4 sp_phone = (int_4)-1;
    
  // create the sp phone
  //
  int_4* phn = new int_4[ph_size];
  phn[0] = (int_4)0;
  phn[1] = HT_SP_PHONE;
  phn[2] = (int_4)0;
  sp_phone = phone_map[get_nphone_ind_cc(ph_size, num_monophones,
					 phn,context_mode, num_sph,
					 sph_index)];
  
  // get the model params
  //
  Train_Model* model = phones[sp_phone]->get_model_cc();
  int_4 num_st = model->get_num_states_cc();
  
  // temp variables
  //
  int_4 ntrans = (int_4)0;

  // find all possible start states for this phone
  //
  model->get_next_states_cc((int_4)0, ntrans, trans_states, trans_scores,
			    trans_states_size);
  
  // get the transition score to exit state
  //
  for (int_4 kk = 0; kk < ntrans; kk++) {
    
    // check for stop state
    //
    if (trans_states[kk] == num_st - (int_4)1) {
      sp_score = (float_8)trans_scores[kk];
      break;
    }
  }
  
  // set the phone for the start trace
  //
  phn[0] = (int_4)0;
  phn[1] = HT_SILENCE_PHONE;
  phn[2] = (int_4)0;
  int_4 start_phn = phone_map[get_nphone_ind_cc(ph_size, num_monophones,
						phn,context_mode, num_sph,
						sph_index)];
  delete [] phn;
  phn = (int_4*)NULL;

  // set the sentence start word to be the !NULL word
  //
  Train_Word* start_word = (Train_Word*)((word_table->hash_lookup_cc(TRAIN_WRD_NULL))->
			     get_item_cc());
  
  // create variable to read input feature data
  //
  float_8* features = new float_8[num_features];
  
  // create a separate link list of tokens for each phone
  //
  Train_Link_list** state_toklist = new Train_Link_list*[num_phones];

  // create a list of active phones
  //
  int_4 num_active_ph = (int_4)0;
  int_4* active_phones = new int_4[num_phones];

  // create a link list of active traces at the phone level for each
  // phone
  //
  Train_Link_list** phone_trlist = new Train_Link_list*[num_phones];
  Train_Link_node** phmarker = new Train_Link_node*[num_phones];
  Train_Link_node** prev_phmark = new Train_Link_node*[num_phones];

  // create a list of active words
  //
  int_4 num_active_wd = (int_4)0;
  int_4* active_words = new int_4[num_words];

  // create a link list of active traces at the word level for each
  // word
  //
  Train_Link_list** word_trlist = new Train_Link_list*[num_words];
  Train_Link_node** wdmarker = new Train_Link_node*[num_words];
  Train_Link_node** prev_wdmark = new Train_Link_node*[num_words];

  // create a link list of active lexical trees
  //
  Train_Link_list* lextree_list = (Train_Link_list*)NULL;

  // TRAIN open file
  //
  FILE* flat = (FILE*)NULL;
  FILE* fin = (FILE*)NULL;
  FILE* fout = (FILE*)NULL;
  FILE* fpl = (FILE*)NULL;
  FILE* fpi = (FILE*)NULL;

  // variable to count the number of feature vectors in the current input
  // file
  //
  int_4 num_vect = (int_4)0;
  float_8** vectors = (float_8**)NULL;
  char_1* temp_vect = new char_1[ISIP_MAX_STRING_LENGTH];
  
  // open the files containing the lists of input files, mlf_file
  // and output files respectively
  //
  flat = fopen((char*)mlf_file, "r");
  if (flat == (FILE*)NULL) {
    fprintf(stdout, "Error : cannot open mlf file %s\n",
	    mlf_file);
    exit(ISIP_PROTO_ERROR);
  }
  
  fin = fopen((char*)input_file, "r");
  if (fin == (FILE*)NULL) {
    fprintf(stdout, "Error : cannot open input file %s\n", input_file);
    exit(ISIP_PROTO_ERROR);
  }

  if (op_out == ISIP_TRUE) {
    
    // for debugging use
    //
    fout = fopen((char*)output_file, "r");
    if (fout == (FILE*)NULL) {
      fprintf(stdout, "Error : cannot open output file %s\n", output_file);
      exit(ISIP_PROTO_ERROR);
    }
  }
  
  // open the transcription file
  //
  fpl = fopen((char*)mlf_file, "r");
  expand_filename_cc(new_states_file);
  expand_filename_cc(new_trans_file);

  // read the corresoponding data file from each list and process one
  // by one
  // 
  while (fgets((char*)mfcc_file, ISIP_MAX_STRING_LENGTH, fin) !=
	 (char*)NULL) {

    // some incremental output
    //
    fprintf (stdout, "processing file: %s", (char*)mfcc_file);
    
    // for phone_level input transcriptions
    //
    if (mlf_mode == HT_MODEL_TRANS) {

      // make sure these lists have been freed before reallocating memory
      // at the end of each utterance
      //
      if (phn_list != (int_4*)NULL) {
	delete [] phn_list;
	phn_list = (int_4*)NULL;
      }
      if (phns != (int_4*)NULL) {
	delete [] phns;
	phns = (int_4*)NULL;
      }

      phn_list = new int_4[ISIP_MAX_STRING_LENGTH];
      phns = new int_4[ISIP_MAX_STRING_LENGTH];
      num_ph = (int_4)0;

      // make sure there is a corresponding transcription and output file for
      // each input file
      //
      get_phlist_cc(flat, phn_list, monophones, num_monophones, ph_size,
		    context_mode, phone_map, phns, num_ph, num_sph, sph_index);

      if (phn_list == (int_4*)NULL) {
	fprintf(stdout,
		"Error : mismatch in the number of input mfcc files, ");
	fprintf(stdout, "and transcription\n");
	exit(ISIP_PROTO_ERROR);
      }

      //get_phn_index_cc(phn_list, monophones, num_monophones, ph_size,
      //	       context_mode, phone_map, phn_ind, phns, num_ph);      
    }
    
    if (op_out == ISIP_TRUE) {
      if (fgets((char*)output_file, ISIP_MAX_STRING_LENGTH, fout) ==
	  (char*)NULL) {
	fprintf(stdout,
		"Error : mismatch in the number of input mfcc files, ");
	fprintf(stdout, "and output files.\n");
	exit(ISIP_PROTO_ERROR);
      }
    }

    // strip newline characters and expand the filenames
    //
    expand_filename_cc(mfcc_file);

    // check the mfcc file. we close it immediately, so we don't care what
    // mode it is in.
    //
    fmfc = fopen((char*)mfcc_file, "r");
    
    if (fmfc == (FILE*)NULL) {
      num_warn++;
      fprintf(stdout, "Warning %ld : missing mfcc file %s\n", num_warn,
	      mfcc_file);
      continue;
    }
    fclose(fmfc);
    
    if (op_out == ISIP_TRUE) {
      expand_filename_cc(output_file);
    }
    
    // initialize the word-level lists
    //
    for (int_4 i = 0; i < num_words; i++) {
      active_words[i] = (int_4)-1;
      word_trlist[i] = (Train_Link_list*)NULL;
      wdmarker[i] = (Train_Link_node*)NULL;
      prev_wdmark[i] = (Train_Link_node*)NULL;
    }
    
    // initialize the phone and state level lists
    //
    for (int_4 i = 0; i < num_phones; i++) {
      active_phones[i] = (int_4)-1;
      phone_trlist[i] = (Train_Link_list*)NULL;
      phmarker[i] = (Train_Link_node*)NULL;
      prev_phmark[i] = (Train_Link_node*)NULL;
      phones[i]->set_active_cc(ISIP_FALSE);
      state_toklist[i] = (Train_Link_list*)NULL;
    }
    
    // initialize the nbest lists
    //
    for (int_4 i = 0; i < num_nbest; i++) {
      n_best_array[i] = (Train_Trace*)NULL;
    }
    
    // initialize counts
    //
    current_frame = (int_4)0;
    num_active_ph = (int_4)0;
    num_active_wd = (int_4)0;
    num_hyps = (int_4)0;
    for (int_4 i = 0; i < num_levels; i++) {
      num_traces_total[i] = (int_4)0;
      num_traces_gen[i] = (int_4)0;
      num_traces_del[i] = (int_4)0;
      total_gen[i] = (int_4)0;
      total_del[i] = (int_4)0;
    }    
    
    // create a list to store the active lexical trees
    //
    lextree_list = new Train_Link_list();
    
    // mark all words as inactive
    //
    Train_Word* word = (Train_Word*)NULL;
    Train_Hash_cell** hash_cells = word_table->get_cells_cc();
    int_4 hash_size = word_table->get_size_cc();
    for (int_4 k = 0; k < hash_size; k++) {
      for (Train_Hash_cell* cell = hash_cells[k]; cell != (Train_Hash_cell*)NULL;
	   cell = cell->get_next_cc()) {
	word = (Train_Word*)(cell->get_item_cc());
	word->set_active_cc(ISIP_FALSE);
      }
    }
    
    // create the lattice
    //
    Train_Lattice* lattice = new Train_Lattice();

    // for word_level input transcriptions open the mlf file to read
    //
    if(mlf_mode == HT_WORD_TRANS) {
      if (fpl == (FILE*)NULL) {
	fprintf(stdout, "Cannot open file %s\n", mlf_file);
	exit(ISIP_PROTO_ERROR);
      }      
      lattice->read_trans_cc(fpl, word_table);
    }

    // else for phone_level alignment
    //
    else {
      
      // dummy lattice for phone alignment during training
      // 
      build_dm_lat_cc(lattice, word_table);    
      
      // test for built a tree for training
      //
      Train_Lattice_node* lat_node = lattice->get_start_node_cc();
      Train_Link_list* next_list = lat_node->get_next_nodes_cc();
      Train_Link_node* node = next_list->get_head_cc();
      Train_Lattice_node* next_node = (Train_Lattice_node*)(node->get_item_cc());
      
      next_list = next_node->get_next_nodes_cc();
      node = next_list->get_head_cc();
      next_node = (Train_Lattice_node*)(node->get_item_cc());
      
      // build tree with phones in the alignment
      //
      traintree = new Train_Lex_tree;
      
      traintree->build_traintree_cc(phn_list, phns, num_ph,
				    num_monophones, next_node);
      
      // set the lex_tree for !SENT_START
      //  
      lat_node = lattice->get_start_node_cc();
      next_list = lat_node->get_next_nodes_cc();
      node = next_list->get_head_cc();
      next_node = (Train_Lattice_node*)(node->get_item_cc());
      next_node->set_lex_tree_cc(traintree);
    }

    // get the word_penalty from lattice
    //
    float_4 word_penalty = lattice->get_word_penalty_cc();
       
    // open input feature data file to read
    //
    if (input_mode == HT_ASCII_MODE) {
      fpi = fopen((char*)mfcc_file, "r");
    }
    else {
      fpi = fopen((char*)mfcc_file, "rb");
    }
    if (fpi == (FILE*)NULL) {
      fprintf(stdout, "Cannot open file %s\n", mfcc_file);
      exit(ISIP_PROTO_ERROR);
    }
    
    // create a sentence start trace
    //
    Train_Trace* start_trace = manager->new_trace_cc();    
    start_trace->set_level_cc(HT_WORD_LEVEL);
    
    // set the triphone to the start triphone (silence)
    //
    start_trace->set_phone_ind_cc((int_4)start_phn);
    
    // initialize the active word list
    //
    int_4 start_wd = start_word->get_index_cc();
    start_word->set_active_cc(ISIP_TRUE);
    active_words[num_active_wd++] = start_wd;
    
    // insert the start trace in the correct list
    //
    word_trlist[start_wd] = new Train_Link_list();
    word_trlist[start_wd]->insert_cc(start_trace);
    wdmarker[start_wd] = word_trlist[start_wd]->get_curr_cc();
    num_traces_gen[HT_WORD_LEVEL]++;
    
    // set the lattice information for the start trace
    //
    Train_Lattice_node* latnode = lattice->get_start_node_cc();
    start_trace->set_lat_node_cc(latnode);
    
    // set the lexical information for the start trace
    // create a new tree if one doesn't exist and insert it in the list
    // of active trees
    //
    Train_Hash_cell* hcell = (Train_Hash_cell*)NULL;
    lattice->get_lat_node_cc(latnode, hcell);
    Train_Lex_tree* start_tree = latnode->get_lex_tree_cc();
    if (start_tree == (Train_Lex_tree*)NULL) {
      start_tree = new Train_Lex_tree(hcell);
      latnode->set_lex_tree_cc(start_tree);
      lextree_list->insert_cc(start_tree);
    }
    
    // update counts
    //
    for (int_4 i = 0; i < num_levels; i++) {
      total_gen[i] += num_traces_gen[i];
      total_del[i] += num_traces_del[i];
      num_traces_total[i] += (num_traces_gen[i] - num_traces_del[i]);
    }
    
    // reset counts
    //
    for (int_4 i = 0; i < num_levels; i++) {
      num_traces_gen[i] = (int_4)0;
      num_traces_del[i] = (int_4)0;
    }    
    num_mapmi = (int_4)0;
    num_steps = (int_4)0;
    
    num_vect = (int_4)0;
    
    // counting number of features in file    
    //

    // if ascii mode
    //
    if (input_mode == HT_ASCII_MODE) {
      while (fgets((char*)temp_vect, ISIP_MAX_STRING_LENGTH, fpi) !=
	     (char*)NULL) {
	num_vect++;
      }
    }

    // if binary mode
    //
    else {
      float_8* temp_array = new float_8[num_features];
      while ((int_4)fread(temp_vect, sizeof(float_8), num_features, fpi)
	     == num_features) {
	num_vect++;
      }
      delete [] temp_array;
    }
    fclose(fpi);
    
    // open input feature data file to read
    //
    if (input_mode == HT_ASCII_MODE) {
      fpi = fopen((char*)mfcc_file, "r");
    }
    else {
      fpi = fopen((char*)mfcc_file, "rb");
    }
    if (fpi == (FILE*)NULL) {
      fprintf(stdout, "Cannot open file %s\n", mfcc_file);
      exit(ISIP_PROTO_ERROR);
    }
    
    // array used to store the input data
    //
    vectors = new float_8*[num_vect];
    for (int_4 i = 0; i < num_vect; i++) {
      vectors[i] = new float_8[num_features];
    }
    
    // initiallize the vectors array
    //
    for (int_4 i = 0; i < num_vect; i++) {
      for (int_4 j = 0; j < num_features; j++) {
	vectors[i][j] = 0.0;
      }
    }
    
    // main loop: this will loop over all traces for each frame of data
    // the traces are taken care of differently depending on which level
    // they belong to -- word or phone or state
    //
    
    // read the input data
    //
    while (read_input_cc(fpi, num_features, input_mode, features)
	   == ISIP_TRUE) {
      
      // store the current frame data
      //
      for (int_4 i = 0; i < num_features; i++) {
	vectors[current_frame][i] = features[i];
      }
      
      // reset the beam thresholds
      //
      for (int_4 i = 0; i < num_levels; i++) {
	beam_thresh[i] = HT_DEFAULT_SCORE;
	max_score[i] = HT_DEFAULT_SCORE;
      }
      
      // loop over all active state-level traces
      // evaluate the state and update the score for each trace
      // then find all possible transitions and advance traces
      //
      project_states_cc(state_toklist, current_frame, features, sp_phone,
			phone_trlist, active_phones, num_active_ph,
			align_mode, phones, num_traces_gen, num_traces_del,
			num_mapmi,max_score, phmarker);
      
      // compute the MAPMI pruning threshold score
      //
      if (mapmi_limit > (int_4)0) {
	
	// reset mapmi threshold
	//
	mapmi_thresh = HT_DEFAULT_SCORE;
	
	// sort all the active traces according to score
	//
	if (num_mapmi > mapmi_limit) {
	  sort_traces_cc(phmarker, active_phones, num_active_ph,
			 num_mapmi, mapmi_limit, mapmi_thresh);
	}
      }
      
      // find the phone level beam pruning threshold
      //
      if (beam_width[HT_PHONE_LEVEL] != (float_8)0) {
	beam_thresh[HT_PHONE_LEVEL] = max_score[HT_PHONE_LEVEL] +
	  beam_width[HT_PHONE_LEVEL];
      }
      
      // if mapmi pruning works better make that the phone-level beam
      // threshold
      //
      if (beam_thresh[HT_PHONE_LEVEL] < mapmi_thresh) {
	beam_thresh[HT_PHONE_LEVEL] = mapmi_thresh;
      }
      
      // mark all phone level traces below the beam width for pruning
      //
      trace_prune_cc(phone_trlist, phmarker, active_phones, num_active_ph,
		     beam_thresh[HT_PHONE_LEVEL], HT_PHONE_LEVEL,
		     num_traces_del[HT_PHONE_LEVEL]);
      
      // create word-level traces for the word-end phone-level traces
      // make sure that the inactive traces are no longer kept
      //
      active_trace_cc(phone_trlist, phmarker, prev_phmark,
		      word_trlist, wdmarker, prev_wdmark,
		      active_phones, num_active_ph,
		      active_words, num_active_wd,
		      sp_score, word_penalty,
		      num_traces_gen, num_traces_del,
		      max_score[HT_WORD_LEVEL]);
      
      // find the word level beam pruning threshold
      //
      if (beam_width[HT_WORD_LEVEL] != (float_8)0) {
	beam_thresh[HT_WORD_LEVEL] = max_score[HT_WORD_LEVEL] +
	  beam_width[HT_WORD_LEVEL];
      }
      
      // if mapmi pruning works better make that the word-level beam
      // threshold
      //
      if (beam_thresh[HT_WORD_LEVEL] < mapmi_thresh) {
	beam_thresh[HT_WORD_LEVEL] = mapmi_thresh;
      }
      
      // mark all word level traces below the beam width for pruning
      // also update the status of active lexical trees
      //
      trace_prune_cc(word_trlist, wdmarker, active_words, num_active_wd,
		     beam_thresh[HT_WORD_LEVEL], HT_WORD_LEVEL,
		     num_traces_del[HT_WORD_LEVEL]);
      
      // loop over all active word and phone-level traces in the
      // current time frame and grow the next phone
      //
      project_phones_cc(wdmarker, phmarker, state_toklist, phones,
			lextree_list, lattice, num_monophones, ph_size,
			phone_map, active_words, num_active_wd,
			active_phones, num_active_ph, num_traces_gen,
			max_score[HT_STATE_LEVEL], context_mode,
			mlf_mode, num_sph, sph_index);
	
      // find the beam pruning score for state-level pruning
      //
      if (beam_width[HT_STATE_LEVEL] != (float_8)0) {
	beam_thresh[HT_STATE_LEVEL] = max_score[HT_STATE_LEVEL] +
	  beam_width[HT_STATE_LEVEL];
      }
      
      // if mapmi pruning works better make that the state-level beam
      // threshold
      //
      if (beam_thresh[HT_STATE_LEVEL] < mapmi_thresh) {
	beam_thresh[HT_STATE_LEVEL] = mapmi_thresh;
      }
      
      // perform beam pruning at state tokens
      //
      token_prune_cc(state_toklist, active_phones, num_active_ph,
		     beam_thresh[HT_STATE_LEVEL], num_traces_del);
      
      // reset the status flags and update the list of lexical trees
      //
      update_trees_cc(lextree_list);
      
      // store counts
      //
      for (int_4 i = 0; i < num_levels; i++) {
	total_gen[i] += num_traces_gen[i];
	total_del[i] += num_traces_del[i];
	num_traces_total[i] += (num_traces_gen[i] - num_traces_del[i]);
      }
      
      // reset counts
      //
      for (int_4 i = 0; i < num_levels; i++) {
	num_traces_gen[i] = (int_4)0;
	num_traces_del[i] = (int_4)0;
      }    
      num_mapmi = (int_4)0;
      num_hyps = 0;
      
      // reset word-level marker positions 
      //
      for (int_4 i = 0; i < num_active_wd; i++) {
	prev_wdmark[active_words[i]] = wdmarker[active_words[i]];
	wdmarker[active_words[i]] = (Train_Link_node*)NULL;
      }
      
      // reset phone-level marker positions 
      //
      for (int_4 i = 0; i < num_active_ph; i++) {
	prev_phmark[active_phones[i]] = phmarker[active_phones[i]];
	phmarker[active_phones[i]] = (Train_Link_node*)NULL;
      }
      
      // increment the current time frame
      //
      current_frame++;
      
    } // end while loop over all frames
    
    // close file
    //
    fclose(fpi);

    // for phone_level input transcriptions, delete the phn_list and
    // utterance id
    //
    if (mlf_mode == HT_MODEL_TRANS) {
      delete [] phn_list;
      phn_list = (int_4*)NULL;
      delete [] phns;
      phns = (int_4*)NULL;
      utterance = (char_1*)NULL;
    }
    
    // reset the n-best array
    //
    for (int_4 ff = 0; ff < num_nbest; ff++) {
      n_best_array[ff] = (Train_Trace*)NULL;
    }
    
    // reset word-level marker positions 
    //
    for (int_4 i = 0; i < num_active_wd; i++) {
      wdmarker[active_words[i]] = prev_wdmark[active_words[i]];
      prev_wdmark[active_words[i]] = (Train_Link_node*)NULL;
    }
    
    // reset phone-level marker positions 
    //
    for (int_4 i = 0; i < num_active_ph; i++) {
      phmarker[active_phones[i]] = prev_phmark[active_phones[i]];
      prev_phmark[active_phones[i]] = (Train_Link_node*)NULL;
    }
    
    // reset the current time frame
    //
    current_frame--;
    
    // we have now finished the state sequences so we need to backtrace
    // i.e. find an emitting word level trace with the highest score and
    // follow backpointers to the start trace
    //
    backtrace_cc(wdmarker[HT_SENT_END_WORD], num_nbest, num_hyps,
		 n_best_array);
    
    // if no sentence end is found print the best word end anyway
    //
    if (num_hyps == 0) {
      
      // print error message
      //
      fprintf(stdout, "%s :: No sentence-end hypotheses were found.\n",
	      mfcc_file);
      
      // find the best word end
      //
      find_nbest_cc(word_trlist, wdmarker, (int_4)1, active_words,
		    num_active_wd, num_hyps, n_best_array);
    }

    if (op_out == ISIP_TRUE) {
      
      // print the n-best list of sentence-hypotheses to file
      //
      print_hyps_cc(num_hyps, output_file, n_best_array, current_frame,
		    phones, ph_size, total_gen, total_del, num_traces_total);
    }
    
    if (num_hyps != 0) {
      
      // count the valid file
      //
      file_count++;
      para_estimate_cc(num_hyps, n_best_array, vectors, states, models,
		       train_mean, train_covar, count, num_features,
		       num_states, num_mix, trans_count, mod_map,
		       st_map, trans_map, sp_phn_ind);
    }
    
    // free the vectors array TRAIN
    //
    for (int_4 i = 0; i < num_vect; i++) {
      delete [] vectors[i];
    }
    delete [] vectors;
    vectors = (float_8**)NULL;
    
    // dummy variables for deleting memory
    //
    Train_Link_node* nd = (Train_Link_node*)NULL;
    Train_Link_node* pd = (Train_Link_node*)NULL;
    
    // free memory in word-level trace lists
    //
    Train_Trace* tr = (Train_Trace*)NULL;
    for (int_4 i = 0; i < num_active_wd; i++) {
      if (word_trlist[active_words[i]] != (Train_Link_list*)NULL) {
	for (nd = word_trlist[active_words[i]]->get_head_cc();
	     nd != (Train_Link_node*)NULL; nd = pd) {
	  pd = nd->get_next_cc();
	  tr = (Train_Trace*)(nd->get_item_cc());
	  word_trlist[active_words[i]]->remove_cc(nd);
	  if (tr != (Train_Trace*)NULL) {
	    manager->delete_cc(tr);
	  }
	}
      }
      delete word_trlist[active_words[i]];
    }
    
    // free memory in phone-level trace lists
    //
    tr = (Train_Trace*)NULL;
    for (int_4 i = 0; i < num_active_ph; i++) {
      if (phone_trlist[active_phones[i]] != (Train_Link_list*)NULL) {
	for (nd = phone_trlist[active_phones[i]]->get_head_cc();
	     nd != (Train_Link_node*)NULL; nd = pd) {
	  pd = nd->get_next_cc();
	  tr = (Train_Trace*)(nd->get_item_cc());
	  phone_trlist[active_phones[i]]->remove_cc(nd);
	  if (tr != (Train_Trace*)NULL) {
	    manager->delete_cc(tr);
	  }
	}
      }
      delete phone_trlist[active_phones[i]];
    }
    
    // free memory in the token lists
    //
    Train_Token* tok = (Train_Token*)NULL;
    for (int_4 i = 0; i < num_active_ph; i++) {
      if (state_toklist[active_phones[i]] != (Train_Link_list*)NULL) {
	for (nd = state_toklist[active_phones[i]]->get_head_cc();
	     nd != (Train_Link_node*)NULL; nd = pd) {
	  pd = nd->get_next_cc();
	  tok = (Train_Token*)(nd->get_item_cc());
	    state_toklist[active_phones[i]]->remove_cc(nd);
	    if (tok != (Train_Token*)NULL) {
	      delete tok;
	    }
	}
      }
      delete state_toklist[active_phones[i]];
    }
    
    // free memory in active lexical tree list
    //
    Train_Lex_tree* tree = (Train_Lex_tree*)NULL;
    for (nd = lextree_list->get_head_cc(); nd != (Train_Link_node*)NULL; nd = pd) {
      pd = nd->get_next_cc();
      tree = (Train_Lex_tree*)(nd->get_item_cc());
      lextree_list->remove_cc(nd);
      if (tree != (Train_Lex_tree*)NULL) {
	delete tree;
      }
    }
    delete lextree_list;
    lextree_list = (Train_Link_list*)NULL;

    // delete the lex tree
    //
    delete traintree;

    // free memory
    //
    delete lattice;
    lattice = (Train_Lattice*)NULL;
    
  } // end while loop over all files

  // close transcription file
  //
  fclose(fpl);
  
  // for TRAIN update the states parameter and transition prob.
  //
  if (file_count > (int_4)0 && train_mode == HT_NORMAL) {
    update_cc(transitions, states, train_mean, train_covar, count,
	      trans_count, trans_size, num_features, num_states, num_mix,
	      num_trans, var_floor);
    
    // print out the training result
    //
    print_state_cc(new_states_file, states, num_states, num_features,
		   output_mode);
    print_trans_cc(new_trans_file, num_trans, trans_size, transitions);

    // determine if we need to output the occupancy
    //
    if (state_occ_mode == HT_OPT_ON) {

      // open the occupancy file
      //
      FILE* fp_occ_file = fopen((char*)state_occ_file, "w");
      if (fp_occ_file == (FILE*)NULL) {
	fprintf(stdout, "Error: unable to open state occupancy file %s\n",
		state_occ_file);
	exit (ISIP_PROTO_ERROR);
      }

      // call the print method
      //
      print_occ_cc(fp_occ_file, count, num_states,
		   states[1]->get_num_mixtures_cc());
      
      fclose(fp_occ_file);
    }
    
    fprintf(stdout, "Number of valid mfcc files: %ld\n", file_count);
  }

  else if (file_count > (int_4)0 && train_mode == HT_BATCH) {

    // open file to dump the stats for this batch
    //
    FILE* fp_batch = fopen((char*)batch_stats_file, "wb");

    // exit if file does not exist
    //
    if (fp_batch == (FILE*)NULL) {
      fprintf(stdout, "Cannot open file %s\n", batch_stats_file);
      exit(ISIP_PROTO_ERROR);
    }
    
    // dump the stats
    // 
    dump_stats_cc(fp_batch, train_mean, train_covar, count, trans_count,
		  trans_size, num_states, num_mix, num_features, num_trans);

    // close file and deallocate memory for the file name
    //
    fclose(fp_batch);
  }

  else {
    fprintf(stdout, "Warning: no valid mfcc file found\n");
  }
  
  // free memory for TRAIN
  //
  for (int_4 i = 0; i < num_states; i++) {
    for (int_4 j =0; j < num_mix; j++) {
      delete [] train_mean[i][j];
      delete [] train_covar[i][j];
    }
    delete [] train_mean[i];
    delete [] train_covar[i];
    delete [] count[i];
  }
  delete [] train_mean;
  delete [] train_covar;
  delete [] count;
  
  for (int_4 i = 0; i < num_trans; i++) {
    for (int_4 j = 0; j < trans_size[i]; j++) {
      delete [] trans_count[i][j];
    }
    delete [] trans_count[i];
  }
  delete [] trans_count;

  for (int_4 i = 0; i < num_trans; i++) {
    for (int_4 j = 0; j < trans_size[i]; j++) {
      delete [] transitions[i][j];
    }
    delete [] transitions[i];
  }
  delete [] transitions;

  delete [] trans_size;

  for (int_4 i = 0; i < num_models; i++) {
    delete [] st_map[i];
  }
  delete [] st_map;
  delete [] trans_map;
  delete [] mod_map;
  delete [] phone_map;
  
  // free memory
  //
  delete [] max_score;
  delete [] beam_width;
  delete [] beam_thresh;

  // free memory
  //
  delete [] num_traces_total;
  delete [] num_traces_gen;
  delete [] num_traces_del;
  delete [] total_gen;
  delete [] total_del;

  // free memory
  //
  delete [] features;
  delete [] active_phones;
  delete [] active_words;
    
  // free memory
  //
  delete [] wdmarker;
  delete [] prev_wdmark;
  delete [] word_trlist;
  delete [] phmarker;
  delete [] prev_phmark;
  delete [] phone_trlist;
  delete [] n_best_array;
  delete [] state_toklist;
    
  // close files
  //
  fclose(flat);
  fclose(fin);
  if (op_out == ISIP_TRUE)
     fclose(fout);
  
  // free memory in filenames
  //
  delete [] input_file;
  delete [] mfcc_file;
  delete [] output_file;
  delete [] mlf_file;
  delete [] temp_vect;
  delete [] new_states_file;
  delete [] new_trans_file;
  delete [] acc_list_file;
  delete [] batch_stats_file;
  delete [] state_occ_file;
  batch_stats_file = (char_1*)NULL;


  // free memory in the word hash table
  //
  Train_Word* word = (Train_Word*)NULL;
  Train_Hash_cell* next = (Train_Hash_cell*)NULL;
  Train_Hash_cell** hash_cells = word_table->get_cells_cc();
  int_4 hash_size = word_table->get_size_cc();
  for (int_4 k = 0; k < hash_size; k++) {
    for (Train_Hash_cell* cell = hash_cells[k]; cell != (Train_Hash_cell*)NULL;
	 cell = next) {
      next = cell->get_next_cc();
      word = (Train_Word*)(cell->get_item_cc());
      delete word;
      manager->delete_cc(cell);
    }
    hash_cells[k] = (Train_Hash_cell*)NULL;
  }
  delete word_table;


  // free memory in data structures
  //
  for (int_4 i = 0; i < num_monophones; i++) {
    delete [] monophones[i];
  }
  delete [] monophones;
  
  for (int_4 i = 0; i < num_states; i++) {
    delete states[i];
  }
  delete [] states;
  
  for (int_4 i = 0; i < num_models; i++) {
    delete models[i];
  }
  delete [] models;
  
  for (int_4 i = 0; i < num_phones; i++) {
    delete phones[i];
  }
  delete [] phones;

  if(num_sph>0) {
    delete [] sph_index;
  }
  
  delete [] phn_ind;
  delete [] phn_str;
  
  // free the memory manager
  //
  delete manager;    

  // exit gracefully
  //
  exit(ISIP_NO_ERROR);
}
