// file: $isip/util/speech/isip_model_converter/imc_lexicon_0.cc
// version: $Id: imc_lexicon_0.cc 8977 2003-01-12 06:37:52Z jelinek $
//
// isip include files
//
#include "isip_model_converter.h"

// method: read_lexicon_cc
//
// arguments:
//  String& lexicon_filename : (input) file containing prototype lexicon
//  File& lexicon_file : (output) lexicon file
//  Vector<SearchSymbol>& word_syms : (output) words in the lexicon
//  Vector<DiGraph<Ulong> >& pron_graph : (output) pronunciation graphs
//  HashTable<String, Ulong>& lex_hash : (output) lexicon hash table
//  HashTable<String, Ulong>& phone_hash : (output) phone hash table
//
// return: a logical flag to indicate success
//
// this method reads the input lexicon file and convert it production
// format
//
boolean read_lexicon_cc(String& lexicon_filename_a, File& lexicon_file_a,
			Vector<SearchSymbol>& word_syms_a,
			Vector<DiGraph<Ulong> >& pron_graph_a,
			HashTable<String, Ulong>& lex_hash_a,
			HashTable<String, Ulong>& phone_hash_a) {

  // declare local variables
  //
  ulong num_words = 0;
  Ulong* index = (Ulong*)NULL;
  Ulong dummy = 0;
  String token;
  String subword;

  // check if the file exists
  //
  if (!lexicon_file_a.open(lexicon_filename_a)) {
    String out(L"error opening ");
    out.concat(lexicon_filename_a);
    Console::put(out);
    Integral::exit();
  }

  SearchSymbol tmp_str(L"");
  SearchSymbol tmp;

  // count the number of words in the lexicon
  //
  while(!lexicon_file_a.eof()) {

    num_words++;
    // get the next line
    //
    if(!lexicon_file_a.get(tmp_str)) {
      break;
    }
  }
  lexicon_file_a.close();

  // set the max length for the graph lists
  //
  word_syms_a.setLength(num_words);
  pron_graph_a.setLength(num_words);

  // reset the number of words
  //
  num_words = 0;
  
  // check if the file exists
  //
  if (!lexicon_file_a.open(lexicon_filename_a)) {
    String out(L"error opening ");
    out.concat(lexicon_filename_a);
    Console::put(out);
    Integral::exit();
  }

  // read the lexicon file
  //
  while(!lexicon_file_a.eof()) {

    // get the next line
    //
    if(!lexicon_file_a.get(tmp_str)) {
      break;
    }

    // trim the line and check for empty lines and comment lines
    //
    tmp_str.trim();
    if (tmp_str.length() <= 0) {
      continue;
    }
    if (tmp_str(0) == IMC_COMMENT_CHAR) {
      continue;
    }

    // tokenize the line based on space
    //
    long pos = 0;
    tmp_str.tokenize(token, pos, IMC_DELIMITER_CHAR);
    
    // make an entry to the hash table
    // check if the word is already entered
    //
    if (!lex_hash_a.containsKey(token)) {

      index = new Ulong(num_words);
	  
      // add it to the hash table and
      // increment the num words count
      //
      lex_hash_a.insert(token, index);
      num_words++;

      // assign it to the vector of SearchSymbol
      //
      word_syms_a(num_words - 1).assign(token);

      pron_graph_a(num_words - 1).setAllocationMode(DstrBase::USER);
      
      // construct the pronunciation graph
      //
      GraphVertex<Ulong>* prev_node = pron_graph_a(num_words - 1).getStart();
      GraphVertex<Ulong>* next_node = (GraphVertex<Ulong>*)NULL;


      // add the phone index as node items
      //
      while (tmp_str.tokenize(subword, pos, IMC_DELIMITER_CHAR)) {
	
	// create the string to store and place it in a new GraphVertex
	//
	
	// get the subword unit and then index it
	// index of the sub-word unit
	//
	Ulong* ind = phone_hash_a.get(subword);
	if (ind == (Ulong*)NULL) {
	  String out(L"Error in pronunciation of: ");
	  out.concat(token);
	  String out2(L": contains sub-word unit: ");
	  out.concat(out2);
	  out.concat(subword);
	  Console::put(out);
	  exit(Integral::exit());
	}
	
	// set the index
	//
	next_node = pron_graph_a(num_words-1).insertVertex(ind);
	
	// add an arc between the previous symbol and this one
	//
	pron_graph_a(num_words - 1).insertArc(prev_node, next_node,
					      true,
					      (float)log((double)1.0));
	
	// update the previous pointer to the symbol just added
	//
	prev_node = next_node;
      }

      // connect the last symbol added to the terminal vertex
      //
      pron_graph_a(num_words - 1).insertArc(prev_node,
					 pron_graph_a(num_words - 1).getTerm(),
					    true, 
					    (float)log((double)1.0));
    }
  
    // if the word has already been encountered
    // 
    else {
    
      // get the index for the graph from the hash table
      //
      Ulong* item = lex_hash_a.get(token);
      item->debug(L"Item is = ");
      ulong ind_item = (ulong)(*item);
      
      printf("%lu\n", ind_item);
      
      // construct the pronunciation graph
      //
      GraphVertex<Ulong>* prev_node = pron_graph_a(ind_item).getStart();
      GraphVertex<Ulong>* next_node = (GraphVertex<Ulong>*)NULL;
      
      // add the phone index as node items
      //
      while (tmp_str.tokenize(subword, pos, IMC_DELIMITER_CHAR)) {
	
	// create the string to store and place it in a new GraphVertex
	//
	next_node = new GraphVertex<Ulong>;
	
	// get the index of the sub-word unit
	//
	Ulong* ind = phone_hash_a.get(subword);
	if (ind == (Ulong*)NULL) {
	  String out(L"Error in pronunciation of: ");
	  out.concat(token);
	  String out2(L": contains sub-word unit: ");
	  out.concat(out2);
	  out.concat(tmp_str);
	  Console::put(out);
	  exit(Integral::exit());
	}
	
	// set the index
	//
	next_node = pron_graph_a(ind_item).insertVertex(ind);
	
	// add an arc between the previous symbol and this one
	//
	pron_graph_a(ind_item).insertArc(prev_node, next_node,
					 true,
				      (float)log((double)1.0));
	
	// update the previous pointer to the symbol just added
	//
	prev_node = next_node;
      }
      
      // connect the last symbol added to the terminal vertex
      //
      pron_graph_a(ind_item).insertArc(prev_node,
				       pron_graph_a(ind_item).getTerm(),
				       true, 
				       (float)log((double)1.0));
    }
  }
  
  pron_graph_a.setLength(num_words);
  word_syms_a.setLength(num_words);
  return lexicon_file_a.close();
}




