// file: $isip/util/asr/isip_lm_converter.cc
// version: $Id: isip_network_converter.cc 10618 2006-08-24 16:03:40Z dom5 $
//

// isip include files
//
#include <LanguageModel.h>
#include <LanguageModelIHD.h>

#include "isip_network_converter.h"

// isip_net_converter: language model converter utility
//
// this is the language model converter utility. it can be used for
// convert the output of network builder to the model file that can
// be used for the isip_recognizer.
//
int main(int argc, const char** argv) {
 
 // set the sdb
 //
 Sdb sdb;
 CommandLine cmdl(sdb);
 
 cmdl.setUsage(
#include "usage_message.text"
                );
 cmdl.setHelp(
#include "help_message.text"
               );
 
 cmdl.setIdent("$Revision: 10618 $",
	       "$Name: $",
	       "$Date: 2006-08-24 11:03:40 -0500 (Thu, 24 Aug 2006) $");

 // add a command line option for the output mode
 //

 String input_format;
 cmdl.addOptionParam(input_format, OPTION_INPUT_FORMAT, L"NATIVE");

 // add a command line option for the input mode
 //
 String output_format;
 cmdl.addOptionParam(output_format, OPTION_OUTPUT_FORMAT, L"JSGF");

 // add a command line option for output type
 //
 String output_type;
 cmdl.addOptionParam(output_type, OPTION_OUTPUT_TYPE, L"TEXT");
 
 // add a command line option for graph minimization
 //
 Boolean graph_minimization;
 cmdl.addFlagParam(graph_minimization, OPTION_MINIMIZE);

 // add a command line option for the debug level
 //
 DebugLevel debug_level;
 cmdl.addOptionParam(debug_level, OPTION_DEBUG_LEVEL, DEBUG_LEVEL_DEFAULT);

 // parse the command line
 //
 if (!cmdl.parse(argc, argv)) {
   cmdl.printUsage();
 };

 // input file and output file
 //
 Filename input_lm_file;
 Filename output_lm_file;
 Filename output_sm_file;

 // get the input file
 //
 if ( !sdb.gotoFirst() ){
   cmdl.printUsage();
 }
 sdb.getName(input_lm_file);

 // get the output language model file
 //
 if ( !sdb.gotoNext() ){
   cmdl.printUsage();
 }
 sdb.getName(output_lm_file);
 
 // get the output statistical model file
 //
 if ( !sdb.gotoNext() ){
   cmdl.printUsage();
 }
 sdb.getName(output_sm_file);
 
 // print the command line
 //
 cmdl.printCommandLine();
 cmdl.printVersion();

 // check the input file arguments
 //
 if(input_lm_file.eq(NULL)) {
   String msg(L"Error: no input file specified ");
   Console::put(msg);
   Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		 __FILE__, __LINE__);
 };

 if(output_lm_file.eq(NULL)) {
   String msg(L"Error: no output file specified ");
   Console::put(msg);
   Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		 __FILE__, __LINE__);
 };

 // set the options to upper case
 //
 input_format.toUpper();
 output_format.toUpper();
 output_type.toUpper();

 // set the output type
 //
 File::TYPE out_type = File::TEXT;
 if( output_type.eq(L"TEXT")) {
   out_type = File::TEXT;
 }
 else if( output_type.eq(L"BINARY")) {
   out_type = File::BINARY;
 }
 else {
   cmdl.printUsage();
 }

 // input is the native java network builder format
 //
 if (input_format.eq(L"NATIVE")) {
   if (output_format.eq(L"IHD")) {
     if (!nativeToModel(input_lm_file, output_lm_file, output_sm_file,
			LanguageModel::IHD, out_type)) {
       return Error::handle(PROG_NAME, L"NATIVE->IHD conversion error",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
   else if (output_format.eq(L"JSGF")) {
     if (!nativeToModel(input_lm_file, output_lm_file, output_sm_file,
			LanguageModel::JSGF, out_type)) {
       return Error::handle(PROG_NAME, L"NATIVE->JSGF conversion error",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
   else if (output_format.eq(L"XML")) {
     if (!nativeToModel(input_lm_file, output_lm_file, output_sm_file,
			LanguageModel::XML, out_type)) {
       return Error::handle(PROG_NAME, L"NATIVE->XML conversion error",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
   else if (output_format.eq(L"BNF")) {
     if (!nativeToModel(input_lm_file, output_lm_file, output_sm_file,
			LanguageModel::BNF, out_type)) {
       return Error::handle(PROG_NAME, L"NATIVE->BNF conversion error",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
   else if (output_format.eq(L"ABNF")) {
     if (!nativeToModel(input_lm_file, output_lm_file, output_sm_file,
			LanguageModel::ABNF, out_type)) {
       return Error::handle(PROG_NAME, L"NATIVE->ABNF conversion error",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
 }
 else if ( input_format.eq(L"IHD") || input_format.eq(L"JSGF") ||
	   input_format.eq(L"DIGRAPH") || input_format.eq(L"XML") ||
	   input_format.eq(L"BNF") || input_format.eq(L"ABNF")){

   if ( output_format.eq(L"NATIVE") ) {
     if(!modelToNative(input_lm_file, output_lm_file)) {
       return Error::handle(PROG_NAME, L"error while converting to NATIVE",
			   Error::ARG, __FILE__, __LINE__);
     }
   }

   else if (output_format.eq(L"JSGF")) {
     if (!modelToModel(input_lm_file, output_lm_file,
		      LanguageModel::JSGF, out_type,
		      graph_minimization)) {
       return Error::handle(PROG_NAME, L"error while converting to JSGF",
			    Error::ARG, __FILE__, __LINE__);
     }
   }

   else if (output_format.eq(L"IHD")) {
     if (!modelToModel(input_lm_file, output_lm_file,
		       LanguageModel::IHD, out_type,
		       graph_minimization)) {
       return Error::handle(PROG_NAME, L"error while converting to IHD",
			    Error::ARG, __FILE__, __LINE__);
     }
   }

   else if (output_format.eq(L"XML")) {
     if (!modelToModel(input_lm_file, output_lm_file,
		       LanguageModel::XML, out_type,
		       graph_minimization)) {
       return Error::handle(PROG_NAME, L"error while converting to XML",
			    Error::ARG, __FILE__, __LINE__);
     }
   }

   else if (output_format.eq(L"BNF")) {
     if (!modelToModel(input_lm_file, output_lm_file,
		       LanguageModel::BNF, out_type,
		       graph_minimization)) {
       return Error::handle(PROG_NAME, L"error while converting to BNF",
			    Error::ARG, __FILE__, __LINE__);
     }
   }

   else if (output_format.eq(L"ABNF")) {
     if (!modelToModel(input_lm_file, output_lm_file,
		       LanguageModel::ABNF, out_type,
		       graph_minimization)) {
       return Error::handle(PROG_NAME, L"error while converting to ABNF",
			    Error::ARG, __FILE__, __LINE__);
     }
   }
   
   else {
     cmdl.printUsage();
   }
 }

else{
   cmdl.printUsage();
 }
 
 // exit gracefully
 //
 return Integral::exit();

}

// method: modelToModel
//
// arguments:
//  FileName&  input_file_name_a: (input) file name
//  FileName& output_file_name_a: (output) file name
//
// return: boolean value indicating status
//
// this method reads the data into a hash table
//
boolean modelToModel(Filename& input_file_name_a,
		     Filename& output_file_name_a,
		     LanguageModel::ALGORITHM out_format_a,
		     File::TYPE out_type_a,
		     Boolean minimize_a){

  // the input language model file
  //
  LanguageModel input_lm;
  LanguageModelIHD ihd_model;
  
  // hierarchical digraph to be used in case the
  // lm file is in the old digraph format
  //
  HierarchicalDigraph h_digraph;

  // if the model is in IHD format, check to make sure it's in the
  // updated IHD format and not the older DIGRAPH format.  if it's
  // in the older DIGRAPH format, load the HierachicalDigraph
  // object the old way
  //
  if (isOldDigraph(input_file_name_a)) {

    // load old digraph language model
    //
    loadOldDigraph(input_file_name_a, h_digraph);

    ihd_model.setIHD(h_digraph);

    // set up the language model object
    //
    input_lm.setFormat(LanguageModel::IHD);
    input_lm.setRuleModel(ihd_model.getRuleModel());
 
  }
  else {
    
    // input language model file
    //
    Sof lm_file_in;
    lm_file_in.open(input_file_name_a, File::READ_ONLY);
        
    // read the input language model file
    //
    if (!input_lm.read(lm_file_in)) {
      return Error::handle(input_file_name_a,
			   L"error reading language model",
			   Error::TEST, __FILE__, __LINE__);
    }
    
    // close the input language model file
    //
    lm_file_in.close();
  }
  
  // output language model file
  //
  Sof lm_file_out;
  lm_file_out.open(output_file_name_a, File::WRITE_ONLY, out_type_a);  
  
  // set the format and HierarchicalDigraph of the output language model
  // and write it out to a file
  //
  input_lm.convert(out_format_a);

  // minimize the graph if user has specified minimize option
  //
  if (minimize_a) {

    input_lm.minimizeGraph();
  }
  
  input_lm.write(lm_file_out);
  
  // close output language model sof file
  //
  lm_file_out.close();
  
  // exit gracefully
  //
  return true;  
}

// method: modelToNative
//
// arguments:
//  FileName&  input_file_name_a: (input) file name
//  FileName& output_file_name_a: (output) file name
//
// return: boolean value indicating status
//
// this method reads the data into a hash table
//
boolean modelToNative(Filename& input_file_name_a,
		      Filename& output_file_name_a){


  // the input language model file
  //
  LanguageModel input_lm;
  LanguageModelIHD ihd_model;
  
  // the search levels, sub-graphs are maintained at each SearchLevel 
  //
  HierarchicalDigraph h_digraph;

  // if the model is in IHD format, check to make sure it's in the
  // updated IHD format and not the older DIGRAPH format.  if it's
  // in the older DIGRAPH format, load the HierachicalDigraph
  // object the old way
  //
  if (isOldDigraph(input_file_name_a)) {

    // load old digraph into HierarchicalDigraph object
    //
    if (!loadOldDigraph(input_file_name_a, h_digraph)) {
      return Error::handle(input_file_name_a, L"error loading old digraph file",
			   Error::ARG, __FILE__, __LINE__);
    }

    // set up LanguageModel object
    //
    ihd_model.setIHD(h_digraph);    
    input_lm.setFormat(LanguageModel::IHD);
    input_lm.setRuleModel(ihd_model.getRuleModel());

    
  }
  else {
    
    // input language model file
    //
    Sof lm_file_in;
    lm_file_in.open(input_file_name_a, File::READ_ONLY);
        
    // read the input language model file
    //
    input_lm.read(lm_file_in);

    // close the input language model file
    //
    lm_file_in.close();

    // convert the language model to IHD
    //
    ihd_model.setRuleModel(input_lm.getRuleModel());
    
  }
  
  // get the hierarchical digraph for use in conversion to java native
  //
  //h_digraph = input_lm.getHDigraph();
 
  // store the number of levels
  //
  HashTable<String, String>  hash_data;
  String hash_key;
  long num_of_levels = ihd_model.getIHD().length();
  String num_levels;
  num_levels.assign(num_of_levels);
  hash_key.assign(IO_CDATA_LEVEL_SIZE);
  hash_data.insert(hash_key, &num_levels);
 
  // read each level
  //
  for (long i = 0 ; i < num_of_levels; i++){
    
    // set the prefix
    //
    String curr_level;
    curr_level.assign((long)i);
   
    String level_prefix;
    level_prefix.assign(IO_SEARCHLEVEL_PREFIX);
    level_prefix.concat(curr_level);

    // read one level
    //
    Vector<SearchSymbol> graph_name;
    if ( i == 0 ){
      graph_name.concat(LEVEL_TREE_TOP_GRAMMAR);
    }
    else{
      if ( ihd_model.getIHD()(i-1).getContextMap().length() > 0 ){
	convertContextMapping(ihd_model.getIHD()(i-1).getContextMap(), graph_name);
      }
      else {
	graph_name = ihd_model.getIHD()(i-1).getSymbolTable();
      }
    }
    storeSearchLevel(ihd_model.getIHD()(i),  hash_data, graph_name, level_prefix);
   
  }

  // write to the file
  //
  File output_file;
  if (!output_file.open(output_file_name_a, File::WRITE_ONLY)) {
    return Error::handle(output_file_name_a,
			 L"open error", Error::TEST, __FILE__, __LINE__);
  }

  // set the string
  //
  String output;
  //writeHashTable(hash_data, output, L"test");
  writeHashTable(hash_data, output_file, L"test");
  //Console::put(output);
  //output_file.put(output);
  output_file.close();
 
  // exit gracefully
  //
  return true;
  
}

// method: nativeToModel
//
// arguments:
//  FileName&  input_file_name_a: (input) file name
//  FileName& output_file_name_a: (output) file name
//
// return: boolean value indicating status
//
// this method reads the data into a hash table
//
boolean nativeToModel(Filename& input_file_name_a,
		      Filename& output_file_name_a,
		      Filename& output_sm_file_a,
		      LanguageModel::ALGORITHM out_format_a,
		      File::TYPE out_type_a){

  // read the file line by line and parse the input data
  // the data will be stored in a HashTable
  //
  HashTable<String, String>  hash_data;
  readHashTable(input_file_name_a,  hash_data);
  
  // read the number of levels
  //
  long num_of_levels = 0;
  hash_data.get(IO_CDATA_LEVEL_SIZE)->get(num_of_levels);
  
  Long level_num(num_of_levels);

  // the search levels, sub-graphs are maintained at each SearchLevel 
  //
  HierarchicalDigraph h_digraph;
  h_digraph.setLength(num_of_levels);
  
  // read each level
  //
  for (long i = 0 ; i < num_of_levels; i++) {
    
    // set the prefix
    //
    String curr_level;
    curr_level.assign((long)i);
   
    String level_prefix;
    level_prefix.assign(IO_SEARCHLEVEL_PREFIX);
    level_prefix.concat(curr_level);
    
    readSearchLevel(h_digraph, i, hash_data, level_prefix);
  }

  Sof lm_file_out;
  lm_file_out.open(output_file_name_a, File::WRITE_ONLY, out_type_a);  

  // use the language model class to output different types
  //
  LanguageModel output_lm;
  LanguageModelIHD ihd_model;

  ihd_model.setIHD(h_digraph);

  output_lm.setFormat(LanguageModel::IHD);
  output_lm.setRuleModel(ihd_model.getRuleModel());

  output_lm.convert(out_format_a);
  output_lm.write(lm_file_out);

  // close sof file
  //
  lm_file_out.close();
  
  // write the statistical model file
  //
  if(!output_sm_file_a.eq(NULL)) {

    // print messages
    //
    String msg(L"store the statistical file");
    Console::put(msg);

    // store the sm symbols
    //
    Sof sof_sm;
    sof_sm.open(output_sm_file_a, File::WRITE_ONLY);
      

    // read each level
    //
    for (long i = 0 ; i < num_of_levels; i++){
    
      // set the prefix
      //
      String curr_level;
      curr_level.assign((long)i);
   
      String level_prefix;
      level_prefix.assign(IO_SEARCHLEVEL_PREFIX);
      level_prefix.concat(curr_level);

      // write the symbols to the sm file
      //
      Vector<SearchSymbol> sm_symbols;
      HashTable<SearchSymbol, Long>  sm_table;

      // read one level
      //
      readSMSymbols(sm_symbols,  hash_data, level_prefix);

      for ( int j = 0; j < sm_symbols.length(); j++ ){

	// no multiple copies
	//
	if ( sm_table.get(sm_symbols(j)) == NULL){
	  
	  Long index(j);
	  // put the data into the hash table
	  //
	  sm_table.insert(sm_symbols(j), &index);       
	}
      }

      // debug message
      //
      if (debug_level >= Integral::BRIEF) {
	sm_table.debug(L"sm_table");
      }

      // write the hashtable from the input sof file
      //
      if (!sm_table.isEmpty()) {
	if(!sm_table.write(sof_sm, i,
			   SearchLevel::PARAM_STAT_HASH)) {
	  return Error::handle(L"isip_network_converter", L"store symbols with statistical models", Error::ARG, __FILE__, __LINE__);
	}
      }
    }

    // close the sm file
    //
    sof_sm.close();
    
  }

  // exit gracefully
  //
  return true;
  
}

// method: isOldDigraph
//
// arguments:
//  FileName&: (input) file name
//
// return: boolean value indicating whether or not IHD file is old DIGRAPH
//
// checks to see if the input IHD file is in the old DIGRAPH format by looking
// for the search_symbols tag
//
boolean isOldDigraph(Filename& input_file_name_a) {

  // open the language model file
  //
  Sof lm_file;
  lm_file.open(input_file_name_a, File::READ_ONLY);

  // look for a search_symbols tag.  if it exists, language model
  // must be in old Digraph format
  //
  long tag = lm_file.first(L"search_symbols");

  // close the language model file
  //
  lm_file.close();

  // return true if tag was found
  //
  if (tag != Sof::NO_TAG) {
    return true;
  }

  // exit gracefully
  //
  return false;
}

// method: loadOldDigraph
//
// arguments:
//  FileName&: (input) file name
//  LanguageModel&: lm
//
// return: boolean value indicating status
//
// loads a language model file using SearchLevel's load method.  this is the
// old method of loading a language model.
//
boolean loadOldDigraph(Filename& input_file_name_a, HierarchicalDigraph& hd_a) {

  // Open the old language model file
  //
  Sof old_lm_file;
  old_lm_file.open(input_file_name_a, File::READ_ONLY);
    
  // number of levels in language model
  //
  long num_levels = 0;
  
  // load each of the search levels individually
  //
  for(long tag = old_lm_file.first(PARAM_SEARCH_SYMBOL); tag != Sof::NO_TAG;
      tag = old_lm_file.next(PARAM_SEARCH_SYMBOL, tag)) {
    
    // count the number of JSGF tags
    //
    num_levels++;
  }

  // set the length (number of levels) of the HierarchicalDigraph
  //
  hd_a.setLength(num_levels);
  
  for (long i = 0; i < num_levels; i++) {
    
    // load the search level according to the input level index
    //
    if (!hd_a(i).load(old_lm_file, i)) {
      return Error::handle(SearchLevel::name(), L"load: loading search level",
			   Error::ARG, __FILE__, __LINE__);      
    }
    
    // set the level index of this particular level
    //
    hd_a(i).setLevelIndex(i);
  }

  // close the language model sof file
  //
  old_lm_file.close();

  // exit gracefully
  //
  return true;
}


// method: readHashTable
//
// arguments:
//  FileName&: (input) file name
//  HashTable&: (output) the data will be put into the hash table
//
// return: boolean value indicating status
//
// this method reads the data into a hash table
//
boolean readHashTable(Filename& input_file_name_a,
		      HashTable<String, String>& hash_table_a) {

  // set local variable
  //
  File input_file;
  String line;

  input_file.open(input_file_name_a, File::READ_ONLY);
  
  while ( input_file.get(line) ){
    
    // parse the line
    //
    line.trim();
    
    if ( line.firstChr(COMMENT_CHAR) == 0 ){
     continue;
    }
    else{
      
      // find the parameter name and the value
      //
     long count = line.countTokens(DELIMITER_CHAR);
     
     if ( count != 2 ){
       return Error::handle(L"read", L"parsing error in model file",
			    Error::ARG, __FILE__, __LINE__);  
     }
     
     long pos = 0;
     Vector<String> values(count);
     
     // get each word by tokenizing using multiple spaces as a delimiter
     //
     for (long i = 0; i < count; i++) {
       line.tokenize(values(i), pos, DELIMITER_CHAR);
     }
     
     // put the data into the hash table
     //
     hash_table_a.insert(values(0), &values(1));       

     // debug information
     //
     if (debug_level >= Integral::BRIEF) {
       Console::put(line);
     }
     
    }
  }

  // close the input file
  //
  input_file.close();

  // exit gracefully
  //
  return true;
  
}

// method: readSMSymbols
//
// arguments:
//  SearchLevel&: (output) construct the search level
//  HashTable<String, String>&: (intput) the hash table which stored data
//  String& prefix_a: (output) the pefix
//
// return: boolean value indicating status
//
// this method reads symbols with statistical model
//
boolean readSMSymbols(Vector<SearchSymbol>& sm_symbols_a,
		      HashTable<String, String>& hash_table_a,
		      String& prefix_a) {
  // local variable
  //
  String hash_key;
  String symbol_prefix;

  // find the number of symbols in SM symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SM_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SM_SYMBOL);
  readSymbols(sm_symbols_a,
	      hash_table_a, hash_key, symbol_prefix);

  // exit gracefully
  //
  return true;
  
}

// method: readSearchLevel
//
// arguments:
//  SearchLevel&: (output) construct the search level
//  HashTable<String, String>&: (intput) the hash table which stored data
//  String& prefix_a: (output) the pefix
//
// return: boolean value indicating status
//
// this method reads the data into a SearchLevel
//
boolean readSearchLevel(HierarchicalDigraph& h_digraph_a, long level_a,
			HashTable<String, String>& hash_table_a,
			String& prefix_a) {
  // local variable
  //
  String hash_key;

  // set the name for this level
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NAME);
  String level_tag;
  level_tag.assign(*hash_table_a.get(hash_key));

  // debug information
  //
  if (debug_level >= Integral::BRIEF) {
    level_tag.debug(L"level_tag");
  }

  // set the index of the level
  //
  h_digraph_a(level_a).setLevelIndex(level_a);

  // set the level tag (level name)
  //
  h_digraph_a(level_a).setLevelTag(level_tag);
  
  // set symbol table and all other symbols
  // including : non_speech, exclude_symbols,
  //

  // find the number of symbols in the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SYMBOL_SIZE);
  String symbol_prefix;
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SYMBOL);
  readSymbols(h_digraph_a(level_a).getSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);

  // find the number of symbols in the context symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_CONTEXT_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_CONTEXT);
  Vector<SearchSymbol> context_symbols;
  readSymbols(context_symbols, 
	      hash_table_a, hash_key, symbol_prefix);
  convertContextMapping(context_symbols,
			h_digraph_a(level_a).getContextMap());

  // find the number of symbols in the non-speech boundary symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NON_SPEECH_BOUNDARY_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_NON_SPEECH_BOUNDARY_SYMBOL);
  readSymbols(h_digraph_a(level_a).getNonSpeechBoundarySymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);

  // debug message
  //
  if (debug_level >= Integral::BRIEF) {
    h_digraph_a(level_a).debug( L"search_level");
  }
  
  // find the number of symbols in the non-speech internal symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NON_SPEECH_INTERNAL_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_NON_SPEECH_INTERNAL_SYMBOL);
  readSymbols(h_digraph_a(level_a).getNonSpeechInternalSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);

  // debug message
  //
  if (debug_level >= Integral::BRIEF) {
    h_digraph_a(level_a).debug( L"search_level");
  }
  
  // find the number of symbols in the spenalty exclude symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SPENALTY_EXCLUDE_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SPENALTY_EXCLUDE_SYMBOL);
  readSymbols(h_digraph_a(level_a).getSPenaltyExcludeSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);

  // debug message
  //
  if (debug_level >= Integral::BRIEF) {
    h_digraph_a(level_a).debug( L"search_level");
  }

  // find the number of symbols in the context-less symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_CONTEXTLESS_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_CONTEXTLESS_SYMBOL);
  readSymbols(h_digraph_a(level_a).getContextLessSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);
  
  // find the number of symbols in the dummy symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_DUMMY_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_DUMMY_SYMBOL);
  readSymbols(h_digraph_a(level_a).getDummySymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);

  // find the number of symbols in the skip symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SKIP_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SKIP_SYMBOL);
  readSymbols(h_digraph_a(level_a).getSkipSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);
  
  // find the number of symbols in the exclude symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_EXCLUDE_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_EXCLUDE_SYMBOL);
  readSymbols(h_digraph_a(level_a).getExcludeSymbolTable(),
	      hash_table_a, hash_key, symbol_prefix);
  
  // find the number of graphs
  //
  long num_of_graphs = 0;
  long total_graphs = 0;

  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_GRAPH_SIZE);
  
  hash_table_a.get(hash_key)->get(num_of_graphs);
  Long graph_num(num_of_graphs);

  // get the symbol table(s) from the previous (higher) level
  //
  Vector<SearchSymbol> prev_symbol_table;
  Vector<SearchSymbol> prev_dummy_symbol_table;
  
  if (level_a == 0) {
    total_graphs = num_of_graphs;
  }
  else {
    prev_symbol_table = h_digraph_a(level_a - 1).getSymbolTable();
    prev_dummy_symbol_table
      = h_digraph_a(level_a - 1).getDummySymbolTable();    
    total_graphs = prev_symbol_table.length();
  }

  // set the number of graphs
  //
  if (total_graphs > 0) {
    h_digraph_a(level_a).setNumSubGraphs(total_graphs);
  }
  
  // read graphs for a paticular level
  //
  long k = 0;
  for (long j = 0; j < total_graphs; j++) {
    
    // check if the symbol corresponding to this sub-graph at the
    // higher level is a dummy symbol
    //
    dummy_check = false;
    
    for (long i = 0; i < prev_dummy_symbol_table.length(); i++) {
      if (prev_symbol_table(j).eq(prev_dummy_symbol_table(i))) {
	dummy_check = true;
      }
    }

    // set the prefix
    //
    String curr_graph;
    curr_graph.assign((long)k);
    k++;
    
    String graph_prefix;
    graph_prefix.assign(prefix_a);
    graph_prefix.concat(IO_DIGRAPH_PREFIX);
    graph_prefix.concat(curr_graph);
    
    // read the graph
    //
    readSubGraph(h_digraph_a(level_a),
		 h_digraph_a(level_a).getSubGraph(j),
		 hash_table_a, graph_prefix);
  }
  
  // exit gracefully
  //
  return true;
  
}

// method: readSubGraph
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph<SearchNode>
//
boolean readSubGraph(SearchLevel& search_level_a,
		     DiGraph<SearchNode>& sub_graph_a,
		     HashTable<String, String>& hash_table_a,
		     String& prefix_a) {
  // local variable
  //
  String hash_key;
  SingleLinkedList<SingleLinkedList<GraphVertex<SearchNode> > >
    vertices_list(DstrBase::SYSTEM);

  // get the number of vertices
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_DIGRAPH_VERTEX_SIZE);
  long num_of_vertices = 0;
  hash_table_a.get(hash_key)->get(num_of_vertices);

  // get the number of arcs
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_DIGRAPH_ARC_SIZE);
  long num_of_arcs = 0;
  hash_table_a.get(hash_key)->get(num_of_arcs);

  // read graphs for a paticular level
  //
  for (long i = 0; i < num_of_vertices; i++){

   // set the prefix
   //
   String curr_vertex;
   curr_vertex.assign((long)i);
   
   String vertex_prefix;
   vertex_prefix.assign(prefix_a);
   vertex_prefix.concat(IO_VERTEX_PREFIX);
   vertex_prefix.concat(curr_vertex);

   // create a new list to hold a list of vertices
   //
   SingleLinkedList<GraphVertex<SearchNode> >* vertices_on_node =
     new SingleLinkedList<GraphVertex<SearchNode> >(DstrBase::USER);
   
   // read the vertex
   //
   readVertices(search_level_a, sub_graph_a,
		*vertices_on_node, hash_table_a, vertex_prefix);
   vertices_list.insert(vertices_on_node);
   
  }

  // detect the self loops
  //
  for (long j = 0; j < num_of_arcs; j++){

   // set the prefix
   //
   String curr_arc;
   curr_arc.assign((long)j);
   
   String arc_prefix;
   arc_prefix.assign(prefix_a);
   arc_prefix.concat(IO_ARC_PREFIX);
   arc_prefix.concat(curr_arc);

   // read the arcs
   //
   detectSelfLoop(search_level_a, sub_graph_a,
		  vertices_list, hash_table_a, arc_prefix);
  }

  // read graphs for a paticular level
  //
  for (long j = 0; j < num_of_arcs; j++){

   // set the prefix
   //
   String curr_arc;
   curr_arc.assign((long)j);
   
   String arc_prefix;
   arc_prefix.assign(prefix_a);
   arc_prefix.concat(IO_ARC_PREFIX);
   arc_prefix.concat(curr_arc);

   // read the arcs
   //
   readArcs(sub_graph_a, vertices_list, hash_table_a, arc_prefix);
  }

  // exit gracefully
  //
  return true;
  
}

// method: readVertices
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  SingleLinkedList<GraphVertex<SearchNode>>& vertices_list_a:
//                  (input) hold the pointers of Vertices
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph and store pointers of Vertices
//
boolean readVertices(SearchLevel& search_level_a,
		     DiGraph<SearchNode>& sub_graph_a,
		     SingleLinkedList<GraphVertex<SearchNode> > &
		     vertices_list_a,
		     HashTable<String, String>& hash_table_a,
		     String& prefix_a) {
  // local variable
  //
  String hash_key;
  Vector<SearchSymbol> symbol_list;
  GraphVertex<SearchNode>* curr_vert = NULL;

  // read the type of the node
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_VERTEX_TYPE);
  long vertex_type = 0;
  hash_table_a.get(hash_key)->get(vertex_type);

  // branch on the vertex type
  //
  switch ( vertex_type ){

  case START_VERTEX:
    curr_vert = sub_graph_a.getStart();
    vertices_list_a.insert(curr_vert);  
    return true;
    
  case TERM_VERTEX:
    curr_vert = sub_graph_a.getTerm();
    vertices_list_a.insert(curr_vert);  
    return true;

  case NODE_VERTEX:
    break;
    
  default:
    return Error::handle(L"readVertices", L"vertex type",
			 Error::ARG, __FILE__, __LINE__);
    
  }
  
  // find the number of symbols in the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_VERTEX_SYMBOL_SIZE);
  String symbol_prefix;
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_VERTEX_SYMBOL);
  
  //Check if the symbol is dummy. If so dont read the symbols.
  //
  if ( !dummy_check ) {
    
    readSymbols(symbol_list, hash_table_a, hash_key, symbol_prefix);
    
  }
  
  // set the search nodes for the graph
  //
  for ( int i = 0; i < symbol_list.length(); i++ ){
    
    // create a new search node
    //
    SearchNode* curr_node = new SearchNode();
    curr_node->setSearchLevel(&search_level_a);

    long symbol_index = search_level_a.getSymbolIndex(symbol_list(i));

    // check if this index is in the range of symbols
    //
    if ( symbol_index < 0 ) {
      Long index(symbol_index);
      return Error::handle(L"readVertices", L"invalid symbol index",
			   Error::ARG, __FILE__, __LINE__);
    }
    curr_node->setSymbolId(symbol_index);
    curr_vert = sub_graph_a.insertVertex(curr_node);
    vertices_list_a.insert(curr_vert);  
  }
  
  // exit gracefully
  //
  return true;
  
}

// method: detectSelfLoop
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  SingleLinkedList<GraphVertex<SearchNode>>& vertices_list_a:
//                  (output) hold the pointers to vertices
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph and store pointers of vertices
//
boolean detectSelfLoop(SearchLevel& search_level_a,
		       DiGraph<SearchNode>& sub_graph_a,
		 SingleLinkedList<SingleLinkedList<GraphVertex<SearchNode> > >&
		     vertices_list_a,
		     HashTable<String, String>& hash_table_a,
		     String& prefix_a) {
    
  // local variable
  //
  String hash_key;

  // find the from_id
  //
  long from_id;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_FROM_ID);
  hash_table_a.get(hash_key)->get(from_id);

  // find the from_id
  //
  long to_id;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_TO_ID);
  hash_table_a.get(hash_key)->get(to_id);

  // check with the self loop
  //
  if (from_id == to_id){
  
    // get number of vertex in the from node
    //
    long num_of_from_vertices = vertices_list_a(from_id).length();

    if ( num_of_from_vertices > 1 ){

      long symbol_index = search_level_a.getSymbolIndex(dummy_symbol_g);

      // check if this index is in the range of symbols
      //
      if ( !search_level_a.isDummySymbol(symbol_index)) {
	
	// add this dummy symbol
	//
	search_level_a.getSymbolTable().concat(dummy_symbol_g);
	search_level_a.getDummySymbolTable().concat(dummy_symbol_g);
	//DiGraph<SearchNode>* dummy_graph = new DiGraph<SearchNode>();
	//search_level_a.getSubGraphs().concat(*dummy_graph);
      }

      // create two dummy search node
      //
      symbol_index = search_level_a.getSymbolIndex(dummy_symbol_g);

      SearchNode* dummy_node1 = new SearchNode();
      SearchNode* dummy_node2 = new SearchNode();
      GraphVertex<SearchNode>* curr_vert1 = NULL;
      GraphVertex<SearchNode>* curr_vert2 = NULL;

      dummy_node1->setSearchLevel(&search_level_a);
      dummy_node2->setSearchLevel(&search_level_a);

      num_of_from_vertices = vertices_list_a(from_id).length();
      dummy_node1->setSymbolId(symbol_index);
      curr_vert1 = sub_graph_a.insertVertex(dummy_node1);
      dummy_node2->setSymbolId(symbol_index);
      curr_vert2 = sub_graph_a.insertVertex(dummy_node2);

      vertices_list_a(from_id).insert(curr_vert1);
      vertices_list_a(from_id).insert(curr_vert2);
      
    }
    
  }

  // exit gracefully
  //
  return true;
  
}

// method: readArcs
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  SingleLinkedList<GraphVertex<SearchNode>>& vertices_list_a:
//                  (output) hold the pointers to vertices
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph and store pointers of vertices
//
boolean readArcs(DiGraph<SearchNode>& sub_graph_a,
		 SingleLinkedList<SingleLinkedList<GraphVertex<SearchNode> > >&
		 vertices_list_a,
		 HashTable<String, String>& hash_table_a,
		 String& prefix_a) {
    
  // local variable
  //
  String hash_key;

  // find the from_id
  //
  long from_id;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_FROM_ID);
  hash_table_a.get(hash_key)->get(from_id);

  // find the from_id
  //
  long to_id;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_TO_ID);
  hash_table_a.get(hash_key)->get(to_id);

  // get the epsilon flag
  //
  boolean is_epsilon;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_EPSILON);
  hash_table_a.get(hash_key)->get(is_epsilon);

  // get the transition
  //
  float weight;
  hash_key.assign(prefix_a);
  hash_key.concat(IO_ARC_WEIGHTS);
  hash_table_a.get(hash_key)->get(weight);  

  // get number of vertex in the from node
  //
  long num_of_from_vertices = vertices_list_a(from_id).length();

  // get number of vertex in the to node
  //
  long num_of_to_vertices = vertices_list_a(to_id).length();

  // if this is a self loop
  //
  if ( from_id == to_id && num_of_from_vertices > 3 ){

    // get the from vertex and dest vertex
    //
    GraphVertex<SearchNode>* from_vertex =
      &vertices_list_a(from_id)(num_of_from_vertices - 2);
	
    GraphVertex<SearchNode>* to_vertex =
      &vertices_list_a(to_id)(num_of_from_vertices - 1);
	
    // connect the self loop back
    //
    sub_graph_a.insertArc(to_vertex, from_vertex,
			  is_epsilon,
			  weight);

    weight = 1.0 / ( num_of_to_vertices - 2);

    // insert the arcs from first dummy node to other symbols
    //
    for ( int j = 0 ; j < num_of_to_vertices - 2 ; j++ ){
	
      GraphVertex<SearchNode>* mid_vertex =
	&vertices_list_a(to_id)(j);
	
	// connect the vertices in the searchnode graph
	//
	sub_graph_a.insertArc(from_vertex, mid_vertex,
			      false,
			      weight);

	// connect the vertices in the searchnode graph
	//
	sub_graph_a.insertArc(mid_vertex, to_vertex,
			      false,
			      1.0);
    }
    
    // exit gracefully
    //
    return true;

  }

  int from_index = 0;
  int from_index_end = num_of_from_vertices;
  int to_index = 0;
  int to_index_end = num_of_to_vertices;

  // set from index
  //
  if ( num_of_from_vertices > 3 ){
    
    // get the from vertex and dest vertex
    //
    GraphVertex<SearchNode>* from_vertex =
      &vertices_list_a(from_id)(num_of_from_vertices - 2);

    SearchNode* from_node = from_vertex->getItem();

    if ( from_node->isDummyNode() ){
      from_index = num_of_from_vertices - 1 ;
      from_index_end = from_index + 1;
    }
  }

  // set to index
  //
  if ( num_of_to_vertices > 3 ){
    
    // get the to vertex and dest vertex
    //
    GraphVertex<SearchNode>* to_vertex =
      &vertices_list_a(to_id)(num_of_to_vertices - 2);

    SearchNode* to_node = to_vertex->getItem();

    if ( to_node->isDummyNode() ){
      to_index = num_of_to_vertices - 2 ;
      to_index_end = to_index + 1;
    }
  }

  // insert the arc into graphs
  //
  for ( int i= from_index; i < from_index_end; i++ ){
    for ( int j = to_index ; j < to_index_end; j++ ){

      // get the from vertex and dest vertex
      //
      GraphVertex<SearchNode>* from_vertex =
	&vertices_list_a(from_id)(i);

      GraphVertex<SearchNode>* to_vertex =
	&vertices_list_a(to_id)(j);
      
      // connect the vertices in the searchnode graph
      //
      sub_graph_a.insertArc(from_vertex, to_vertex,
			    is_epsilon,
			    weight);
    }
  }
  
  // exit gracefully
  //
  return true;
  
}


// method: readSymbols
//
// arguments:
//  Vector<SearchSymbol>& Symbols_list_a:
//                  (input) hold the pointers of symbols
//  long : (input) total number of symbols to read
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph and store pointers of Symbols
//
boolean readSymbols(Vector<SearchSymbol>& symbol_list_a,
		    HashTable<String, String>& hash_table_a,
		    String& size_prefix_a,
		    String& prefix_a) {
  // local variable
  //
  long num_of_symbols = 0;
  hash_table_a.get(size_prefix_a)->get(num_of_symbols);

  // read each symbol
  //
  for (long j = 0; j < num_of_symbols; j++){

   // set the hash_key
   //
   String curr_symbol;
   curr_symbol.assign((long)j);
   
   String hash_key;
   hash_key.assign(prefix_a);
   hash_key.concat(curr_symbol);
   curr_symbol.assign(*hash_table_a.get(hash_key));
   String new_symbol;

   for ( long k = 0; k < curr_symbol.length(); k ++ ){
     if ( curr_symbol(k) != '\\' ){
       new_symbol.concat(curr_symbol(k));
     }
     else {
       k++;
       if ( k < curr_symbol.length() ){
	 new_symbol.concat(curr_symbol(k));
       }
     }
   }

   // debug information
   //
   if (debug_level >= Integral::BRIEF) {
     Console::put(new_symbol);
   }
  
   // put the symbol into list
   //
   symbol_list_a.concat(new_symbol);
  }

  // exit gracefully
  //
  return true;
  
}

// method: storeSearchLevel
//
// arguments:
//  SearchLevel&: (output) construct the search level
//  HashTable<String, String>&: (intput) the hash table which stored data
//  String& prefix_a: (output) the pefix
//
// return: boolean value indicating status
//
// this method reads the data into a SearchLevel
//
boolean storeSearchLevel(SearchLevel& search_level_a,
			HashTable<String, String>& hash_table_a,
			 Vector<SearchSymbol>& graph_name_a,
			String& prefix_a) {

  // local variable
  //
  String hash_key;

  // set the name for this level
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NAME);
  String level_tag;
  level_tag.assign(search_level_a.getLevelTag());
  hash_table_a.insert(hash_key, &level_tag);

  // debug information
  //
  if (debug_level >= Integral::BRIEF) {
    level_tag.debug(L"level_tag");
  }
  
  // set symbol table and all other symbols
  // including : non_speech, exclude_symbols,
  //

  // find the number of symbols in the symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SYMBOL_SIZE);
  String symbol_prefix = prefix_a;
  symbol_prefix.concat(IO_LEVEL_SYMBOL);

  // remove the NO_LEFT_CONTEXT and NO_RIGHT_CONTEXT symbols in case
  // these symbols exist in the symbol-table
  //
  long j = 0;
  Vector<SearchSymbol>& symbol_table_t = search_level_a.getSymbolTable();
  long len = symbol_table_t.length();
  Vector<SearchSymbol> symbol_table(len);  

  for( long i = 0; i < len; i++) {
    if (!((symbol_table_t(i).eq(SearchSymbol::NO_LEFT_CONTEXT)) ||
	(symbol_table_t(i).eq(SearchSymbol::NO_RIGHT_CONTEXT)))) {
      symbol_table.setLength(j + 1);
      symbol_table(j) = symbol_table_t(i);
      j++;
    }
  }
  
  symbol_table_t.assign(symbol_table);
  
  storeSymbols(search_level_a.getSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);

  // find the number of contexts in the contexts table
  // and store the context mapping table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_CONTEXT_SIZE);
  Vector<SearchSymbol> context_table;
  convertContextMapping(search_level_a.getContextMap(), context_table);
  String context_prefix = prefix_a;
  context_prefix.concat(IO_LEVEL_CONTEXT);
  storeSymbols(context_table, hash_table_a, hash_key, context_prefix);  
  
  // find the number of symbols in the non-speech internal symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NON_SPEECH_INTERNAL_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_NON_SPEECH_INTERNAL_SYMBOL);
  storeSymbols(search_level_a.getNonSpeechInternalSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);


  // find the number of symbols in the spenalty exclude symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SPENALTY_EXCLUDE_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SPENALTY_EXCLUDE_SYMBOL);
  storeSymbols(search_level_a.getSPenaltyExcludeSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);


  // find the number of symbols in the dummy symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_DUMMY_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_DUMMY_SYMBOL);
  storeSymbols(search_level_a.getDummySymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);

  // find the number of symbols in the non-speech boundary symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_NON_SPEECH_BOUNDARY_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_NON_SPEECH_BOUNDARY_SYMBOL);
  storeSymbols(search_level_a.getNonSpeechBoundarySymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);
  

  
  // find the number of symbols in the contextLess symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_CONTEXTLESS_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_CONTEXTLESS_SYMBOL);
  storeSymbols(search_level_a.getContextLessSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);

  // find the number of symbols in the skip symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_SKIP_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_SKIP_SYMBOL);
  storeSymbols(search_level_a.getSkipSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);

  // find the number of symbols in the exclude symbol table
  // store the symbol table
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_EXCLUDE_SYMBOL_SIZE);
  symbol_prefix.assign(prefix_a);
  symbol_prefix.concat(IO_LEVEL_EXCLUDE_SYMBOL);
  storeSymbols(search_level_a.getExcludeSymbolTable(), hash_table_a,
	       hash_key, symbol_prefix);

  // find the number of graphs
  //
  long num_of_graphs = search_level_a.getNumSubGraphs();
  String num_graphs;
  num_graphs.assign(num_of_graphs);
  hash_key.assign(prefix_a);
  hash_key.concat(IO_LEVEL_GRAPH_SIZE);
  hash_table_a.insert(hash_key, &num_graphs);
  
  // store graphs for a paticular level
  //
  for (long j = 0; j < num_of_graphs; j++){

   // set the prefix
   //
   String curr_graph_index;
   curr_graph_index.assign((long)j);
   
   String graph_prefix;
   graph_prefix.assign(prefix_a);
   graph_prefix.concat(IO_DIGRAPH_PREFIX);
   graph_prefix.concat(curr_graph_index);

   // store the graph name
   //
   hash_key.assign(prefix_a);
   hash_key.concat(IO_LEVEL_GRAPH_NAME);
   hash_key.concat(curr_graph_index);
   hash_table_a.insert(hash_key, &graph_name_a(j));

   // debug information
   //
   if (debug_level >= Integral::BRIEF) {
     String msg;
     msg.assign(hash_key);
     msg.concat(L"=");
     msg.concat(graph_name_a(j));
     Console::put(msg);
   }
  
   // read the graph
   //
   storeSubGraph(search_level_a, search_level_a.getSubGraph(j),
		 hash_table_a, graph_prefix);
  }
  
  // exit gracefully
  //
  return true;
  
}

// method: storeSubGraph
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph<SearchNode>
//
boolean storeSubGraph(SearchLevel& search_level_a,
		     DiGraph<SearchNode>& sub_graph_a,
		     HashTable<String, String>& hash_table_a,
		     String& prefix_a) {
  // local variable
  //
  String hash_key;



  // find the length of the digraph , number of vertices
  //
  String num_of_vertices;
  num_of_vertices.assign(sub_graph_a.length()+2);

  // get the number of vertices
  //
  hash_key.assign(prefix_a);
  hash_key.concat(IO_DIGRAPH_VERTEX_SIZE);
  hash_table_a.insert(hash_key, &num_of_vertices);

  // create local variables
  //
  Vector<Long> y_index;
  long x_index = 0;
  long num_of_arcs = 0;

  // call store vertex recursively
  //
  sub_graph_a.makeColorHash();
  sub_graph_a.setColor(Integral::WHITE);
  storeVertex(*sub_graph_a.getStart(),
	      sub_graph_a,
	      hash_table_a,
	      num_of_arcs,
	      y_index,
	      x_index,
	      prefix_a);

  // set the position for the term
  //
  x_index = y_index.length();
  storeVertex(*sub_graph_a.getTerm(),
	      sub_graph_a,
	      hash_table_a,
	      num_of_arcs,
	      y_index,
	      x_index,
	      prefix_a);
  
  // store the the num_of_arcs
  //
  String num_arcs;
  num_arcs.assign(num_of_arcs);
  hash_key.assign(prefix_a);
  hash_key.concat(IO_DIGRAPH_ARC_SIZE);
  hash_table_a.insert(hash_key, &num_arcs);  
  
  // exit gracefully
  //
  return true;
  
}

// method: storeVertices
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  SingleLinkedList<GraphVertex<SearchNode>>& vertices_list_a:
//                  (input) hold the pointers of Vertices
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads store vertices of a DiGraph into a hash_table
//
boolean storeVertex(GraphVertex<SearchNode>& vertex_a,
		    DiGraph<SearchNode>& sub_graph_a,
		    HashTable<String, String>& hash_table_a,
		    long& num_of_arcs_a,
		    Vector<Long>& y_index_a,
		    long x_index_a,
		    String& prefix_a) {
  
  // local variable
  //
  long vertex_index = indexOf(sub_graph_a, &vertex_a);
  long from_index = vertex_index;
  long to_index = 0;
  String hash_key;
  SearchSymbol symbol_name;
  long vertex_type = 0;
  String num_of_symbols(L"0");
  
  // set the vertex prefix
  //
  String vertex_prefix;
  vertex_prefix.assign(prefix_a);
  vertex_prefix.concat(IO_VERTEX_PREFIX);
  vertex_prefix.concat(vertex_index);
      
  // store the vertex type
  //
  if ( vertex_a.isStart() ){
    vertex_type = START_VERTEX;
    symbol_name.assign(START_VERTEX_NAME);
  }
  else if ( vertex_a.isTerm() ){
    vertex_type = TERM_VERTEX;
    symbol_name.assign(TERM_VERTEX_NAME);
  }
  else {
    vertex_type = NODE_VERTEX;    
    vertex_a.getItem()->getSymbol(symbol_name);
    
    // store the symbol, only one symbol for each vertex
    //
    hash_key.assign(vertex_prefix);
    hash_key.concat(IO_VERTEX_SYMBOL);
    hash_key.concat(L"0");
    hash_table_a.insert(hash_key, &symbol_name);

    // set the number of symbols, only one symbol on each node
    //
    num_of_symbols.assign(L"1");
  }

  // set the symbol size ( number of symbols on ths vertex )
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_SYMBOL_SIZE);
  hash_table_a.insert(hash_key, &num_of_symbols);

  // set the vertex name
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_NAME);
  hash_table_a.insert(hash_key, &symbol_name);

  // store the vertex type
  //
  String type;
  type.assign(vertex_type);
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_TYPE);
  hash_table_a.insert(hash_key, &type);
  
  // set the width
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_WIDTH);
  String width;
  width.assign((long)VERTEX_WIDTH);
  hash_table_a.insert(hash_key, &width);

  // set the height
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_HEIGHT);
  String height;
  height.assign((long)VERTEX_HEIGHT);
  hash_table_a.insert(hash_key, &height);

  // set the location and width and height
  //
  long x_pos = x_index_a;
  long y_pos = 0;

  // set the positions
  //
  if ( x_pos == y_index_a.length() ){
    Long y(0);
    y_index_a.concat(y);
  }else if ( x_pos < y_index_a.length() ){
    y_pos = y_index_a(x_pos);
  }
  else {
    return Error::handle(L"storeVertices", L"invalide x_pos",
			 Error::ARG, __FILE__, __LINE__);
  }

  // compute the x, y position
  //
  String x_loc;
  String y_loc;
  x_loc.assign(VERTEX_INITIAL_X + VERTEX_OFFSET_X * x_pos);
  y_loc.assign(VERTEX_INITIAL_Y + VERTEX_OFFSET_Y * y_pos);
  
  // set the x position
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_LOC_X);
  hash_table_a.insert(hash_key, &x_loc);

  // set the y position
  //
  hash_key.assign(vertex_prefix);
  hash_key.concat(IO_VERTEX_LOC_Y);
  hash_table_a.insert(hash_key, &y_loc);

  // increase the y index
  //
  y_index_a(x_pos) += 1;

  // set the color of this vertex to black
  //
  // store the vertex type
  //
  if ( (!vertex_a.isStart()) && (!vertex_a.isTerm())){
    sub_graph_a.setColor(&vertex_a, Integral::BLACK);
  }

  // loop over all arcs starting from this node
  //
  for (boolean more = vertex_a.gotoFirst(); more; more = vertex_a.gotoNext()) {

    // if this vertex is white, expand it
    //
    GraphArc<SearchNode>* tmp_arc = vertex_a.getCurr();
    GraphVertex<SearchNode>* vertex = tmp_arc->getVertex();
    to_index = indexOf(sub_graph_a, vertex);

    // store the arc
    //
    storeArc(num_of_arcs_a, from_index, to_index, tmp_arc,
	     hash_table_a, prefix_a);

    // error detecting
    //
    if ( vertex->isStart() ){
      return Error::handle(L"storeVertices", L"loop back to the Start vertex",
			   Error::ARG, __FILE__, __LINE__);
    }
    
    // store vertex
    //
    if ( (!vertex->isStart()) && (!vertex->isTerm())){
      if (sub_graph_a.getColor(vertex) != Integral::BLACK) {
	storeVertex(*vertex, sub_graph_a,hash_table_a, num_of_arcs_a,
		    y_index_a, x_index_a+1, prefix_a);
      }
    }
  }

  // exit gracefully
  //
  return true;
  
}

// method: storeArc
//
// arguments:
//  DiGraph<SearchNode>&: (output) construct this DiGraph
//  SingleLinkedList<GraphVertex<SearchNode>>& vertices_list_a:
//                  (output) hold the pointers to vertices
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method reads the data into a DiGraph and store pointers of vertices
//
boolean storeArc(long& arc_index_a,
		 long from_index_a,
		 long to_index_a,
		 GraphArc<SearchNode>*& arc_a,
		 HashTable<String, String>& hash_table_a,
		 String& prefix_a) {
    
  // local variable
  //
  String hash_key;
  String arc_prefix;

  // set the arc prefix
  //
  arc_prefix.assign(prefix_a);
  arc_prefix.concat(IO_ARC_PREFIX);
  arc_prefix.concat(arc_index_a);

  // store the from_id
  //
  String from_id;
  from_id.assign(from_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_FROM_ID);
  hash_table_a.insert(hash_key, &from_id);

  // store the to_id
  //
  String to_id;
  to_id.assign(to_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_TO_ID);
  hash_table_a.insert(hash_key, &to_id);

  // store width
  //
  String width(L"0");
  to_id.assign(to_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_WIDTH);
  hash_table_a.insert(hash_key, &width);

  // store height
  //
  String height(L"0");
  to_id.assign(to_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_HEIGHT);
  hash_table_a.insert(hash_key, &height);

  // store loc_x
  //
  String loc_x(L"0");
  to_id.assign(to_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_LOC_X);
  hash_table_a.insert(hash_key, &loc_x);

  // store loc_y
  //
  String loc_y(L"0");
  to_id.assign(to_index_a);
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_LOC_Y);
  hash_table_a.insert(hash_key, &loc_y);
  
  // get the epsilon flag
  //
  String is_epsilon;
  if ( arc_a->getEpsilon() ){
    is_epsilon.assign(L"true");
  }
  else {
    is_epsilon.assign(L"false");
  }
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_EPSILON);
  hash_table_a.insert(hash_key, &is_epsilon);

  // get the transition
  //
  String weight;
  weight.assign(arc_a->getWeight());
  hash_key.assign(arc_prefix);
  hash_key.concat(IO_ARC_WEIGHTS);
  hash_table_a.insert(hash_key, &weight);  

  // increase the arc index
  //
  arc_index_a++;
  
  // exit gracefully
  //
  return true;
  
}

// method: storeSymbols
//
// arguments:
//  Vector<SearchSymbol>& Symbols_list_a:
//                  (input) hold the pointers of symbols
//  long : (input) total number of symbols to read
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method store symbols into a hash table
//
boolean storeSymbols(Vector<SearchSymbol>& symbol_list_a,
		    HashTable<String, String>& hash_table_a,
		     String size_prefix_a,
		    String& prefix_a) {
  // local variable
  //
  long num_of_symbols = symbol_list_a.length();
  String num_symbols;
  num_symbols.assign(num_of_symbols);
  hash_table_a.insert(size_prefix_a, &num_symbols);

  // read each symbol
  //
  for (long j = 0; j < symbol_list_a.length(); j++){

   // set the hash_key
   //
   String curr_symbol;
   curr_symbol.assign((long)j);
   
   String hash_key;
   hash_key.assign(prefix_a);
   hash_key.concat(curr_symbol);

   // put the symbol into the symbol table
   //
   hash_table_a.insert(hash_key, &symbol_list_a(j));

  }

  // exit gracefully
  //
  return true;
  
}

// method: indexOf
//
// arguments:
//  Vector<SearchSymbol>& Symbols_list_a:
//                  (input) hold the pointers of symbols
//  long : (input) total number of symbols to read
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method find the index of a vertex in DiGraph
//
long indexOf(DiGraph<SearchNode>& sub_graph_a,
		GraphVertex<SearchNode>* vertex_a) {

  // special case
  //
  long size = sub_graph_a.length();
  
  if ( vertex_a->isStart() ){
    return size;
  }
  if ( vertex_a->isTerm() ){
    return size+1;
  }
  
  // search from the beginning for the item
  //
  long count = 0;
  for (boolean more = sub_graph_a.gotoFirst();
       more && ( sub_graph_a.getCurr() != vertex_a);
       more = sub_graph_a.gotoNext()) {
    count++;
  }

  // exit gracefully
  //
  return count;
  
}

// method: writeHashTable
//
// arguments:
//  Vector<SearchSymbol>& Symbols_list_a:
//                  (input) hold the pointers of symbols
//  long : (input) total number of symbols to read
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method write the hash table in a format of property=value
//
boolean writeHashTable(HashTable<String, String>& hash_table_a,
		       String & output_a,
		       String  description_a ) {

  // local variable
  //
  output_a.assign(L"# ");
  output_a.concat(description_a);

  // get the keys
  //
  Vector<String> keys;
  Vector<String> values;
  hash_table_a.keys(keys);
  hash_table_a.values(values);

  // put all values
  //
  for ( int i = 0 ; i < keys.length(); i ++ ){
    output_a.concat(L"\n");
    output_a.concat(keys(i));
    output_a.concat(L"=");
    //output_a.concat(*hash_table_a.get(keys(i)));
    output_a.concat(values(i));
  }

  // exit gracefully
  //
  return true;
  
}

// method: writeHashTable
//
// arguments:
//  Vector<SearchSymbol>& Symbols_list_a:
//                  (input) hold the pointers of symbols
//  long : (input) total number of symbols to read
//  HashTable<String, String>&: (intput) the hash table which holds the data
//  String& prefix_a: (input) the prefix
//
// return: boolean value indicating status
//
// this method write the hash table in a format of property=value
//
boolean writeHashTable(HashTable<String, String>& hash_table_a,
		       File & output_a,
		       String  description_a ) {

  // local variable
  //
  String output;
  output.assign(L"# ");
  output.concat(description_a);
  output.concat(L"\n");
  output_a.put(output);

  // get the keys
  //
  Vector<String> keys;
  Vector<String> values;
  hash_table_a.keys(keys);
  hash_table_a.values(values);

  for ( int i = 0 ; i < keys.length(); i ++ ){
    output.assign(keys(i));
    output.concat(L"=");
    output.concat(values(i));
    output.concat(L"\n");
    output_a.put(output);
  }  
  // exit gracefully
  //
  return true;
  
}

// method: storeContextMapping
//
// arguments:
//
// return: a boolean value indicating status
//
// convert the context map to a vector of search symbols
// 
//
boolean convertContextMapping(Vector<ContextMap>& context_map_a,
			      Vector<SearchSymbol>& context_symbol_a){
  
  // local variable
  //
  long len = context_map_a.length();
  context_symbol_a.setCapacity(len);
  context_symbol_a.setLength(len);
  
  // loop over all the context_maps and parse string in each loop
  //
  for (long k = 0; k < len; k++) {
    
    // local variable
    //
    String rule_name;
    SearchSymbol context_symbol;
    Vector<SearchSymbol> context;
    Ulong index;
    
    // get the context and the context-index from the current
    // context-map
    //
    context = context_map_a(k).getContext();
    index = context_map_a(k).getContextIndex();
    
    // set the rule name
    //
    for (long i = 0; i < context.length(); i++){
      
      // read the rule name
      //
      context_symbol.concat(context(i));
      if ( i != (context.length() - 1)){
	context_symbol.concat(L"-");
      }
    }

    // set the context list
    //
    context_symbol_a(index).assign(context_symbol);
  }
  
  // exit gracefully
  //
  return true;
}

// method: convertContextMapping
//
// arguments:
//
// return: a boolean value indicating status
//
// convert the context map to a vector of search symbols
// 
//
boolean convertContextMapping(Vector<SearchSymbol>& context_symbol_a,
			      Vector<ContextMap>& context_map_a){
  
  // local variable
  //
  long len = context_symbol_a.length();
  context_map_a.setCapacity(len);
  context_map_a.setLength(len);
  
  // loop over all the context_maps and parse string in each loop
  //
  for (long k = 0; k < len; k++) {
    
    // local variable
    //
    SearchSymbol context_symbol = context_symbol_a(k);
    SearchSymbol symbol;
    Vector<SearchSymbol> context;

    // get the data string
    //
    long context_length = context_symbol.countTokens(L"-");
    
    // get the context
    //
    long j=0;
    for (long i = 0; i < context_length; i++) {
      context_symbol.tokenize(symbol, j, L"-");
      context.concat(symbol);
    }
    
    // set this context
    //
    if (!context_map_a(k).setContext(context)) {
      return Error::handle(L"convertContextMapping", L"can set context", Error::READ, __FILE__, __LINE__, Error::WARNING);    
    }

    context_map_a(k).setContextIndex(k);
  }
  
  // exit gracefully
  //
  return true;
}
