// file: $isip/util/speech/isip_lm_tester/isip_lm_tester.cc
// version: $Id: isip_lm_tester.cc 10614 2006-08-24 16:01:38Z dom5 $
//

// isip include files
//
#include "isip_lm_tester.h"

// isip_lm_tester: language model tester utility
//
// this is the general language model tester utility. it can be used for
// randomly generated sentences based on the language model file
//
int main(int argc, const char** argv) {

  // ccommand line parser and an Sdb object.
  //
  CommandLine cmdl;
  
  cmdl.setUsage(
#include "usage_message.text"
                );
  cmdl.setHelp(
#include "help_message.text"
               );

  cmdl.setIdent("$Revision: 10614 $",
		"$Name: $",
		"$Date: 2006-08-24 11:01:38 -0500 (Thu, 24 Aug 2006) $");


 // add a command line option for the mode type
  //
  String select_mode;
  cmdl.addOptionParam(select_mode, PARAM_MODE, DEF_MODE);

  // add a command line option for the identifier list
  //
  Filename identifier_list;
  cmdl.addOptionParam(identifier_list, PARAM_ID_LIST,
		      (Filename)EMPTY);

  // add a command line option for the transcription database file 
  //
  Filename transcription_db_file;
  cmdl.addOptionParam(transcription_db_file, PARAM_TRANS_INPUT_NAME,
		      (Filename)EMPTY);  
  
  // add a command line option for the output file for parse output
  //
  Filename parse_output;
  cmdl.addOptionParam(parse_output, PARAM_PARSE_OUTPUT, (Filename)EMPTY); 

  // add a command line option for the grammar file
  //
  Filename grammar_file;
  cmdl.addOptionParam(grammar_file, PARAM_GRAMMAR_NAME, (Filename)EMPTY);  

  // add a command line option for the output type
  //
  String output_type_str;
  cmdl.addOptionParam(output_type_str, PARAM_OUTPUT_TYPE,
		      DEF_OUTPUT_TYPE);
  
  // add a command line option for the output database name
  //
  String db_name;
  cmdl.addOptionParam(db_name, PARAM_DB_NAME, (Filename)EMPTY);  

  // add a command line option for the debug level
  //
  DebugLevel debug_level;
  cmdl.addOptionParam(debug_level);
  
  // add a command line option for random seed
  //
  Long random_seed;
  cmdl.addOptionParam(random_seed, PARAM_RANDOM_SEED, DEF_RANDOM_SEED);
  
  // add a command line option for number of sentences
  //
  Long sent_number;
  cmdl.addOptionParam(sent_number, PARAM_SENTENCE_NUMBER, DEF_SENTENCE_NUMBER);

  // add a command line option for the sentence length
  //
  Long sent_length;
  cmdl.addOptionParam(sent_length, PARAM_SENTENCE_LENGTH, DEF_SENTENCE_LENGTH);
  
  // add a command line option for the output level
  //
  String output_level_tag;
  cmdl.addOptionParam(output_level_tag, PARAM_OUTPUT_LEVEL, EMPTY);

  // add a flag for using exclude symbols
  //
  Boolean exclude;
  cmdl.addFlagParam(exclude, PARAM_EXCLUDE);

  // parse the command line
  //
  if (!cmdl.parse(argc, argv)) {
    cmdl.printUsage();
  };

  // check the grammar file argument
  //
  if(grammar_file.eq(NULL)) {
    cmdl.printUsage();
  };

  // HierarchicalSearch object which contains all the methods
  // for generating and parsing transcriptions
  //
  HierarchicalSearch search_engine;
    
  // generate the grammar
  //
  if(select_mode.eq(PARAM_GEN_MODE)) {
    
    // input identifier list is a sdb object
    //
    Sdb sdb;
    if(!identifier_list.eq(NULL)) {  
      sdb.append(identifier_list, true);
    }
    
    // verify that the user has specified the output file type
    //
    File::TYPE output_type = DEF_OUTPUT_TYPE;
    if (output_type_str.eq(File::TYPE_TEXT)) {
      output_type = File::TEXT;
    }
    else if (output_type_str.eq(File::TYPE_BINARY)) {
      output_type = File::BINARY;
    }
    
    // declare the input transcription database
    //
    Filename output_file;
    
    // get number of unused arguments (this is the input transcription
    // database)
    //
    long num_arg = (long)cmdl.numArguments();

    // get the output database file
    //
    String arg;
    Sof out_fp;
    if (num_arg >= 1) {
      cmdl.getArgument(arg, (long)0);
      output_file.concat(arg);
      
      if (!out_fp.open(output_file, File::WRITE_ONLY, output_type)) {
	String msg(L"Error: Cannot open the output transcription DB file! ");
	Console::put(msg);
	Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		      __FILE__, __LINE__);
      }
      
      // check if the DB name is told
      //
      if(db_name.eq(NULL)) {
	String msg(L"Error: Transcription DB name is specified! ");
	Console::put(msg);
	Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		      __FILE__, __LINE__);
      }
    }
    
    // initialize random generator
    //
    Random random_gen;
    random_gen.setAlgorithm(Random::UNIFORM);
    random_gen.setImplementation(Random::SUBTRACTIVE);
    
    // check if the random seed is input, otherwise generate it from time
    //
    if (random_seed == DEF_RANDOM_SEED) {
      random_seed = (Long)Integral::time();
      random_gen.seed((long)random_seed);    
      random_seed.rand(random_gen);
    }
    
    // seed the random variable
    //
    random_gen.seed((long)random_seed);
    
    // load grammar file
    //
    String model_file_a;
    model_file_a.assign(grammar_file); 
    
    // temporary language model used to load a language
    // model of arbitrary format.  once loaded, the
    // language model is converted to IHD and assigned
    // to the language_model object
    //
    LanguageModel temp_language_model;
    
    // IHD language model.  the utility will use this
    // object directly
    //
    LanguageModelIHD language_model;
    
    Sof lm_file;
    
    lm_file.open(model_file_a, File::READ_ONLY);
    
    temp_language_model.read(lm_file);
    
    lm_file.close();
    
    language_model.setRuleModel(temp_language_model.getRuleModel());

    search_engine.setHDigraph(language_model.getIHD());
    
  // get the output level
  //
    Long output_level;
    
    // 
    if (output_level_tag.eq(NULL)) {
      output_level = language_model.getIHD().length() - 1;
      output_level_tag.assign(language_model.getIHD()(output_level).getLevelTag());
    }
    else {
      for (output_level = 0; output_level < language_model.getIHD().length();
	   output_level++) {
	if (output_level_tag.eq(language_model.getIHD()(output_level).getLevelTag())) {
	  break;
	}
      }
      
      // verify if the output level is valid
      //
      if (output_level.eq(language_model.getIHD().length())) {
	String msg(L"Error: the specified output level is invalid! ");
	Console::put(msg);
	Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		      __FILE__, __LINE__);      
      }
    }
    
    // local variables
    //
    Vector<String> symbol_table;
    TranscriptionDatabase transdb;
    String id;
    String sentence;
    boolean is_id_file;
    
    // set database name
    //
    transdb.setDataBaseName(db_name);
    
    // if id list file exists
    //
    if (sdb.gotoFirst()) {
      is_id_file = true;
      sent_number = sdb.length();
      sdb.gotoFirst();
    }
    else {
      is_id_file = false;
    }
    
    // create a vector table for identifiers
    //
    symbol_table.setLength((long)sent_number);
    
    // display debug information
    //
    if (debug_level >= Integral::BRIEF) {
      random_seed.debug(L"random_seed");
      sent_number.debug(L"sentence_count"); 
      sent_length.debug(L"max_length");
      output_level.debug(L"output_level");
      output_level_tag.debug(L"output_level_tag");
      db_name.debug(L"db_name");
      String value;
      value.assign(output_type);
      value.debug(L"output_type");
    }
    
    // generate sentence specified by the grammar and the probabilty
    // assigned to each corresponding arc
    //
    for (long i = 0; i < (long)sent_number; i++) {
      
      //GenRandomSentences((long)sent_length, random_gen, output_level, language_model.getIHD(), sentence, exclude_a);

      search_engine.genRandomSentences((long)sent_length, random_gen,
				       output_level,
				       sentence, (boolean)exclude);
      
      // save the identifier in the symbol-table
      //
      if (out_fp.isOpen()) {
	if (is_id_file) {
	  
	  sdb.getName((Filename&)id);
	  sdb.gotoNext();
	}
	else {
	  id.assign(i);
	}
	symbol_table(i).assign(id);
	transdb.storePartial(sentence, db_name, output_level_tag,
			     i, out_fp, 0);
      }
      
      if (!out_fp.isOpen() || debug_level >= Integral::BRIEF) {
	Console::put(sentence);
      }
    }
    
    // write the symbol table and close output file
    //
    if (out_fp.isOpen()) {
      transdb.storePartial(out_fp, (long)0, symbol_table);
      out_fp.close();
    }
  }
  else if (select_mode.eq(PARAM_PARSE_MODE)) {
    
    // declare local variable
    //
    // temporary language model used to load a language
    // model of arbitrary format.  once loaded, the
    // language model is converted to IHD and assigned
    // to the language_model object
    //
    LanguageModel temp_language_model;
    
    // IHD language model.  the utility will use this
    // object directly
    //
    LanguageModelIHD lm;
    
    // load grammar file`
    //
    Sof lm_file;
    
    lm_file.open(grammar_file, File::READ_ONLY);
    
    temp_language_model.read(lm_file);
    
    lm_file.close();

    lm.setRuleModel(temp_language_model.getRuleModel());

    search_engine.setHDigraph(lm.getIHD());
    
    // get the output level
    //
    Long output_level;   
    if (output_level_tag.eq(NULL)) {
      output_level = lm.getIHD().length() - 1;
      output_level_tag.assign(lm.getIHD()(output_level).getLevelTag());
    }
    else {
      for (output_level = 0; output_level < lm.getIHD().length();
	   output_level++) {
	if (output_level_tag.eq(lm.getIHD()(output_level).getLevelTag())) {
	  break;
	}
      }
      
      // verify if the output level is valid
      //
      if (output_level.eq(lm.getIHD().length())) {
	String msg(L"Error: the specified output level is invalid! ");
	Console::put(msg);
	Error::handle(PROG_NAME, L"main", Error::NO_PARAM_FILE,
		      __FILE__, __LINE__);      
      }
    }

    // output file to write the result of parse
    //
    Sof transcription_parse_output;
    if( transcription_parse_output.open(parse_output, File::WRITE_ONLY) != true) {
      return Error::handle(PROG_NAME, L"main", Error::TEST, __FILE__, __LINE__);
    }
    
    // load the transcription file
    //
    TranscriptionDatabase transcription_db;
    Filename transcription_db_file(transcription_db_file);
    if (!transcription_db.open(transcription_db_file)) {
      transcription_db_file.debug(L"trans_db_file_a");
      return Error::handle(PROG_NAME, L"error opening transcription database file", Error::ARG, __FILE__, __LINE__);  
    }
    
    // read transcription db and get identifier to run each transcription
    //
    Vector<String> identifiers;
    transcription_db.getIdentifiers(identifiers);

    // the output parsing result will be stored as pair object
    //
    Vector<Pair<String, Boolean> > pair(identifiers.length());
    boolean permissible;
    Boolean permissible_w;

    // display debug information
    //
    if (debug_level >= Integral::BRIEF) {
      select_mode.debug(L"mode");
      exclude.debug(L"exclude symbol");
      output_level.debug(L"output_level");
      output_level_tag.debug(L"output_level_tag");
    }
    
    for ( long i = 0; i <  identifiers.length(); i++) {

      permissible = parseTranscription(transcription_db_file,
				       (boolean)exclude, i, output_level,
				       search_engine);
      permissible_w = permissible;
      pair(i).assign(identifiers(i), permissible_w);
     
    }
    pair.write(transcription_parse_output,0);
    transcription_db.close();
    transcription_parse_output.close();
  }
  // exit gracefully
  //
  return Integral::exit();
}

// method: parseTranscription
//
//  TranscriptionDatabase& trans_db_file: (input) transcription file
//  boolean exclude_a: (input) user specifies include exclude symbol or not
//  boolean permissible: (output) tell right or wrong grammar of trans
//
// return: a boolean value indicating status
//
// this methods parse the transcription based on the grammar file
//
boolean parseTranscription(Filename& trans_db_file_a,
			   boolean exclude_a,
			   long i, Long& output_level_a,
			   HierarchicalSearch& search_engine_a) {

  // after read identifier, need to check the sentence length for each
  // identifier to loop again
  
  Sof read_anno;
  SysString file_name;
  AnnotationGraph ag;
  DoubleLinkedList<Annotation> anno_data;
  String anno_type;
  long output_level_temp;
  boolean permissible;

  output_level_temp = output_level_a;
  permissible = false;

  file_name.assign(trans_db_file_a);
  read_anno.open(file_name, File::READ_ONLY, File::TEXT);
  ag.read((read_anno), (long)i);
  read_anno.close();
  ag.getAnnotationSet(anno_data);
  anno_data.gotoFirst();
  anno_type.assign(anno_data.getCurr() -> getType());
   
  // use while loop to process each word
  //
  long pos = 0;
  String symbol_string;
  SearchSymbol symbol;

  // create StartNode
  //
  Long start_symbol = -1;
  Long end_symbol = -2;

  Vector<Long> input_node;
  Vector<SearchSymbol> vec_transcription(0);
  
  // set the length of input_node vector with length of search level
  // and initialize it
  //
  input_node.setLength(output_level_temp+1);
  input_node.assign(start_symbol);
  
  while (anno_type.tokenize(symbol_string, pos)) {
    
    Vector<SearchSymbol> temp_vec_string(1);
    symbol_string.trim();
    symbol.assign(symbol_string);
    temp_vec_string(0).assign(symbol);
    vec_transcription.concat(temp_vec_string);
  }

  
  permissible = search_engine_a.parseGraph(input_node, vec_transcription,
					   0, output_level_temp,
					   output_level_a, exclude_a);

  return permissible;
  
}
