// file: $isip/util/speech/isip_run/isip_run.cc
// version: $Id: isip_run.cc 10621 2006-08-24 16:04:27Z dom5 $
//
// isip include files
//
#include "isip_run.h"

// isip_run: utility that distributes jobs in parallel on different machines
// specified bu the machine list (attributes of the machines are specified
// in the machine database which is optional)
//
int main(int argc, const char** argv) {
  
  // declare local variables
  //
  FILE* fp;
  char* exec_command;
  
  String output;
  String command;  
  Long cpu_number;
  Sdb sdb_list;
  Sdb sdb_split_lists;
  
  // declare miscellaneous local variables
  //
  Filename input_file;
  
  // setup the command line
  //
  CommandLine cmdl(sdb_list);
  cmdl.setUsage(
#include "usage_message.text"
                );
  cmdl.setHelp(
#include "help_message.text"
               );
  
  cmdl.setIdent("$Revision: 10621 $",
		"$Name: $",
		"$Date: 2006-08-24 11:04:27 -0500 (Thu, 24 Aug 2006) $");
  
  // add a command line option for the parameters file
  //
  Filename params_file;
  cmdl.addOptionParam(params_file, OPTION_PARAMS_FILE, (Filename)EMPTY);
  
  // add a command line option for the identifier file
  //
  Filename identifier_file;
  cmdl.addOptionParam(identifier_file, OPTION_IDENTIFIER_FILE, (Filename)EMPTY);
  
  // add a command line option for the identifier list
  //
  Filename identifier_list;
  cmdl.addOptionParam(identifier_list, OPTION_IDENTIFIER_LIST, (Filename)EMPTY);
  // add a command line option for the machine list
  //
  Filename machine_list;
  cmdl.addOptionParam(machine_list, OPTION_MACHINE_LIST, (Filename)EMPTY);
  
  // add a command line option for the intermediate scripts
  //
  String walltime;
  cmdl.addOptionParam(walltime, OPTION_ESTIMATED_WALLTIME, EMPTY);
  
  // add a command line option for the machine mode(Cluster/server)
  //
  String mode;
  cmdl.addOptionParam(mode, OPTION_MACHINE_MODE, EMPTY);
  
  // add a command line option for the number of cpus required
  //
  Filename cpus;
  cmdl.addOptionParam(cpus, OPTION_CPUS_NUMBERS, (Filename)EMPTY);
  
  // add a command line option for accepting a command from the user
  //
  String command_to_execute;
  cmdl.addOptionParam(command_to_execute, OPTION_COMMAND_EXECUTE, EMPTY);
  
  // add a command line option for accepting the cluster name
  //
  String cluster_name;
  cmdl.addOptionParam(cluster_name,OPTION_CHOOSE_CLUSTER, DEF_CLUSTER_NAME);
  
  // add a command line option for accepting the command for submitting to
  // the cluster
  //
  String batch_command;
  cmdl.addOptionParam(batch_command, OPTION_BATCH_COMMAND, EMPTY);
  
  // add a command line option for the log list
  //
  Filename log_list;
  cmdl.addOptionParam(log_list, OPTION_LOG_LIST, (Filename)EMPTY);
  
  // add a command line option for the accumulator list
  //
  Filename accumulator_list;
  cmdl.addOptionParam(accumulator_list, OPTION_ACCUMULATOR_LIST, (Filename)EMPTY);
  
  // add a command line option for the machine database
  //
  Filename database_file;
  cmdl.addOptionParam(database_file, OPTION_DATABASE_FILE, (Filename)EMPTY);
  
  // add a command line option for the split mode
  //
  String split_mode;
  cmdl.addOptionParam(split_mode, OPTION_SPLIT_MODE, EMPTY);
  
  // add a command line option for the debug level
  //
  DebugLevel debug_level;
  cmdl.addOptionParam(debug_level, OPTION_DEBUG_LEVEL,
		      DEBUG_LEVEL_DEFAULT);
  
  // add a command line option for the log verbosity level
  //
  DebugLevel verbosity;
  cmdl.addOptionParam(verbosity, OPTION_VERBOSITY_LEVEL,
		      VERBOSITY_LEVEL_DEFAULT);
  
  // add verify command line flag
  //
  Boolean verify;
  cmdl.addFlagParam(verify, OPTION_FLAG_VERIFY);
  
  // parse the command line
  //
  if (!cmdl.parse(argc, argv)) {
    cmdl.printUsage();
  };
  
  // print the command line
  //
  if (verbosity >= Integral::BRIEF) {
    cmdl.printCommandLine();
    cmdl.printVersion();
  }
  
  // check the mode of operation, if the mode is cluster then call the
  // appropriate function else skip and proceede in a conventional manner.
  //
  if (!mode.eq(EMPTY)) {
    mode.toLower();
    if (mode.eq(DEF_MACHINE_MODE_CLUSTER)) {
      
      Splitter cluster_split;

      // set the default modes for the Splitter object
      //
      if (!split_mode.eq(EMPTY)) {
	split_mode.toLower();
	if (split_mode.eq(DEF_MODE_SPEED)) {
	  cluster_split.setSplitMode(Splitter::SPEED);
	}
	if (split_mode.eq(DEF_MODE_MEMORY)) {
	  cluster_split.setSplitMode(Splitter::MEMORY);
	}
      }
      
      // check for the command_to_execute option
      //
      if(command_to_execute.eq((String)EMPTY)) {
	cmdl.printUsage();
      };
      
      // call the function that splits the job list and submits it to the
      // cluster
      //
      cluster_split.splitJobList(input_file,cpus,command_to_execute,
				 walltime,batch_command,sdb_list,cluster_name);
      
      // exit gracefully
      //
      return Integral::exit();
    }
    
  }
  
  // entering conventional isip_run
  //

  // check the param file argument
  //
  if(params_file.eq((Filename)EMPTY)) {
    cmdl.printUsage();
  };
  
  // Display that the server mode is selected
  //
  Console::put(L"Server mode selected");
  
  // check the param file argument
  //
  if(params_file.eq((Filename)EMPTY)) {
    
    // print debugging information
    //
    output.assign(L"parameter file was not specified... bailing out...");
    Console::put(output);
    
    return Integral::exit();    
  };
  
  // check the identifier file argument
  //
  if(identifier_file.eq((Filename)EMPTY)) {
    
    // print debugging information
    //
    output.assign(L"identifier file was not specified... bailing out...");
    Console::put(output);
    
    return Integral::exit();        
  };
  
  // check the machine list argument
  //
  if(machine_list.eq((Filename)EMPTY)) {
    
    // print debugging information
    //
    output.assign(L"machine list was not specified... bailing out...");
    Console::put(output);
    
    return Integral::exit();            
  };
  
  // check the accumulator list argument
  //
  if(accumulator_list.eq((Filename)EMPTY)) {
    
    // print debugging information
    //
    output.assign(L"accumulator list was not specified... bailing out...");
    Console::put(output);
    
    return Integral::exit();                
  };
  
  // check the identifier list argument
  //
  if(identifier_list.eq((Filename)EMPTY)) {
    Integral::makeTemp(identifier_list);
  };
  
  // read the elements in the machine list
  //
  Sdb machines;
  Sof machine_sof;
  
  if (machine_sof.open(machine_list)) {
    if (!machines.read(machine_sof, (long)0)) {
      
      // print debugging information
      //
      output.assign(L"unable to read machine list... bailing out...");
      Console::put(output);
      
      return Integral::exit();                      
    }
    machine_sof.close();
  }
  else {
    
    // print debugging information
    //
    output.assign(L"cannot open machine list... bailing out...");
    Console::put(output);
    
    return Integral::exit();                          
  }  
  
  // read the elements in the log list if available
  //
  Sdb log;
  Sof log_sof;
  
  if (log_list.ne((Filename)EMPTY)) {
    if (log_sof.open(log_list)) {
      if (!log.read(log_sof, (long)0)) {
	
	// print debugging information
	//
	output.assign(L"unable to read log list... bailing out...");
	Console::put(output);	
	return Integral::exit();                      
      }
      log_sof.close();
    }    
  }
  
  // read the elemenst in the accumulator list
  //  
  Sdb identifiers;
  Sof identifier_sof;
  
  if (identifier_sof.open(identifier_list)) {
    if (!identifiers.read(identifier_sof, (long)0)) {
      
      // print debugging information
      //
      output.assign(L"unable to read identifier list... bailing out...");
      Console::put(output);
      
      return Integral::exit();                                
    }
    identifier_sof.close();
  }
  else {
    
    // print debugging information
    //
    output.assign(L"cannot open identifier list... bailing out...");
    Console::put(output);
    
    return Integral::exit();                                    
  }
  
  // elements in the lists must be equal
  //
  if (machines.length() != identifiers.length()) {
    
    // print debugging information
    //
    output.assign(L"machine and identifier lists are not the same size... bailing out...");
    Console::put(output);
    
    return Integral::exit();
  }
  
  // read the elemenst in the accumulator list
  //
  Sdb accumulators;
  Sof accumulator_sof;
  
  if (accumulator_sof.open(accumulator_list)) {
    if (!accumulators.read(accumulator_sof, (long)0)) {
      
      // print debugging information
      //
      output.assign(L"unable to read accumulator list... bailing out...");
      Console::put(output);
      
      return Integral::exit();
    }
    accumulator_sof.close();
  }
  else {
    
    // print debugging information
    //
    output.assign(L"cannot open accumulator list... bailing out...");
    Console::put(output);
    
    return Integral::exit();
  }
  
  // elements in the lists must be equal
  //
  if (machines.length() != accumulators.length()) {
    
    // print debugging information
    //
    output.assign(L"machine and accumulator lists are not the same size... bailing out...");
    Console::put(output);
    
    return Integral::exit();    
  }
  
  // set the split mode of the object
  //
  Splitter splt;
  
  if (!split_mode.eq(EMPTY)) {
    split_mode.toLower();
    if (split_mode.eq(DEF_MODE_SPEED)) {
      splt.setSplitMode(Splitter::SPEED);
    }
    if (split_mode.eq(DEF_MODE_MEMORY)) {
      splt.setSplitMode(Splitter::MEMORY);
    }
  }
  
  // read the machine database if present
  //
  Sof tmp_sof;
  if (!database_file.eq(String::EMPTY)) {
    
    MachineDatabase& mdb = splt.getMachineDatabase();
    
    if (tmp_sof.open(database_file)) {
      mdb.read(tmp_sof, (long)0);
      tmp_sof.close();
    }
  }
  
  // split the identifiers based on the machine list
  //  
  Vector<SingleLinkedList<Filename> > items;  
  
  splt.divideAndConquer(identifier_file, machine_list, items);
  
  // generate new identifier lists based on the split lists
  //
  Vector<Filename> identifier_lookup;
  Vector<Machine> machine_lookup;
  Vector<Filename> accumulator_lookup;
  Vector<Filename> log_lookup;
  
  splt.createUtterLists(identifier_file, identifier_list, machine_list,
			accumulator_list, items, identifier_lookup,
			machine_lookup,	accumulator_lookup);
  
  // convert the sdb of log to a vector of log
  //
  if (log.length() > (long)0) {
    log_lookup.setLength(log.length());
    long i = (long)0;
    for (boolean m = log.gotoFirst(); m; m = log.gotoNext()) {
      log.getName(log_lookup(i++));
    }
  }
  
  // expand the recognizer executable
  //
  SysString recognizer;
  
  if (!Integral::expandName(recognizer, DEF_RECOGNIZER_EXECUTABLE)) {
    
    // print debugging information
    //
    output.assign(L"executable not found: ");
    output.concat(DEF_RECOGNIZER_EXECUTABLE);
    Console::put(output);
    
    return Integral::exit();    
  }
  
  // get the number of child processes that we need to spawn
  //
  int num_processes = items.length();
  
  // create a buffer of shared memory for inter-process communication
  //
  Vector<String> shared_memory;
  shared_memory.setLength(num_processes);
  
  for (long i=0; i < num_processes; i++) {    
    Integral::makeTemp(shared_memory(i));
  }
  
  // read the contents of the parameter file
  //
  HiddenMarkovModel hmm_obj;
  if (tmp_sof.open(params_file)) {
    if (!hmm_obj.read(tmp_sof, (long)0)) {
      
      // print debugging information
      //
      output.assign(L"unable to read parameter file... bailing out...");
      Console::put(output);
      
      return Integral::exit();      
    }
    tmp_sof.close();
  }
  else {
    
    // print debugging information
    //
    output.assign(L"cannot open parameter file... bailing out...");
    Console::put(output);
    
    return Integral::exit();          
  }
  
  // get the required parameters
  //
  long num_iterations = hmm_obj.getNumIterations();  
  Filename lm_file = hmm_obj.getLanguageModelFile();
  Filename lm_output_file = hmm_obj.getLanguageModelUpdateFile();  
  Filename smp_file = hmm_obj.getStatisticalModelPoolFile();
  Filename smp_output_file = hmm_obj.getStatisticalModelPoolUpdateFile(); 
  
  Filename lm_file_accum(lm_file);
  Filename smp_file_accum(smp_file);
  
  // loop over the number of iterations
  //
  for (long iter = 0; iter < num_iterations; iter++) {
    
    // on the second iteration and there after set the input models to be
    // output of the previous iteration
    //
    if (iter > 0) {
      lm_file_accum.assign(lm_output_file);
      smp_file_accum.assign(smp_output_file);
    }
    
    // print the current iteration
    //
    if (verbosity >= Integral::BRIEF) {
      output.assign(L"\nstarting iteration: ");
      output.concat(iter);
      Console::put(output);
    }
    
    // clear the buffer of shared memory
    //
    for (long i=0; i < num_processes; i++) {    
      fp = fopen((char*)(byte*)shared_memory(i), "w");
      fprintf(fp, "%d", 0);
      fclose(fp);
    }
    
    // loop until we have spawned the required number of precesses
    //
    int pid = 0;
    int processes = 0;
    while(processes < num_processes) {
      
      // create a new child process
      //
      pid = fork();
      
      // process id is not zero - parent process
      //
      if (pid != 0) {
	
	// increment the number of child processes generated
	//
	processes++;
      }
      
      // process id is zero - child process
      //
      else {
	
	// setup the command
	//
	command.assign(DEF_REMOTE_SHELL_PROTOCOL);
	command.concat(L" ");
	command.concat(machine_lookup(processes).getNodeName());
	command.concat(L" ");
	command.concat(DEF_NICE_LEVEL);
	command.concat(L" ");
	command.concat(recognizer);
	command.concat(L" ");
	command.concat(OPTION_PARAM_FLAG);
	command.concat(L" ");
	command.concat(params_file);
	command.concat(L" ");
	command.concat(OPTION_LIST_FLAG);
	command.concat(L" ");
	command.concat(identifier_lookup(processes));
	command.concat(L" ");	
	command.concat(OPTION_ACCUMULATOR_FILE_FLAG);
	command.concat(L" ");
	command.concat(accumulator_lookup(processes));
	command.concat(L" ");	
	command.concat(OPTION_LANGUAGE_MODEL_FLAG);
	command.concat(L" ");
	command.concat(lm_file_accum);
	command.concat(L" ");	
	command.concat(OPTION_STATISTICAL_MODEL_POOL_FLAG);
	command.concat(L" ");
	command.concat(smp_file_accum);	
	command.concat(L" ");      
	command.concat(OPTION_FUNCTION_FLAG);
	command.concat(L" ");
	command.concat(OPTION_FUNCTION_ACCUMULATE);
	command.concat(L" ");
	command.concat(OPTION_VERBOSITY_LEVEL_FLAG);
	command.concat(L" ");
	command.concat(verbosity.getName());	  	
	
	// setup the log files in the command if available
	//
	if (log_lookup.length() > (long)0) {
	  command.concat(L" ");
	  command.concat(OPTION_LOG_FILE_FLAG);
	  command.concat(L" ");
	  command.concat(log_lookup(processes));	  
	}
	
	// convert the wide character string to a character string
	//
	exec_command = (char*)(byte*)command;
	
	// print debugging information
	//
	if (verbosity >= Integral::ALL) {
	  output.assign(L"command line on: ");
	  output.concat(machine_lookup(processes).getNodeName());
	  output.concat(L"\n");
	  output.concat(command);
	  Console::put(output);
	}
	
	// execute the command
	//
	if (system(exec_command) < 0) {
	  
	  // print debugging information
	  //
	  output.assign(L"error executing command: ");
	  output.concat(command);
	  Console::put(output);	
	  
	  return Integral::exit();
	}
	
	// signal the parent process that the child process has finished
	//
	fp = fopen((char*)(byte*)shared_memory(processes), "w");      
	fprintf(fp, "%d", 1);
	fclose(fp);
	
	// print debugging information
	//
	if (verbosity >= Integral::BRIEF) {
	  output.assign(L"child process on ");
	  output.concat(machine_lookup(processes).getNodeName());
	  output.concat(L" has finished...");
	  output.concat(L"\n");
	  Console::put(output);
	}
	
	// kill the child process after completion
	//
	exit(0);      
      }
    }
    
    // print debugging information
    //
    if (verbosity >= Integral::BRIEF) {
      output.assign(L"waiting on child processes...");
      output.concat(L"\n");
      Console::put(output);
    }
    
    // wait on all processes to complete execuation
    //
    int val = 0;
    int tmp_val = 0;
    while (val < num_processes) {
      
      // read the shared memory
      //
      val = 0;    
      for (long i=0; i < num_processes; i++) {    
	fp = fopen((char*)(byte*)shared_memory(i), "r");
	fscanf(fp, "%d", &tmp_val);
	fclose(fp);
	val += tmp_val;
      }
      
      // sleep for some time and then try again
      //
      Integral::sleep(DEF_SLEEP_DURATION);
    }
    
    // setup the command
    //
    command.assign(recognizer);
    command.concat(L" ");
    command.concat(OPTION_PARAM_FLAG);
    command.concat(L" ");
    command.concat(params_file);
    command.concat(L" ");
    command.concat(OPTION_ACCUMULATOR_LIST_FLAG);
    command.concat(L" ");
    command.concat(accumulator_list);
    command.concat(L" ");	
    command.concat(OPTION_LANGUAGE_MODEL_FLAG);
    command.concat(L" ");
    command.concat(lm_file_accum);
    command.concat(L" ");	
    command.concat(OPTION_STATISTICAL_MODEL_POOL_FLAG);
    command.concat(L" ");
    command.concat(smp_file_accum);	    
    command.concat(L" ");      
    command.concat(OPTION_FUNCTION_FLAG);
    command.concat(L" ");
    command.concat(OPTION_FUNCTION_UPDATE);
    
    // convert the wide character string to a character string
    //
    exec_command = (char*)(byte*)command;
    
    // print debugging information
    //
    if (verbosity >= Integral::ALL) {
      output.assign(L"command line: ");	  
      output.concat(command);
      Console::put(output);
    }
    
    // execute the command
    //
    if (system(exec_command) < 0) {
      
      // print debugging information
      //
      output.assign(L"error executing command: ");
      output.concat(command);
      Console::put(output);	
      
      return Integral::exit();    
    }
    
    // print debugging information
    //
    if (verbosity >= Integral::BRIEF) {
      output.assign(L"parent process has finished...");
      output.concat(L"\n");
      Console::put(output);  
    }
  }
  
  // exit gracefully
  //
  return Integral::exit();
  
}
