// file: $isip/util/speech/isip_posterior_creator/isip_posterior_creator.cc
//
// isip include files
//
#include "isip_posterior_creator.h"

// method: sigmoidEstimateCallback
//
// arguments:
//  double& y: (output) f(x)
//  VectorDouble& derivatives: (output) df/dparam(i) (x)
//  const double x: (input) test point
//  const VectorDouble& params: (input) parameters to test
//
// return: boolean value indicating status
//
// method to compute a sigmoid and its derivatives
//
boolean sigmoidEstimateCallback(double& y_a, VectorDouble& derivatives_a,
				const double x_a,
				const VectorDouble& params_a) {
  // setup the sigmoid
  //
  Sigmoid sig;
  sig.set(1, params_a(0), params_a(1), 0);

  // find f(x) and the derivatives 
  //
  double tmp;
  sig.compute(y_a, x_a);
  sig.derivativeSlope(tmp, x_a);
  derivatives_a(0) = tmp;
  sig.derivativeXOffset(tmp, x_a);
  derivatives_a(1) = tmp;

  // exit gracefully
  //
  return true;
}

// isip_posterior_creator: computes the posterior model that transforms the
// support vector distances to a probability estimate
//
int main(int argc, const char** argv) {

  // setup the command line
  //
  Sdb sdb;
  CommandLine cmdl(sdb);
  cmdl.setUsage(
#include "usage_message.text"
                );
  cmdl.setHelp(
#include "help_message.text"
               );

  cmdl.setIdent("$Revision: 10619 $",
		"$Name: $",
		"$Date: 2006-08-24 11:04:00 -0500 (Thu, 24 Aug 2006) $");
  
  // add a command line option for the model file
  //
  Filename model_file;
  cmdl.addOptionParam(model_file, OPTION_MODEL_FILE, (Filename)EMPTY);

  // add a command line option for the output file
  //
  Filename output_file;
  cmdl.addOptionParam(output_file, OPTION_OUTPUT_FILE, (Filename)EMPTY);
  
  // add a command line option for the output type
  //
  String output_type;
  cmdl.addOptionParam(output_type, OPTION_OUTPUT_TYPE, OPTION_TYPE_BINARY);
  
  // add a command line option for the positive example list
  //
  Filename pos_example_list;
  cmdl.addOptionParam(pos_example_list, OPTION_POS_EXAMPLE_LIST, (Filename)EMPTY);

  // add a command line option for the positive prior
  //
  Double pos_prior;
  cmdl.addOptionParam(pos_prior, OPTION_POS_PRIOR, DEF_PRIORI_PROBABILITY);

  // add a command line option for the negative example list
  //
  Filename neg_example_list;
  cmdl.addOptionParam(neg_example_list, OPTION_NEG_EXAMPLE_LIST, (Filename)EMPTY);    

  // add a command line option for the negative prior
  //
  Double neg_prior;
  cmdl.addOptionParam(neg_prior, OPTION_NEG_PRIOR,  DEF_PRIORI_PROBABILITY);

  // add a command line option for the histogram bin width
  //
  Double bin_width;
  cmdl.addOptionParam(bin_width, OPTION_BIN_WIDTH, DEF_HISTOGRAM_BIN_WIDTH);
  
  // add a command line option for the debug level
  //
  DebugLevel debug_level;
  cmdl.addOptionParam(debug_level, OPTION_DEBUG_LEVEL,
		      DEBUG_LEVEL_DEFAULT);

  // add a command line option for the log verbosity level
  //
  DebugLevel verbosity;
  cmdl.addOptionParam(verbosity, OPTION_VERBOSITY_LEVEL,
		      VERBOSITY_LEVEL_DEFAULT);

  // add a command line option for the log file
  //
  Filename log_file;
  cmdl.addOptionParam(log_file, OPTION_LOG_FILE, (Filename)LOG_FILE_DEFAULT);
  
  // parse the command line
  //
  if (!cmdl.parse(argc, argv)) {
    cmdl.printUsage();
  };

  // open the log file
  //
  if (log_file.ne(L"-")) {
    Console::open(log_file);
    Console::setLineWrap(File::NO_WRAP);
  }

  // print the command line
  //
  if (verbosity >= Integral::BRIEF) {
    cmdl.printCommandLine();
    cmdl.printVersion();
  }

  // check the model file argument
  //
  if(model_file.eq((Filename)EMPTY)) {
    cmdl.printUsage();
  };

  // check the output file argument
  //  
  if(output_file.eq((Filename)EMPTY)) {
    cmdl.printUsage();
  };

  // check the positive example list
  //  
  if(pos_example_list.eq((Filename)EMPTY)) {
    cmdl.printUsage();
  };

  // check the negative example list
  //  
  if(neg_example_list.eq((Filename)EMPTY)) {
    cmdl.printUsage();
  };    

  // declare local variables
  //
  Sof plist;
  Sof nlist;
  Sof mfile;
  Sof ofile;  
  
  File pfile;
  File nfile;

  Sdb pos_sdb;
  Sdb neg_sdb;

  String output;
  StatisticalModel stat_model;
  
  VectorDouble pos_examples;
  VectorDouble neg_examples;

  // open the output file for writing
  //
  if (verbosity >= Integral::BRIEF) {
    Console::increaseIndention();    
    output.assign(L"\nopening output file: ");
    output.concat(output_file);
    Console::put(output);
    Console::decreaseIndention();    
  }
  
  if (output_type.eq(OPTION_TYPE_TEXT)) {
    
    // open the output file in text mode
    //    
    if (!ofile.open(output_file, File::WRITE_ONLY, File::TEXT)) {
      output.assign(L"\ncannot open file: ");
      output.concat(output_file);
      output.concat(L", bailing out...");
      Console::put(output);
      return Integral::exit();
    }
  }
  
  else {
    
    // open the output file in binary mode
    //        
    if (!ofile.open(output_file, File::WRITE_ONLY, File::BINARY)) {
      output.assign(L"\ncannot open file: ");
      output.concat(output_file);
      output.concat(L", bailing out...");
      Console::put(output);
      return Integral::exit();
    }
  }
  
  // open the support vector model
  //  
  if (!mfile.open(model_file, File::READ_ONLY)) {
    output.assign(L"\ncannot open file: ");
    output.concat(model_file);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();
  }
  
  // read the support vector model
  //
  if (verbosity >= Integral::BRIEF) {
    Console::increaseIndention();    
    output.assign(L"\nloading support vector model: ");
    output.concat(model_file);
    Console::put(output);
    Console::decreaseIndention();    
  }
  
  if (!stat_model.read(mfile, (long)0)) {
    output.assign(L"\nerror reading model: ");
    output.concat(model_file);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }
  
  // open the positive examples list
  //
  if (verbosity >= Integral::BRIEF) {
    Console::increaseIndention();    
    output.assign(L"\nopening positive examples list: ");
    output.concat(pos_example_list);
    Console::put(output);
    Console::decreaseIndention();    
  }
  
  if (!plist.open(pos_example_list, File::READ_ONLY, File::TEXT)) {
    output.assign(L"\nerror opening the positive examples list: ");
    output.concat(pos_example_list);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }

  if (!pos_sdb.read(plist, (long)0)) {
    output.assign(L"\nerror reading the positive examples list: ");
    output.concat(pos_example_list);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }  

  // open the negative examples list
  //
  if (verbosity >= Integral::BRIEF) {
    Console::increaseIndention();    
    output.assign(L"\nopening negative examples list: ");
    output.concat(neg_example_list);
    Console::put(output);
    Console::decreaseIndention();    
  }
  
  if (!nlist.open(neg_example_list, File::READ_ONLY, File::TEXT)) {
    output.assign(L"\nerror opening the negative examples list: ");
    output.concat(neg_example_list);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }

  if (!neg_sdb.read(nlist, (long)0)) {
    output.assign(L"\nerror reading the negative examples list: ");
    output.concat(neg_example_list);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }

  // close the file descriptors
  //
  plist.close();
  nlist.close();
  mfile.close();
  
  // accumulate the negative and positive examples
  //
  Char chr;
  Double val;
  String buffer;  
  Filename input_file_name;  
  long current_file_num = 0;

  // loop over the positive examples
  //
  pos_examples.setCapacity(DEF_CAPACITY);
  
  for (pos_sdb.gotoFirst(); pos_sdb.getName(input_file_name);
       pos_sdb.gotoNext()) {

    current_file_num++;

    // open the input file
    //
    if (!pfile.open(input_file_name, File::READ_ONLY, File::TEXT)) {
      output.assign(L"\nerror opening the positive example file: ");
      output.concat(input_file_name);
      output.concat(L", bailing out...");
      Console::put(output);
      Integral::exit();    
    }

    if (verbosity >= Integral::BRIEF) {
      Console::increaseIndention();
      output.assign(L"\nprocessing file ");
      output.concat(current_file_num);
      output.concat(L" (");
      output.concat(L"pos");
      output.concat(L")");
      output.concat(L": ");
      output.concat(input_file_name);
      Console::put(output);
      Console::decreaseIndention();	    
    }
    
    // read the contents of the parameter file line-by-line
    //
    long num_pos_examples = 0;
    
    while (pfile.get(buffer, Sof::BUFFER_SIZE)) {

      // get the first char
      //
      buffer.trim();    
      chr = buffer(0);
      
      // make sure that the line is valid
      //    
      if (!buffer.eq(EMPTY) && !chr.eq(L'#')) {
	
	if (debug_level >= Integral::DETAILED) {
	  buffer.debug(L"file line");
	}

	num_pos_examples++;
	val.assign(buffer);

	if (pos_examples.length() < num_pos_examples) {
	  pos_examples.setLength(num_pos_examples);
	  pos_examples(num_pos_examples - 1).assign(val);
	}
      }
    }
    
    // close the file descriptor
    //
    pfile.close();
  }

  // loop over the negative examples
  //
  neg_examples.setCapacity(DEF_CAPACITY);
  
  for (neg_sdb.gotoFirst(); neg_sdb.getName(input_file_name);
       neg_sdb.gotoNext()) {

    current_file_num++;

    // open the input file
    //
    if (!nfile.open(input_file_name, File::READ_ONLY, File::TEXT)) {
      output.assign(L"\nerror opening the negative example file: ");
      output.concat(input_file_name);
      output.concat(L", bailing out...");
      Console::put(output);
      Integral::exit();    
    }

    if (verbosity >= Integral::BRIEF) {
      Console::increaseIndention();
      output.assign(L"\nprocessing file ");
      output.concat(current_file_num);
      output.concat(L" (");
      output.concat(L"neg");
      output.concat(L")");
      output.concat(L": ");
      output.concat(input_file_name);
      Console::put(output);
      Console::decreaseIndention();	    
    }
    
    // read the contents of the parameter file line-by-line
    //
    long num_neg_examples = 0;
    
    while (nfile.get(buffer, Sof::BUFFER_SIZE)) {

      // get the first char
      //
      buffer.trim();    
      chr = buffer(0);
      
      // make sure that the line is valid
      //    
      if (!buffer.eq(EMPTY) && !chr.eq(L'#')) {
	
	if (debug_level >= Integral::DETAILED) {
	  buffer.debug(L"file line");
	}

	num_neg_examples++;
	val.assign(buffer);

	if (neg_examples.length() < num_neg_examples) {
	  neg_examples.setLength(num_neg_examples);
	  neg_examples(num_neg_examples - 1).assign(val);
	}
      }
    }
    
    // close the file descriptor
    //
    nfile.close();
  }

  // get the data bounds
  //
  double data_min = ((pos_examples.min() < neg_examples.min()) ?
		     pos_examples.min() : neg_examples.min());
  double data_max = ((pos_examples.max() > neg_examples.max()) ?
		     pos_examples.max() : neg_examples.max());

  // generate bins for histogram
  //
  VectorDouble bins;
  bins.ramp(data_min, bin_width, data_max);
  
  // generate histogram
  //
  Histogram pos_hist;
  Histogram neg_hist;

  pos_hist.setScale(Histogram::USER_DEFINED);
  pos_hist.setMode(Histogram::CENTERS);
  neg_hist.setScale(Histogram::USER_DEFINED);
  neg_hist.setMode(Histogram::CENTERS);
  
  pos_hist.setBins(bins);
  neg_hist.setBins(bins);

  if (debug_level >= Integral::DETAILED) {
    bins.debug(L"bins"); 
  }

  Vector<VectorDouble> tmp_pos_examples(1), tmp_neg_examples(1);
  tmp_pos_examples(0).assign(pos_examples);
  tmp_neg_examples(0).assign(neg_examples);
  
  pos_hist.compute(tmp_pos_examples);
  neg_hist.compute(tmp_neg_examples);

  // get the pdf from each histogram
  //
  VectorDouble pos_pdf;
  VectorDouble neg_pdf;

  pos_hist.pdf(pos_pdf);
  neg_hist.pdf(neg_pdf);

  if (debug_level >= Integral::DETAILED) {
    pos_pdf.debug(L"pos_pdf");
    neg_pdf.debug(L"neg_pdf");
  }
  
  // use discounting to fill the empty bins
  //
  pos_pdf.limitMin(DEF_MIN_PROB);
  neg_pdf.limitMin(DEF_MIN_PROB);
  
  // renormalize
  //
  pos_pdf.div(pos_pdf.sum());
  neg_pdf.div(neg_pdf.sum());

  // generate the posterior: numerator
  //
  VectorDouble posterior = pos_pdf;
  posterior.mult(pos_prior);

  // generate the posterior: denominator
  //
  neg_pdf.mult(neg_prior);
  neg_pdf.add(posterior);

  // numerator / denominator
  //
  posterior.div(neg_pdf);

  // setup the optimization data
  //
  VectorDouble params(2);
  params.assign(1.0);

  double chi_square = 0;
  VectorDouble stddev(posterior.length());
  stddev.assign(1.0);

  // run the optimization
  //
  if (!NonlinearOptimization::levenbergMarquardt(params, chi_square, bins,
						 posterior, stddev,
						 sigmoidEstimateCallback,
						 DEF_CONVERGENCE_THRESHOLD)) {
    output.assign(L"\nnon-linear optimization failed, bailing out...");
    Console::put(output);
    Integral::exit();    
  }

  // setup the output parameters
  //
  Sigmoid& sigmoid = stat_model.getSupportVectorModel().getSigmoid();
  sigmoid.set(1, params(0), params(1), 0);

  // write the updated model
  //
  if (!stat_model.write(ofile, (long)0)) {
    output.assign(L"\nerror writing the to the output file: ");
    output.concat(output_file);
    output.concat(L", bailing out...");
    Console::put(output);
    Integral::exit();    
  }

  // close the file descriptor
  //
  ofile.close();

  if (verbosity >= Integral::BRIEF) {    
    output.assign(L"\nprocessed ");
    output.concat(current_file_num);
    output.concat(L" file(s) successfully, attempted ");
    output.concat(current_file_num);
    output.concat(L" file(s)\n");
    Console::put(output);
  }
  
  // close the log file
  //
  if (log_file.ne(L"-")) {
    Console::close();
  }

  // exit gracefully
  //
  return Integral::exit();
}
