// file: $isip/class/search/HierarchicalSearch/HierarchicalSearch.h
// version: $Id: HierarchicalSearch.h 10636 2007-01-26 22:18:09Z tm334 $
//

// make sure definitions are only made once
//
#ifndef ISIP_HIERARCHICAL_SEARCH
#define ISIP_HIERARCHICAL_SEARCH

// isip include files:
//
#ifndef ISIP_FRONT_END
#include <FrontEnd.h>
#endif

#ifndef ISIP_VECTOR_FLOAT
#include <VectorFloat.h>
#endif

/* #ifndef ISIP_STACK */
/* #include <Stack.h> */
/* #endif */

#ifndef ISIP_QUEUE
#include <Queue.h>
#endif

#ifndef ISIP_TRACE
#include <Trace.h>
#endif

#ifndef ISIP_INSTANCE
#include <Instance.h>
#endif

#ifndef ISIP_CONTEXT_POOL
#include <ContextPool.h>
#endif

#ifndef ISIP_HISTORY_POOL
#include <HistoryPool.h>
#endif

#ifndef ISIP_HASH_KEY
#include <HashKey.h>
#endif

#ifndef ISIP_HIERARCHICAL_DIGRAPH
#include <HierarchicalDigraph.h>
#endif

#ifndef ISIP_SEARCH_NODE
#include <SearchNode.h>
#endif

#ifndef ISIP_BI_GRAPH
#include <BiGraph.h>
#endif

#ifndef NGRAM_CACHE
#include <NGramCache.h>
#endif

#ifndef ISIP_SYMBOL_GRAPH
#include <SymbolGraph.h>
#endif

#ifndef ISIP_SYMBOL_GRAPH_NODE
#include <SymbolGraphNode.h>
#endif

// HierarchicalSearch:	A hierarchical, synchronous, Viterbi search engine.
//
// The basic premise behind this is as follows:
// 
//      1) The search space is hierarchical in that each level has nodes and
//         each node is either:
// 	   a) composed of a sub-graph + probability evaluation
// 	   b) composed of probability evaluation.
// 
//      2) At each level you will be able to do the following: 
// 	   a) beam prune
// 	   b) instance prune
// 	   c) generate context-dependent scoring models on the fly including
//            cross-symbol models
// 	   d) specify whether search information is to be maintained -
//            otherwise we can prune away redundant information. For instance,
//            once a word is completed, it is no longer necessary to maintain
//            the sub-word units that correspond to that word.
// 	   e) turn on/off sub-level evaluation
//
//      3) Each graph position will be capable of the following: 
// 	   a) generate a probability of being in that graph position given the 
// 	      input stream. For acoustic models, this will involve direct 
// 	      evaluation of the data stream. For levels like language models,
//            this will not involve manipulation of the data stream at all.
//            Any level can access the data stream as it will be passed from
//            the highest level all the way down to the lowest via pointer
//            reference.
// 	   b) store the overlapping paths in the sub-graph as a "lexical" tree.
// 	   c) use N-symbol models related to the possible entries on that level
//
//      4) We will revolve around the concept of a "trace" where the 
// 	   trace moves through the graph and keeps track of the paths/scores
//         through the search space
// 
class HierarchicalSearch {
  
  //---------------------------------------------------------------------------
  //
  // public constants
  //
  //---------------------------------------------------------------------------
public:

  // define the class name
  //
  static const String CLASS_NAME;

  //----------------------------------------
  //
  // i/o related constants
  //
  //----------------------------------------  

  static const String DEF_PARAM;
  
  //----------------------------------------
  //
  // other important constants
  //
  //----------------------------------------  

  // define the search mode choices
  //
  enum SEARCH_MODE { DECODE = 0, TRAIN, DEF_SEARCH_MODE = DECODE };

  // define the start frame index
  //
  static const llong DEF_START_FRAME = -1;
  static const long DEF_CONTEXT_LEVEL = 0;
  static const boolean DEF_CONTEXT_MODE = false;  

  //----------------------------------------
  //
  // default values and arguments
  //
  //----------------------------------------  

  // default values
  //
  static const long DEF_NUM_LEVELS = 1;
  static const long DEF_CAPACITY = 12000;
  
  // default arguments to methods
  //
  static const long DEF_NUM_FRAMES = -1;
  static const long DEF_INITIAL_LEVEL = 0;
  static const long ALL_LEVELS = -1;
  
  //---------------------------------------
  //
  // error codes
  //
  //---------------------------------------

  static const long ERR = (long)90700;
  static const long ERR_LEVEL = (long)90701;
  
  //---------------------------------------------------------------------------
  //
  // protected data
  //
  //---------------------------------------------------------------------------
protected:

  // define a search mode flag
  //
  SEARCH_MODE search_mode_d;

  // current frame's feature vector
  //
  VectorFloat features_d;
  
  // HierarchicalDigraph, sub-graphs are maintained at each SearchLevel 
  //
  HierarchicalDigraph* h_digraph_d;

  // initial search level
  //
  Long initial_level_d;
  
  // current frame in the search
  //
  Long current_frame_d;

  // the lexical tree list
  //
  SingleLinkedList<LexicalTree> lex_tree_list_d;

  // current maximal trace scores
  //
  Vector<Float> max_trace_scores_d;

  // current maximal instance scores
  //
  Vector<Float> max_instance_scores_d;  

  // symbol graph generation structure
  //
  SymbolGraph symbol_graph_d;
  
  // n-symbol probability lookup variables
  //
  NGramCache nsymbol_cache_d;
  Vector<Long> nsymbol_indices_d;

  // lists to keep track of instances during the search
  //
  Vector<DoubleLinkedList<Instance> > instance_lists_d;
  DoubleLinkedList<Instance> instance_valid_hyps_d;
  
  // lists to keep track of traces during the search
  //
  Vector<DoubleLinkedList<Trace> > trace_lists_d;
  DoubleLinkedList<Trace> trace_valid_hyps_d;

  // define a bi-graph structure that stores the entire search path
  //
  BiGraph<TrainNode> trellis_d;

  // define a mapping from the hypothesis path to the vertices
  //
  HashTable<HashKey<Instance>, BiGraphVertex<TrainNode> > instance_mapping_d;

  // define the context generation level
  //
  Long context_level_d;
  boolean context_generation_mode_d;
  
  // define the context generation list
  //
  Vector<String> context_list_d;  
  HashTable<String, Ulong> context_hash_d;

  // history and context pools
  //
  HistoryPool history_pool_d;
  ContextPool context_pool_d;
  
  // define a static debug level
  //
  static Integral::DEBUG debug_level_d;

  // define a static memory manager
  //
  static MemoryManager mgr_d;
  
  //---------------------------------------------------------------------------
  //
  // required public methods
  //
  //---------------------------------------------------------------------------
public:

  // method: name
  //
  static const String& name() {
    return CLASS_NAME;
  }
  static boolean diagnose(Integral::DEBUG debug_level);
  
  // debug methods:
  //  the setDebug method for this class is static because the debug_level is
  //  shared across all objects of this class
  //
  boolean debug(const unichar* message) const;

  // method: setDebug
  //
  static boolean setDebug(Integral::DEBUG debug_level) {
    debug_level_d = debug_level;
    return true;
  }

  // destructor/constructor(s)
  //
  ~HierarchicalSearch();
  HierarchicalSearch();
  HierarchicalSearch(const HierarchicalSearch& copy_search);
  
  // method: assign
  //
  boolean assign(const HierarchicalSearch& copy_search) {
    return Error::handle(name(), L"assign", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }

  // method: sofSize
  //
  long sofSize() const {
    return Error::handle(name(), L"sofSize", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }
  
  // method: read
  //
  boolean read(Sof& sof, long tag, const String& cname = CLASS_NAME) {
    return Error::handle(name(), L"read", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }

  // method: write
  //
  boolean write(Sof& sof, long tag, const String& cname = CLASS_NAME) const {
    return Error::handle(name(), L"write", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }

  // method: readData
  //
  boolean readData(Sof& sof, const String& pname = DEF_PARAM,
		   long size = SofParser::FULL_OBJECT, boolean param = true,
                   boolean nested = false) {
    return Error::handle(name(), L"readData", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }

  // method: writeData
  //
  boolean writeData(Sof& sof, const String& pname = DEF_PARAM) const {
    return Error::handle(name(), L"writeData", Error::NOT_IMPLEM, __FILE__,
			 __LINE__);
  }

  // method: eq
  //
  boolean eq(const HierarchicalSearch& compare_search) const {
    return h_digraph_d->eq(*compare_search.h_digraph_d);
  }
  
  // method: new
  //
  static void* operator new(size_t size) {
    return mgr_d.get();
  }

  // method: new[]
  //
  static void* operator new[](size_t size) {
    return mgr_d.getBlock(size);
  }

  // method: delete
  //
  static void operator delete(void* ptr) {
    mgr_d.release(ptr);
  }

  // method: delete[]
  //
  static void operator delete[](void* ptr) {
    mgr_d.releaseBlock(ptr);
  }

  // method: setGrowSize
  //
  static boolean setGrowSize(long grow_size) {
    return mgr_d.setGrow(grow_size);
  }

  // clear method
  //
  boolean clear(Integral::CMODE ctype = Integral::DEF_CMODE);

  //---------------------------------------------------------------------------
  //
  // class-specific public methods
  //
  //---------------------------------------------------------------------------

  // method: setSearchMode
  //
  boolean setSearchMode(SEARCH_MODE arg) {
    search_mode_d = arg;
    return true;
  }

  // method: getSearchMode
  //
  SEARCH_MODE getSearchMode() {
    return search_mode_d;
  }

  // method: setInitialLevel
  //
  boolean setInitialLevel(long init_level) {
    initial_level_d = init_level;
    return true;
  }

  // method: getInitialLevel
  //
  long getInitialLevel() const {
    return (long)initial_level_d;
  }
  
  // search level manipulation
  //
  boolean setNumLevels(long num_levels);

  // method: getNumLevels
  //
  long getNumLevels() const {
    return h_digraph_d->length();
  }

  // method: getContextLevel
  //  
  long getContextLevel() {
    return (long)context_level_d;
  }
  
  // method: setContextLevel
  //
  boolean setContextLevel(long arg) {
    return context_level_d.assign(arg);
  }

  // method: getContextList
  //  
  Vector<String>& getContextList() {
    return context_list_d;
  }
  
  // method: setContextList
  //
  boolean setContextList(Vector<String>& arg) {
    return context_list_d.assign(arg);
  }    
  
  // method: setFeatures
  //
  boolean setFeatures(Vector<VectorFloat>& features) {
    current_frame_d = DEF_START_FRAME;
    return true;
  }

  // method: getSearchLevel
  //
  SearchLevel& getSearchLevel(long level) {
    return (*h_digraph_d)(level);
  }
  
  // method: getHDigraph
  //
  HierarchicalDigraph& getHDigraph() {
    return *h_digraph_d;
  }

  // method: setHDigraph
  //
  boolean setHDigraph(HierarchicalDigraph& h_digraph) {
    h_digraph_d = &h_digraph;
    return true;
  }

  // method: getContextPool
  //
  ContextPool& getContextPool() {
    return context_pool_d;
  }

  // method: setContextPool
  //
  boolean setContextPool(ContextPool& arg) {
    return context_pool_d.assign(arg);
  }  

  // method: getHistoryPool
  //
  HistoryPool& getHistoryPool() {
    return history_pool_d;
  }

  // method: setHistoryPool
  //
  boolean setHistoryPool(HistoryPool& arg) {
    return history_pool_d.assign(arg);
  }  
  
  // search initialization methods
  //
  boolean initializeLinearDecoder();
  boolean initializeLinearPartial();  

  boolean initializeGrammarDecoder();
  boolean initializeGrammarPartial();  
  boolean initializeNetworkDecoder();
  boolean initializeContextGeneration();
  
  // decoding methods
  //
  boolean linearDecoder(Vector<FrontEnd>& vector_fe,
			long num_frames = DEF_NUM_FRAMES);
  boolean linearDecoder(Vector<VectorFloat>& fe, long num_frames = DEF_NUM_FRAMES);

  boolean grammarDecoder(Vector<FrontEnd>& vector_fe,
			 long num_frames = DEF_NUM_FRAMES);
  boolean grammarDecoder(Vector<VectorFloat>& fe, long num_frames = DEF_NUM_FRAMES);

  boolean networkDecoder(Vector<FrontEnd>& vector_fe,
			 long num_frames = DEF_NUM_FRAMES);
  boolean networkDecoder(Vector<VectorFloat>& fe, long num_frames = DEF_NUM_FRAMES);    
  
  // hypothesis methods
  //
  boolean getHypotheses(String& output_hyp, long level, double& total_score,
			long& num_frames, DoubleLinkedList<Trace>& trace_path);
  boolean getHypotheses(String& output_hyp, long level, double& total_score,
		     long& num_frames, DoubleLinkedList<Instance>& trace_path);

  boolean convertLexInstances( DoubleLinkedList<Instance>&
			       instance_path);
  boolean printInstances( DoubleLinkedList<Instance>&
			  instance_path);
  
  boolean getLexHypotheses(String& output_hyp, long level,
			   float& total_score,
			   long& num_frames,
			   DoubleLinkedList<Instance>& trace_path);
  
  boolean forcedAlignment(String& alignment, Long level,
			  DoubleLinkedList<Trace>& trace_path,
			  boolean cumulative = false);
  
  // trace manipulation methods
  //
  boolean connectValidHypothesis();
  BiGraphVertex<TrainNode>* insertTrace(Trace* arg);
  BiGraphVertex<TrainNode>* insertInstance(Instance* arg);  
  
  // trellis state occupancy computation routines
  //
  BiGraph<TrainNode>* computeForwardBackward(Vector<VectorFloat>& data,
					     float beta_threshold);

  // method: getSymbolGraph
  //
  SymbolGraph& getSymbolGraph() {
    return symbol_graph_d;
  }
  
  // symbol graph generation methods
  //
  boolean generateSymbolGraph();      
  boolean initializeSymbolGraphTrace(double& score, long& num_frames);
  boolean initializeSymbolGraphInstance(double& score, long& num_frames);  
  
  // methods for parsing and generating transcriptions
  //
  boolean parseGraph(Vector<Long>& node_a,
		     const Vector<SearchSymbol>& transcription_a,
		     long index_a, long search_level_a,
		     Long& operation_level_a, boolean exclude_a);

  boolean genRandomSentences(long sent_len, Random& random_gen,
			     long output_level,
			     String& sentence, boolean exclude);
  
  //---------------------------------------------------------------------------
  //
  // private methods
  //
  //---------------------------------------------------------------------------
private:

  // symbol graph generation methods
  //
  boolean generateSymbolGraph(SymbolGraphNode* prev_node,
			      SymbolGraphNode* curr_node,
			      float score, float lm_score);
  
  // search initialization methods
  //
  boolean clearTraceStorage();
  boolean clearSearchNodesTraceLists();
  boolean clearValidHypsTrace();
  
  boolean clearInstanceStorage();
  boolean clearSearchNodesInstanceLists();  
  boolean clearValidHypsInstance();
  boolean clearValidHypsLexInstance();
  
  // search traversal methods
  //
  boolean traverseTraceLevels();
  boolean pathsRemainTrace();

  boolean traverseInstanceLevels();
  boolean pathsRemainInstance();  
  
  // trace propagation methods
  //
  boolean propagateTraces();
  boolean propagateTracesDown(long level_num);
  boolean propagateTracesUp(long level_num);
  boolean evaluateTraceModels();

  // instance propagation methods
  //  
  boolean propagateInstances();
  boolean propagateInstancesDown(long level_num);
  boolean propagateInstancesUp(long level_num);
  boolean evaluateInstanceModels();  
  boolean evaluateLexInstanceModels();  

  // trace bookkeeping methods
  //
  long getActiveTraces(long search_level = ALL_LEVELS);

  // instance bookkeeping methods
  //
  long getActiveInstances(long search_level = ALL_LEVELS);  
  
  // debugging methods
  //
  boolean printNewPath(Trace* new_trace, Trace* old_trace);
  boolean printDeletedPath(Trace* new_trace, Trace* old_trace);  

  boolean printNewPath(Instance* new_instance, Instance* old_instance);
  boolean printDeletedPath(Instance* new_instance, Instance* old_instance);
  
  // trellis insertion methods
  //
  boolean insertNewPath(Trace* parent, Trace* child, float weight);
  boolean insertOldPath(Trace* parent, Trace* child, float weight);

  boolean insertNewPath(Instance* parent, Instance* child, float weight);
  boolean insertOldPath(Instance* parent, Instance* child, float weight);    
  
  // method: getPosteriorScore (not implemented yet)
  //
  float getPosteriorScore(Trace* curr_trace) {
    return 0.0;
  }

  // method to compute the n-symbol probability
  //
  float getPosteriorScore(Context** context, long level);

  // pruning method
  //
  boolean beamPruneTrace(long level_num);
  boolean beamPruneInstance(long level_num);

  boolean instancePruneTrace(long level_num);
  boolean instancePruneInstance(long level_num);    

  boolean beamPruneLexInstance(long level_num);  
  boolean instancePruneLexInstance(long level_num);
  
  // initialization routine
  //
  boolean initializeForwardBackward();

  // backward probability computation routines
  //
  boolean computeBackward(Vector<VectorFloat>& data,
			  float beta_threshold);  
  boolean computeBeta(Vector<VectorFloat>& data,
		      BiGraphVertex<TrainNode>* vertex,
		      float** model_cache);

  // forward probability computation routines
  //
  boolean computeForward(Vector<VectorFloat>& data);  
  boolean computeAlpha(Vector<VectorFloat>& data, 
		       BiGraphVertex<TrainNode>* vertex);

  // cross-word context related methods
  //
  boolean generateRightContexts(DoubleLinkedList<Context>& context_list,
				DoubleLinkedList<Float>& score_list,
				Context* initial_context,
				long depth, long level);
  boolean initializeRightContexts(Instance* curr_instance,
				  Context& init_context, long level_num,
				  float curr_weight, long depth);
  boolean extendRightContexts(Instance* curr_instance, long level_num,
			      Context* prev_symbol = (Context*)NULL);

  boolean printHistory(const History* history);

  boolean isTerminal(Instance* curr_instance, long level_num);  

  boolean propagateUp(long curr_level, long level,
		      long curr_depth, long depth,
		      DoubleLinkedList<Instance>& inst_list,
		      DoubleLinkedList<History>& curr_hist,
		      DoubleLinkedList<History>& prev_hist,
		      DoubleLinkedList<Float>& score_list);
  boolean propagateDown(Context* symbol,
			Context* symbol1, 
			GraphVertex<SearchNode>* curr_vertex,
			long curr_level, long level,
			long curr_depth, long depth,
			DoubleLinkedList<Instance>& inst_list,
			DoubleLinkedList<History>& curr_hist,
			DoubleLinkedList<History>& prev_hist,
			DoubleLinkedList<Float>& score_list);

  boolean ascend(Instance* instance);
  boolean descend(Instance* instance);
  
  boolean lookAhead(long level, long depth,
		    DoubleLinkedList<Instance>& inst_list,
		    DoubleLinkedList<Float>& score_list);
  boolean lookAheadHelper(GraphVertex<SearchNode>* curr_vertex,
			  long curr_level, long level,
			  long curr_depth, long depth,
			  DoubleLinkedList<Instance>& inst_list,
			  DoubleLinkedList<History>& curr_hist,
			  DoubleLinkedList<History>& prev_hist,
			  DoubleLinkedList<Float>& score_list);
  
  // context related methods
  //
  boolean generateRightLexContexts(DoubleLinkedList<Context>& context_list,
				   const Context& initial_context,
				   long depth,
				   long level);

  // start search engine from linear structure
  //
  boolean linearStart();

  // start search engine from lexical tree structure
  //
  boolean lexicalTreeStart();
  
  // initialize the lexical tree
  //
  boolean initializeLexicalTree();
  
  // lexical tree related methods
  //
  boolean printInstance(Instance* new_instance,
			long level_num = -1,
			boolean recursive = false);
  
  boolean changeHistory(Instance*& curr_instance,
			boolean ascend,
			long level_num = 0,
			GraphVertex<SearchNode>* start_vert =
			(GraphVertex<SearchNode>*)NULL);

  boolean addHypothesisPath(Trace*& new_instance, Trace*& old_instance,
			    GraphArc<SearchNode>*& arc, long level_num);
  boolean addHypothesisPath(Instance*& new_instance, Instance*& old_instance,
			    long level_num, float weight);  
  
  boolean addInstance(long level_num,
		      Instance*& curr_instance,
		      Instance*& next_instance,
		      boolean pruning);

  boolean ascendInstance(long level_num,
			 Instance*& curr_instance);

  boolean descendInstance(long level_num,
			  Instance*& curr_instance);

  boolean traverseLexInstanceLevels();
  boolean propagateLexInstances();
  
  boolean propagateLexInstancesUp(long level_num);
  boolean propagateLexInstancesDown(long level_num);

  boolean networkLexStart();
  boolean expandLexicalTree(long level_num);

  // convert hypotheses
  //
  boolean addToPath( Instance*& curr_inst,
		     Instance*& prev_inst,
		     Vector< DoubleLinkedList<Instance> >& inst_levels,
		     Vector< DoubleLinkedList<Instance> >& prev_levels,
		     long level_num,
		     long& frame_ind,
		     DoubleLinkedList<Instance>& instance_path);

  // n-symbol probability related methods
  //
  boolean shiftNSymbol(Trace*& trace, long level,
		       GraphVertex<SearchNode>* vertex);
  boolean shiftNSymbol(Instance*& instance, long level,
		       GraphVertex<SearchNode>* vertex);
  
  boolean initializeNsymbolInstance(Instance*& instance);
  boolean initializeNsymbolTrace(Trace*& trace);
  
  boolean applyNSymbolScore(Instance*& curr_instance,
			    long level_num);
  
  boolean addInstanceScore(Instance*& curr_instance,
			   float score,
			   long level_num,
			   boolean weight_score);

  // methods for generation and parsing of transcriptions
  // given a hierarchical digraph
  //
  boolean getNextNodes(Vector<Long>& input_node_a,
		       Vector< Vector<Long> >& next_nodes_a,
		       long current_level_a, Long& operation_level_d,
		       boolean exclude_a);
  
  Long getSymbolIndex(long current_level_a, Vector<Long>& input_node_a);
  
  boolean getVectorSymbol(long output_level_a,
			  Vector<SearchSymbol>& vec_symbol_a);
  
  boolean getVertexArc(long output_level_a, Long& symbol_index_a,
		       Vector<Ulong>& vec_vertex_a,
		       Vector< Pair<Long, Long> >& vec_arc_a);
  
  
};

// end of include file
//
#endif
