// file: $isip/class/pr/LanguageModelXML/LanguageModelXML.h
// version: $Id: LanguageModelXML.h 10636 2007-01-26 22:18:09Z tm334 $

// make sure definitions are only made once
//
#ifndef ISIP_LANGUAGEMODELXML
#define ISIP_LANGUAGEMODELXML

//---------------------------------------------------------------------------
//
// ISIP include files
//
//---------------------------------------------------------------------------  

#if defined(HAVE_EXPAT)

#ifndef ISIP_XMLPARSER
#include <XMLParser.h>
#endif

#endif

#ifndef ISIP_XMLTOKEN
#include <XMLToken.h>
#endif

#ifndef ISIP_FILENAME
#include <Filename.h>
#endif

#ifndef ISIP_DIGRAPH
#include <DiGraph.h>
#endif

#ifndef ISIP_ULONG
#include <Ulong.h>
#endif

#ifndef ISIP_HIERARCHICAL_DIGRAPH
#include <HierarchicalDigraph.h>
#endif

#ifndef ISIP_SHORT
#include <Short.h>
#endif

#ifndef ISIP_TRIPLE
#include <Triple.h>
#endif

#ifndef ISIP_PAIR
#include <Pair.h>
#endif

#ifndef ISIP_SDB
#include <Sdb.h>
#endif

#ifndef ISIP_SOF
#include <Sof.h>
#endif

#ifndef ISIP_LANGUAGE_MODEL_BASE
#include <LanguageModelBase.h>
#endif

// rule type definition
//
typedef Vector<XMLToken> Rule;

// LanguageModelXML: a class used for adding XML support to
//   LanguageModel. This class transparently loads XML grammars
//   and converts them into DiGraph format, and 
//   performs the conversion in the other direction.
//
//   GLOSSARY
//
//   "structure":
//   An XML "structure" is any unit of xml code which is
//   properly nested. For example:
//     "<item>"
//     is NOT an xml structure, because it has not close tag.
//     "<item/>"
//     is an XML structure, for it counts as both an open and close tag.
//     "<item> content </item>"
//     is an XML structure
//     "content"
//     is an XML structure
//     <item> content
//     is NOT an XML structure.
//
//  "nested":
//  A nested XML structure is any XMLToken or set of XMLTokens which
//  are properly nested within an open and close tag. For example:
//  "<one-of><item> content </item></one-of>"
//  Here, the structure "content" is nested within the structure
//  "<item> content </item>", which is in turn nested within the
//  structure "<one-of><item> content </item></one-of>"
//
//  "properly nested":
//  A structure is properly nested if every XML structure within it
//  is complete. That is, every XML open tag contained in this
//  structure must have a matching close tag. Consider every open
//  and close tag within this structure as a structure as well, and
//  apply the definition recursively.
//
//  "circumscribing"
//  All structures in which another structure is nested circumbscribe that
//  structure.
//
//  "following":
//  A following tag or structure is one that occurs later in an XML
//  document when traversing the document from start to end.
//  "<item/><metadata/>"
//  Here the metadata tag follows the item tag.
//
//  "preceding":
//  A preceding tag or structure is one that occurs earlier in an XML
//  document when traversing the document from start to end.
//  "<item/><metadata/>"
//  Here the item tag precedes the metadata tag.
//
//  "depth":
//  The depth of an XML tag or structure is a measure of the number of
//  structure in which a tag or structure is nested. When traversing
//  the document from start to end, starting at 0, every start tag
//  increases the depth by one, ever end tag reduces the depth by one,
//  and character data does not affect the depth. See the example below:
//
//  depth  XMLToken
//  0
//  1      <grammar>
//  2        <item>
//  2          character data
//  1        </item>
//  0      </grammar>
//
//  "start/end/open/close tags/tokens":
//  "<item>" is an XML tag or token (terms used interchangeably)
//  "<item>" is a start or open tag (terms used interchangeably)
//  "</item>" is an end or close tag (terms used interchangeably)
//
//
class LanguageModelXML : public LanguageModelBase {
  
  
  //---------------------------------------------------------------------------
  //
  // public constants
  //
  //---------------------------------------------------------------------------  
public:
  
  // constant: class name
  //
  static const String CLASS_NAME;
  
  // constants: file parameters
  //
  static const String DEF_PARAM;
  static const long DEF_LEVEL;
  static const String PARAM_GRAMMARS;
  static const String PARAM_UNDERSCORE;
  static const long DEF_TAG;
  
  // constants: defaut attribute values
  //
  static const float DEF_WEIGHT;
  
  // constants: delimiters used when parsing contexts
  //
  static const String CONTEXT_LABEL_PREFIX;
  static const String CONTEXT_SYMBOL_DELIM;
  
  // constants: Special rule definitions
  //
  static const String SPECIAL_GARBAGE;
  static const String SPECIAL_VOID;
  static const String SPECIAL_NULL;
  
  // constants: handled XML token definitions
  //
  static const String ONE_OF;
  static const String ITEM;
  static const String RULEREF;
  static const String RULE;
  static const String GRAMMAR;
  
  // constants: handled XML token attributes
  //
  static const String WEIGHT;
  static const String REPEAT;
  static const String REPEAT_PROB;
  static const String ROOT;
  static const String URI;
  static const String SPECIAL;
  static const String ID;
  
  // constants: handled XML attribute values
  //
  static const String RULEREF_URI_LOCAL_DELIM;
  static const String REPEAT_RANGE_DELIM;
  static const String REPEAT_LOOP_BACK;
  
  // constant: isip dummy tag
  // 
  static const String ISIP_DUMMY_NODE;

  // constant: output constants
  //
  static const String XML_VERSION_TAG;
  static const String ROOT_ATTRIB;
  static const String ID_ATTRIB;
  static const String ALGORITHM;
  static const String XML;
  static const String IMPLEMENTATION;
  static const String IHD;
  static const long LOOP_BACK;
  static const long TERMINAL_INDEX;
  static const long START_INDEX;
  
  //---------------------------------------
  //
  // error codes
  //
  //---------------------------------------
  //
  static const long ERR = 100800;
  static const long ERR_CTXT_NO_SYM_TAB = 100801;
  static const long ERR_CTXT_INV_LENGTHS = 100802;
  static const long ERR_CTXT_INV_GRAPH_ALIGN = 100803;
  static const long ERR_CTXT_INV_SYM = 100804;
  static const long ERR_READ_SYM_TYPE = 100805;
  static const long ERR_CTXT_SCV = 100806;
  static const long ERR_CTXT_INV_SYM_TAB = 100807;
  static const long ERR_CTXT_INV_TOKEN_INDEX = 100808;
  static const long ERR_CTXT_ESHG = 100809;
  static const long ERR_CTXT_EXISTS_DIF_INDEX = 100810;
  static const long ERR_CTXT_EXISTS_SAME_INDEX = 100811;
  static const long ERR_DEBG_TEMP_FILE = 100812;
  static const long ERR_DEBG_DSM_NOGR = 100813;
  static const long ERR_DEBG_DTM_NOGR = 100814;
  static const long ERR_REPT_IAV = 100815;
  static const long ERR_GRAMMAR_NO_START = 100816;
  static const long ERR_GRAMMAR_NO_END = 100817;
  static const long ERR_GRAMMAR_EMPTY = 100818;
  static const long ERR_GRAMMAR_FORMAT = 100819;
  static const long ERR_RULEREF_NOT_FOUND = 100820;
  static const long ERR_RULEREF_NON_LOCAL = 100821;
  static const long ERR_ONE_OF_EMPTY= 100822;
  static const long ERR_RULE_EMPTY = 100823;
  static const long ERR_TOKEN_UNHANDLED = 100824;
  static const long ERR_RULE_SPECIAL_INV = 100825;
  static const long ERR_VERTEX_STORE = 100826;
  
  //---------------------------------------------------------------------------
  //
  // protected data
  //
  //---------------------------------------------------------------------------  
protected:

  // these members hold an XML format grammar in memory
  //
  HierarchicalDigraph hg_d;
  Vector< Vector < Vector<XMLToken> > > grammars_d;

  // variable to temporarily store the rule being parsed.
  // when the rule is complete, it will be added to the
  // vector of rules, rules_d.
  // 
  Rule temp_rule_d;
  
  // vector to store all parsed rules.
  //
  Vector<Rule> rules_d;
  
  // variables to store the start and end grammar tags
  // the start grammar tag will have many useful attributes.
  //
  XMLToken grammar_start_tag_d;
  XMLToken grammar_end_tag_d;

  // keeps track of nesting level of rules.
  // 
  long rule_nesting_level_d;

  // create a vector of strings to store the symbol table if conversion
  // is requested.
  //
  Vector< String > symbol_table_d;

  // ABNF RuleModel 
  //
  RuleModel abnf_model_d;
  
  // debug level
  //
  // debug_level_d inherited from LanguageModelBase
  
  //---------------------------------------------------------------------------
  //
  // required public methods
  //
  //---------------------------------------------------------------------------  
public:
  
  // method: name
  //
  static const String& name() {
    return CLASS_NAME;
  } 
  
  // method: diagnose
  //
  static boolean diagnose(Integral::DEBUG level);
  
  // method: debug
  //
  boolean debug(const unichar* msg) const;
  
  // constructor
  //
  LanguageModelXML ();
  
  // method: read
  //
  boolean read(Sof& sof, long tag = LanguageModelXML::DEF_TAG,
	       const String& cname = CLASS_NAME);
  
  // method: readData
  //
  boolean readData(Sof& sof, const String& pname = DEF_PARAM,
		   long size = SofParser::FULL_OBJECT, boolean param = true,
		   boolean nested = false);
  
  // method: write
  //
  boolean write(Sof& sof, long tag, const String& cname = CLASS_NAME) const;
  
  // method: writeData
  //
  boolean writeData(Sof& sof, const String& pname = DEF_PARAM) const;
  
  // method: sofSize
  //
  long sofSize() const {
    return hg_d.sofSize();
  }
  
  // method: assign
  //
  boolean assign(const LanguageModelXML& lmxml); 
  
  // method: operator=
  //
  boolean operator=(const LanguageModelXML& lmxml);
  
  // method: eq
  //
  boolean eq(const LanguageModelXML& lmxml) const;
  
  // method: operator==
  //
  boolean operator==(const LanguageModelXML& lmxml) const;
  
  // method: clear
  //
  boolean clear(Integral::CMODE ctype);
  
  //---------------------------------------------------------------------------
  //
  // class-specific public methods required by interface contract
  // defined in LanguageModelBase.h
  //
  //---------------------------------------------------------------------------
  
  // method: assign
  //
  boolean assign(const LanguageModelBase& arg);

  // method: getRuleModel
  //
  RuleModel getRuleModel();
  
  // method: setRuleModel
  //
  boolean setRuleModel(const RuleModel& rm_a);

  // method: getABNFRuleModel
  //
  RuleModel getABNFRuleModel();
  
  // method: setABNFRuleModel
  //
  boolean setABNFRuleModel(const RuleModel& rm_a) {
    return abnf_model_d.assign(rm_a);
  }
  
  // method: eq
  //
  boolean eq(const LanguageModelBase& arg) const;  
  
  // method: className
  //
  const String& className() const {
    return CLASS_NAME;
  }
  
  //---------------------------------------------------------------------------
  //
  // class-specific public methods
  //
  //---------------------------------------------------------------------------
public: 
  
  //--------------------------------------------------------
  //
  // methods used to read/write the language model
  //
  //-------------------------------------------------------- 
  
  // method: readXMLGrammars
  //
  boolean readXMLGrammars(Sof& sof, const String& gname,
			  long level, boolean use_context,
			  Vector<SearchSymbol>* symbol_table,
			  SofParser& parser);
 
  // method: parseXMLGrammar
  //
  boolean parseXMLGrammar(String grammar_a,
			  Vector<String>& sub_symbol_list_a,
			  Vector<XMLToken>& token_vector_a,
			  String& grammar_name_a);
  
  // method: alignGraphs
  //
  boolean alignGraphs(Vector< Vector<XMLToken> >& grammar_list_a,
		      Vector<SearchSymbol>& symbol_table_a,
		      Vector<String>& graph_name_list_a);
  
  // method: readSymbolType
  //
  boolean readSymbolType(Sof& sof, long level,
			 const String& pname,
			 Vector<SearchSymbol>& symbol_list,
			 SofParser& parser);
  
  // method: readContextMapping
  //
  boolean readContextMapping(Sof& sof, const String& pname, long level,
			     SofParser&parser);
  
  //---------------------------------------------------------------------------
  //
  // class-specific protected methods
  //
  //---------------------------------------------------------------------------
protected:

  //--------------------------------------------------------
  //
  // output methods
  //
  //-------------------------------------------------------- 

  // method: getXMLModel()
  // 
  Pair< Vector< Vector< Rule > >, HierarchicalDigraph> getXMLModel();
  
  // method: setXMLModel()
  //
  boolean setXMLModel(Pair< Vector< Vector< Rule > >, HierarchicalDigraph>
		      xml_model_a);
  
  // method: writeSymbols
  //
  boolean writeSymbols(Sof& sof_a, long level_a, const String& pname_a,
		       Vector<SearchSymbol> symbol_list_a) const;

  // method: writeGrammars
  //
  boolean writeGrammars(Sof& sof_a, long level_a,
			HierarchicalDigraph& h_digraph_a) const;

  // method: writeLevelTag
  //
  boolean writeLevelTag(Sof& sof_a, long level_a, const String& tag_a) const;

  // method: addRuleRef
  //
  boolean addRuleRef(Vector<XMLToken>& token_vector_a,
		     String rule_name_a,
		     long& depth_a) const;
  
  // method: addDummyItem
  //
  boolean addDummyItem(Vector<XMLToken>& token_vector_a,
		       long& depth_a) const;

  // method: addDummyItem
  //
  boolean addCDATA(Vector<XMLToken>& token_vector_a,
		   const String& cdata_a,
		   long& depth_a) const;
  
  // method: addStartItemTag
  //
  boolean addStartItemTag(Vector<XMLToken>& token_vector_a,
			  long& depth_a,
			  float weight_a=1,
			  boolean repeat_a=0,
			  float repeat_prob_a=1) const;

  // method: addEndItemTag
  //
  boolean addEndItemTag(Vector<XMLToken>& token_vector_a,
			long& depth_a) const;

  // method: addStartBranchTag
  //
  boolean addStartBranchTag(Vector<XMLToken>& token_vector_a,
			    long& depth_a) const;

  // method: addEndBranchTag
  //
  boolean addEndBranchTag(Vector<XMLToken>& token_vector_a,
			  long& depth_a) const;

  // method: addStartRuleTag
  //
  boolean addStartRuleTag(Vector<XMLToken>& token_vector_a,
			  String rule_name_a,
			  long& depth_a) const;

  // method: addEndRuleTag
  //
  boolean addEndRuleTag(Vector<XMLToken>& token_vector_a,
			long& depth_a) const;
  
  // method: tokensToString
  //
  String tokensToString(Vector<XMLToken>& token_vector_a) const;

  // method: removeRedundantItemTags
  //
  Vector<XMLToken> removeRedundantItemTags(Vector<XMLToken>& token_vector_a)
    const;

  // method: constFindFollowingTagAtDepth
  //
  long constFindFollowingTagAtDepth(
					     Vector<XMLToken>& token_vector_a,
					     long start_index_a,
					     long base_depth_a,
					     XMLToken::TYPE type_a,
					     String value_a) const;
  //--------------------------------------------------------
  //
  // BNF to XML conversion methods
  //
  //--------------------------------------------------------
  
  // method: convertBNFtoXML
  //  
  Vector<XMLToken> convertBNFtoXML(ProductionRuleSet prset_a);

  // method: extractRule
  //  
  ProductionRuleSet extractRule(const String& rule_name_a,
				ProductionRuleSet& prset_a) const;

  // method: addRule
  //  
  boolean addRule(ProductionRuleSet& prset_a,
		  Vector<XMLToken>& token_vector_a) const;

  // method: addAlternative
  //
  boolean addAlternative(ProductionRule& pr_a,
			 Vector<XMLToken>& token_vector_a,
			 long depth_a) const;
  
  //--------------------------------------------------------
  //
  // XML to ABNF conversion methods
  //
  //-------------------------------------------------------- 

  // method: convertXMLtoABNF
  //  
  ProductionRule convertXMLtoABNF(Vector<XMLToken> token_vector_a);

  // method: appendProduction
  //
  boolean appendProduction(ProductionRule& prod1_a,
			   ProductionRule prod2_a);

  // method: getRange
  //
  Vector<XMLToken> getRange(long start_a, long end_a, Vector<XMLToken> vector_a);

  // method: findNextItem
  //
  long findNextItem(long start_a, Vector<XMLToken> vector_a);
  
  // method: preProcessXMLGrammar
  //
  boolean preProcessXMLGrammar(Vector<XMLToken>& token_vector);
  
  // method: validateGrammar
  //
  boolean validateXMLGrammar();
  
  // method: partitionGrammar
  //
  boolean partitionGrammar(Vector<XMLToken>& token_vector);
  
  // method: handleStartAndEndElement
  //
  boolean handleStartAndEndElement (XMLToken xml_token);
  
  // method: handleStartElement
  //
  boolean handleStartElement (XMLToken xml_token);
  
  // method: handleEndElement
  //
  boolean handleEndElement (XMLToken xml_token);
  
  // method: handleCharacterData
  //
  boolean handleCharacterData (XMLToken xml_token);
  
  // method: handleSpecialRules
  //
  boolean handleSpecialRules(Vector<XMLToken>& token_vector);
  
  // method: handleRepeatRequests
  //
  boolean handleRepeatRequests(Vector<XMLToken>& token_vector);
  
  // method: addOptionalRepeatSections
  //
  boolean addOptionalRepeatSections(Vector<XMLToken>& token_vector,
				    Long m);
  
  // method: addManditoryRepeatSections
  //
  boolean addManditoryRepeatSections(Vector<XMLToken>& token_vector,
				     Long m);
  
  //--------------------------------------------------------
  //
  // methods for searching an XML grammar
  //
  //-------------------------------------------------------- 
  
  // method: findFirstItemTagOfDATA
  //
  long findFirstItemTagOfCDATA(Vector<XMLToken>& token_vector,
			       long index);
  
  // method: findFollowingStructure
  //
  long findFollowingStructure(Vector<XMLToken>& token_vector, long index);
  
  // method: findForwardTargetStructure
  //
  long findForwardTargetStructure(Vector<XMLToken>& token_vector, long index);
  
  // findImmediateNesting
  //
  long findImmediateNesting(Vector<XMLToken> token_vector, long index);
  
  // method: findInternalStructure
  //
  long findInternalStructure(Vector<XMLToken>& token_vector, long index);
  
  // method: findMatchingStartTag
  //
  long findMatchingStartTag(Vector<XMLToken>& token_vector, long index);
  
  // method: findMatchingEndTag
  //
  long findMatchingEndTag(Vector<XMLToken>& token_vector, long index);
  
  // method: findPrecedingStructure
  //
  long findPrecedingStructure(Vector<XMLToken>& token_vector, long index);
  
  // method: findPrecedingTagAtDepth
  //
  long findPrecedingTagAtDepth(Vector<XMLToken>& token_vector,
			       long start_index,
			       long base_depth,
			       XMLToken::TYPE type = XMLToken::NULL_TAG,
			       String value = String::EMPTY);
  
  // method: findFollowingTagAtDepth
  //
  long findFollowingTagAtDepth(Vector<XMLToken>& token_vector,
			       long start_index,
			       long base_depth,
			       XMLToken::TYPE type = XMLToken::NULL_TAG,
			       String value = String::EMPTY);
  
  //--------------------------------------------------------
  //
  // worker methods
  //
  //-------------------------------------------------------- 
  
  // method: clearXMLGrammar
  //
  boolean clearXMLGrammar();
  
  // method: displayVector
  //
  boolean displayVector(Vector<String>& targets);

  // method displayXMLModel
  boolean displayXMLModel();
  
  
  // method: debugSearchMethod
  //
  boolean debugSearchMethod(long (LanguageModelXML::*func_ptr)(long),
			    Vector<XMLToken> token_vector_a);
  
  // method: debugTargetMethod
  //
  boolean debugTargetMethod
  (long (LanguageModelXML::*func_ptr)(long, Vector<Long>&),
   Vector<XMLToken> token_vector_a);
  
  // method: indexIsA
  //
  boolean indexIsA(Vector<XMLToken>& token_vector,
		   long index, String value);
  
  // method: indexIsA
  //
  boolean indexIsA(Vector<XMLToken>& token_vector,
		   long index, XMLToken::TYPE type);
  
  // method: isInBounds
  //
  boolean isInBounds(Vector<XMLToken>& token_vector, long index);
  
  // method: getDummySymbol
  //
  String getDummySymbol();
  
  // method: getGrammarName
  //
  String getGrammarName();
  
  // method: getHandledValues
  //
  Vector<String> getHandledValues();
  
  // method: getRuleID
  //
  String getRuleID(Rule rule);
  
  // method: getRuleIndexByID
  //
  long getRuleIndexByID(String id);
  
  // method: getSymbolList
  //
  Vector<String> getSymbolList(){
    return symbol_table_d;
  }
  
  // method: updateSymbolTable
  //
  boolean updateSymbolTable(XMLToken xml_token);
  
  // method: tokenizeRepeatValue
  //
  Triple<Long, Long, Boolean> tokenizeRepeatValue(XMLToken xml_token);
};

#endif
