#include <OgreCompiler2Pass.h>
Inheritance diagram for Ogre::Compiler2Pass:
Public Member Functions | |
Compiler2Pass () | |
constructor | |
virtual | ~Compiler2Pass () |
bool | compile (const String &source, const String &sourceName) |
compile the source - performs 2 passes. | |
virtual const String & | getClientBNFGrammer (void) const =0 |
gets BNF Grammer. | |
virtual const String & | getClientGrammerName (void) const =0 |
get the name of the BNF grammer. | |
Protected Types | |
typedef std::vector< TokenRule > | TokenRuleContainer |
typedef TokenRuleContainer::iterator | TokenRuleIterator |
typedef std::vector< LexemeTokenDef > | LexemeTokenDefContainer |
typedef LexemeTokenDefContainer::iterator | LexemeTokenDefIterator |
typedef std::map< std::string, size_t > | LexemeTokenMap |
typedef LexemeTokenMap::iterator | TokenKeyIterator |
typedef std::vector< TokenInst > | TokenInstContainer |
typedef TokenInstContainer::iterator | TokenInstIterator |
typedef std::map< size_t, String > | LabelContainer |
storage container for string labels defined in source container uses Token index as a key associated with a label | |
enum | OperationType { otUNKNOWN, otRULE, otAND, otOR, otOPTIONAL, otREPEAT, otDATA, otNOT_TEST, otINSERT_TOKEN, otEND } |
enum | SystemRuleToken { _no_token_ = SystemTokenBase, _character_, _value_, _no_space_skip_ } |
enum | BNF_ID { BNF_UNKOWN = 0, BNF_SYNTAX, BNF_RULE, BNF_IDENTIFIER, BNF_IDENTIFIER_RIGHT, BNF_IDENTIFIER_CHARACTERS, BNF_ID_BEGIN, BNF_ID_END, BNF_CONSTANT_BEGIN, BNF_SET_RULE, BNF_EXPRESSION, BNF_AND_TERM, BNF_OR_TERM, BNF_TERM, BNF_TERM_ID, BNF_CONSTANT, BNF_OR, BNF_TERMINAL_SYMBOL, BNF_TERMINAL_START, BNF_REPEAT_EXPRESSION, BNF_REPEAT_BEGIN, BNF_REPEAT_END, BNF_SET, BNF_SET_BEGIN, BNF_SET_END, BNF_NOT_TEST, BNF_NOT_TEST_BEGIN, BNF_CONDITIONAL_TOKEN_INSERT, BNF_OPTIONAL_EXPRESSION, BNF_NOT_EXPRESSION, BNF_NOT_CHK, BNF_OPTIONAL_BEGIN, BNF_OPTIONAL_END, BNF_NO_TOKEN_START, BNF_SINGLEQUOTE, BNF_SINGLE_QUOTE_EXC, BNF_SET_END_EXC, BNF_ANY_CHARACTER, BNF_SPECIAL_CHARACTERS1, BNF_SPECIAL_CHARACTERS2, BNF_WHITE_SPACE_CHK, BNF_LETTER, BNF_LETTER_DIGIT, BNF_DIGIT, BNF_WHITE_SPACE, BNF_ALPHA_SET, BNF_NUMBER_SET, BNF_SPECIAL_CHARACTER_SET1, BNF_SPECIAL_CHARACTER_SET2, BNF_SPECIAL_CHARACTER_SET3, BNF_NOT_CHARS, BNF_AUTOTOKENSTART } |
Protected Member Functions | |
bool | doPass1 () |
perform pass 1 of compile process scans source for lexemes that can be tokenized and then performs general semantic and context verification on each lexeme before it is tokenized. | |
bool | doPass2 () |
performs Pass 2 of compile process which is execution of the tokens | |
virtual void | executeTokenAction (const size_t tokenID)=0 |
execute the action associated with the token pointed to by the Pass 2 token instruction position. | |
virtual size_t | getAutoTokenIDStart () const =0 |
Get the start ID for auto generated token IDs. | |
virtual void | setupTokenDefinitions (void)=0 |
setup client token definitions. | |
const TokenInst & | getNextToken (const size_t expectedTokenID=0) const |
Gets the next token from the instruction que. | |
const TokenInst & | getCurrentToken (const size_t expectedTokenID=0) const |
Gets the current token from the instruction que. | |
bool | testNextTokenID (const size_t expectedTokenID) const |
If a next token instruction exist then test if its token ID matches. | |
bool | testCurrentTokenID (const size_t expectedTokenID) const |
If a current token instruction exist then test if its token ID matches. | |
void | skipToken (void) const |
skip to the next token in the pass2 queue. | |
void | replaceToken (void) |
go back to the previous token in the pass2 queue. | |
float | getNextTokenValue (void) const |
Gets the next token's associated floating point value in the instruction que that was parsed from the text source. | |
float | getCurrentTokenValue (void) const |
Gets the current token's associated floating point value in the instruction que that was parsed from the text source. | |
const String & | getNextTokenLabel (void) const |
Gets the next token's associated text label in the instruction que that was parsed from the text source. | |
const String & | getCurrentTokenLabel (void) const |
Gets the next token's associated text label in the instruction que that was parsed from the text source. | |
size_t | getNextTokenID (void) const |
Get the next token's ID value. | |
size_t | getCurrentTokenID (void) const |
Get the current token's ID value. | |
const String & | getNextTokenLexeme (void) const |
Get the next token's lexeme string. | |
const String & | getCurrentTokenLexeme (void) const |
Get the current token's lexeme string. | |
size_t | getPass2TokenQueCount (void) const |
Gets the number of tokens waiting in the instruction que that need to be processed by an token action in pass 2. | |
size_t | getRemainingTokensForAction (void) const |
Get the number of tokens not processed by action token. | |
void | setPass2TokenQuePosition (size_t pos, const bool activateAction=false) |
Manualy set the Pass2 Token que position. | |
size_t | getPass2TokenQuePosition (void) const |
Get the current position in the Pass2 Token Que. | |
bool | setNextActionQuePosition (size_t pos, const bool search=false) |
Set the position of the next token action in the Pass2 Token Que. | |
size_t | addLexemeToken (const String &lexeme, const size_t token, const bool hasAction=false, const bool caseSensitive=false) |
Add a lexeme token association. | |
void | setClientBNFGrammer (void) |
Sets up the parser rules for the client based on the BNF Grammer text passed in. | |
void | findEOL () |
find the eol charater | |
bool | isFloatValue (float &fvalue, size_t &charsize) const |
check to see if the text at the present position in the source is a numerical constant | |
bool | isCharacterLabel (const size_t rulepathIDX) |
Check if source at current position is supposed to be a user defined character label. | |
bool | isLexemeMatch (const String &lexeme, const bool caseSensitive) const |
check to see if the text is in the lexeme text library | |
bool | isEndOfSource () const |
Check if pass 1 has parsed to the end of the source. | |
bool | positionToNextLexeme () |
position to the next possible valid sysmbol | |
bool | processRulePath (size_t rulepathIDX) |
process input source text using rulepath to determine allowed tokens | |
void | setActiveContexts (const uint contexts) |
setup ActiveContexts - should be called by subclass to setup initial language contexts | |
void | skipComments () |
comment specifiers are hard coded | |
void | skipEOL () |
find end of line marker and move past it | |
void | skipWhiteSpace () |
skip all the white space which includes spaces and tabs | |
bool | ValidateToken (const size_t rulepathIDX, const size_t activeRuleID) |
check if current position in source has the lexeme text equivalent to the TokenID | |
void | verifyTokenRuleLinks (const String &grammerName) |
scan through all the rules and initialize token definition with index to rules for non-terminal tokens. | |
void | checkTokenActionTrigger (void) |
Checks the last token instruction and if it has an action then it triggers the action of the previously found token having an action. | |
String | getBNFGrammerTextFromRulePath (size_t ruleID, const size_t level=0) |
Get the text representation of the rule path. | |
Protected Attributes | |
TokenState * | mClientTokenState |
TokenState * | mActiveTokenState |
Active token que, definitions, rules currntly being used by parser. | |
size_t | mPass2TokenQuePosition |
the location within the token instruction container where pass 2 is | |
size_t | mPreviousActionQuePosition |
the que position of the previous token that had an action. | |
size_t | mNextActionQuePosition |
the que position for the next token that has an action. | |
const String * | mSource |
pointer to the source to be compiled | |
String | mSourceName |
name of the source to be compiled | |
size_t | mEndOfSource |
size_t | mCurrentLine |
size_t | mCharPos |
current line number in source being tokenized | |
size_t | mErrorCharPos |
position in current line in source being tokenized | |
std::map< size_t, float > | mConstants |
storage container for constants defined in source container uses Token index as a key associated with a float constant | |
LabelContainer | mLabels |
bool | mLabelIsActive |
flag indicates when a label is being parsed. | |
size_t | mActiveLabelKey |
the key of the active label being built during pass 1. | |
String * | mActiveLabel |
The active label that is receiving characters during pass 1. | |
bool | mNoSpaceSkip |
flag being true indicates that spaces are not to be skipped automatically gets set to false when mLabelIsActive goes to false | |
bool | mNoTerminalToken |
if flag is true then next terminal token is not added to token que if found but does effect rule path flow | |
size_t | mInsertTokenID |
TokenID to insert if next rule finds a terminal token if zero then no token inserted. | |
uint | mActiveContexts |
Active Contexts pattern used in pass 1 to determine which tokens are valid for a certain context. | |
Static Protected Attributes | |
static const size_t | SystemTokenBase = 1000 |
Private Types | |
typedef std::map< String, TokenState > | TokenStateContainer |
Private Member Functions | |
void | activatePreviousTokenAction (void) |
if a previous token action was setup then activate it now | |
void | initBNFCompiler (void) |
initialize token definitions and rule paths | |
void | buildClientBNFRulePaths (void) |
Convert BNF grammer token que created in pass 1 into a BNF rule path. | |
void | modifyLastRule (const OperationType pendingRuleOp, const size_t tokenID) |
modify the last rule in the container. An end operation is added to the rule path. | |
size_t | getClientLexemeTokenID (const String &lexeme, const bool isCaseSensitive=false) |
get the token ID for a lexeme in the client state. | |
void | extractNonTerminal (const OperationType pendingRuleOp) |
Extract a Non Terminal identifier from the token que. | |
void | extractTerminal (const OperationType pendingRuleOp, const bool notoken=false) |
Extract a Terminal lexeme from the token que and add to current rule expression. | |
void | extractSet (const OperationType pendingRuleOp) |
Extract a set from the token que and add to current rule expression. | |
void | extractNumericConstant (const OperationType pendingRuleOp) |
Extract a numeric constant from the token que and add it to the current rule expression. | |
void | setConditionalTokenInsert (void) |
changes previous terminal token rule into a conditional terminal token insert rule | |
String | getLexemeText (size_t &ruleID, const size_t level=0) |
get the lexeme text of a rule. | |
Static Private Attributes | |
static TokenState | mBNFTokenState |
static TokenStateContainer | mClientTokenStates |
Classes | |
struct | LexemeTokenDef |
structure used to build lexeme Type library More... | |
struct | TokenInst |
structure for Token instructions that are constructed during first pass More... | |
struct | TokenRule |
structure used to build rule paths More... | |
struct | TokenState |
During Pass1, if a terminal token is identified as having an action then that action gets triggered when the next terminal token is uncountered that has an action.
PASS 2 - generate application specific instructions ie native instructions based on the tokens in the instruction container.
<identiefier> appears on both sides of the ::= so (?!"::=") test to make sure that ::= is not on the right which would indicate that a new rule was being formed.
Works on both terminals and non-terminals. Note: lookahead failure causes the whole rule to fail and rollback to occur
any character but
or is accepted in the input.
In the example, <last_resort> gets processed when all other rules fail to parse. if <special_label> parses (reads in any character but
) then the terminal token 'external_command' is inserted prior to the Special_Label for pass 2 processing. 'external_command' does not have have an explicit text representation but based on the context of no other rules matching and <special_label> parsing, 'external_command' is considered parsed.
Definition at line 152 of file OgreCompiler2Pass.h.
|
storage container for string labels defined in source container uses Token index as a key associated with a label
Definition at line 287 of file OgreCompiler2Pass.h. |
|
Definition at line 228 of file OgreCompiler2Pass.h. |
|
Definition at line 229 of file OgreCompiler2Pass.h. |
|
Definition at line 231 of file OgreCompiler2Pass.h. |
|
Definition at line 246 of file OgreCompiler2Pass.h. |
|
Definition at line 247 of file OgreCompiler2Pass.h. |
|
Definition at line 232 of file OgreCompiler2Pass.h. |
|
Definition at line 174 of file OgreCompiler2Pass.h. |
|
Definition at line 175 of file OgreCompiler2Pass.h. |
|
Definition at line 607 of file OgreCompiler2Pass.h. |
|
Definition at line 185 of file OgreCompiler2Pass.h. |
|
Definition at line 158 of file OgreCompiler2Pass.h. |
|
Definition at line 178 of file OgreCompiler2Pass.h. |
|
constructor
|
|
Definition at line 640 of file OgreCompiler2Pass.h. |
|
if a previous token action was setup then activate it now
|
|
Add a lexeme token association.
|
|
Convert BNF grammer token que created in pass 1 into a BNF rule path.
|
|
Checks the last token instruction and if it has an action then it triggers the action of the previously found token having an action.
|
|
compile the source - performs 2 passes. First pass is to tokinize, check semantics and context. The second pass is performed by using tokens to look up function implementors and executing them which convert tokens to application specific instructions.
|
|
perform pass 1 of compile process scans source for lexemes that can be tokenized and then performs general semantic and context verification on each lexeme before it is tokenized. A tokenized instruction list is built to be used by Pass 2. A rule path can trigger Pass 2 execution if enough tokens have been generated in Pass 1. Pass 1 will then pass control to pass 2 temporarily until the current tokens have been consumed. |
|
performs Pass 2 of compile process which is execution of the tokens
|
|
execute the action associated with the token pointed to by the Pass 2 token instruction position.
Implemented in Ogre::CompositorScriptCompiler, and Ogre::MaterialScriptCompiler. |
|
Extract a Non Terminal identifier from the token que.
|
|
Extract a numeric constant from the token que and add it to the current rule expression.
|
|
Extract a set from the token que and add to current rule expression.
|
|
Extract a Terminal lexeme from the token que and add to current rule expression.
|
|
find the eol charater
|
|
Get the start ID for auto generated token IDs. This is also one pass the end of manually set token IDs. Manually set Token ID are usually setup in the client code through an enum type so its best to make the last entry the auto ID start position and return this enum value. This method gets called automatically just prior to setupTokenDefinitions() to ensure that any tokens that are auto generated are placed after the manually set ones. Implemented in Ogre::CompositorScriptCompiler, and Ogre::MaterialScriptCompiler. |
|
Get the text representation of the rule path. This is a good way to way to visually verify that the BNF grammer did compile correctly.
|
|
gets BNF Grammer. Gets called when BNF grammer has to be compiled for the first time. Implemented in Ogre::CompositorScriptCompiler, and Ogre::MaterialScriptCompiler. |
|
get the name of the BNF grammer.
Implemented in Ogre::CompositorScriptCompiler, and Ogre::MaterialScriptCompiler. |
|
get the token ID for a lexeme in the client state. If the lexeme is not found then it is added to the map and definition container and a new tokenID created.
|
|
Gets the current token from the instruction que.
|
|
Get the current token's ID value.
Definition at line 431 of file OgreCompiler2Pass.h. |
|
Gets the next token's associated text label in the instruction que that was parsed from the text source. If an unkown token is found or no associated label was found then an exception is raised. The subclass should catch the exception, provide an error message, and attempt recovery. |
|
Get the current token's lexeme string. Handy when you don't want the ID but want the string representation. |
|
Gets the current token's associated floating point value in the instruction que that was parsed from the text source.
|
|
get the lexeme text of a rule.
|
|
Gets the next token from the instruction que.
Definition at line 356 of file OgreCompiler2Pass.h. |
|
Get the next token's ID value.
Definition at line 428 of file OgreCompiler2Pass.h. |
|
Gets the next token's associated text label in the instruction que that was parsed from the text source.
Definition at line 416 of file OgreCompiler2Pass.h. |
|
Get the next token's lexeme string. Handy when you don't want the ID but want the string representation. Definition at line 435 of file OgreCompiler2Pass.h. |
|
Gets the next token's associated floating point value in the instruction que that was parsed from the text source. If an unkown token is found or no associated value was found then an exception is raised but the instruction pointer is still moved passed the unknown token. The subclass should catch the exception, provide an error message, and attempt recovery. Definition at line 397 of file OgreCompiler2Pass.h. |
|
Gets the number of tokens waiting in the instruction que that need to be processed by an token action in pass 2.
|
|
Get the current position in the Pass2 Token Que.
Definition at line 465 of file OgreCompiler2Pass.h. |
|
Get the number of tokens not processed by action token. Client Actions should use this method to retreive the number of parameters(tokens) remaining to be processed in the action. |
|
initialize token definitions and rule paths
|
|
Check if source at current position is supposed to be a user defined character label. A new label is processed when previous operation was not _character_ otherwise the processed character (if match was found) is added to the current label. This allows _character_ operations to be chained together to form a crude regular expression to build a label.
|
|
Check if pass 1 has parsed to the end of the source.
Definition at line 533 of file OgreCompiler2Pass.h. |
|
check to see if the text at the present position in the source is a numerical constant
|
|
check to see if the text is in the lexeme text library
|
|
modify the last rule in the container. An end operation is added to the rule path.
|
|
position to the next possible valid sysmbol
|
|
process input source text using rulepath to determine allowed tokens
|
|
go back to the previous token in the pass2 queue.
|
|
setup ActiveContexts - should be called by subclass to setup initial language contexts
Definition at line 561 of file OgreCompiler2Pass.h. |
|
Sets up the parser rules for the client based on the BNF Grammer text passed in.
|
|
changes previous terminal token rule into a conditional terminal token insert rule
|
|
Set the position of the next token action in the Pass2 Token Que.
|
|
Manualy set the Pass2 Token que position.
|
|
setup client token definitions. Gets called when BNF grammer is being setup. Implemented in Ogre::CompositorScriptCompiler, and Ogre::MaterialScriptCompiler. |
|
comment specifiers are hard coded
|
|
find end of line marker and move past it
|
|
skip to the next token in the pass2 queue.
|
|
skip all the white space which includes spaces and tabs
|
|
If a current token instruction exist then test if its token ID matches.
Definition at line 382 of file OgreCompiler2Pass.h. |
|
If a next token instruction exist then test if its token ID matches.
|
|
check if current position in source has the lexeme text equivalent to the TokenID
|
|
scan through all the rules and initialize token definition with index to rules for non-terminal tokens. Gets called when internal grammer is being verified or after client grammer has been parsed.
|
|
Active Contexts pattern used in pass 1 to determine which tokens are valid for a certain context.
Definition at line 308 of file OgreCompiler2Pass.h. |
|
The active label that is receiving characters during pass 1.
Definition at line 296 of file OgreCompiler2Pass.h. |
|
the key of the active label being built during pass 1. a new key is calculated when mLabelIsActive switches from false to true Definition at line 294 of file OgreCompiler2Pass.h. |
|
Active token que, definitions, rules currntly being used by parser.
Definition at line 261 of file OgreCompiler2Pass.h. |
|
Definition at line 605 of file OgreCompiler2Pass.h. |
|
current line number in source being tokenized
Definition at line 279 of file OgreCompiler2Pass.h. |
|
Definition at line 258 of file OgreCompiler2Pass.h. |
|
Definition at line 608 of file OgreCompiler2Pass.h. |
|
storage container for constants defined in source container uses Token index as a key associated with a float constant
Definition at line 284 of file OgreCompiler2Pass.h. |
|
Definition at line 278 of file OgreCompiler2Pass.h. |
|
Definition at line 276 of file OgreCompiler2Pass.h. |
|
position in current line in source being tokenized
Definition at line 280 of file OgreCompiler2Pass.h. |
|
TokenID to insert if next rule finds a terminal token if zero then no token inserted.
Definition at line 305 of file OgreCompiler2Pass.h. |
|
flag indicates when a label is being parsed. It gets set false when a terminal token not of _character_ is encountered Definition at line 291 of file OgreCompiler2Pass.h. |
|
Definition at line 288 of file OgreCompiler2Pass.h. |
|
the que position for the next token that has an action.
Definition at line 270 of file OgreCompiler2Pass.h. |
|
flag being true indicates that spaces are not to be skipped automatically gets set to false when mLabelIsActive goes to false
Definition at line 299 of file OgreCompiler2Pass.h. |
|
if flag is true then next terminal token is not added to token que if found but does effect rule path flow
Definition at line 302 of file OgreCompiler2Pass.h. |
|
the location within the token instruction container where pass 2 is
Definition at line 263 of file OgreCompiler2Pass.h. |
|
the que position of the previous token that had an action. A token's action is fired on the next token having an action. Definition at line 267 of file OgreCompiler2Pass.h. |
|
pointer to the source to be compiled
Definition at line 273 of file OgreCompiler2Pass.h. |
|
name of the source to be compiled
Definition at line 275 of file OgreCompiler2Pass.h. |
|
Definition at line 177 of file OgreCompiler2Pass.h. |
Copyright © 2000-2005 by The OGRE Team
This work is licensed under a Creative Commons Attribution-ShareAlike 2.5 License.
Last modified Sun Jul 8 15:20:28 2007