import GrammarCompilerHelper from "./helper/GrammarCompilerHelper";
import GRule, { GElement } from "./grammar/GRule";
import IGRuleContext from "./grammar/IGRuleContext";
import GRuleContextHelper from "./grammar/GRuleContextHelper";
import Assert from "@cafetextual/util/dist/src/assert/Assert";


export default class GrammarCompiler 
{
	
	private static WS:string = " ";
	

	static mergeTokens(existing:Array<string> = null, v1:any= null, v2:any = null):Array<string>  {
		return GrammarCompilerHelper.mergeTokens(existing, v1, v2);
	}
	
	
	/**
	 * Recursive iteration across rules and elements to calculate next element tokens, ie tokens that cause the parser
	 * to skip to a further rule.
	 * 
	 * <p>
	 * 	 Which, recall, differers from possible next tokens is that it's only valid as the first character of a match
	 * 	 
	 *   ie (var1)?, X(var2)  // upon hitting (var1), if we encounter an X, skip to X(var2)
	 * 	 
	 * 
	 * </p>
	 * 
	 * @param isOR 
	 * @param parentTokens 
	 * @param asChild
	 * @param parentAllowNone
	 * @param suffixTokens - used only when we have a `(var) construct
	 */  
	static setUpNextTokens(r:GRule, isOR:boolean, stack:IGRuleContext, parentElementIndex:number = -1,
									parentTokens:Array<string> = null,  asChild:boolean = false, 
									parentAllowNone:boolean = false, parentAllowMany:boolean = false, 
									suffixTokens:Array<string> = null,
									// --- cf merging initialization algorithm
									parentIsOptional:boolean = false,  				
									// -- for context calculation - duplicates functionality, but is simpler  
									parentContextTokens:Array<string> = null
							):boolean {
		
		
		// tokens will keep track of non-blocked tokens as we move backwards
		var tokens:Array<string> = null;  
		
		stack.push(r, parentElementIndex);

		// ruleOrChildTokens will similarly keep track of non-blocked tokens if this is an OR rule
		//   - these are the tokens that sibling of the parent rule might see, not used in the individual elements
		r.ruleORChildTokens = new Array<string>();
		
		var origionalParentTokens:Array<string> = parentTokens;
		var repeatValueTokens:boolean
		
		
		// --- context tokens: simple calculation for the relevant context of the items.
		
		var contextTokens:Array<string> = parentContextTokens
		
		// 1. backwards iteration
		for (var i:number = r.elements.length - 1; i >= 0; i--) {
			var e:GElement = r.elements[i];
			var isLast:boolean = (i == (r.elements.length - 1));
			var isFirst:boolean = (i == 0);
			var allowNone:boolean = e.allowNone;
		
			// TODO - collapse allowNone and ParentIsPotionsl		
			var elementOptional:boolean = parentIsOptional || e.lineAllowNone || e.allowNone || e.isOR

			var hasChild:boolean = e.childRule != null || (e.childRuleName != null);
			var tokensFromChildRule:Array<string> = null;
			var allowMany:boolean = e.allowMany;
			var skipToToken:boolean = e.skipToToken
			
			
			// element is effectively optional when 
			// 	 i)  optional           (var)  X? 
			//   ii) parent is optional  [(var) X]? 
			//   iii) in an OR construct  [(var) | X]
			// var effectivelyOptionalElement:boolean = (allowNone || parentAllowNone || isOR)  -- can't actually use this because  ...
			
			
			// 1. last element consumes parent tokens (unless a token or non optional element blocks them)
			if (isLast) {
				if (parentTokens && !e.prefixTokenValue &&   // a prefix token would block propagations   
					(skipToToken || allowNone || parentAllowNone || isOR)      // so would an element that is  optional in a the larger context
				) {
					//  if this is the last element, then we need to set the possible next tokens on this itself,
					//   ( recall that in the rest of the algorithm we're calculating tokens for the previous element)
					e.nextElementTokens = this.mergeTokens(null, parentTokens);  // need to do this here as it's used when recursing on child
				}
				
				// (but note that if these conditions aren't met, parent tokens may still be propagated
				
				// 1.x merge in suffix token in a situation like   PAREN[ X `(var)  ]  
				
			}	
		
			// -------- calculate context tokens ----------  
			
			
			
			
			
			
			// 2. recurse on child rule 
			//    a) this will calculate e.childRule.ruleNextTokens (as well as values for child elements) 
			//        (we may use these later as tokensFromChildRule)
			//	  b) do not recurse is child rule is injected by reference. The algorithm at the grammar level
			//       will have ensured that the tokens have been previously calculated however.
			if (hasChild) {
				var elementContext:Array<string>;
				var elementSuffixContext:Array<string> 

				// ----- calculate context -----
				if (e.suffixTokenValue) {
					// x.  /w a suffix, no other context is relevant
					elementContext = null
					elementSuffixContext = this.mergeTokens(null, e.suffixTokenValue);
				} else if (this.isPrefixRepeat(e)) {
					elementContext = null;
					elementSuffixContext = this.mergeTokens(null, e.prefixTokenValue);

				} else {
					elementContext = contextTokens;    // contextTokens are the cumulative tokens
					elementSuffixContext = suffixTokens;
				}

				if (e.allowWS) {
					elementContext = this.mergeTokens(elementContext, this.WS);
				}


				var requiresInit:boolean = false;
				var requiresInjection:boolean = (e.childRule == null);
				if (requiresInjection) {
					var ok:boolean = stack.doInject(r, e, i, parentIsOptional, elementOptional, elementContext , elementSuffixContext);
					if (!ok) {
						stack.pop();
						return false;
					}
					requiresInit = stack.requiresInit(e);
				}
				
				var recurseOnRule:boolean = !e.ruleIsReference || requiresInit
				if (recurseOnRule) {
					//  suffix will block tokens, but otherwise simply propageate the tokens that we have.
					var parentTokensToPropagate:Array<string> = null
					
					if (e.suffixTokenValue == null) {
						// if is OR 
						parentTokensToPropagate = isOR ? origionalParentTokens : e.nextElementTokens;  // if element has a suffix token, don't propagate
																										// (recall that e.nextElementTokens c.f. previous element)
					}
					
					// repeated rule tokens - if a rule is repeated, we need to add it's own possible next tokend to it
					//   ie   [X?, :ruleX,  (var)?]*,  is eqivalent (from a token perspective) to [X?, :ruleX, (var)], [X?, :rule (var)]?  
					// 
					// note:
					//	- suffix, prefix, content tokens or require ws mean that there's no need to dwell on it
					//  - repeated rule tokens do not depend on other parent tokens, 
					//     which is why we can calcaulte them simply w/o needing to recurse in context
					//  - 
					if (       (skipToToken || (e.allowMany  && !e.requireWS) ) 
							&& (e.suffixTokenValue == null) && (e.contentTokens == null)   ) {
						
						parentTokensToPropagate = parentTokensToPropagate ? parentTokensToPropagate : new Array<string>()
						
						if (e.prefixTokenValue) {
							this.mergeTokens(parentTokensToPropagate, e.prefixTokenValue)
						} else if (e.contentTokens) {
							this.mergeTokens(parentTokensToPropagate, e.contentTokens);
						} else {
							var repeatedRuleTokens:Array<string> = GRuleContextHelper.calculateRepeatedRuleTokens(e.childRule, false,  stack.getRuleFn() );
							this.mergeTokens(parentTokensToPropagate, repeatedRuleTokens);
						}
					}
					
					//var orElementsToPropagate:Array<string> = isOR ? ruleORChildTokens : null;
					
					
					
					if (e.childRule == null) {
						Assert.assert(r.isValid() == false);
						stack.pop();
						return false;
					}
					
					ok = this.setUpNextTokens(e.childRule, e.childRule.isOR, stack, i, parentTokensToPropagate, true /*as Child*/, 
						(allowNone || parentAllowNone || isOR),   // effectively optional --> parentAllowNone
						allowMany 
						//orElementsToPropagate
						, e.suffixTokenValue ? this.mergeTokens(null, e.suffixTokenValue) : suffixTokens
						, elementOptional, contextTokens
					);			
					
					if (!ok) {
						stack.pop();
						return false;
					}
					
					if (requiresInit) {
						// setPossibleNextTokens(e.childRule);	
					}
					
					
				} else {
				// rule is reference, so we don't recurse here, tokens will be extracted by alternate means below
				
					
					if (!e.childRule.isValid()) {
						return false;
					}
				}
			}
			
			
			
			
			// X. --- update general context ---- 
			// X.i tokens from this element only
			var eTokens:Array<string> = this.getElementTokens(e, stack.getRuleFn());   // (note that convergence of this algroithm is guaranteed only following the 
																				//      validation in the injection algorithm) 
			
			if (isOR) {
				if (eTokens) {
					contextTokens = this.mergeTokens(contextTokens, eTokens); // recalling that parent tokens will be in here also
				} 
				
			} else {
				if (allowNone)   {
					contextTokens = this.mergeTokens(contextTokens, eTokens)
				} else {
					contextTokens = eTokens;
				}
				
			}
			
			
			// 3. If rule has a prefix token, or content tokens, then add this to the cumulative tokens
			if (e.prefixTokenValue || e.contentTokens) { 
				//  note if we have a prefix token:
				// 		a) we can ignore tokens from the child rule
				//      b) we're not going to be adding tokens to this element,
				//          but if its optional, we still need to proagate tokens
				//      c) consume parent tokens (into tokens, not the element) if they haven't been consumed yet 
				
				// 3a. consume or disgard parentTokens
				if (skipToToken || allowNone || isOR) {
					Assert.assert(!tokens || !parentTokens); // if we 've already got tokens, they should have already consumed the parent tokens
					tokens = this.mergeTokens(tokens, parentTokens); 
					// note - or tokens do not 
				} else {
					// only use previous one :: ---> 
					tokens = this.mergeTokens();  // ie create empty vector
				}
				
				parentTokens = null;  // parent tokens are now either consumed or disgarded
				
				// 3b  consume prefix/content tokens
				// 		--> also note: that ruleORChildTokens get the same 
				if (e.prefixTokenValue) {
					tokens = this.mergeTokens(tokens, e.prefixTokenValue);
					r.ruleORChildTokens = this.mergeTokens(r.ruleORChildTokens, e.prefixTokenValue);
				} else {
					Assert.assert(e.contentTokens.length > 0);
					tokens = this.mergeTokens( tokens, e.contentTokens.reverse());
					r.ruleORChildTokens = this.mergeTokens(r.ruleORChildTokens, e.contentTokens.reverse());   // 
					//for (var j:number = e.contentTokens.length -1; j >=0; j--) {
					//	tokens.push(e.contentTokens[j]);
					//	r.ruleORChildTokens = mergeTokens(r.ruleORChildTokens, e.contentTokens[j]);   // 
					//}
				}
				
			} else if (hasChild && (e.childRule.ruleNextElementTokens || e.ruleIsReference)  ) {  
				// 4.  no prefix/content token(s), but child has tokens, so add them
				//    a) first add the childRule's tokens (calculated in the above recursive call)
				
				if (!e.ruleIsReference) {
					// 4a. regular rule, consume tokens from child rule
					tokensFromChildRule = e.childRule.ruleNextElementTokens;  
					if (!tokensFromChildRule) {
						Assert.assert(tokensFromChildRule.length > 0);				  // require null if zero	
					}
					// 4a.i. merge child OR tokens
					this.mergeTokens(r.ruleORChildTokens, e.childRule.ruleORChildTokens);   // For OR, all tokens, but w/o parentTokens mixed in
				} else {
					// 4b. reference rule. 
					
					// child rule is referenced, so can't recurse, convergence requirements of the grammar  ensure we can calcualte tokens like this.
					// recall that if this child rule is here by reference, then either 
					// 		i.  it will be a simple rule ie PAREN(var)[...] independant of context
					// 		ii. it will have be a copy of rule higher in the call chain, which won't have 
					//          tokens calculated yet as this is (necessarily) a depth first algorithm.
					//          In which case, we need to have constrained such rules so that we readily calculate it's tokens
					
					tokensFromChildRule = new Array<string>();
					this.getDeterminateTokensFromReferencedRule(e.childRule, tokensFromChildRule, false, stack.getRuleFn()); // note that there is no dependancy on parentTokens										
					// 4b.i merge chidlTokens to childORTokens 
					//  note that we can use  tokensFromChildRule here beacuse we're guaranteed no dependancy of parentTokens 
					
					r.ruleORChildTokens = this.mergeTokens(r.ruleORChildTokens, tokensFromChildRule);
				}
				
				if (allowNone || parentAllowNone || skipToToken) {
					// 4c.  if element is optional, may still need to consume parentTokens
					Assert.assert(!parentTokens || !tokens);
					tokens = this.mergeTokens(tokens, tokensFromChildRule, parentTokens); // propagate any tokens that propagate through child rule
				} else if (e.childRule.isOR) {
					// 4d. if rule is OR,  may need to consume the child rules ORTokens
					tokens = this.mergeTokens(null, e.childRule.ruleORChildTokens);  // if child is non-optional OR, do not propagate parent tokens, as something in that child rule must match.
				} else {
					// 4c. by default 
					tokens = this.mergeTokens(tokens, tokensFromChildRule, parentTokens);
				}
				
				parentTokens = null;  
				
			} else if (parentTokens) {
				// 5. no prefix token, nor child tokens to consume, but still may need to consume parentTokens
				if (allowNone || skipToToken) {
					tokens = this.mergeTokens(tokens, parentTokens);
					// again note that OR tokens aren't consuming parentTokens
				}
				parentTokens = null;
			} else {
				// 6. no prefix token, no content tokens, no child tokens, no parent tokens ==> no action required
			}
			
			// 7. use parentAllowNone to track whether we've hit a non-optional element since the parent rule
			// ie as soon as we hit a no optional element, then turn parentAllowNone off  
			//     [(var) X (var2)?]? Y(var3)   <-- X will turn off parent allow none   ( [(var) (varX) (var2?]? would yield the same) 
			//                                       elements previous to X have a determine  element 
			//    unless
			//     [(var) `(varX)  (var2)?]? Y   <-- here Y propagates thorough (var2) b/c its optional, and `(varX) b/c because of the skipToToken mechanism, regardless of whether it's optional    
			if ((!e.allowNone  && !e.skipToToken) && !isOR) {
				parentAllowNone = false;
				suffixTokens = null;
			}
			
			// 8. finalise which the nextElementTokens that the previous element will see
			//   note: if !parentAllowNone then need to use tokens as they will have have consumed parentTokens
			//         wheras a non-optional OR rule must match something.
			var elementTokens:Array<string>  = this.cloneTokens( (isOR  && !parentAllowNone) ?  r.ruleORChildTokens  : tokens);
			elementTokens = elementTokens.length > 0  ? elementTokens : null;
			
			// x. if we have somethign like PAREN[ X `(var) Y? ] then (var) get the parent suffix tokens
			//    (note that suffix tokens are cleared upon a non non-optional, not ` element
			if (skipToToken && suffixTokens) {
				e.nextElementTokens = this.mergeTokens(e.nextElementTokens ,suffixTokens)
			}
			
			
			// 9.  set calculated tokens
			var previousElement:GElement = isFirst ? null : r.elements[i-1];  // previous element in definition, actually the next element in our iteration
			
			if ( elementTokens && (elementTokens.length > 0 ) ) { 
				if (isFirst) {
					// 9a. no more elements in this rule, so what we've calculated is set of next element tokens for the entire rule
					//     which may then be picked up by the parent element. 
					
					if (isOR ||  allowNone ) {
						r.ruleNextElementTokens = elementTokens   // tokens 
					} else if (eTokens) { 
						r.ruleNextElementTokens = eTokens         // these are tokens local to this one one
					}
				
					//r.ruleORChildTokens = r.ruleORChildTokens; //  this is an alternate version of possible next tokens for an ORed rule
				} else if (isOR) {
					previousElement.nextElementTokens = origionalParentTokens && origionalParentTokens.length > 0 ? this.cloneTokens(origionalParentTokens) : null;
				} else if (this.requiresNextTokens(previousElement)) {            //        - or explicitly requests to collide /w invoming tokens `(varX)
					// 9b. simple assign to previos element 
					previousElement.nextElementTokens = elementTokens; 
					
				}
				if (isOR && !isLast && e.skipToToken) {
					e.nextElementTokens = this.mergeTokens(e.nextElementTokens, elementTokens);
				} 
				
			}
			
		} // backwards iteration over child elements
		
		// 10. Finally, at the top level, recurse over all elements and invert token orders 
		if (!asChild) {
			this.invertTokenOrders(r);	
		}
		stack.pop();
		
		return true
	} // setUpNextTokens		
	
	
	
	static requiresNextTokens(e:GElement):boolean {
		return ((e.childRule != null ) ||                 //   [X (var1)?] Z   <-- FIX_THIS - what does this mean?? probably that the algorithm has previous taken careof this by calculuating what propagates through?
					(  (e.prefixTokenValue == null && e.contentTokens == null)    &&    //  b/c  X(var) would block tokensly
						(e.allowNone                     // and    - (var)?    is optional (which includes or)
							|| e.skipToToken
						)  )             //      or `(var)     skipping mechanism   
					
				);
	}
	static getElementTokens(e:GElement, getRule:Function):Array<string> {
		if (e.prefixTokenValue) {   // TODO - cache - this is a universal propery of rules. So long as it's called after rules have been recursively injected
			return  [e.prefixTokenValue] as Array<string>;
		}
		if (e.contentTokens) {
			return this.mergeTokens(null, e.contentTokens);
		}
		if (e.childRule) {
			var out:Array<string> = new Array<string>()
			this.getDeterminateTokensFromReferencedRule(e.childRule, out, true, getRule);
			return out;
		}
		return null;
	}
	
	/**
	 * Calculated the possible next token of a variable.
	 * 
	 * <p> Used when an element's prefix has already been matched. Output is vector of tokens of characters that
	 *  indicate the start of a new rule.</p>
	 * 
	 *  examples:
	 * 
	 *    X(var1).[Y | Z]   // <-- possible next tokens are [Y, Z, ' ' ];  
	 * 	  PAREN[(var1)]      // <-- possible next token with be [')', ' '], as suffix of parent needs to propagate down.
	 * ,
	 * 
	 * <p> Differs from nextElementToken in that 
	 *      a) Used when matching (or attempting to match) a variable within an element (ie after a prefix token, if present has 
	 *         been matched. Wheras nextElement token is used to detetermin which element to attempt to match by 
	 *         looking at possible prefix tokens 
	 *
	 *      b) It therfore pays attention to whitespace, and it's own suffix  and prefix tokens, wheras nextElementTtoken
	 *         only cares about tokes of possible next prefix (or content) token
	 * 
	 * </p>
	 * 
	 * @param parentTokens tokens from subsequent elements (as we iterate backwards)
	 * 		ie   [(var1) | Y].  Z?. Q?.   <-- Z and Q will are parent tokens     
	 *  
	 * @param ixTokens parent prefix and suffix(ie *ix) tokens  
	 * 		  ie PAREN[X[(var1) | Y?]*]   <--- 'X' is repeated prefix token that needs to propagate down
	 * 									  <--- ')' also needs to propagate   
	 * 
	 * 
	 */
	static setPossibleNextTokens(rule:GRule, parentTokens_pnt:Array<string>= null, ixTokens:Array<string> = null):Array<string> {
		// TODO - extract to seep gramar. This is not a run time case
		
		var origionalIXTokens:Array<string> = ixTokens;
		var origionalParentTokens_pnt:Array<string> = parentTokens_pnt 
		var tokens_pnt:Array<string> = this.mergeTokens(null, parentTokens_pnt);
		
		
		
		var isOR:boolean = rule.isOR;   
		
		// prefix & suffix tokens
		ixTokens = ixTokens ? ixTokens : this.mergeTokens(null); // create if necessary
		
		// iterate backwards over elements
		for  (var i:number = rule.elements.length-1; i>=0 ; i--) {
			
			var element:GElement = rule.elements[i];
			// element token are 
			//      - for an OR rule, include *ix token, local suffix. In general tokens that won't necessarily propagate on to
			//        previous elements like parent rules
			var elementTokens_pnt:Array<string> = new Array<string>();
			var isLeaf:boolean = (element.childRule == null); 
			//Assert.assert(isOR == element.isOR); 
			
			// indicate that a prefix is repeated and there is no WS or suffix to get in the way ie X(var).* 
			var repeatedPrefix:boolean = this.isPrefixRepeat(element);
			
			if (element.requireWS && isOR) {
				Assert.fail(); // grammar validation should catch this
				//Assert.assert(!(element.requireWS && isOR) ); // OR cannot be combined /w 
			}
			
			if (element.requireWS && !isOR) {
				tokens_pnt = new Array<string>();        // reset tokens completely if whitespace required  (but not if this is an ORed rule) 			
				ixTokens = new Array<string>();      // xsimilarly parent tokens 
			}
			
			if (isLeaf) {
				// 1. a leaf element , ie  X(var1)  or PAREN(var1)
				
				
				if (element.suffixTokenValue) {  
					// 1a. suffix on a leaf -  a suffix value trumps all other values as the only possible next token
					this.mergeTokens(elementTokens_pnt ,element.suffixTokenValue);
				} else {
					// 1b. add suffix and parent tokens 
					
					if (isOR) {
						this.mergeTokens(elementTokens_pnt, origionalIXTokens);  
						this.mergeTokens(elementTokens_pnt, origionalParentTokens_pnt);
					} else {
						this.mergeTokens(elementTokens_pnt, ixTokens);  
						this.mergeTokens(elementTokens_pnt, tokens_pnt);   //  no suffix, so need to consume history of tokens						
					}
					
					// 1c. if item repeats, has prefix, doesn't require ws etc 
					if (repeatedPrefix) { //   ie X(var0)*.  or X(var0)*,  
						this.mergeTokens(elementTokens_pnt, element.prefixTokenValue);
					} 
					
				}
				
				// 1d. add ws 
				elementTokens_pnt = this.mergeTokens(elementTokens_pnt, this.WS); // <---- currently ws is always recognised as a possible next token - although not necessarily a valid one 
				this.reverseTokens(elementTokens_pnt);
				element.possibleNextTokens = elementTokens_pnt;
				
			} else {
				
				// 2. non-leaf element
				
				// recurse on child rule
				var childTokens_pnt:Array<string>;	// childTokens are returned from recursing on the child rule				
				var childIXTokens:Array<string>;  // childIXTokens are
				
				// referenceElementTokens  - tokens 
				var referenceElementTokens:Array<string> = null;
				var referenceElementsCompleted:boolean = false;
				
				// 2a. if this is a reference element, first calculate contextless possible next tokens 
				if (element.ruleIsReference) {
					referenceElementTokens = new Array<string>();
					if (element.prefixTokenValue || element.contentTokens) {
						referenceElementsCompleted = false;
						this.mergeTokens(referenceElementTokens, element.prefixTokenValue, element.contentTokens)
					} else {
						referenceElementsCompleted = this.getDeterminateTokensFromReferencedRule(element.childRule, referenceElementTokens, true);
					}
				} 
				
				
				if (element.suffixTokenValue) {
					
					childIXTokens = new Array<string>(); 
					
					if (!element.ruleIsReference) {
						// 2c non leaf, but /w suffix token only need to downwardly propagate that single suffix token
						childIXTokens = this.mergeTokens(childIXTokens, element.suffixTokenValue);  			// to only valid toke to be sent is the prefix of the current element
						childTokens_pnt = this.setPossibleNextTokens(element.childRule, childIXTokens);  // <--- child IXtokens become context tokens
						
						
					} else {
						// 2d. non-leaf /w suffix token, but rule is reference, and therefore at least pottentiall recursive.  
						//
						// case1: 
						//       rule0:   PAREN(var)[:rule1]
						//       rule1:   X  (var2):rule0     	
						//
						//     - here the grammar validation will have ensured that rule0 is context independant, so it's always safe to inject by reference
						//     - previously we could just take the prefix token of the referenced rule (which is the example would be '(' )
						// 
						// case2: 
						//    ie rule0:   ...  (var)  ... PAREN(var):rule0?  ..  <-- we only require one preceeding non-optional element
						//     or rule0: ... [ X  :rule0]  ...                   <-- may also be in a unnamed child rule     	
						
						//     - currently a rule may only be injected into itself					
						//     - the acutal next tokens of the child rule are calculated elsewhere, ie
						
						//            rule0: X(var1) (var2):rule0?   -->  rule0: X(var1) (var2)[X(var1) (var2)[X(var1) (var2):rule0]? ]?  
						//      										   ^ 			  ^              ^
						//												   |			  |              +---  not cloned. this is the same instance of the previous one
						//         										   |			  +-- This is the first instance in the chain will cloned. When we get the the point of dealing /w a reference, we recursing through this (cloned) instance 
						// 												   +--- the root instance. 					
						// 
						//    - the actual next value tokens will be calculated elsewhere, so we needn't calcuate them here.
						
						// if there's a path throught the child rule, then also add the suffix token]
						// ie 
						//      rule0:      X  PAREN:rule0?]? )
						
						if (referenceElementsCompleted) {
							// this should never happen
							//  the conditions set upon a recursive rule prevent it
							var offendingValue:boolean = this.getDeterminateTokensFromReferencedRule(element.childRule, referenceElementTokens, true);
							Assert.fail();
							childTokens_pnt.push(element.suffixTokenValue);
						}
						
						// need to iterate to get next possilble tokens 
						//   ie rule0: [X | Y]?  Z (var1):rule0*   --> possible next tokens are X, Y & Z 
						//             [X | (var3)]? [Z (var1):rule0*]  --> X and whitespace only. 
						
						childTokens_pnt = new Array<string>();
						this.mergeTokens(childTokens_pnt, referenceElementTokens); // as referenced child rule must have prefix and suffix tokens, the only possible child result     
						
					} 
				} else {
					if (!element.ruleIsReference) { 
						// 3a. non-leaf, no suffix token. Need here to propagate all parent prefix and suffix tokens
						
						childIXTokens = this.mergeTokens(null, ixTokens);
						
						if (repeatedPrefix) {
							childIXTokens.push(element.prefixTokenValue);    // ie DOT(var)*  ==> add the prefix to
						}
						
						var baseTokens:Array<string> = isOR ? origionalParentTokens_pnt : tokens_pnt
						
						if (element.allowMany && !element.requireWS && element.prefixTokenValue == null 
							&& element.suffixTokenValue == null && element.contentTokens == null) {
							
							var repeatedRuleTokens:Array<string> =  GRuleContextHelper.calculateRepeatedRuleTokens(element.childRule, false);
							
							var allChildTokens:Array<string> = this.mergeTokens(null, baseTokens, repeatedRuleTokens);
							childTokens_pnt = this.setPossibleNextTokens(element.childRule, allChildTokens, childIXTokens)
						} else {
							childTokens_pnt = this.setPossibleNextTokens(element.childRule, baseTokens, childIXTokens);// only parent tokens propagate to child rule in an OR 
						}
					} else {
						// don't recurse if child is injected by reference. Grammar enforces constraints to ensure we can calculate tokens w/o recursion.
						// 3b.
						
						// 3c. repeated prefix - for a reference element there's no need for special treatment 
						//if (repeatedPrefix) {   // rule0: X (var)? Y(var):rule0*. 
						//
						// }
						
						// 3d. if there's no definative blocking element in the child rule, we need to explicitly 
						//     add them to child tokens
						childTokens_pnt = new Array<string>();
						
						
						
						
						
						
						// 3e. and finally, add the elements 
						this.mergeTokens(childTokens_pnt, referenceElementTokens);
						
						//childTokens.push(" ");
						this.reverseTokens(childTokens_pnt);
						
					}			
					
				} 
				
				
			} // end recurse on child
			
			
			
			// 4.  --- from this point on, we've completed the calculation for this element, and 
			//         look to what elements return that might be propagated to the next (ie previous) element ---
			//    (recall: tokens has not been altered in the previous  
			
			//  4a) conditions in which we need to abandon previous tokens
			
			var resetTokens:boolean = (!element.allowNone || element.requireWS) && !isOR 
			
			if (resetTokens && isLeaf)  {
				tokens_pnt = new Array<string>();
			}
			
			// 5)  add  prefix or content tokens for next item
			if (element.prefixTokenValue || element.contentTokens) {
				// 5a.  reset tokens ( ie  
				
				if (resetTokens) {    // X[(var1)] Y(var) or X[(var)]?Y(var)  => drop later tokens
					tokens_pnt = new Array<string>();  // reset tokens 
				} 
				
				// 5. add the prefix/content tokens 
				if(element.prefixTokenValue) {
					tokens_pnt.push(element.prefixTokenValue );
				} else if (element.contentTokens) { 									    // append this elements prefix or content token(s)
					GrammarCompilerHelper.mergeArray(tokens_pnt, element.contentTokens, true /*backwards*/);	  				
				}
				
			} else if (!isLeaf) {
				// 6. non-leaf, no prefix/content tokens
				// keep old tokens if:
				if (element.allowNone ||    // <next-element>.   [X? Y]?. Z     <-- next element *may* depend on Z if [X? Y] not matched
					element.requireWS) {	// <next-element>.   [X? Y] Z          
					
					// 5a. 
					tokens_pnt = this.mergeTokens(tokens_pnt, childTokens_pnt);   //  [X(var1)? Z(varx)]?Y(var2)  - remove keep the Y from tokens, and add the X, Z from childTokens  
					
					
				} else if (element.childRule.isOR) {
					
					
					// we can do this even on a reference variable becase it has previously been calculated
					tokens_pnt = this.cloneTokens(element.childRule.ruleORChildTokens);   
					
				} else { 
					tokens_pnt = childTokens_pnt;						  //[X(var1)]Z(var2)    disgard the Z 
				}
			} 
			
			
		} // iteration over elements in this rule  
		
		// could also save the tokens on the rule, maybe for testing.
		// rule.rulePossibleNextTokens = mergeTokens(null, tokens); // this is 
		
		
		// 3. finally, return tokens, which is tokens inherited 
		return tokens_pnt;  
		
		
	} // setPossibleNextTokens		

	static isPrefixRepeat(e:GElement):boolean {
		return (!e.requireWS && e.allowMany && e.prefixTokenValue && !e.suffixTokenValue)
	}
	
	
	
	
	
	/**
	 * forward iteration  
	 * 
	 * Note that this is non-context dependant, so could readily be cached, including on parents if cloned.
	 * 
	 * @param stopOnWS whether required ws should halt iteration if it hits ws (ie (var)~ :rule).
	 *        (True when calculaating possible next token as opposed to next element token).
	 * 
	 *  
	 * 
	 * @return true if completed  (and false if we've hit a token that we ca
	 */
	static getDeterminateTokensFromReferencedRule(rule:GRule, out:Array<string>, stopOnWS:boolean = false, getRule:Function = null):boolean {
		return this.calculateReferencedTokens_low(rule, out, stopOnWS, getRule);
	}
	
	
	
	/**
	 * Context independant forward iteration over tokens. 
	 */
	static calculateReferencedTokens_low(rule:GRule, out:Array<string>, stopOnWS:boolean  = false, getRule:Function = null):boolean {
		return GRuleContextHelper.calculateReferenceTokens(rule, out, stopOnWS, true, getRule);
	} // calculateReferencedTokens_low 
	
	
	
	
	
	
	
	
	
	static cloneTokens(v:Array<string>):Array<string> {
		return this.mergeTokens(null, v)
	}
	
			/**
	 * recurse through rules and their elements inverting the order of the calculated tokens
	 */
	static invertTokenOrders(rule:GRule):void {
		this.reverseTokens(rule.ruleNextElementTokens);
		var e:GElement
		for (e of rule.elements) {
			this.reverseTokens(e.nextElementTokens);
			if (e.childRule && !e.ruleIsReference) { 
				this.invertTokenOrders(e.childRule);
			}
		}
		
	} // invertTokenOrders
	
	
	/**
	 * swaps the order of 
	 */
	static reverseTokens(v:Array<string>):void {
		if (v) {
			for (var i:number = 0; i < (v.length/2); i++) {
				var endIndex:number = v.length - 1 - i;
				var temp:string  = v[endIndex];
				v[endIndex] = v[i];
				v[i] = temp;
			} 
		}
	} // swapOrder 
			
	
} // class