Added Lexer state change

sly/lex.py

@@ -32,7 +32,7 @@
 # -----------------------------------------------------------------------------
 
 __version__    = '0.1'
-__all__ = ['Lexer']
+__all__ = ['Lexer', 'LexerStateChange']
 
 import re
 from collections import OrderedDict
@@ -62,6 +62,14 @@ class LexerBuildError(Exception):
     '''
     pass
 
+class LexerStateChange(Exception):
+    '''
+    Exception raised to force a lexing state change
+    '''
+    def __init__(self, newstate, tok=None):
+        self.newstate = newstate
+        self.tok = tok
+
 class Token(object):
     '''
     Representation of a single token.
@@ -192,6 +200,7 @@ class Lexer(metaclass=LexerMeta):
             raise LexerBuildError('literals must be specified as strings')
 
     def tokenize(self, text, lineno=1, index=0):
+        while True:
             # Local copies of frequently used values
             _ignored_tokens = self._ignored_tokens
             _master_re = self._master_re
@@ -207,7 +216,7 @@ class Lexer(metaclass=LexerMeta):
                             index += 1
                             continue
                     except IndexError:
-                    break
+                        return
 
                     tok = Token()
                     tok.lineno = lineno
@@ -246,6 +255,11 @@ class Lexer(metaclass=LexerMeta):
                             index = self.index
                             lineno = self.lineno
 
+            except LexerStateChange as e:
+                self.__class__ = e.newstate
+                if e.tok:
+                    yield e.tok
+
             # Set the final state of the lexer before exiting (even if exception)
             finally:
                 self.text = text