Add support for third-party regex module

Fixes #26.

tests/test_lex.py

@@ -1,6 +1,11 @@
 import pytest
 from sly import Lexer
 
+try:
+    import regex
+except ImportError:
+    regex = None
+
 class CalcLexer(Lexer):
     # Set of token names.   This is always required
     tokens = {
@@ -56,6 +61,29 @@ class CalcLexer(Lexer):
     def __init__(self):
         self.errors = []
 
+if regex is not None:
+    class RegexModuleCalcLexer(Lexer):
+        regex_module = regex
+
+        tokens = { 'ID', 'PLUS', 'MINUS' }
+
+        literals = { '(', ')' }
+        ignore = ' \t'
+
+        ID      = r'\p{Ll}+'  # Unicode lowercase letters, regex module feature
+        PLUS    = r'\+'
+        MINUS   = r'-'
+
+        ignore_comment = r'\#.*'
+
+        @_(r'\n+')
+        def newline(self, t):
+            self.lineno += t.value.count('\n')
+
+        def ID(self, t):
+            t.value = t.value.upper()
+            return t
+
 # Test basic recognition of various tokens and literals
 def test_tokens():
     lexer = CalcLexer()
@@ -65,6 +93,17 @@ def test_tokens():
     assert types == ['ID','NUMBER','PLUS','MINUS','TIMES','DIVIDE','ASSIGN','LT','LE','(',')']
     assert vals == ['ABC', 123, '+', '-', '*', '/', '=', '<', '<=', '(', ')']
 
+# Test third-party regex module support
+@pytest.mark.skipif(regex is None,
+                    reason="third-party regex module not installed")
+def test_3rd_party_regex_module():
+    lexer = RegexModuleCalcLexer()
+    toks = list(lexer.tokenize('a + b - c'))
+    types = [t.type for t in toks]
+    vals = [t.value for t in toks]
+    assert types == ['ID','PLUS','ID','MINUS','ID']
+    assert vals == ['A', '+', 'B', '-', 'C']
+
 # Test ignored comments and newlines
 def test_ignored():
     lexer = CalcLexer()