sst/sly
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvements to decorators. Change to yacc grammar specification

Browse Source
This commit is contained in:
David Beazley 2016-09-07 13:10:58 -05:00
parent 7bfadaaab3
commit a2a345e689
3 changed files with 82 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
example/calc/calc.py
View File

@ -42,19 +42,19 @@ class CalcParser(Parser):
def __init__(self): def __init__(self):
self.names = { } self.names = { }
@_('statement : NAME "=" expression') @_('NAME "=" expression')
def statement_assign(self, p): def statement(self, p):
self.names[p[1]] = p[3] self.names[p[1]] = p[3]
@_('statement : expression') @_('expression')
def statement_expr(self, p): def statement(self, p):
print(p[1]) print(p[1])
@_('''expression : expression '+' expression @_('expression "+" expression',
| expression '-' expression 'expression "-" expression',
| expression '*' expression 'expression "*" expression',
| expression '/' expression''') 'expression "/" expression')
def expression_binop(self, p): def expression(self, p):
if p[2] == '+': if p[2] == '+':
p[0] = p[1] + p[3] p[0] = p[1] + p[3]
elif p[2] == '-': elif p[2] == '-':
@ -64,20 +64,20 @@ class CalcParser(Parser):
elif p[2] == '/': elif p[2] == '/':
p[0] = p[1] / p[3] p[0] = p[1] / p[3]
@_('expression : "-" expression %prec UMINUS') @_('"-" expression %prec UMINUS')
def expression_uminus(self, p): def expression(self, p):
p[0] = -p[2] p[0] = -p[2]
@_('expression : "(" expression ")"') @_('"(" expression ")"')
def expression_group(self, p): def expression(self, p):
p[0] = p[2] p[0] = p[2]
@_('expression : NUMBER') @_('NUMBER')
def expression_number(self, p): def expression(self, p):
p[0] = p[1] p[0] = p[1]
@_('expression : NAME') @_('NAME')
def expression_name(self, p): def expression(self, p):
try: try:
p[0] = self.names[p[1]] p[0] = self.names[p[1]]
except LookupError: except LookupError:

15
sly/lex.py
View File

@ -84,15 +84,16 @@ class LexerMeta(type):
@classmethod @classmethod
def __prepare__(meta, *args, **kwargs): def __prepare__(meta, *args, **kwargs):
d = NoDupeDict() d = NoDupeDict()
def _(pattern): def _(*patterns):
def decorate(func): def decorate(func):
if hasattr(func, 'pattern'): for pattern in patterns:
if isinstance(pattern, str): if hasattr(func, 'pattern'):
func.pattern = ''.join(['(', pattern, ')|(', func.pattern, ')']) if isinstance(pattern, str):
func.pattern = ''.join(['(', pattern, ')|(', func.pattern, ')'])
else:
func.pattern = b''.join([b'(', pattern, b')|(', func.pattern, b')'])
else: else:
func.pattern = b''.join([b'(', pattern, b')|(', func.pattern, b')']) func.pattern = pattern
else:
func.pattern = pattern
return func return func
return decorate return decorate
d['_'] = _ d['_'] = _

110
sly/yacc.py
View File

@ -383,18 +383,15 @@ class Grammar(object):
# Look for literal tokens # Look for literal tokens
for n, s in enumerate(syms): for n, s in enumerate(syms):
if s[0] in "'\"": if s[0] in "'\"" and s[0] == s[-1]:
try: c = s[1:-1]
c = eval(s) if (len(c) != 1):
if (len(c) > 1): raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' %
raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' % (file, line, s, prodname))
(file, line, s, prodname)) if c not in self.Terminals:
if c not in self.Terminals: self.Terminals[c] = []
self.Terminals[c] = [] syms[n] = c
syms[n] = c continue
continue
except SyntaxError:
pass
# Determine the precedence level # Determine the precedence level
if '%prec' in syms: if '%prec' in syms:
@ -1502,43 +1499,54 @@ class LRTable(object):
return '\n'.join(out) return '\n'.join(out)
# ----------------------------------------------------------------------------- # Collect grammar rules from a function
# _parse_grammar_rule() def _collect_grammar_rules(func):
#
# This takes a raw grammar rule string and parses it into production data
# -----------------------------------------------------------------------------
def _parse_grammar_rule(doc, file, line):
grammar = [] grammar = []
# Split the doc string into lines while func:
pstrings = doc.splitlines() prodname = func.__name__
lastp = None unwrapped = inspect.unwrap(func)
for dline, ps in enumerate(pstrings, start=line): filename = unwrapped.__code__.co_filename
p = ps.split() lineno = unwrapped.__code__.co_firstlineno
if not p: for rule, lineno in zip(func.rules, range(lineno+len(func.rules)-1, 0, -1)):
continue syms = rule.split()
try: if syms[1:2] == [':'] or syms[1:2] == ['::=']:
if p[0] == '|': grammar.append((func, filename, lineno, syms[0], syms[2:]))
# This is a continuation of a previous rule
if not lastp:
raise SyntaxError("%s:%d: Misplaced '|'" % (file, dline))
prodname = lastp
syms = p[1:]
else: else:
prodname = lastp = p[0] grammar.append((func, filename, lineno, prodname, syms))
syms = p[2:] func = getattr(func, 'next_func', None)
assign = p[1]
if assign != ':' and assign != '::=':
raise SyntaxError("%s:%d: Syntax error. Expected ':'" % (file, dline))
grammar.append((file, dline, prodname, syms))
except SyntaxError:
raise
except Exception:
raise SyntaxError('%s:%d: Syntax error in rule %r' % (file, dline, ps.strip()))
return grammar return grammar
class Parser(metaclass=RuleMeta): class OverloadDict(OrderedDict):
'''
Dictionary that allows decorated grammar rule functions to be overloaded
'''
def __setitem__(self, key, value):
if key in self and callable(value) and hasattr(value, 'rules'):
value.next_func = self[key]
super().__setitem__(key, value)
class ParserMeta(type):
@classmethod
def __prepare__(meta, *args, **kwargs):
d = OverloadDict()
def _(*rules):
def decorate(func):
if hasattr(func, 'rules'):
func.rules.extend(rules[::-1])
else:
func.rules = list(rules[::-1])
return func
return decorate
d['_'] = _
return d
def __new__(meta, clsname, bases, attributes):
del attributes['_']
cls = super().__new__(meta, clsname, bases, attributes)
cls._build(list(attributes.items()))
return cls
class Parser(metaclass=ParserMeta):
# Logging object where debugging/diagnostic messages are sent # Logging object where debugging/diagnostic messages are sent
log = PlyLogger(sys.stderr) log = PlyLogger(sys.stderr)
@ -1625,21 +1633,17 @@ class Parser(metaclass=RuleMeta):
fail = True fail = True
for name, func in rules: for name, func in rules:
# Possible validation of function arguments?
try: try:
file = inspect.getsourcefile(func) parsed_rule = _collect_grammar_rules(func)
line = func.__code__.co_firstlineno for pfunc, rulefile, ruleline, prodname, syms in parsed_rule:
parsed_rule = _parse_grammar_rule(func.rule, file, line)
for rulefile, ruleline, prodname, syms in parsed_rule:
try: try:
grammar.add_production(prodname, syms, func, rulefile, ruleline) grammar.add_production(prodname, syms, pfunc, rulefile, ruleline)
except GrammarError as e: except GrammarError as e:
cls.log.error(str(e)) cls.log.error(str(e))
fail = True fail = True
except SyntaxError as e: except SyntaxError as e:
cls.log.error(str(e)) cls.log.error(str(e))
fail = True fail = True
try: try:
grammar.set_start(getattr(cls, 'start', None)) grammar.set_start(getattr(cls, 'start', None))
except GrammarError as e: except GrammarError as e:
@ -1717,7 +1721,7 @@ class Parser(metaclass=RuleMeta):
Collect all of the tagged grammar rules Collect all of the tagged grammar rules
''' '''
rules = [ (name, value) for name, value in definitions rules = [ (name, value) for name, value in definitions
if callable(value) and hasattr(value, 'rule') ] if callable(value) and hasattr(value, 'rules') ]
return rules return rules
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------