|
7267
|
1 # lexer for yacc-grammars
|
|
|
2 #
|
|
|
3 # Author: David Beazley (dave@dabeaz.com)
|
|
|
4 # Date : October 2, 2006
|
|
|
5
|
|
|
6 import sys
|
|
|
7 sys.path.append("../..")
|
|
|
8
|
|
|
9 from ply import *
|
|
|
10
|
|
|
11 tokens = (
|
|
|
12 'LITERAL','SECTION','TOKEN','LEFT','RIGHT','PREC','START','TYPE','NONASSOC','UNION','CODE',
|
|
|
13 'ID','QLITERAL','NUMBER',
|
|
|
14 )
|
|
|
15
|
|
|
16 states = (('code','exclusive'),)
|
|
|
17
|
|
|
18 literals = [ ';', ',', '<', '>', '|',':' ]
|
|
|
19 t_ignore = ' \t'
|
|
|
20
|
|
|
21 t_TOKEN = r'%token'
|
|
|
22 t_LEFT = r'%left'
|
|
|
23 t_RIGHT = r'%right'
|
|
|
24 t_NONASSOC = r'%nonassoc'
|
|
|
25 t_PREC = r'%prec'
|
|
|
26 t_START = r'%start'
|
|
|
27 t_TYPE = r'%type'
|
|
|
28 t_UNION = r'%union'
|
|
|
29 t_ID = r'[a-zA-Z_][a-zA-Z_0-9]*'
|
|
|
30 t_QLITERAL = r'''(?P<quote>['"]).*?(?P=quote)'''
|
|
|
31 t_NUMBER = r'\d+'
|
|
|
32
|
|
|
33 def t_SECTION(t):
|
|
|
34 r'%%'
|
|
|
35 if getattr(t.lexer,"lastsection",0):
|
|
|
36 t.value = t.lexer.lexdata[t.lexpos+2:]
|
|
|
37 t.lexer.lexpos = len(t.lexer.lexdata)
|
|
|
38 else:
|
|
|
39 t.lexer.lastsection = 0
|
|
|
40 return t
|
|
|
41
|
|
|
42 # Comments
|
|
|
43 def t_ccomment(t):
|
|
|
44 r'/\*(.|\n)*?\*/'
|
|
|
45 t.lexer.lineno += t.value.count('\n')
|
|
|
46
|
|
|
47 t_ignore_cppcomment = r'//.*'
|
|
|
48
|
|
|
49 def t_LITERAL(t):
|
|
|
50 r'%\{(.|\n)*?%\}'
|
|
|
51 t.lexer.lineno += t.value.count("\n")
|
|
|
52 return t
|
|
|
53
|
|
|
54 def t_NEWLINE(t):
|
|
|
55 r'\n'
|
|
|
56 t.lexer.lineno += 1
|
|
|
57
|
|
|
58 def t_code(t):
|
|
|
59 r'\{'
|
|
|
60 t.lexer.codestart = t.lexpos
|
|
|
61 t.lexer.level = 1
|
|
|
62 t.lexer.begin('code')
|
|
|
63
|
|
|
64 def t_code_ignore_string(t):
|
|
|
65 r'\"([^\\\n]|(\\.))*?\"'
|
|
|
66
|
|
|
67 def t_code_ignore_char(t):
|
|
|
68 r'\'([^\\\n]|(\\.))*?\''
|
|
|
69
|
|
|
70 def t_code_ignore_comment(t):
|
|
|
71 r'/\*(.|\n)*?\*/'
|
|
|
72
|
|
|
73 def t_code_ignore_cppcom(t):
|
|
|
74 r'//.*'
|
|
|
75
|
|
|
76 def t_code_lbrace(t):
|
|
|
77 r'\{'
|
|
|
78 t.lexer.level += 1
|
|
|
79
|
|
|
80 def t_code_rbrace(t):
|
|
|
81 r'\}'
|
|
|
82 t.lexer.level -= 1
|
|
|
83 if t.lexer.level == 0:
|
|
|
84 t.type = 'CODE'
|
|
|
85 t.value = t.lexer.lexdata[t.lexer.codestart:t.lexpos+1]
|
|
|
86 t.lexer.begin('INITIAL')
|
|
|
87 t.lexer.lineno += t.value.count('\n')
|
|
|
88 return t
|
|
|
89
|
|
|
90 t_code_ignore_nonspace = r'[^\s\}\'\"\{]+'
|
|
|
91 t_code_ignore_whitespace = r'\s+'
|
|
|
92 t_code_ignore = ""
|
|
|
93
|
|
|
94 def t_code_error(t):
|
|
|
95 raise RuntimeError
|
|
|
96
|
|
|
97 def t_error(t):
|
|
|
98 print "%d: Illegal character '%s'" % (t.lexer.lineno, t.value[0])
|
|
|
99 print t.value
|
|
|
100 t.lexer.skip(1)
|
|
|
101
|
|
|
102 lex.lex()
|
|
|
103
|
|
|
104 if __name__ == '__main__':
|
|
|
105 lex.runmain()
|
|
|
106
|
|
|
107
|
|
|
108
|
|
|
109
|
|
|
110
|
|
|
111
|
|
|
112
|
