|
7267
|
1 # testlex.py
|
|
|
2
|
|
|
3 import unittest
|
|
|
4 try:
|
|
|
5 import StringIO
|
|
|
6 except ImportError:
|
|
|
7 import io as StringIO
|
|
|
8
|
|
|
9 import sys
|
|
|
10 import os
|
|
|
11 import warnings
|
|
|
12 import platform
|
|
|
13
|
|
|
14 sys.path.insert(0,"..")
|
|
|
15 sys.tracebacklimit = 0
|
|
|
16
|
|
|
17 import ply.lex
|
|
|
18
|
|
|
19 try:
|
|
|
20 from importlib.util import cache_from_source
|
|
|
21 except ImportError:
|
|
|
22 # Python 2.7, but we don't care.
|
|
|
23 cache_from_source = None
|
|
|
24
|
|
|
25
|
|
|
26 def make_pymodule_path(filename, optimization=None):
|
|
|
27 path = os.path.dirname(filename)
|
|
|
28 file = os.path.basename(filename)
|
|
|
29 mod, ext = os.path.splitext(file)
|
|
|
30
|
|
|
31 if sys.hexversion >= 0x3050000:
|
|
|
32 fullpath = cache_from_source(filename, optimization=optimization)
|
|
|
33 elif sys.hexversion >= 0x3040000:
|
|
|
34 fullpath = cache_from_source(filename, ext=='.pyc')
|
|
|
35 elif sys.hexversion >= 0x3020000:
|
|
|
36 import imp
|
|
|
37 modname = mod+"."+imp.get_tag()+ext
|
|
|
38 fullpath = os.path.join(path,'__pycache__',modname)
|
|
|
39 else:
|
|
|
40 fullpath = filename
|
|
|
41 return fullpath
|
|
|
42
|
|
|
43 def pymodule_out_exists(filename, optimization=None):
|
|
|
44 return os.path.exists(make_pymodule_path(filename,
|
|
|
45 optimization=optimization))
|
|
|
46
|
|
|
47 def pymodule_out_remove(filename, optimization=None):
|
|
|
48 os.remove(make_pymodule_path(filename, optimization=optimization))
|
|
|
49
|
|
|
50 def implementation():
|
|
|
51 if platform.system().startswith("Java"):
|
|
|
52 return "Jython"
|
|
|
53 elif hasattr(sys, "pypy_version_info"):
|
|
|
54 return "PyPy"
|
|
|
55 else:
|
|
|
56 return "CPython"
|
|
|
57
|
|
|
58 test_pyo = (implementation() == 'CPython')
|
|
|
59
|
|
|
60 def check_expected(result, expected, contains=False):
|
|
|
61 if sys.version_info[0] >= 3:
|
|
|
62 if isinstance(result,str):
|
|
|
63 result = result.encode('ascii')
|
|
|
64 if isinstance(expected,str):
|
|
|
65 expected = expected.encode('ascii')
|
|
|
66 resultlines = result.splitlines()
|
|
|
67 expectedlines = expected.splitlines()
|
|
|
68
|
|
|
69 if len(resultlines) != len(expectedlines):
|
|
|
70 return False
|
|
|
71
|
|
|
72 for rline,eline in zip(resultlines,expectedlines):
|
|
|
73 if contains:
|
|
|
74 if eline not in rline:
|
|
|
75 return False
|
|
|
76 else:
|
|
|
77 if not rline.endswith(eline):
|
|
|
78 return False
|
|
|
79 return True
|
|
|
80
|
|
|
81 def run_import(module):
|
|
|
82 code = "import "+module
|
|
|
83 exec(code)
|
|
|
84 del sys.modules[module]
|
|
|
85
|
|
|
86 # Tests related to errors and warnings when building lexers
|
|
|
87 class LexErrorWarningTests(unittest.TestCase):
|
|
|
88 def setUp(self):
|
|
|
89 sys.stderr = StringIO.StringIO()
|
|
|
90 sys.stdout = StringIO.StringIO()
|
|
|
91 if sys.hexversion >= 0x3020000:
|
|
|
92 warnings.filterwarnings('ignore',category=ResourceWarning)
|
|
|
93
|
|
|
94 def tearDown(self):
|
|
|
95 sys.stderr = sys.__stderr__
|
|
|
96 sys.stdout = sys.__stdout__
|
|
|
97 def test_lex_doc1(self):
|
|
|
98 self.assertRaises(SyntaxError,run_import,"lex_doc1")
|
|
|
99 result = sys.stderr.getvalue()
|
|
|
100 self.assert_(check_expected(result,
|
|
|
101 "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
|
|
|
102 def test_lex_dup1(self):
|
|
|
103 self.assertRaises(SyntaxError,run_import,"lex_dup1")
|
|
|
104 result = sys.stderr.getvalue()
|
|
|
105 self.assert_(check_expected(result,
|
|
|
106 "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
|
|
|
107
|
|
|
108 def test_lex_dup2(self):
|
|
|
109 self.assertRaises(SyntaxError,run_import,"lex_dup2")
|
|
|
110 result = sys.stderr.getvalue()
|
|
|
111 self.assert_(check_expected(result,
|
|
|
112 "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
|
|
|
113
|
|
|
114 def test_lex_dup3(self):
|
|
|
115 self.assertRaises(SyntaxError,run_import,"lex_dup3")
|
|
|
116 result = sys.stderr.getvalue()
|
|
|
117 self.assert_(check_expected(result,
|
|
|
118 "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
|
|
|
119
|
|
|
120 def test_lex_empty(self):
|
|
|
121 self.assertRaises(SyntaxError,run_import,"lex_empty")
|
|
|
122 result = sys.stderr.getvalue()
|
|
|
123 self.assert_(check_expected(result,
|
|
|
124 "No rules of the form t_rulename are defined\n"
|
|
|
125 "No rules defined for state 'INITIAL'\n"))
|
|
|
126
|
|
|
127 def test_lex_error1(self):
|
|
|
128 run_import("lex_error1")
|
|
|
129 result = sys.stderr.getvalue()
|
|
|
130 self.assert_(check_expected(result,
|
|
|
131 "No t_error rule is defined\n"))
|
|
|
132
|
|
|
133 def test_lex_error2(self):
|
|
|
134 self.assertRaises(SyntaxError,run_import,"lex_error2")
|
|
|
135 result = sys.stderr.getvalue()
|
|
|
136 self.assert_(check_expected(result,
|
|
|
137 "Rule 't_error' must be defined as a function\n")
|
|
|
138 )
|
|
|
139
|
|
|
140 def test_lex_error3(self):
|
|
|
141 self.assertRaises(SyntaxError,run_import,"lex_error3")
|
|
|
142 result = sys.stderr.getvalue()
|
|
|
143 self.assert_(check_expected(result,
|
|
|
144 "lex_error3.py:20: Rule 't_error' requires an argument\n"))
|
|
|
145
|
|
|
146 def test_lex_error4(self):
|
|
|
147 self.assertRaises(SyntaxError,run_import,"lex_error4")
|
|
|
148 result = sys.stderr.getvalue()
|
|
|
149 self.assert_(check_expected(result,
|
|
|
150 "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
|
|
|
151
|
|
|
152 def test_lex_ignore(self):
|
|
|
153 self.assertRaises(SyntaxError,run_import,"lex_ignore")
|
|
|
154 result = sys.stderr.getvalue()
|
|
|
155 self.assert_(check_expected(result,
|
|
|
156 "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
|
|
|
157
|
|
|
158 def test_lex_ignore2(self):
|
|
|
159 run_import("lex_ignore2")
|
|
|
160 result = sys.stderr.getvalue()
|
|
|
161 self.assert_(check_expected(result,
|
|
|
162 "t_ignore contains a literal backslash '\\'\n"))
|
|
|
163
|
|
|
164
|
|
|
165 def test_lex_re1(self):
|
|
|
166 self.assertRaises(SyntaxError,run_import,"lex_re1")
|
|
|
167 result = sys.stderr.getvalue()
|
|
|
168 if sys.hexversion < 0x3050000:
|
|
|
169 msg = "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"
|
|
|
170 else:
|
|
|
171 msg = "Invalid regular expression for rule 't_NUMBER'. missing ), unterminated subpattern at position 0"
|
|
|
172 self.assert_(check_expected(result,
|
|
|
173 msg,
|
|
|
174 contains=True))
|
|
|
175
|
|
|
176 def test_lex_re2(self):
|
|
|
177 self.assertRaises(SyntaxError,run_import,"lex_re2")
|
|
|
178 result = sys.stderr.getvalue()
|
|
|
179 self.assert_(check_expected(result,
|
|
|
180 "Regular expression for rule 't_PLUS' matches empty string\n"))
|
|
|
181
|
|
|
182 def test_lex_re3(self):
|
|
|
183 self.assertRaises(SyntaxError,run_import,"lex_re3")
|
|
|
184 result = sys.stderr.getvalue()
|
|
|
185 # self.assert_(check_expected(result,
|
|
|
186 # "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
|
|
|
187 # "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
|
|
|
188
|
|
|
189 if sys.hexversion < 0x3050000:
|
|
|
190 msg = ("Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
|
|
|
191 "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n")
|
|
|
192 else:
|
|
|
193 msg = ("Invalid regular expression for rule 't_POUND'. missing ), unterminated subpattern at position 0\n"
|
|
|
194 "ERROR: Make sure '#' in rule 't_POUND' is escaped with '\#'")
|
|
|
195 self.assert_(check_expected(result,
|
|
|
196 msg,
|
|
|
197 contains=True), result)
|
|
|
198
|
|
|
199 def test_lex_rule1(self):
|
|
|
200 self.assertRaises(SyntaxError,run_import,"lex_rule1")
|
|
|
201 result = sys.stderr.getvalue()
|
|
|
202 self.assert_(check_expected(result,
|
|
|
203 "t_NUMBER not defined as a function or string\n"))
|
|
|
204
|
|
|
205 def test_lex_rule2(self):
|
|
|
206 self.assertRaises(SyntaxError,run_import,"lex_rule2")
|
|
|
207 result = sys.stderr.getvalue()
|
|
|
208 self.assert_(check_expected(result,
|
|
|
209 "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
|
|
|
210
|
|
|
211 def test_lex_rule3(self):
|
|
|
212 self.assertRaises(SyntaxError,run_import,"lex_rule3")
|
|
|
213 result = sys.stderr.getvalue()
|
|
|
214 self.assert_(check_expected(result,
|
|
|
215 "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
|
|
|
216
|
|
|
217
|
|
|
218 def test_lex_state1(self):
|
|
|
219 self.assertRaises(SyntaxError,run_import,"lex_state1")
|
|
|
220 result = sys.stderr.getvalue()
|
|
|
221 self.assert_(check_expected(result,
|
|
|
222 "states must be defined as a tuple or list\n"))
|
|
|
223
|
|
|
224 def test_lex_state2(self):
|
|
|
225 self.assertRaises(SyntaxError,run_import,"lex_state2")
|
|
|
226 result = sys.stderr.getvalue()
|
|
|
227 self.assert_(check_expected(result,
|
|
|
228 "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
|
|
|
229 "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
|
|
|
230
|
|
|
231 def test_lex_state3(self):
|
|
|
232 self.assertRaises(SyntaxError,run_import,"lex_state3")
|
|
|
233 result = sys.stderr.getvalue()
|
|
|
234 self.assert_(check_expected(result,
|
|
|
235 "State name 1 must be a string\n"
|
|
|
236 "No rules defined for state 'example'\n"))
|
|
|
237
|
|
|
238 def test_lex_state4(self):
|
|
|
239 self.assertRaises(SyntaxError,run_import,"lex_state4")
|
|
|
240 result = sys.stderr.getvalue()
|
|
|
241 self.assert_(check_expected(result,
|
|
|
242 "State type for state comment must be 'inclusive' or 'exclusive'\n"))
|
|
|
243
|
|
|
244
|
|
|
245 def test_lex_state5(self):
|
|
|
246 self.assertRaises(SyntaxError,run_import,"lex_state5")
|
|
|
247 result = sys.stderr.getvalue()
|
|
|
248 self.assert_(check_expected(result,
|
|
|
249 "State 'comment' already defined\n"))
|
|
|
250
|
|
|
251 def test_lex_state_noerror(self):
|
|
|
252 run_import("lex_state_noerror")
|
|
|
253 result = sys.stderr.getvalue()
|
|
|
254 self.assert_(check_expected(result,
|
|
|
255 "No error rule is defined for exclusive state 'comment'\n"))
|
|
|
256
|
|
|
257 def test_lex_state_norule(self):
|
|
|
258 self.assertRaises(SyntaxError,run_import,"lex_state_norule")
|
|
|
259 result = sys.stderr.getvalue()
|
|
|
260 self.assert_(check_expected(result,
|
|
|
261 "No rules defined for state 'example'\n"))
|
|
|
262
|
|
|
263 def test_lex_token1(self):
|
|
|
264 self.assertRaises(SyntaxError,run_import,"lex_token1")
|
|
|
265 result = sys.stderr.getvalue()
|
|
|
266 self.assert_(check_expected(result,
|
|
|
267 "No token list is defined\n"
|
|
|
268 "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
|
|
|
269 "Rule 't_PLUS' defined for an unspecified token PLUS\n"
|
|
|
270 "Rule 't_MINUS' defined for an unspecified token MINUS\n"
|
|
|
271 ))
|
|
|
272
|
|
|
273 def test_lex_token2(self):
|
|
|
274 self.assertRaises(SyntaxError,run_import,"lex_token2")
|
|
|
275 result = sys.stderr.getvalue()
|
|
|
276 self.assert_(check_expected(result,
|
|
|
277 "tokens must be a list or tuple\n"
|
|
|
278 "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
|
|
|
279 "Rule 't_PLUS' defined for an unspecified token PLUS\n"
|
|
|
280 "Rule 't_MINUS' defined for an unspecified token MINUS\n"
|
|
|
281 ))
|
|
|
282
|
|
|
283 def test_lex_token3(self):
|
|
|
284 self.assertRaises(SyntaxError,run_import,"lex_token3")
|
|
|
285 result = sys.stderr.getvalue()
|
|
|
286 self.assert_(check_expected(result,
|
|
|
287 "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
|
|
|
288
|
|
|
289
|
|
|
290 def test_lex_token4(self):
|
|
|
291 self.assertRaises(SyntaxError,run_import,"lex_token4")
|
|
|
292 result = sys.stderr.getvalue()
|
|
|
293 self.assert_(check_expected(result,
|
|
|
294 "Bad token name '-'\n"))
|
|
|
295
|
|
|
296
|
|
|
297 def test_lex_token5(self):
|
|
|
298 try:
|
|
|
299 run_import("lex_token5")
|
|
|
300 except ply.lex.LexError:
|
|
|
301 e = sys.exc_info()[1]
|
|
|
302 self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
|
|
|
303
|
|
|
304 def test_lex_token_dup(self):
|
|
|
305 run_import("lex_token_dup")
|
|
|
306 result = sys.stderr.getvalue()
|
|
|
307 self.assert_(check_expected(result,
|
|
|
308 "Token 'MINUS' multiply defined\n"))
|
|
|
309
|
|
|
310
|
|
|
311 def test_lex_literal1(self):
|
|
|
312 self.assertRaises(SyntaxError,run_import,"lex_literal1")
|
|
|
313 result = sys.stderr.getvalue()
|
|
|
314 self.assert_(check_expected(result,
|
|
|
315 "Invalid literal '**'. Must be a single character\n"))
|
|
|
316
|
|
|
317 def test_lex_literal2(self):
|
|
|
318 self.assertRaises(SyntaxError,run_import,"lex_literal2")
|
|
|
319 result = sys.stderr.getvalue()
|
|
|
320 self.assert_(check_expected(result,
|
|
|
321 "Invalid literals specification. literals must be a sequence of characters\n"))
|
|
|
322
|
|
|
323 import os
|
|
|
324 import subprocess
|
|
|
325 import shutil
|
|
|
326
|
|
|
327 # Tests related to various build options associated with lexers
|
|
|
328 class LexBuildOptionTests(unittest.TestCase):
|
|
|
329 def setUp(self):
|
|
|
330 sys.stderr = StringIO.StringIO()
|
|
|
331 sys.stdout = StringIO.StringIO()
|
|
|
332 def tearDown(self):
|
|
|
333 sys.stderr = sys.__stderr__
|
|
|
334 sys.stdout = sys.__stdout__
|
|
|
335 try:
|
|
|
336 shutil.rmtree("lexdir")
|
|
|
337 except OSError:
|
|
|
338 pass
|
|
|
339
|
|
|
340 def test_lex_module(self):
|
|
|
341 run_import("lex_module")
|
|
|
342 result = sys.stdout.getvalue()
|
|
|
343 self.assert_(check_expected(result,
|
|
|
344 "(NUMBER,3,1,0)\n"
|
|
|
345 "(PLUS,'+',1,1)\n"
|
|
|
346 "(NUMBER,4,1,2)\n"))
|
|
|
347
|
|
|
348 def test_lex_object(self):
|
|
|
349 run_import("lex_object")
|
|
|
350 result = sys.stdout.getvalue()
|
|
|
351 self.assert_(check_expected(result,
|
|
|
352 "(NUMBER,3,1,0)\n"
|
|
|
353 "(PLUS,'+',1,1)\n"
|
|
|
354 "(NUMBER,4,1,2)\n"))
|
|
|
355
|
|
|
356 def test_lex_closure(self):
|
|
|
357 run_import("lex_closure")
|
|
|
358 result = sys.stdout.getvalue()
|
|
|
359 self.assert_(check_expected(result,
|
|
|
360 "(NUMBER,3,1,0)\n"
|
|
|
361 "(PLUS,'+',1,1)\n"
|
|
|
362 "(NUMBER,4,1,2)\n"))
|
|
|
363
|
|
|
364 def test_lex_optimize(self):
|
|
|
365 try:
|
|
|
366 os.remove("lextab.py")
|
|
|
367 except OSError:
|
|
|
368 pass
|
|
|
369 try:
|
|
|
370 os.remove("lextab.pyc")
|
|
|
371 except OSError:
|
|
|
372 pass
|
|
|
373 try:
|
|
|
374 os.remove("lextab.pyo")
|
|
|
375 except OSError:
|
|
|
376 pass
|
|
|
377 run_import("lex_optimize")
|
|
|
378
|
|
|
379 result = sys.stdout.getvalue()
|
|
|
380 self.assert_(check_expected(result,
|
|
|
381 "(NUMBER,3,1,0)\n"
|
|
|
382 "(PLUS,'+',1,1)\n"
|
|
|
383 "(NUMBER,4,1,2)\n"))
|
|
|
384 self.assert_(os.path.exists("lextab.py"))
|
|
|
385
|
|
|
386 p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
|
|
|
387 stdout=subprocess.PIPE)
|
|
|
388 result = p.stdout.read()
|
|
|
389
|
|
|
390 self.assert_(check_expected(result,
|
|
|
391 "(NUMBER,3,1,0)\n"
|
|
|
392 "(PLUS,'+',1,1)\n"
|
|
|
393 "(NUMBER,4,1,2)\n"))
|
|
|
394 if test_pyo:
|
|
|
395 self.assert_(pymodule_out_exists("lextab.pyo", 1))
|
|
|
396 pymodule_out_remove("lextab.pyo", 1)
|
|
|
397
|
|
|
398 p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
|
|
|
399 stdout=subprocess.PIPE)
|
|
|
400 result = p.stdout.read()
|
|
|
401 self.assert_(check_expected(result,
|
|
|
402 "(NUMBER,3,1,0)\n"
|
|
|
403 "(PLUS,'+',1,1)\n"
|
|
|
404 "(NUMBER,4,1,2)\n"))
|
|
|
405
|
|
|
406 if test_pyo:
|
|
|
407 self.assert_(pymodule_out_exists("lextab.pyo", 2))
|
|
|
408 try:
|
|
|
409 os.remove("lextab.py")
|
|
|
410 except OSError:
|
|
|
411 pass
|
|
|
412 try:
|
|
|
413 pymodule_out_remove("lextab.pyc")
|
|
|
414 except OSError:
|
|
|
415 pass
|
|
|
416 try:
|
|
|
417 pymodule_out_remove("lextab.pyo", 2)
|
|
|
418 except OSError:
|
|
|
419 pass
|
|
|
420
|
|
|
421 def test_lex_optimize2(self):
|
|
|
422 try:
|
|
|
423 os.remove("opt2tab.py")
|
|
|
424 except OSError:
|
|
|
425 pass
|
|
|
426 try:
|
|
|
427 os.remove("opt2tab.pyc")
|
|
|
428 except OSError:
|
|
|
429 pass
|
|
|
430 try:
|
|
|
431 os.remove("opt2tab.pyo")
|
|
|
432 except OSError:
|
|
|
433 pass
|
|
|
434 run_import("lex_optimize2")
|
|
|
435 result = sys.stdout.getvalue()
|
|
|
436 self.assert_(check_expected(result,
|
|
|
437 "(NUMBER,3,1,0)\n"
|
|
|
438 "(PLUS,'+',1,1)\n"
|
|
|
439 "(NUMBER,4,1,2)\n"))
|
|
|
440 self.assert_(os.path.exists("opt2tab.py"))
|
|
|
441
|
|
|
442 p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
|
|
|
443 stdout=subprocess.PIPE)
|
|
|
444 result = p.stdout.read()
|
|
|
445 self.assert_(check_expected(result,
|
|
|
446 "(NUMBER,3,1,0)\n"
|
|
|
447 "(PLUS,'+',1,1)\n"
|
|
|
448 "(NUMBER,4,1,2)\n"))
|
|
|
449 if test_pyo:
|
|
|
450 self.assert_(pymodule_out_exists("opt2tab.pyo", 1))
|
|
|
451 pymodule_out_remove("opt2tab.pyo", 1)
|
|
|
452 p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
|
|
|
453 stdout=subprocess.PIPE)
|
|
|
454 result = p.stdout.read()
|
|
|
455 self.assert_(check_expected(result,
|
|
|
456 "(NUMBER,3,1,0)\n"
|
|
|
457 "(PLUS,'+',1,1)\n"
|
|
|
458 "(NUMBER,4,1,2)\n"))
|
|
|
459 if test_pyo:
|
|
|
460 self.assert_(pymodule_out_exists("opt2tab.pyo", 2))
|
|
|
461 try:
|
|
|
462 os.remove("opt2tab.py")
|
|
|
463 except OSError:
|
|
|
464 pass
|
|
|
465 try:
|
|
|
466 pymodule_out_remove("opt2tab.pyc")
|
|
|
467 except OSError:
|
|
|
468 pass
|
|
|
469 try:
|
|
|
470 pymodule_out_remove("opt2tab.pyo", 2)
|
|
|
471 except OSError:
|
|
|
472 pass
|
|
|
473
|
|
|
474 def test_lex_optimize3(self):
|
|
|
475 try:
|
|
|
476 shutil.rmtree("lexdir")
|
|
|
477 except OSError:
|
|
|
478 pass
|
|
|
479
|
|
|
480 os.mkdir("lexdir")
|
|
|
481 os.mkdir("lexdir/sub")
|
|
|
482 open("lexdir/__init__.py","w").write("")
|
|
|
483 open("lexdir/sub/__init__.py","w").write("")
|
|
|
484 run_import("lex_optimize3")
|
|
|
485 result = sys.stdout.getvalue()
|
|
|
486 self.assert_(check_expected(result,
|
|
|
487 "(NUMBER,3,1,0)\n"
|
|
|
488 "(PLUS,'+',1,1)\n"
|
|
|
489 "(NUMBER,4,1,2)\n"))
|
|
|
490 self.assert_(os.path.exists("lexdir/sub/calctab.py"))
|
|
|
491
|
|
|
492 p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
|
|
|
493 stdout=subprocess.PIPE)
|
|
|
494 result = p.stdout.read()
|
|
|
495 self.assert_(check_expected(result,
|
|
|
496 "(NUMBER,3,1,0)\n"
|
|
|
497 "(PLUS,'+',1,1)\n"
|
|
|
498 "(NUMBER,4,1,2)\n"))
|
|
|
499 if test_pyo:
|
|
|
500 self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 1))
|
|
|
501 pymodule_out_remove("lexdir/sub/calctab.pyo", 1)
|
|
|
502
|
|
|
503 p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
|
|
|
504 stdout=subprocess.PIPE)
|
|
|
505 result = p.stdout.read()
|
|
|
506 self.assert_(check_expected(result,
|
|
|
507 "(NUMBER,3,1,0)\n"
|
|
|
508 "(PLUS,'+',1,1)\n"
|
|
|
509 "(NUMBER,4,1,2)\n"))
|
|
|
510 if test_pyo:
|
|
|
511 self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo", 2))
|
|
|
512 try:
|
|
|
513 shutil.rmtree("lexdir")
|
|
|
514 except OSError:
|
|
|
515 pass
|
|
|
516
|
|
|
517 def test_lex_opt_alias(self):
|
|
|
518 try:
|
|
|
519 os.remove("aliastab.py")
|
|
|
520 except OSError:
|
|
|
521 pass
|
|
|
522 try:
|
|
|
523 os.remove("aliastab.pyc")
|
|
|
524 except OSError:
|
|
|
525 pass
|
|
|
526 try:
|
|
|
527 os.remove("aliastab.pyo")
|
|
|
528 except OSError:
|
|
|
529 pass
|
|
|
530 run_import("lex_opt_alias")
|
|
|
531 result = sys.stdout.getvalue()
|
|
|
532 self.assert_(check_expected(result,
|
|
|
533 "(NUMBER,3,1,0)\n"
|
|
|
534 "(+,'+',1,1)\n"
|
|
|
535 "(NUMBER,4,1,2)\n"))
|
|
|
536 self.assert_(os.path.exists("aliastab.py"))
|
|
|
537
|
|
|
538 p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
|
|
|
539 stdout=subprocess.PIPE)
|
|
|
540 result = p.stdout.read()
|
|
|
541 self.assert_(check_expected(result,
|
|
|
542 "(NUMBER,3,1,0)\n"
|
|
|
543 "(+,'+',1,1)\n"
|
|
|
544 "(NUMBER,4,1,2)\n"))
|
|
|
545 if test_pyo:
|
|
|
546 self.assert_(pymodule_out_exists("aliastab.pyo", 1))
|
|
|
547 pymodule_out_remove("aliastab.pyo", 1)
|
|
|
548
|
|
|
549 p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
|
|
|
550 stdout=subprocess.PIPE)
|
|
|
551 result = p.stdout.read()
|
|
|
552 self.assert_(check_expected(result,
|
|
|
553 "(NUMBER,3,1,0)\n"
|
|
|
554 "(+,'+',1,1)\n"
|
|
|
555 "(NUMBER,4,1,2)\n"))
|
|
|
556
|
|
|
557 if test_pyo:
|
|
|
558 self.assert_(pymodule_out_exists("aliastab.pyo", 2))
|
|
|
559 try:
|
|
|
560 os.remove("aliastab.py")
|
|
|
561 except OSError:
|
|
|
562 pass
|
|
|
563 try:
|
|
|
564 pymodule_out_remove("aliastab.pyc")
|
|
|
565 except OSError:
|
|
|
566 pass
|
|
|
567 try:
|
|
|
568 pymodule_out_remove("aliastab.pyo", 2)
|
|
|
569 except OSError:
|
|
|
570 pass
|
|
|
571
|
|
|
572 def test_lex_many_tokens(self):
|
|
|
573 try:
|
|
|
574 os.remove("manytab.py")
|
|
|
575 except OSError:
|
|
|
576 pass
|
|
|
577 try:
|
|
|
578 os.remove("manytab.pyc")
|
|
|
579 except OSError:
|
|
|
580 pass
|
|
|
581 try:
|
|
|
582 os.remove("manytab.pyo")
|
|
|
583 except OSError:
|
|
|
584 pass
|
|
|
585 run_import("lex_many_tokens")
|
|
|
586 result = sys.stdout.getvalue()
|
|
|
587 self.assert_(check_expected(result,
|
|
|
588 "(TOK34,'TOK34:',1,0)\n"
|
|
|
589 "(TOK143,'TOK143:',1,7)\n"
|
|
|
590 "(TOK269,'TOK269:',1,15)\n"
|
|
|
591 "(TOK372,'TOK372:',1,23)\n"
|
|
|
592 "(TOK452,'TOK452:',1,31)\n"
|
|
|
593 "(TOK561,'TOK561:',1,39)\n"
|
|
|
594 "(TOK999,'TOK999:',1,47)\n"
|
|
|
595 ))
|
|
|
596
|
|
|
597 self.assert_(os.path.exists("manytab.py"))
|
|
|
598
|
|
|
599 if implementation() == 'CPython':
|
|
|
600 p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
|
|
|
601 stdout=subprocess.PIPE)
|
|
|
602 result = p.stdout.read()
|
|
|
603 self.assert_(check_expected(result,
|
|
|
604 "(TOK34,'TOK34:',1,0)\n"
|
|
|
605 "(TOK143,'TOK143:',1,7)\n"
|
|
|
606 "(TOK269,'TOK269:',1,15)\n"
|
|
|
607 "(TOK372,'TOK372:',1,23)\n"
|
|
|
608 "(TOK452,'TOK452:',1,31)\n"
|
|
|
609 "(TOK561,'TOK561:',1,39)\n"
|
|
|
610 "(TOK999,'TOK999:',1,47)\n"
|
|
|
611 ))
|
|
|
612
|
|
|
613 self.assert_(pymodule_out_exists("manytab.pyo", 1))
|
|
|
614 pymodule_out_remove("manytab.pyo", 1)
|
|
|
615 try:
|
|
|
616 os.remove("manytab.py")
|
|
|
617 except OSError:
|
|
|
618 pass
|
|
|
619 try:
|
|
|
620 os.remove("manytab.pyc")
|
|
|
621 except OSError:
|
|
|
622 pass
|
|
|
623 try:
|
|
|
624 os.remove("manytab.pyo")
|
|
|
625 except OSError:
|
|
|
626 pass
|
|
|
627
|
|
|
628 # Tests related to run-time behavior of lexers
|
|
|
629 class LexRunTests(unittest.TestCase):
|
|
|
630 def setUp(self):
|
|
|
631 sys.stderr = StringIO.StringIO()
|
|
|
632 sys.stdout = StringIO.StringIO()
|
|
|
633 def tearDown(self):
|
|
|
634 sys.stderr = sys.__stderr__
|
|
|
635 sys.stdout = sys.__stdout__
|
|
|
636
|
|
|
637 def test_lex_hedit(self):
|
|
|
638 run_import("lex_hedit")
|
|
|
639 result = sys.stdout.getvalue()
|
|
|
640 self.assert_(check_expected(result,
|
|
|
641 "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
|
|
|
642 "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
|
|
|
643 "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
|
|
|
644
|
|
|
645 def test_lex_state_try(self):
|
|
|
646 run_import("lex_state_try")
|
|
|
647 result = sys.stdout.getvalue()
|
|
|
648 self.assert_(check_expected(result,
|
|
|
649 "(NUMBER,'3',1,0)\n"
|
|
|
650 "(PLUS,'+',1,2)\n"
|
|
|
651 "(NUMBER,'4',1,4)\n"
|
|
|
652 "Entering comment state\n"
|
|
|
653 "comment body LexToken(body_part,'This is a comment */',1,9)\n"
|
|
|
654 "(PLUS,'+',1,30)\n"
|
|
|
655 "(NUMBER,'10',1,32)\n"
|
|
|
656 ))
|
|
|
657
|
|
|
658
|
|
|
659
|
|
|
660 unittest.main()
|
