lex を試す。状態の管理

everyday parser PLY

begin でなく、push_state, pop_state を使うこともできる。

#!/usr/bin/env python

import ply.lex as lex

states = (
 ('foo', 'exclusive'),
)

tokens = (
  'A',
  'PLUS',
)

t_A          = r'a'
t_ANY_PLUS   = r'\+'
t_ANY_ignore = ' \t\n'

t_foo_A      = r'b'

def t_begin_foo(t):
    r'start_foo'
    t.lexer.push_state('foo')

def t_foo_end(t):
    r'end_foo'
    t.lexer.pop_state()

def t_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

def t_foo_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

lexer = lex.lex()
lexer.input("""
a+a
start_foo
b+b
end_foo
a+a
""")

while 1:
    tok = lexer.token()
    if not tok: break      # No more input
    print tok

で、

LexToken(A,'a',1,1)
LexToken(PLUS,'+',1,2)
LexToken(A,'a',1,3)
LexToken(A,'b',1,15)
LexToken(PLUS,'+',1,16)
LexToken(A,'b',1,17)
LexToken(A,'a',1,27)
LexToken(PLUS,'+',1,28)
LexToken(A,'a',1,29)