lex を試す。状態 inclusive, exclusive

Flex のマニュアルの例を使って

Flex のマニュアルで言うところの包含的スタート状態(%s)

#!/usr/bin/env python

import ply.lex as lex

states = (
 ('state1', 'inclusive'),
)

tokens = (
  'ONE',
  'THREE',
)

def t_state1_ONE(t):
    r'one'
    print "two"
    return t

def t_THREE(t):
    r'three'
    print "four"
    return t

t_ANY_ignore = ' \t\n'

def t_begin_state1(t):
    r'@'
    t.lexer.begin('state1')

def t_ANY_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

lexer = lex.lex()
lexer.input("""
three
@one three
""")

while 1:
    tok = lexer.token()
    if not tok: break      # No more input
    print tok

で、

four
LexToken(THREE,'three',1,1)
two
LexToken(ONE,'one',1,8)
four
LexToken(THREE,'three',1,12)

Flex のマニュアルで言うところの排他的スタート状態(%x)

#!/usr/bin/env python

import ply.lex as lex

states = (
 ('state1', 'exclusive'),
)

tokens = (
  'ONE',
  'THREE',
)

def t_state1_ONE(t):
    r'one'
    print "two"
    return t

def t_THREE(t):
    r'three'
    print "four"
    return t

t_ANY_ignore = ' \t\n'

def t_begin_state1(t):
    r'@'
    t.lexer.begin('state1')

def t_ANY_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)

lexer = lex.lex()
lexer.input("""
three
@one three
""")

while 1:
    tok = lexer.token()
    if not tok: break      # No more input
    print tok

で、

four
LexToken(THREE,'three',1,1)
two
LexToken(ONE,'one',1,8)
Illegal character 't'
Illegal character 'h'
Illegal character 'r'
Illegal character 'e'
Illegal character 'e'