TDPUtils::StringTokenizer は改行を扱える？

#!/usr/bin/env ruby

require 'tdp'
require 'tdp/utils'

parser = TDParser.define {|g|
  g.a = "a"

  g.input = g.a - g.token(:int) - g.token("\n") >> proc {|x| x }

  def parse(str)
    tokenizer = TDPUtils::StringTokenizer[
      /\d+/ => :int,
    ]
    #input.parse(tokenizer.generate(str))
    tg = tokenizer.generate(str)
    p tg
    p tg.shift()
    p tg.shift()
    p tg.shift()
  end
}

p parser.parse("a 1\n")

で、

#<TDParser::TokenGenerator:0x4034c650 @cont_yield=#<Continuation:0x4034bbd8>, @cont_next=nil, @block=#<Proc:0x402c4b0c@/usr/local/lib/site_ruby/1.8/tdp/utils.rb:40>, @buffer=[], @queue=["a"], @cont_endp=nil, @index=0>
"a"
#<TDPUtils::Token:0x4034b96c @value="1", @kind=:int>
nil
nil

無理なのかな？？？

こうすれば、良いのかな？

#!/usr/bin/env ruby

require 'tdp'
require 'tdp/utils'

parser = TDParser.define {|g|
  g.a = "a"

  g.input = g.a - g.token(:int) - g.token("\n") >> proc {|x| x }

  def parse(str)
    tokenizer = TDPUtils::StringTokenizer.new(
      { /\d+/ => :int,
        /\n/  => :EOL },
      /[ \t]/)
    #input.parse(tokenizer.generate(str))
    tg = tokenizer.generate(str)
    p tg
    p tg.shift()
    p tg.shift()
    p tg.shift()
  end
}

p parser.parse("a 1\n")

で、

#<TDParser::TokenGenerator:0xb7c4f8a8 @block=#<Proc:0xb7c53494@/usr/local/ruby-1.8.7/lib/ruby/gems/1.8/gems/tdp4r-1.4.1/lib/tdp/utils.rb:40>, @queue=["a"], @cont_endp=nil, @index=0, @cont_yield=#<Continuation:0xb7c4f77c>, @buffer=[], @cont_next=nil>
"a"
#<TDPUtils::Token:0xb7c4f27c @kind=:int, @value="1">
#<TDPUtils::Token:0xb7c4f074 @kind=:EOL, @value="\n">
nil