64: def add_token(name, regex, options = {}, &block)
65: token = Token.new(regex, name)
66: raise(ArgumentError, "Token #{name} already exists") if @tokens.include?(name)
67: @tokens[token.name] = token
68: if token.string
69: @string_tokens << token
70: @tokens_by_string[token.string] = token
71: else
72: @regex_tokens << token
73: end
74:
75: options.each do |name, option|
76: token.send(name.to_s + "=", option)
77: end
78:
79: token.meta_def(:convert, &block) if block_given?
80:
81: token
82: end