| pub(crate) fn tokenize<F>(input: &str, mut callback: F) -> Result<(), ParserError>
|
| where
|
| F: FnMut(&str, TokenKind),
|
| {
|
|
|
| let mut lexer = Token::lexer(input);
|
|
|
| while let Some(token) = lexer.next() {
|
| // Return parsing errors, if any.
|
| let token = token?;
|
|
|
| let real_token =
|
| match token {
|
| Token::KeywordIfTrue => TokenKind::Keyword(TokenKeyword::IfTrue),
|
| Token::KeywordIfFalse => TokenKind::Keyword(TokenKeyword::IfFalse),
|
| Token::KeywordIfElse => TokenKind::Keyword(TokenKeyword::IfElse),
|
| Token::KeywordIfEnd => TokenKind::Keyword(TokenKeyword::IfEnd),
|
|
|
| Token::KeywordCaseStart => TokenKind::Keyword(TokenKeyword::CaseStart),
|
| Token::KeywordCaseEnd => TokenKind::Keyword(TokenKeyword::CaseEnd),
|
| Token::KeywordCaseWhen => TokenKind::Keyword(TokenKeyword::CaseWhen),
|
| Token::KeywordCaseWildcard => TokenKind::Keyword(TokenKeyword::CaseWildcard),
|
| Token::KeywordCaseRegex => TokenKind::Keyword(TokenKeyword::CaseRegex),
|
| Token::KeywordCaseDefault => TokenKind::Keyword(TokenKeyword::CaseDefault),
|
| Token::KeywordCaseFall => TokenKind::Keyword(TokenKeyword::CaseFall),
|
|
|
| Token::LiteralNone => TokenKind::Literal(TokenLiteral::Nil),
|
| Token::LiteralBoolTrue => TokenKind::Literal(TokenLiteral::True),
|
| Token::LiteralBoolFalse => TokenKind::Literal(TokenLiteral::False),
|
| Token::LiteralStringSingleQuotes
|
| | Token::LiteralStringDoubleQuotes
|
| | Token::LiteralStringBig => TokenKind::Literal(TokenLiteral::String),
|
| Token::LiteralInteger => TokenKind::Literal(TokenLiteral::Integer),
|
| Token::LiteralIntegerBin => TokenKind::Literal(TokenLiteral::IntegerBin),
|
| Token::LiteralIntegerOct => TokenKind::Literal(TokenLiteral::IntegerOct),
|
| Token::LiteralIntegerHex => TokenKind::Literal(TokenLiteral::IntegerHex),
|
| Token::LiteralFloat => TokenKind::Literal(TokenLiteral::Float),
|
| #[cfg(feature = "parse_floathex")]
|
| Token::LiteralFloatHex => TokenKind::Literal(TokenLiteral::FloatHex),
|
|
|
| Token::CommentStart => TokenKind::CommentStart,
|
| Token::CommentEnd => TokenKind::CommentEnd,
|
| Token::Reserved => TokenKind::Reserved,
|
| Token::Extra => TokenKind::Extra,
|
| Token::Special => TokenKind::SpecialWord,
|
| Token::Word => TokenKind::Word,
|
| };
|
|
|
| callback(lexer.slice(), real_token);
|
| }
|
| Ok(())
|
| }
|