jetcrab\lexer/
mod.rs

1//! # JavaScript Lexer Module
2//!
3//! Provides lexical analysis (tokenization) of JavaScript source code,
4//! converting raw text into a stream of tokens for the parser.
5//!
6//! ## Overview
7//!
8//! The lexer module implements a scanner that produces:
9//!
10//! - **Tokens**: Keywords, identifiers, literals, and operators
11//! - **Position Information**: Accurate source location tracking
12//! - **Error Handling**: Graceful handling of invalid input
13//! - **Fallback Mode**: Basic tokenization for error recovery
14//!
15//! ## Token Types
16//!
17//! - **Keywords**: Language keywords like `let`, `function`, `if`
18//! - **Identifiers**: Variable and function names
19//! - **Literals**: Numbers, strings, booleans, and null
20//! - **Operators**: Arithmetic, logical, and assignment operators
21//! - **Punctuation**: Brackets, semicolons, and other symbols
22//!
23//! ## Usage
24//!
25//! ```rust
26//! use jetcrab::lexer::{tokenize, tokenize_fallback};
27//!
28//! // Tokenize with error handling
29//! let tokens = tokenize("let x = 42;")?;
30//!
31//! // Fallback tokenization for invalid code
32//! let tokens = tokenize_fallback("let x = ;");
33//! ```
34
35pub mod core;
36pub mod error;
37pub mod scanners;
38pub mod token;
39pub mod tokens;
40pub mod utils;
41
42pub use core::Lexer;
43pub use error::LexerError;
44pub use token::{Token, TokenKind};
45pub use tokens::{Keyword, Literal, Operator, Punctuation};
46
47pub fn tokenize(source: &str) -> Result<Vec<Token>, LexerError> {
48    let mut lexer = Lexer::new(source);
49    lexer.tokenize()
50}
51
52pub fn tokenize_fallback(source: &str) -> Vec<Token> {
53    match tokenize(source) {
54        Ok(tokens) => tokens,
55        Err(_) => vec![Token::with_positions(TokenKind::Eof, 1, 1, 1, 1)],
56    }
57}
jetcrab\lexer/mod.rs

jetcrab\lexer/
mod.rs
