jetcrab\lexer/
token.rs

1use crate::ast::common::{Position, Span};
2use crate::lexer::tokens::{Keyword, Literal, Operator, Punctuation};
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
6pub enum TokenKind {
7    // Direct variants for backward compatibility
8    Identifier(String),
9    Keyword(Keyword),
10    Operator(Operator),
11    Literal(Literal),
12    Punctuation(Punctuation),
13    Comment(String),
14    Whitespace,
15    Eof,
16
17    // Literal variants
18    Number(f64),
19    String(String),
20    Boolean(bool),
21    Null,
22    Undefined,
23    BigInt(String),
24
25    // Direct token variants for lexer and parser compatibility
26    // Assignment operators
27    Assign,
28    PlusAssign,
29    MinusAssign,
30    StarAssign,
31    SlashAssign,
32    PercentAssign,
33    StarStarAssign,
34    LeftShiftAssign,
35    RightShiftAssign,
36    UnsignedRightShiftAssign,
37    BitwiseAndAssign,
38    BitwiseOrAssign,
39    BitwiseXorAssign,
40
41    // Comparison operators
42    Equal,
43    NotEqual,
44    StrictEqual,
45    StrictNotEqual,
46    LessThan,
47    LessThanEqual,
48    GreaterThan,
49    GreaterThanEqual,
50
51    // Logical operators
52    LogicalAnd,
53    LogicalOr,
54    LogicalNot,
55    NullishCoalescing,
56
57    // Arithmetic operators
58    Plus,
59    Minus,
60    Star,
61    Slash,
62    Percent,
63    StarStar,
64
65    // Bitwise operators
66    BitwiseAnd,
67    BitwiseOr,
68    BitwiseXor,
69    LeftShift,
70    RightShift,
71    UnsignedRightShift,
72    BitwiseNot,
73
74    // Increment/Decrement
75    Increment,
76    Decrement,
77
78    // Special operators
79    Arrow,
80    OptionalChaining,
81    Spread,
82    Rest,
83    PrivateField,
84
85    // Punctuation
86    LeftParen,
87    RightParen,
88    LeftBrace,
89    RightBrace,
90    LeftBracket,
91    RightBracket,
92    Dot,
93    Semicolon,
94    Comma,
95    Colon,
96    Question,
97    Exclamation,
98    Tilde,
99    TemplateStart,
100    TemplateEnd,
101    TemplateExpr,
102    TemplateString(String),
103}
104
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
106pub struct Token {
107    pub kind: TokenKind,
108    pub span: Span,
109}
110
111impl Token {
112    pub fn new(kind: TokenKind, span: Span) -> Self {
113        Self { kind, span }
114    }
115
116    pub fn with_positions(
117        kind: TokenKind,
118        start_line: usize,
119        start_col: usize,
120        end_line: usize,
121        end_col: usize,
122    ) -> Self {
123        let span = Span::from_positions(start_line, start_col, end_line, end_col);
124        Self::new(kind, span)
125    }
126
127    pub fn start(&self) -> Position {
128        self.span.start
129    }
130
131    pub fn end(&self) -> Position {
132        self.span.end
133    }
134
135    pub fn is_keyword(&self) -> bool {
136        matches!(self.kind, TokenKind::Keyword(_))
137    }
138
139    pub fn is_identifier(&self) -> bool {
140        matches!(self.kind, TokenKind::Identifier(_))
141    }
142
143    pub fn is_literal(&self) -> bool {
144        matches!(self.kind, TokenKind::Literal(_))
145    }
146
147    pub fn is_operator(&self) -> bool {
148        matches!(
149            self.kind,
150            TokenKind::Operator(_)
151                | TokenKind::Assign
152                | TokenKind::PlusAssign
153                | TokenKind::MinusAssign
154                | TokenKind::StarAssign
155                | TokenKind::SlashAssign
156                | TokenKind::PercentAssign
157                | TokenKind::StarStarAssign
158                | TokenKind::LeftShiftAssign
159                | TokenKind::RightShiftAssign
160                | TokenKind::UnsignedRightShiftAssign
161                | TokenKind::BitwiseAndAssign
162                | TokenKind::BitwiseOrAssign
163                | TokenKind::BitwiseXorAssign
164                | TokenKind::Equal
165                | TokenKind::NotEqual
166                | TokenKind::StrictEqual
167                | TokenKind::StrictNotEqual
168                | TokenKind::LessThan
169                | TokenKind::LessThanEqual
170                | TokenKind::GreaterThan
171                | TokenKind::GreaterThanEqual
172                | TokenKind::LogicalAnd
173                | TokenKind::LogicalOr
174                | TokenKind::LogicalNot
175                | TokenKind::Plus
176                | TokenKind::Minus
177                | TokenKind::Star
178                | TokenKind::Slash
179                | TokenKind::Percent
180                | TokenKind::StarStar
181                | TokenKind::BitwiseAnd
182                | TokenKind::BitwiseOr
183                | TokenKind::BitwiseXor
184                | TokenKind::LeftShift
185                | TokenKind::RightShift
186                | TokenKind::UnsignedRightShift
187                | TokenKind::BitwiseNot
188                | TokenKind::Increment
189                | TokenKind::Decrement
190                | TokenKind::Arrow
191                | TokenKind::OptionalChaining
192                | TokenKind::Spread
193                | TokenKind::Rest
194                | TokenKind::PrivateField
195        )
196    }
197
198    pub fn is_punctuation(&self) -> bool {
199        matches!(
200            self.kind,
201            TokenKind::Punctuation(_)
202                | TokenKind::LeftParen
203                | TokenKind::RightParen
204                | TokenKind::LeftBrace
205                | TokenKind::RightBrace
206                | TokenKind::LeftBracket
207                | TokenKind::RightBracket
208                | TokenKind::Dot
209                | TokenKind::Semicolon
210                | TokenKind::Comma
211                | TokenKind::Colon
212                | TokenKind::Question
213                | TokenKind::Exclamation
214                | TokenKind::Tilde
215                | TokenKind::TemplateStart
216                | TokenKind::TemplateEnd
217                | TokenKind::TemplateExpr
218                | TokenKind::TemplateString(_)
219        )
220    }
221
222    pub fn is_comment(&self) -> bool {
223        matches!(self.kind, TokenKind::Comment(_))
224    }
225
226    pub fn is_whitespace(&self) -> bool {
227        matches!(self.kind, TokenKind::Whitespace)
228    }
229
230    pub fn is_eof(&self) -> bool {
231        matches!(self.kind, TokenKind::Eof)
232    }
233
234    pub fn as_identifier(&self) -> Option<&str> {
235        match &self.kind {
236            TokenKind::Identifier(s) => Some(s),
237            _ => None,
238        }
239    }
240
241    pub fn as_keyword(&self) -> Option<&Keyword> {
242        match &self.kind {
243            TokenKind::Keyword(k) => Some(k),
244            _ => None,
245        }
246    }
247
248    pub fn as_operator(&self) -> Option<&Operator> {
249        match &self.kind {
250            TokenKind::Operator(op) => Some(op),
251            _ => None,
252        }
253    }
254
255    pub fn as_literal(&self) -> Option<&Literal> {
256        match &self.kind {
257            TokenKind::Literal(lit) => Some(lit),
258            _ => None,
259        }
260    }
261
262    pub fn as_punctuation(&self) -> Option<&Punctuation> {
263        match &self.kind {
264            TokenKind::Punctuation(punct) => Some(punct),
265            _ => None,
266        }
267    }
268
269    pub fn as_comment(&self) -> Option<&str> {
270        match &self.kind {
271            TokenKind::Comment(s) => Some(s),
272            _ => None,
273        }
274    }
275}
jetcrab\lexer/token.rs

jetcrab\lexer/
token.rs
