diff --git a/boa/src/syntax/ast/token.rs b/boa/src/syntax/ast/token.rs index d3cead3b8a..71c1815feb 100644 --- a/boa/src/syntax/ast/token.rs +++ b/boa/src/syntax/ast/token.rs @@ -1,17 +1,19 @@ -//! This module implements all of the [Token]s we use. +//! This module implements all of the [Token]s used in the JavaScript programing language. + use crate::syntax::ast::{keyword::Keyword, pos::Position, punc::Punctuator}; use std::fmt::{Debug, Display, Formatter, Result}; #[cfg(feature = "serde-ast")] use serde::{Deserialize, Serialize}; -/// Represents a token. +/// This represents the smallest individual words, phrases, or characters that JavaScript can understand. #[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq)] pub struct Token { - /// The token Data + /// The token kind, which contains the actual data of the token. pub kind: TokenKind, - /// Token position from original source code + + /// The token position from origina source code. pub pos: Position, } @@ -31,6 +33,7 @@ impl Display for Token { } } +/// A continuous sequence of tokens. pub struct VecToken(Vec); impl Debug for VecToken { @@ -47,25 +50,38 @@ impl Debug for VecToken { #[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))] #[derive(Clone, PartialEq, Debug)] pub enum TokenKind { - /// A boolean literal, which is either `true` or `false` + /// A boolean literal, which is either `true` or `false`. BooleanLiteral(bool), - /// The end of the file + + /// The end of the file. EOF, - /// An identifier + + /// An identifier. Identifier(String), - /// A keyword + + /// A keyword. + /// + /// see: [`Keyword`](../keyword/enum.Keyword.html) Keyword(Keyword), - /// A `null` literal + + /// A `null` literal. NullLiteral, - /// A numeric literal + + /// A numeric literal. NumericLiteral(f64), + /// A piece of punctuation + /// + /// see: [`Punctuator`](../punc/enum.Punctuator.html) Punctuator(Punctuator), - /// A string literal + + /// A string literal. StringLiteral(String), - /// A regular expression, consisting of body and flags + + /// A regular expression, consisting of body and flags. RegularExpressionLiteral(String, String), - /// Indicates the end of a line \n + + /// Indicates the end of a line (`\n`). LineTerminator, }