Added some documentation to `token`

boa/src/syntax/ast/token.rs

@@ -1,17 +1,19 @@
-//! This module implements all of the [Token]s we use.
+//! This module implements all of the [Token]s used in the JavaScript programing language.
+
 use crate::syntax::ast::{keyword::Keyword, pos::Position, punc::Punctuator};
 use std::fmt::{Debug, Display, Formatter, Result};
 
 #[cfg(feature = "serde-ast")]
 use serde::{Deserialize, Serialize};
 
-/// Represents a token.
+/// This represents the smallest individual words, phrases, or characters that JavaScript can understand.
 #[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
-    /// The token Data
+    /// The token kind, which contains the actual data of the token.
     pub kind: TokenKind,
-    /// Token position from original source code
+
+    /// The token position from origina source code.
     pub pos: Position,
 }
 
@@ -31,6 +33,7 @@ impl Display for Token {
     }
 }
 
+/// A continuous sequence of tokens.
 pub struct VecToken(Vec<Token>);
 
 impl Debug for VecToken {
@@ -47,25 +50,38 @@ impl Debug for VecToken {
 #[cfg_attr(feature = "serde-ast", derive(Serialize, Deserialize))]
 #[derive(Clone, PartialEq, Debug)]
 pub enum TokenKind {
-    /// A boolean literal, which is either `true` or `false`
+    /// A boolean literal, which is either `true` or `false`.
     BooleanLiteral(bool),
-    /// The end of the file
+
+    /// The end of the file.
     EOF,
-    /// An identifier
+
+    /// An identifier.
     Identifier(String),
-    /// A keyword
+
+    /// A keyword.
+    ///
+    /// see: [`Keyword`](../keyword/enum.Keyword.html)
     Keyword(Keyword),
-    /// A `null` literal
+
+    /// A `null` literal.
     NullLiteral,
-    /// A numeric literal
+
+    /// A numeric literal.
     NumericLiteral(f64),
+
     /// A piece of punctuation
+    ///
+    /// see: [`Punctuator`](../punc/enum.Punctuator.html)
     Punctuator(Punctuator),
-    /// A string literal
+
+    /// A string literal.
     StringLiteral(String),
-    /// A regular expression, consisting of body and flags
+
+    /// A regular expression, consisting of body and flags.
     RegularExpressionLiteral(String, String),
-    /// Indicates the end of a line \n
+
+    /// Indicates the end of a line (`\n`).
     LineTerminator,
 }