[TemplateLiteral] Basic lexer implementation (#455)

* Basic template literal lexer implementation

* Fix formatting

boa/src/syntax/ast/token.rs

@@ -240,6 +240,8 @@ pub enum TokenKind {
     /// A string literal.
     StringLiteral(Box<str>),
 
+    TemplateLiteral(Box<str>),
+
     /// A regular expression, consisting of body and flags.
     RegularExpressionLiteral(Box<str>, RegExpFlags),
 
@@ -310,6 +312,14 @@ impl TokenKind {
         Self::StringLiteral(lit.into())
     }
 
+    /// Creates a `TemplateLiteral` token type.
+    pub fn template_literal<S>(lit: S) -> Self
+    where
+        S: Into<Box<str>>,
+    {
+        Self::TemplateLiteral(lit.into())
+    }
+
     /// Creates a `RegularExpressionLiteral` token kind.
     pub fn regular_expression_literal<B>(body: B, flags: RegExpFlags) -> Self
     where
@@ -337,6 +347,7 @@ impl Display for TokenKind {
             Self::NumericLiteral(NumericLiteral::BigInt(ref num)) => write!(f, "{}n", num),
             Self::Punctuator(ref punc) => write!(f, "{}", punc),
             Self::StringLiteral(ref lit) => write!(f, "{}", lit),
+            Self::TemplateLiteral(ref lit) => write!(f, "{}", lit),
             Self::RegularExpressionLiteral(ref body, ref flags) => write!(f, "/{}/{}", body, flags),
             Self::LineTerminator => write!(f, "line terminator"),
         }

boa/src/syntax/lexer/mod.rs

@@ -499,6 +499,7 @@ impl<'a> Lexer<'a> {
             self.next_column();
             let ch = self.next();
             match ch {
+                 // StringLiteral
                 '"' | '\'' => {
                     let mut buf = String::new();
                     loop {
@@ -623,6 +624,28 @@ impl<'a> Lexer<'a> {
                     self.move_columns( str_length.wrapping_add(1));
                     self.push_token(TokenKind::string_literal(buf), start_pos);
                 }
+                // TemplateLiteral
+                '`' => {
+                    let mut buf = String::new();
+                    loop {
+                        if self.preview_next().is_none() {
+                            return Err(LexerError::new("Unterminated template literal"));
+                        }
+                        match self.next() {
+                            '`' => {
+                                break;
+                            }
+                            next_ch => buf.push(next_ch),
+                            // TODO when there is an expression inside the literal
+                        }
+                    }
+                    let str_length = buf.len() as u32;
+                    // Why +1? Quotation marks are not included,
+                    // So technically it would be +2, (for both " ") but we want to be 1 less
+                    // to compensate for the incrementing at the top
+                    self.move_columns( str_length.wrapping_add(1));
+                    self.push_token(TokenKind::template_literal(buf), start_pos);
+                }
                 _ if ch.is_digit(10) => self.reed_numerical_literal(ch)?,
                 _ if ch.is_alphabetic() || ch == '$' || ch == '_' => {
                     let mut buf = ch.to_string();

boa/src/syntax/lexer/tests.rs

@@ -37,6 +37,29 @@ fn check_string() {
     assert_eq!(lexer.tokens[1].kind, TokenKind::string_literal("bbb"));
 }
 
+#[test]
+fn check_template_literal_simple() {
+    let s = "`I'm a template literal`";
+    let mut lexer = Lexer::new(s);
+    lexer.lex().expect("failed to lex");
+    assert_eq!(
+        lexer.tokens[0].kind,
+        TokenKind::template_literal("I'm a template literal")
+    );
+}
+
+#[test]
+fn check_template_literal_unterminated() {
+    let s = "`I'm a template";
+    let mut lexer = Lexer::new(s);
+    match lexer.lex() {
+        Ok(_) => panic!("Lexer did not detect end of stream"),
+        Err(e) => {
+            assert_eq!(e.to_string(), "Unterminated template literal");
+        }
+    }
+}
+
 #[test]
 fn check_punctuators() {
     // https://tc39.es/ecma262/#sec-punctuators