boa/boa/src/syntax/lexer/tests.rs

//! Tests for the lexer.
#![allow(clippy::indexing_slicing)]

use super::*;
use crate::syntax::ast::keyword::Keyword;

#[test]
fn check_single_line_comment() {
    let s1 = "var \n//This is a comment\ntrue";
    let mut lexer = Lexer::new(s1);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Var));
    assert_eq!(lexer.tokens[1].kind, TokenKind::LineTerminator);
    assert_eq!(lexer.tokens[2].kind, TokenKind::BooleanLiteral(true));
}

#[test]
fn check_multi_line_comment() {
    let s = "var /* await \n break \n*/ x";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Var));
    assert_eq!(lexer.tokens[1].kind, TokenKind::identifier("x"));
}

#[test]
fn check_string() {
    let s = "'aaa' \"bbb\"";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::string_literal("aaa"));

    assert_eq!(lexer.tokens[1].kind, TokenKind::string_literal("bbb"));
}

#[test]
fn check_punctuators() {
    // https://tc39.es/ecma262/#sec-punctuators
    let s = "{ ( ) [ ] . ... ; , < > <= >= == != === !== \
             + - * % -- << >> >>> & | ^ ! ~ && || ? : \
             = += -= *= &= **= ++ ** <<= >>= >>>= &= |= ^= =>";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(
        lexer.tokens[0].kind,
        TokenKind::Punctuator(Punctuator::OpenBlock)
    );
    assert_eq!(
        lexer.tokens[1].kind,
        TokenKind::Punctuator(Punctuator::OpenParen)
    );
    assert_eq!(
        lexer.tokens[2].kind,
        TokenKind::Punctuator(Punctuator::CloseParen)
    );
    assert_eq!(
        lexer.tokens[3].kind,
        TokenKind::Punctuator(Punctuator::OpenBracket)
    );
    assert_eq!(
        lexer.tokens[4].kind,
        TokenKind::Punctuator(Punctuator::CloseBracket)
    );
    assert_eq!(lexer.tokens[5].kind, TokenKind::Punctuator(Punctuator::Dot));
    assert_eq!(
        lexer.tokens[6].kind,
        TokenKind::Punctuator(Punctuator::Spread)
    );
    assert_eq!(
        lexer.tokens[7].kind,
        TokenKind::Punctuator(Punctuator::Semicolon)
    );
    assert_eq!(
        lexer.tokens[8].kind,
        TokenKind::Punctuator(Punctuator::Comma)
    );
    assert_eq!(
        lexer.tokens[9].kind,
        TokenKind::Punctuator(Punctuator::LessThan)
    );
    assert_eq!(
        lexer.tokens[10].kind,
        TokenKind::Punctuator(Punctuator::GreaterThan)
    );
    assert_eq!(
        lexer.tokens[11].kind,
        TokenKind::Punctuator(Punctuator::LessThanOrEq)
    );
    assert_eq!(
        lexer.tokens[12].kind,
        TokenKind::Punctuator(Punctuator::GreaterThanOrEq)
    );
    assert_eq!(lexer.tokens[13].kind, TokenKind::Punctuator(Punctuator::Eq));
    assert_eq!(
        lexer.tokens[14].kind,
        TokenKind::Punctuator(Punctuator::NotEq)
    );
    assert_eq!(
        lexer.tokens[15].kind,
        TokenKind::Punctuator(Punctuator::StrictEq)
    );
    assert_eq!(
        lexer.tokens[16].kind,
        TokenKind::Punctuator(Punctuator::StrictNotEq)
    );
    assert_eq!(
        lexer.tokens[17].kind,
        TokenKind::Punctuator(Punctuator::Add)
    );
    assert_eq!(
        lexer.tokens[18].kind,
        TokenKind::Punctuator(Punctuator::Sub)
    );
    assert_eq!(
        lexer.tokens[19].kind,
        TokenKind::Punctuator(Punctuator::Mul)
    );
    assert_eq!(
        lexer.tokens[20].kind,
        TokenKind::Punctuator(Punctuator::Mod)
    );
    assert_eq!(
        lexer.tokens[21].kind,
        TokenKind::Punctuator(Punctuator::Dec)
    );
    assert_eq!(
        lexer.tokens[22].kind,
        TokenKind::Punctuator(Punctuator::LeftSh)
    );
    assert_eq!(
        lexer.tokens[23].kind,
        TokenKind::Punctuator(Punctuator::RightSh)
    );
    assert_eq!(
        lexer.tokens[24].kind,
        TokenKind::Punctuator(Punctuator::URightSh)
    );
    assert_eq!(
        lexer.tokens[25].kind,
        TokenKind::Punctuator(Punctuator::And)
    );
    assert_eq!(lexer.tokens[26].kind, TokenKind::Punctuator(Punctuator::Or));
    assert_eq!(
        lexer.tokens[27].kind,
        TokenKind::Punctuator(Punctuator::Xor)
    );
    assert_eq!(
        lexer.tokens[28].kind,
        TokenKind::Punctuator(Punctuator::Not)
    );
    assert_eq!(
        lexer.tokens[29].kind,
        TokenKind::Punctuator(Punctuator::Neg)
    );
    assert_eq!(
        lexer.tokens[30].kind,
        TokenKind::Punctuator(Punctuator::BoolAnd)
    );
    assert_eq!(
        lexer.tokens[31].kind,
        TokenKind::Punctuator(Punctuator::BoolOr)
    );
    assert_eq!(
        lexer.tokens[32].kind,
        TokenKind::Punctuator(Punctuator::Question)
    );
    assert_eq!(
        lexer.tokens[33].kind,
        TokenKind::Punctuator(Punctuator::Colon)
    );
    assert_eq!(
        lexer.tokens[34].kind,
        TokenKind::Punctuator(Punctuator::Assign)
    );
    assert_eq!(
        lexer.tokens[35].kind,
        TokenKind::Punctuator(Punctuator::AssignAdd)
    );
    assert_eq!(
        lexer.tokens[36].kind,
        TokenKind::Punctuator(Punctuator::AssignSub)
    );
    assert_eq!(
        lexer.tokens[37].kind,
        TokenKind::Punctuator(Punctuator::AssignMul)
    );
    assert_eq!(
        lexer.tokens[38].kind,
        TokenKind::Punctuator(Punctuator::AssignAnd)
    );
    assert_eq!(
        lexer.tokens[39].kind,
        TokenKind::Punctuator(Punctuator::AssignPow)
    );
    assert_eq!(
        lexer.tokens[40].kind,
        TokenKind::Punctuator(Punctuator::Inc)
    );
    assert_eq!(
        lexer.tokens[41].kind,
        TokenKind::Punctuator(Punctuator::Exp)
    );
    assert_eq!(
        lexer.tokens[42].kind,
        TokenKind::Punctuator(Punctuator::AssignLeftSh)
    );
    assert_eq!(
        lexer.tokens[43].kind,
        TokenKind::Punctuator(Punctuator::AssignRightSh)
    );
    assert_eq!(
        lexer.tokens[44].kind,
        TokenKind::Punctuator(Punctuator::AssignURightSh)
    );
    assert_eq!(
        lexer.tokens[45].kind,
        TokenKind::Punctuator(Punctuator::AssignAnd)
    );
    assert_eq!(
        lexer.tokens[46].kind,
        TokenKind::Punctuator(Punctuator::AssignOr)
    );
    assert_eq!(
        lexer.tokens[47].kind,
        TokenKind::Punctuator(Punctuator::AssignXor)
    );
    assert_eq!(
        lexer.tokens[48].kind,
        TokenKind::Punctuator(Punctuator::Arrow)
    );
}

#[test]
fn check_keywords() {
    // https://tc39.es/ecma262/#sec-keywords
    let s = "await break case catch class const continue debugger default delete \
             do else export extends finally for function if import in instanceof \
             new return super switch this throw try typeof var void while with yield";

    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Await));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Keyword(Keyword::Break));
    assert_eq!(lexer.tokens[2].kind, TokenKind::Keyword(Keyword::Case));
    assert_eq!(lexer.tokens[3].kind, TokenKind::Keyword(Keyword::Catch));
    assert_eq!(lexer.tokens[4].kind, TokenKind::Keyword(Keyword::Class));
    assert_eq!(lexer.tokens[5].kind, TokenKind::Keyword(Keyword::Const));
    assert_eq!(lexer.tokens[6].kind, TokenKind::Keyword(Keyword::Continue));
    assert_eq!(lexer.tokens[7].kind, TokenKind::Keyword(Keyword::Debugger));
    assert_eq!(lexer.tokens[8].kind, TokenKind::Keyword(Keyword::Default));
    assert_eq!(lexer.tokens[9].kind, TokenKind::Keyword(Keyword::Delete));
    assert_eq!(lexer.tokens[10].kind, TokenKind::Keyword(Keyword::Do));
    assert_eq!(lexer.tokens[11].kind, TokenKind::Keyword(Keyword::Else));
    assert_eq!(lexer.tokens[12].kind, TokenKind::Keyword(Keyword::Export));
    assert_eq!(lexer.tokens[13].kind, TokenKind::Keyword(Keyword::Extends));
    assert_eq!(lexer.tokens[14].kind, TokenKind::Keyword(Keyword::Finally));
    assert_eq!(lexer.tokens[15].kind, TokenKind::Keyword(Keyword::For));
    assert_eq!(lexer.tokens[16].kind, TokenKind::Keyword(Keyword::Function));
    assert_eq!(lexer.tokens[17].kind, TokenKind::Keyword(Keyword::If));
    assert_eq!(lexer.tokens[18].kind, TokenKind::Keyword(Keyword::Import));
    assert_eq!(lexer.tokens[19].kind, TokenKind::Keyword(Keyword::In));
    assert_eq!(
        lexer.tokens[20].kind,
        TokenKind::Keyword(Keyword::InstanceOf)
    );
    assert_eq!(lexer.tokens[21].kind, TokenKind::Keyword(Keyword::New));
    assert_eq!(lexer.tokens[22].kind, TokenKind::Keyword(Keyword::Return));
    assert_eq!(lexer.tokens[23].kind, TokenKind::Keyword(Keyword::Super));
    assert_eq!(lexer.tokens[24].kind, TokenKind::Keyword(Keyword::Switch));
    assert_eq!(lexer.tokens[25].kind, TokenKind::Keyword(Keyword::This));
    assert_eq!(lexer.tokens[26].kind, TokenKind::Keyword(Keyword::Throw));
    assert_eq!(lexer.tokens[27].kind, TokenKind::Keyword(Keyword::Try));
    assert_eq!(lexer.tokens[28].kind, TokenKind::Keyword(Keyword::TypeOf));
    assert_eq!(lexer.tokens[29].kind, TokenKind::Keyword(Keyword::Var));
    assert_eq!(lexer.tokens[30].kind, TokenKind::Keyword(Keyword::Void));
    assert_eq!(lexer.tokens[31].kind, TokenKind::Keyword(Keyword::While));
    assert_eq!(lexer.tokens[32].kind, TokenKind::Keyword(Keyword::With));
    assert_eq!(lexer.tokens[33].kind, TokenKind::Keyword(Keyword::Yield));
}

#[test]
fn check_variable_definition_tokens() {
    let s = "let a = 'hello';";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::Keyword(Keyword::Let));
    assert_eq!(lexer.tokens[1].kind, TokenKind::identifier("a"));
    assert_eq!(
        lexer.tokens[2].kind,
        TokenKind::Punctuator(Punctuator::Assign)
    );
    assert_eq!(lexer.tokens[3].kind, TokenKind::string_literal("hello"));
}

#[test]
fn check_positions() {
    let s = "console.log(\"hello world\"); // Test";
    // ------123456789
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    // The first column is 1 (not zero indexed)
    assert_eq!(lexer.tokens[0].pos.column_number, 1);
    assert_eq!(lexer.tokens[0].pos.line_number, 1);
    // Dot Token starts on column 8
    assert_eq!(lexer.tokens[1].pos.column_number, 8);
    assert_eq!(lexer.tokens[1].pos.line_number, 1);
    // Log Token starts on column 9
    assert_eq!(lexer.tokens[2].pos.column_number, 9);
    assert_eq!(lexer.tokens[2].pos.line_number, 1);
    // Open parenthesis token starts on column 12
    assert_eq!(lexer.tokens[3].pos.column_number, 12);
    assert_eq!(lexer.tokens[3].pos.line_number, 1);
    // String token starts on column 13
    assert_eq!(lexer.tokens[4].pos.column_number, 13);
    assert_eq!(lexer.tokens[4].pos.line_number, 1);
    // Close parenthesis token starts on column 26
    assert_eq!(lexer.tokens[5].pos.column_number, 26);
    assert_eq!(lexer.tokens[5].pos.line_number, 1);
    // Semi Colon token starts on column 27
    assert_eq!(lexer.tokens[6].pos.column_number, 27);
    assert_eq!(lexer.tokens[6].pos.line_number, 1);
}

#[test]
#[ignore]
fn test_two_divisions_in_expression() {
    let s = "    return a !== 0 || 1 / a === 1 / b;";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    // dbg!(&lexer.tokens);

    assert_eq!(lexer.tokens[11].pos.column_number, 37);
    assert_eq!(lexer.tokens[11].pos.line_number, 1);
}

#[test]
fn check_line_numbers() {
    let s = "x\ny\n";

    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");

    assert_eq!(lexer.tokens[0].pos.column_number, 1);
    assert_eq!(lexer.tokens[0].pos.line_number, 1);

    assert_eq!(lexer.tokens[1].pos.column_number, 2);
    assert_eq!(lexer.tokens[1].pos.line_number, 1);

    assert_eq!(lexer.tokens[2].pos.column_number, 1);
    assert_eq!(lexer.tokens[2].pos.line_number, 2);

    assert_eq!(lexer.tokens[3].pos.column_number, 2);
    assert_eq!(lexer.tokens[3].pos.line_number, 2);
}

// Increment/Decrement
#[test]
fn check_decrement_advances_lexer_2_places() {
    // Here we want an example of decrementing an integer
    let s = "let a = b--;";
    let mut lexer = Lexer::new(s);
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[4].kind, TokenKind::Punctuator(Punctuator::Dec));
    // Decrementing means adding 2 characters '--', the lexer should consume it as a single token
    // and move the curser forward by 2, meaning the next token should be a semicolon
    assert_eq!(
        lexer.tokens[5].kind,
        TokenKind::Punctuator(Punctuator::Semicolon)
    );
}

#[test]
fn numbers() {
    let mut lexer = Lexer::new(
        "1 2 0x34 056 7.89 42. 5e3 5e+3 5e-3 0b10 0O123 0999 1.0e1 1.0e-1 1.0E1 1E1 0.0 0.12",
    );

    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1));
    assert_eq!(lexer.tokens[1].kind, TokenKind::numeric_literal(2));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(52));
    assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(46));
    assert_eq!(lexer.tokens[4].kind, TokenKind::numeric_literal(7.89));
    assert_eq!(lexer.tokens[5].kind, TokenKind::numeric_literal(42.0));
    assert_eq!(lexer.tokens[6].kind, TokenKind::numeric_literal(5000.0));
    assert_eq!(lexer.tokens[7].kind, TokenKind::numeric_literal(5000.0));
    assert_eq!(lexer.tokens[8].kind, TokenKind::numeric_literal(0.005));
    assert_eq!(lexer.tokens[9].kind, TokenKind::numeric_literal(2));
    assert_eq!(lexer.tokens[10].kind, TokenKind::numeric_literal(83));
    assert_eq!(lexer.tokens[11].kind, TokenKind::numeric_literal(999));
    assert_eq!(lexer.tokens[12].kind, TokenKind::numeric_literal(10.0));
    assert_eq!(lexer.tokens[13].kind, TokenKind::numeric_literal(0.1));
    assert_eq!(lexer.tokens[14].kind, TokenKind::numeric_literal(10.0));
    assert_eq!(lexer.tokens[15].kind, TokenKind::numeric_literal(10.0));
    assert_eq!(lexer.tokens[16].kind, TokenKind::numeric_literal(0.0));
    assert_eq!(lexer.tokens[17].kind, TokenKind::numeric_literal(0.12));
}

#[test]
fn implicit_octal_edge_case() {
    let mut lexer = Lexer::new("044.5 094.5");

    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(36));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(5));

    assert_eq!(lexer.tokens[3].kind, TokenKind::numeric_literal(94.5));
}

#[test]
fn hexadecimal_edge_case() {
    let mut lexer = Lexer::new("0xffff.ff 0xffffff");

    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(0xffff));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot));
    assert_eq!(
        lexer.tokens[2].kind,
        TokenKind::Identifier(String::from("ff"))
    );

    assert_eq!(
        lexer.tokens[3].kind,
        TokenKind::numeric_literal(0x00ff_ffff)
    );
}

#[test]
fn test_single_number_without_semicolon() {
    let mut lexer = Lexer::new("1");
    lexer.lex().expect("failed to lex");
}

#[test]
fn test_number_followed_by_dot() {
    let mut lexer = Lexer::new("1..");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1.0));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Dot));
}

#[test]
fn test_regex_literal() {
    let mut lexer = Lexer::new("/(?:)/");
    lexer.lex().expect("failed to lex");
    assert_eq!(
        lexer.tokens[0].kind,
        TokenKind::regular_expression_literal("(?:)", "")
    );
}

#[test]
fn test_regex_literal_flags() {
    let mut lexer = Lexer::new(r"/\/[^\/]*\/*/gmi");
    lexer.lex().expect("failed to lex");
    assert_eq!(
        lexer.tokens[0].kind,
        TokenKind::regular_expression_literal("\\/[^\\/]*\\/*", "gmi")
    );
}

#[test]
fn test_addition_no_spaces() {
    let mut lexer = Lexer::new("1+1");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1));
}

#[test]
fn test_addition_no_spaces_left_side() {
    let mut lexer = Lexer::new("1+ 1");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1));
}

#[test]
fn test_addition_no_spaces_right_side() {
    let mut lexer = Lexer::new("1 +1");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1));
}

#[test]
fn test_addition_no_spaces_e_number_left_side() {
    let mut lexer = Lexer::new("1e2+ 1");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(100.0));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1));
}

#[test]
fn test_addition_no_spaces_e_number_right_side() {
    let mut lexer = Lexer::new("1 +1e3");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(lexer.tokens[2].kind, TokenKind::numeric_literal(1000.0));
}

#[test]
fn test_addition_no_spaces_e_number() {
    let mut lexer = Lexer::new("1e3+1e11");
    lexer.lex().expect("failed to lex");
    assert_eq!(lexer.tokens[0].kind, TokenKind::numeric_literal(1000.0));
    assert_eq!(lexer.tokens[1].kind, TokenKind::Punctuator(Punctuator::Add));
    assert_eq!(
        lexer.tokens[2].kind,
        TokenKind::numeric_literal(100_000_000_000.0)
    );
}