From e4d9887d56f48650d23ade8beb4eca14b59226b1 Mon Sep 17 00:00:00 2001
From: jasonwilliams <jase.williams@gmail.com>
Date: Fri, 10 Aug 2018 17:06:37 +0100
Subject: [PATCH] giving lexer utf-8 capability

---
 src/lib/syntax/lexer.rs | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/lib/syntax/lexer.rs b/src/lib/syntax/lexer.rs
index 26228411e1..1373617d63 100644
--- a/src/lib/syntax/lexer.rs
+++ b/src/lib/syntax/lexer.rs
@@ -1,3 +1,4 @@
+use syntax::ast::punc::Punctuator;
 use syntax::ast::token::{Token, TokenData};
 
 /// A javascript Lexer
@@ -26,4 +27,28 @@ impl Lexer {
         self.tokens
             .push(Token::new(tk, self.line_number, self.column_number))
     }
+
+    /// Push a punctuation token
+    fn push_punc(&mut self, punc: Punctuator) {
+        self.push_token(TokenData::TPunctuator(punc));
+    }
+
+    /// Processes an input stream from a string into an array of tokens
+    pub fn lex_str(script: String) -> Vec<Token> {
+        let mut lexer = Lexer::new(script);
+        lexer.tokens
+    }
+
+    fn next(&mut self) -> Option<char> {
+        self.buffer.chars().next()
+    }
+
+    pub fn lex(&mut self) -> Result<(), &str> {
+        loop {
+            let ch = match self.next() {
+                Some(ch) => ch,
+                None => return Err("oh my days"),
+            };
+        }
+    }
 }