giving lexer utf-8 capability

src/lib/syntax/lexer.rs

@@ -1,3 +1,4 @@
+use syntax::ast::punc::Punctuator;
 use syntax::ast::token::{Token, TokenData};
 
 /// A javascript Lexer
@@ -26,4 +27,28 @@ impl Lexer {
         self.tokens
             .push(Token::new(tk, self.line_number, self.column_number))
     }
+
+    /// Push a punctuation token
+    fn push_punc(&mut self, punc: Punctuator) {
+        self.push_token(TokenData::TPunctuator(punc));
+    }
+
+    /// Processes an input stream from a string into an array of tokens
+    pub fn lex_str(script: String) -> Vec<Token> {
+        let mut lexer = Lexer::new(script);
+        lexer.tokens
+    }
+
+    fn next(&mut self) -> Option<char> {
+        self.buffer.chars().next()
+    }
+
+    pub fn lex(&mut self) -> Result<(), &str> {
+        loop {
+            let ch = match self.next() {
+                Some(ch) => ch,
+                None => return Err("oh my days"),
+            };
+        }
+    }
 }