Lexer

- Implemented ::token_split, which performs a full lex, and doesn't require
  white space like ::word_split does.
- Added unit tests.

src/Lexer.h

@@ -66,6 +66,7 @@ public:
   static const std::string type_name (const Type&);
   static bool is_ws (int);
   static void word_split (std::vector <std::string>&, const std::string&);
+  static void token_split (std::vector <std::string>&, const std::string&);
 
 private:
   bool is_punct (int) const;