Lexer

- Implemented an overload of ::token_split that preserveѕ types.

src/Lexer.cpp

@@ -662,6 +662,19 @@ void Lexer::token_split (std::vector <std::string>& words, const std::string& in
     words.push_back (word);
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// Split 'input' into 'tokens', preserving type.
+void Lexer::token_split (std::vector <std::pair <std::string, Lexer::Type> >& lexemes, const std::string& input)
+{
+  lexemes.clear ();
+
+  std::string word;
+  Lexer::Type type;
+  Lexer lex (input);
+  while (lex.token (word, type))
+    lexemes.push_back (std::pair <std::string, Lexer::Type>(word, type));
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 bool Lexer::is_punct (int c) const
 {

src/Lexer.h

@@ -67,6 +67,7 @@ public:
   static bool is_ws (int);
   static void word_split (std::vector <std::string>&, const std::string&);
   static void token_split (std::vector <std::string>&, const std::string&);
+  static void token_split (std::vector <std::pair <std::string, Lexer::Type> >&, const std::string&);
 
 private:
   bool is_punct (int) const;