Lexer

- Implemented an overload of ::token_split that preserveѕ types.
2014-06-14 13:46:10 -04:00
parent a892abf366
commit 7d4e166277
2 changed files with 14 additions and 0 deletions
--- a/src/Lexer.h
+++ b/src/Lexer.h
@@ -67,6 +67,7 @@ public:
  static bool is_ws (int);
  static void word_split (std::vector <std::string>&, const std::string&);
  static void token_split (std::vector <std::string>&, const std::string&);
+  static void token_split (std::vector <std::pair <std::string, Lexer::Type> >&, const std::string&);

 private:
  bool is_punct (int) const;