Lexer2

- Implemented ::split for typeless tokenization.
2015-02-21 09:28:56 -08:00
parent 0d23511cee
commit d6e3430e0d
2 changed files with 69 additions and 0 deletions
--- a/src/Lexer2.h
+++ b/src/Lexer2.h
@@ -51,6 +51,7 @@ public:
  ~Lexer2 ();
  bool token (std::string&, Lexer2::Type&);
  static std::vector <std::pair <std::string, Lexer2::Type>> tokens (const std::string&);
+  static std::vector <std::string> split (const std::string&);
  static std::string typeToString (Lexer2::Type);

  // Static helpers.