Helpers

- Implemented isTokenEnd, as a special case of isWordEnd, but considers
  consecutive punctuation to be a set of individual tokens.

src/text.cpp

@@ -601,7 +601,7 @@ bool isWordStart (const std::string& input, std::string::size_type pos)
 // Result for pos: ....y......y
 bool isWordEnd (const std::string& input, std::string::size_type pos)
 {
-  // Short circuit: no input means no word start.
+  // Short circuit: no input means no word end.
   if (input.length () == 0)
     return false;
 
@@ -618,6 +618,25 @@ bool isWordEnd (const std::string& input, std::string::size_type pos)
   return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+//          Input: hello, world
+// Result for pos: ....y......y
+//
+//          Input: (one) two
+// Result for pos: y..yy...y
+bool isTokenEnd (const std::string& input, std::string::size_type pos)
+{
+  // Delegate.
+  if (isWordEnd (input, pos))
+    return true;
+
+  // Punctuation divides tokens.
+  if (pos < input.length () && isPunctuation (input[pos]))
+    return true;
+
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Override of ispunct, that considers #, $ and @ not to be punctuation.
 //