Quoting

- Removed automatic dequoting by the Lexer.
- Implemented Lexer::dequote for manual control.
- Variant dequotes string values when appropriate.
- Fixed some unit tests that became wrong.

src/CLI.cpp

@@ -1637,7 +1637,9 @@ void CLI::desugarFilterPlainArgs ()
       op.tag ("FILTER");
       reconstructed.push_back (op);
 
-      A rhs ("argPattern", "'" + a->attribute ("raw") + "'");
+      std::string pattern = a->attribute ("raw");
+      Lexer::dequote (pattern);
+      A rhs ("argPattern", "'" + pattern + "'");
       rhs.tag ("LITERAL");
       rhs.tag ("FILTER");
       reconstructed.push_back (rhs);

src/Lexer.cpp

@@ -87,6 +87,7 @@ bool Lexer::token (std::string& result, Type& type)
       {
         type = typeString;
         quote = _n0;
+        result += utf8_character (_n0);
         shift ();
       }
       else if (_n0 == '0' &&
@@ -189,6 +190,7 @@ bool Lexer::token (std::string& result, Type& type)
     case typeString:
       if (_n0 == quote)
       {
+        result += utf8_character (_n0);
         shift ();
         quote = 0;
         return true;
@@ -247,6 +249,7 @@ bool Lexer::token (std::string& result, Type& type)
       else
       {
         type = quote ? typeString : typeIdentifier;
+        result += utf8_character (quote);
         result += utf8_character (_n0);
         shift ();
       }
@@ -265,7 +268,8 @@ bool Lexer::token (std::string& result, Type& type)
       }
       else
       {
-        result += decode_escape (_n0);
+        result += '\\';
+        result += utf8_character (_n0);
         type = quote ? typeString : typeIdentifier;
         shift ();
       }
@@ -444,6 +448,7 @@ bool Lexer::word (std::string& token, Type& type)
       {
         type = typeString;
         quote = _n0;
+        token += utf8_character (_n0);
         shift ();
       }
       else
@@ -457,6 +462,7 @@ bool Lexer::word (std::string& token, Type& type)
     case typeString:
       if (_n0 == quote)
       {
+        token += utf8_character (_n0);
         shift ();
         quote = 0;
         return true;
@@ -491,7 +497,8 @@ bool Lexer::word (std::string& token, Type& type)
       }
       else
       {
-        token += decode_escape (_n0);
+        token += '\\';
+        token += utf8_character (_n0);
         type = typeString;
         shift ();
       }
@@ -709,6 +716,18 @@ void Lexer::token_split (std::vector <std::pair <std::string, Lexer::Type> >& le
     lexemes.push_back (std::pair <std::string, Lexer::Type>(word, type));
 }
 
+////////////////////////////////////////////////////////////////////////////////
+void Lexer::dequote (std::string& input)
+{
+  int quote = input[0];
+  size_t len = input.length ();
+  if ((quote == '\'' || quote == '"') &&
+      quote == input[len - 1])
+  {
+    input = input.substr (1, len - 2);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 bool Lexer::is_date (std::string& result)
 {
@@ -830,7 +849,6 @@ int Lexer::decode_escape (int c) const
   case 'v':  return 0x0B;
   case '\'': return 0x27;
   case '"':  return 0x22;
-  case '\\': return 0x5C;
   default:   return c;
   }
 }

src/Lexer.h

@@ -84,6 +84,7 @@ public:
   static void word_split (std::vector <std::string>&, const std::string&);
   static void token_split (std::vector <std::string>&, const std::string&);
   static void token_split (std::vector <std::pair <std::string, Lexer::Type> >&, const std::string&);
+  static void dequote (std::string&);
 
 private:
   bool is_date (std::string&);

src/Variant.cpp

@@ -31,6 +31,7 @@
 #include <stdlib.h>
 #include <Variant.h>
 #include <ISO8601.h>
+#include <Lexer.h>
 #include <Date.h>
 #include <Duration.h>
 #include <RX.h>
@@ -193,6 +194,12 @@ bool Variant::operator&& (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   left.cast (type_boolean);
   right.cast (type_boolean);
 
@@ -205,6 +212,12 @@ bool Variant::operator|| (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   left.cast (type_boolean);
   right.cast (type_boolean);
 
@@ -217,6 +230,12 @@ bool Variant::operator_xor (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   left.cast (type_boolean);
   right.cast (type_boolean);
 
@@ -230,6 +249,12 @@ bool Variant::operator< (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -369,6 +394,12 @@ bool Variant::operator<= (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -509,6 +540,12 @@ bool Variant::operator> (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -647,6 +684,12 @@ bool Variant::operator>= (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -787,6 +830,12 @@ bool Variant::operator== (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -911,12 +960,21 @@ bool Variant::operator_match (const Variant& other, const Task& task) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   left.cast (type_string);
   right.cast (type_string);
 
+  std::string pattern = right._string;
+  Lexer::dequote (pattern);
+
   if (searchUsingRegex)
   {
-    RX r (right._string, searchCaseSensitive);
+    RX r (pattern, searchCaseSensitive);
     if (r.match (left._string))
       return true;
 
@@ -935,7 +993,7 @@ bool Variant::operator_match (const Variant& other, const Task& task) const
   }
   else
   {
-    if (find (left._string, right._string, searchCaseSensitive) != std::string::npos)
+    if (find (left._string, pattern, searchCaseSensitive) != std::string::npos)
       return true;
 
     // If the above did not match, and the left source is "description", then
@@ -947,7 +1005,7 @@ bool Variant::operator_match (const Variant& other, const Task& task) const
 
       std::map <std::string, std::string>::iterator a;
       for (a = annotations.begin (); a != annotations.end (); ++a)
-        if (find (a->second, right._string, searchCaseSensitive) != std::string::npos)
+        if (find (a->second, pattern, searchCaseSensitive) != std::string::npos)
           return true;
     }
   }
@@ -972,6 +1030,12 @@ bool Variant::operator_partial (const Variant& other) const
   Variant left (*this);
   Variant right (other);
 
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (left._type)
   {
   case type_unknown:
@@ -1155,6 +1219,7 @@ bool Variant::operator_hastag (const Variant& other, const Task& task) const
 {
   Variant right (other);
   right.cast (type_string);
+  Lexer::dequote (right._string);
   return task.hasTag (right._string);
 }
 
@@ -1168,6 +1233,10 @@ bool Variant::operator_notag (const Variant& other, const Task& task) const
 bool Variant::operator! () const
 {
   Variant left (*this);
+
+  if (left._type == type_string)
+    Lexer::dequote (left._string);
+
   left.cast (type_boolean);
   return ! left._bool;
 }
@@ -1330,6 +1399,9 @@ Variant& Variant::operator+= (const Variant& other)
 {
   Variant right (other);
 
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (_type)
   {
   case type_unknown:
@@ -1439,6 +1511,9 @@ Variant& Variant::operator*= (const Variant& other)
 {
   Variant right (other);
 
+  if (right._type == type_string)
+    Lexer::dequote (right._string);
+
   switch (_type)
   {
   case type_unknown:
@@ -1893,6 +1968,9 @@ Variant::operator std::string () const
 ////////////////////////////////////////////////////////////////////////////////
 void Variant::sqrt ()
 {
+  if (_type == type_string)
+    Lexer::dequote (_string);
+
   cast (type_real);
   if (_real < 0.0)
     throw std::string (STRING_VARIANT_SQRT_NEG);
@@ -1967,6 +2045,7 @@ void Variant::cast (const enum type new_type)
     break;
 
   case type_string:
+    Lexer::dequote (_string);
     switch (new_type)
     {
     case type_unknown:                                                        break;