Fixed null-ref for case-(in)sensitive null-search (#165)

* Fixed null-ref for case-insensitive null-search
Added null-escaping sequence (to distinguish between prop==null (as null) and prop==\null (as string))

* Added null-search case-insensitive test

* Code style

* Added escape-sequences description to README.md

Co-authored-by: Nikita Prokhorov <nikita.prokhorov@grse.de>

README.md

@@ -159,7 +159,10 @@ More formally:
 * `pageSize` is the number of items returned per page 
 
 Notes:
-* You can use backslashes to escape commas and pipes within value fields
+* You can use backslashes to escape special characters and sequences:
+    * commas: `Title@=some\,title` makes a match with "some,title"
+    * pipes: `Title@=some\|title` makes a match with "some|title"
+    * null values: `Title@=\null` will search for items with title equal to "null" (not a missing value, but "null"-string literally)
 * You can have spaces anywhere except *within* `{Name}` or `{Operator}` fields
 * If you need to look at the data before applying pagination (eg. get total count), use the optional paramters on `Apply` to defer pagination (an [example](https://github.com/Biarity/Sieve/issues/34))
 * Here's a [good example on how to work with enumerables](https://github.com/Biarity/Sieve/issues/2)

Sieve/Models/FilterTerm.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
 
@@ -6,50 +7,49 @@ namespace Sieve.Models
 {
     public class FilterTerm : IFilterTerm, IEquatable<FilterTerm>
     {
-        public FilterTerm() { }
+        private const string EscapedPipePattern = @"(?<!($|[^\\]|^)(\\\\)*?\\)\|";
+        private const string OperatorsRegEx = @"(!@=\*|!_=\*|!=\*|!@=|!_=|==\*|@=\*|_=\*|==|!=|>=|<=|>|<|@=|_=)";
+        private const string EscapeNegPatternForOper = @"(?<!\\)" + OperatorsRegEx;
+        private const string EscapePosPatternForOper = @"(?<=\\)" + OperatorsRegEx;
 
-        private const string EscapedPipePattern = @"(?<!($|[^\\])(\\\\)*?\\)\|";
-        private const string PipeToEscape = @"\|";
-
-        private static readonly string[] Operators = new string[] {
-                    "!@=*",
-                    "!_=*",
-                    "!=*",
-                    "!@=",
-                    "!_=",
-                    "==*",
-                    "@=*",
-                    "_=*",
-                    "==",
-                    "!=",
-                    ">=",
-                    "<=",
-                    ">",
-                    "<",
-                    "@=",
-                    "_="
+        private static readonly HashSet<string> _escapedSequences = new HashSet<string>
+        {
+            @"\|",
+            @"\\"
         };
 
         public string Filter
         {
             set
             {
-                var filterSplits = value.Split(Operators, StringSplitOptions.RemoveEmptyEntries)
-                    .Select(t => t.Trim()).ToArray();
+                var filterSplits = Regex.Split(value,EscapeNegPatternForOper).Select(t => t.Trim()).ToArray();
+                
                 Names = Regex.Split(filterSplits[0], EscapedPipePattern).Select(t => t.Trim()).ToArray();
-                Values = filterSplits.Length > 1
-                    ? Regex.Split(filterSplits[1], EscapedPipePattern)
-                        .Select(t => t.Replace(PipeToEscape, "|").Trim())
-                        .ToArray()
-                    : null;
-                Operator = Array.Find(Operators, o => value.Contains(o)) ?? "==";
+
+                if (filterSplits.Length > 2)
+                {
+                    foreach (var match in Regex.Matches(filterSplits[2],EscapePosPatternForOper))
+                    {
+                        var matchStr = match.ToString();
+                        filterSplits[2] = filterSplits[2].Replace('\\' + matchStr, matchStr);
+                    }
+
+                    Values = Regex.Split(filterSplits[2], EscapedPipePattern)
+                        .Select(UnEscape)
+                        .ToArray();
+                }
+ 
+                Operator = Regex.Match(value,EscapeNegPatternForOper).Value;
                 OperatorParsed = GetOperatorParsed(Operator);
                 OperatorIsCaseInsensitive = Operator.EndsWith("*");
                 OperatorIsNegated = OperatorParsed != FilterOperator.NotEquals && Operator.StartsWith("!");
             }
-
         }
 
+        private string UnEscape(string escapedTerm)
+            => _escapedSequences.Aggregate(escapedTerm,
+                (current, sequence) => Regex.Replace(current, $@"(\\)({sequence})", "$2"));
+
         public string[] Names { get; private set; }
 
         public FilterOperator OperatorParsed { get; private set; }

Sieve/Services/SieveProcessor.cs

@@ -68,6 +68,7 @@ namespace Sieve.Services
         where TSortTerm : ISortTerm, new()
     {
         private const string NullFilterValue = "null";
+        private const char EscapeChar = '\\';
         private readonly ISieveCustomSortMethods _customSortMethods;
         private readonly ISieveCustomFilterMethods _customFilterMethods;
         private readonly SievePropertyMapper _mapper = new SievePropertyMapper();
@@ -199,7 +200,7 @@ namespace Sieve.Services
                                 ? Expression.Constant(null, property.PropertyType)
                                 : ConvertStringValueToConstantExpression(filterTermValue, property, converter);
 
-                            if (filterTerm.OperatorIsCaseInsensitive)
+                            if (filterTerm.OperatorIsCaseInsensitive && !isFilterTermValueNull)
                             {
                                 propertyValue = Expression.Call(propertyValue,
                                     typeof(string).GetMethods()
@@ -311,6 +312,10 @@ namespace Sieve.Services
         private static Expression ConvertStringValueToConstantExpression(string value, PropertyInfo property,
             TypeConverter converter)
         {
+            // to allow user to distinguish between prop==null (as null) and prop==\null (as "null"-string)
+            value = value.Equals(EscapeChar + NullFilterValue, StringComparison.InvariantCultureIgnoreCase) 
+                ? value.TrimStart(EscapeChar) 
+                : value;
             dynamic constantVal = converter.CanConvertFrom(typeof(string))
                 ? converter.ConvertFrom(value)
                 : Convert.ChangeType(value, property.PropertyType);

SieveUnitTests/General.cs

@@ -73,7 +73,7 @@ namespace SieveUnitTests
                     CategoryId = 2,
                     TopComment = new Comment { Id = 1, Text = "D1" },
                     FeaturedComment = new Comment { Id = 7, Text = "D2" }
-                },
+                }
             }.AsQueryable();
 
             _comments = new List<Comment>
@@ -684,15 +684,116 @@ namespace SieveUnitTests
                     DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
                     Text = "Here is | another comment"
                 },
+                new Comment
+                {
+                    Id = 2,
+                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
+                    Text = @"Here is \| another comment(1)"
+                }
             }.AsQueryable();
 
-            var model = new SieveModel()
+            var model = new SieveModel
             {
-                Filters = "Text==Here is \\| a comment|Here is \\| another comment",
+                Filters = @"Text==Here is \| a comment|Here is \| another comment|Here is \\\| another comment(1)",
             };
 
             var result = _processor.Apply(model, comments);
-            Assert.Equal(2, result.Count());
+            Assert.Equal(3, result.Count());
         }
+        
+        [Theory]
+        [InlineData("CategoryId==1,(CategoryId|LikeCount)==50")]
+        [InlineData("(CategoryId|LikeCount)==50,CategoryId==1")]
+        public void CanFilterWithEscape(string filter)
+        {
+            var model = new SieveModel
+            {
+                Filters = filter
+            };
+
+            var result = _processor.Apply(model, _posts);
+            var entry = result.FirstOrDefault();
+            var resultCount = result.Count();
+
+            Assert.NotNull(entry);
+            Assert.Equal(1, resultCount);
+        }
+        
+        [Theory]
+        [InlineData(@"Title@=\\")]
+        public void CanFilterWithEscapedBackSlash(string filter)
+        {
+            var posts = new List<Post>
+            {
+                new Post
+                {
+                    Id = 1,
+                    Title = "E\\",
+                    LikeCount = 4,
+                    IsDraft = true,
+                    CategoryId = 1,
+                    TopComment = new Comment { Id = 1, Text = "E1" },
+                    FeaturedComment = new Comment { Id = 7, Text = "E2" }
+                }
+            }.AsQueryable();
+            
+            var model = new SieveModel
+            {
+                Filters = filter
+            };
+
+            var result = _processor.Apply(model, posts);
+            var entry = result.FirstOrDefault();
+            var resultCount = result.Count();
+
+            Assert.NotNull(entry);
+            Assert.Equal(1, resultCount);
+        }
+        
+        [Theory]
+        [InlineData(@"Title@=\== ")]
+        [InlineData(@"Title@=\!= ")]
+        [InlineData(@"Title@=\> ")]
+        [InlineData(@"Title@=\< ")]
+        [InlineData(@"Title@=\<= ")]
+        [InlineData(@"Title@=\>= ")]
+        [InlineData(@"Title@=\@= ")]
+        [InlineData(@"Title@=\_= ")]
+        [InlineData(@"Title@=!\@= ")]
+        [InlineData(@"Title@=!\_= ")]
+        [InlineData(@"Title@=\@=* ")]
+        [InlineData(@"Title@=\_=* ")]
+        [InlineData(@"Title@=\==* ")]
+        [InlineData(@"Title@=\!=* ")]
+        [InlineData(@"Title@=!\@=* ")]
+        public void CanFilterWithEscapedOperators(string filter)
+        {
+            var posts = new List<Post>
+            {
+                new Post
+                {
+                    Id = 1,
+                    Title = @"Operators: == != > < >= <= @= _= !@= !_= @=* _=* ==* !=* !@=* !_=* ",
+                    LikeCount = 1,
+                    IsDraft = true,
+                    CategoryId = 1,
+                    TopComment = new Comment { Id = 1, Text = "F1" },
+                    FeaturedComment = new Comment { Id = 7, Text = "F2" }
+                }
+            }.AsQueryable();
+            
+            var model = new SieveModel
+            {
+                Filters = filter,
+            };
+
+            var result = _processor.Apply(model, posts);
+            var entry = result.FirstOrDefault();
+            var resultCount = result.Count();
+
+            Assert.NotNull(entry);
+            Assert.Equal(1, resultCount);
+        }
+        
     }
 }

SieveUnitTests/StringFilterNullTests.cs

@@ -38,7 +38,7 @@ namespace SieveUnitTests
                 {
                     Id = 2, 
                     DateCreated = DateTimeOffset.UtcNow, 
-                    Text = "Regular comment without n*ll.",
+                    Text = "Regular comment without n*ll",
                     Author = "Mouse",
                 },
                 new Comment
@@ -47,24 +47,28 @@ namespace SieveUnitTests
                     DateCreated = DateTimeOffset.UtcNow, 
                     Text = null,
                     Author = "null",
-                },
+                }
             }.AsQueryable();
         }
 
-        [Fact]
-        public void Filter_Equals_Null()
+        [Theory]
+        [InlineData("Text==null")]
+        [InlineData("Text==*null")]
+        public void Filter_Equals_Null(string filter)
         {
-            var model = new SieveModel {Filters = "Text==null"};
+            var model = new SieveModel {Filters = filter};
 
             var result = _processor.Apply(model, _comments);
 
             Assert.Equal(100, result.Single().Id);
         }
 
-        [Fact]
-        public void Filter_NotEquals_Null()
+        [Theory]
+        [InlineData("Text!=null")]
+        [InlineData("Text!=*null")]
+        public void Filter_NotEquals_Null(string filter)
         {
-            var model = new SieveModel {Filters = "Text!=null"};
+            var model = new SieveModel {Filters = filter};
 
             var result = _processor.Apply(model, _comments);
 
@@ -101,6 +105,22 @@ namespace SieveUnitTests
             Assert.Equal(expectedIds, result.Select(p => p.Id));
         }
 
+        [Theory]
+        [InlineData(@"Author==\null", 100)]
+        [InlineData(@"Author==*\null", 100)]
+        [InlineData(@"Author==*\NuLl", 100)]
+        [InlineData(@"Author!=*\null", 0, 1, 2)]
+        [InlineData(@"Author!=*\NulL", 0, 1, 2)]
+        [InlineData(@"Author!=\null", 0, 1, 2)]
+        public void SingleFilter_Equals_NullStringEscaped(string filter, params int[] expectedIds)
+        {
+            var model = new SieveModel {Filters = filter};
+
+            var result = _processor.Apply(model, _comments);
+
+            Assert.Equal(expectedIds, result.Select(p => p.Id));
+        }
+
         [Theory]
         [InlineData("Text_=null")]
         [InlineData("Text_=*null")]