stop excluding null values when filtering using notEqual (#114)

* stop excluding null values when filtering using notEqual
* add IgnoreNullsOnNotEqual config field, to enable/disable the new feature

Co-authored-by: AnasZakarneh <a.zakarneh@foothillsolutions.com>

README.md

@@ -127,7 +127,8 @@ Then you can add the configuration:
         "CaseSensitive": "boolean: should property names be case-sensitive? Defaults to false",
         "DefaultPageSize": "int number: optional number to fallback to when no page argument is given. Set <=0 to disable paging if no pageSize is specified (default).",
         "MaxPageSize": "int number: maximum allowed page size. Set <=0 to make infinite (default)",
-        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false"
+        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false",
+        "IgnoreNullsOnNotEqual": "boolean: ignore null values when filtering using is not equal operator? Default to true"
     }
 }
 ```

Sieve.Sample/appsettings.Development.json

@@ -1,6 +1,5 @@
 {
   "Logging": {
-    "IncludeScopes": false,
     "LogLevel": {
       "Default": "Debug",
       "System": "Information",

Sieve.Sample/appsettings.json

@@ -4,10 +4,10 @@
   },
   "Sieve": {
     "CaseSensitive": false,
-    "DefaultPageSize": 10
+    "DefaultPageSize": 10,
+    "IgnoreNullsOnNotEqual": true
   },
   "Logging": {
-    "IncludeScopes": false,
     "Debug": {
       "LogLevel": {
         "Default": "Warning"

Sieve/Models/FilterTerm.cs

@@ -9,6 +9,7 @@ namespace Sieve.Models
         public FilterTerm() { }
 
         private const string EscapedPipePattern = @"(?<!($|[^\\])(\\\\)*?\\)\|";
+        private const string PipeToEscape = @"\|";
 
         private static readonly string[] Operators = new string[] {
                     "!@=*",
@@ -36,7 +37,11 @@ namespace Sieve.Models
                 var filterSplits = value.Split(Operators, StringSplitOptions.RemoveEmptyEntries)
                     .Select(t => t.Trim()).ToArray();
                 Names = Regex.Split(filterSplits[0], EscapedPipePattern).Select(t => t.Trim()).ToArray();
-                Values = filterSplits.Length > 1 ? Regex.Split(filterSplits[1], EscapedPipePattern).Select(t => t.Trim()).ToArray() : null;
+                Values = filterSplits.Length > 1
+                    ? Regex.Split(filterSplits[1], EscapedPipePattern)
+                        .Select(t => t.Replace(PipeToEscape, "|").Trim())
+                        .ToArray()
+                    : null;
                 Operator = Array.Find(Operators, o => value.Contains(o)) ?? "==";
                 OperatorParsed = GetOperatorParsed(Operator);
                 OperatorIsCaseInsensitive = Operator.EndsWith("*");
@@ -90,6 +95,5 @@ namespace Sieve.Models
                 && Values.SequenceEqual(other.Values)
                 && Operator == other.Operator;
         }
-
     }
 }

Sieve/Models/SieveModel.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.Linq;

Sieve/Models/SieveOptions.cs

@@ -2,27 +2,14 @@
 {
     public class SieveOptions
     {
-        /// <summary>
-        /// If flag is set, property names have to match including case sensitivity.
-        /// </summary>
         public bool CaseSensitive { get; set; } = false;
 
-        /// <summary>
-        /// Fallback value of no page size is specified in the request.
-        /// </summary>
-        /// <remarks>Values less or equal to 0 disable paging.</remarks>
         public int DefaultPageSize { get; set; } = 0;
 
-        /// <summary>
-        /// Specifies the upper limit of a page size to be requested.
-        /// </summary>
-        /// <remarks>Values less or equal to 0 are ignored.</remarks>
         public int MaxPageSize { get; set; } = 0;
 
-        /// <summary>
-        /// If flag is set, Sieve throws exception otherwise exceptions are caught and the already processed
-        /// result is returned.
-        /// </summary>
-        public bool ThrowExceptions { get; set; } = true;
+        public bool ThrowExceptions { get; set; } = false;
+
+        public bool IgnoreNullsOnNotEqual { get; set; } = true;
     }
 }

Sieve/Services/SieveProcessor.cs

@@ -68,7 +68,6 @@ namespace Sieve.Services
         where TSortTerm : ISortTerm, new()
     {
         private const string NullFilterValue = "null";
-        private readonly IOptions<SieveOptions> _options;
         private readonly ISieveCustomSortMethods _customSortMethods;
         private readonly ISieveCustomFilterMethods _customFilterMethods;
         private readonly SievePropertyMapper _mapper = new SievePropertyMapper();
@@ -78,7 +77,7 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customSortMethods = customSortMethods;
             _customFilterMethods = customFilterMethods;
         }
@@ -87,7 +86,7 @@ namespace Sieve.Services
             ISieveCustomSortMethods customSortMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customSortMethods = customSortMethods;
         }
 
@@ -95,16 +94,18 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customFilterMethods = customFilterMethods;
         }
 
         public SieveProcessor(IOptions<SieveOptions> options)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
         }
 
+        protected IOptions<SieveOptions> Options { get; }
+
         /// <summary>
         /// Apply filtering, sorting, and pagination parameters found in `model` to `source`
         /// </summary>
@@ -148,7 +149,7 @@ namespace Sieve.Services
             }
             catch (Exception ex)
             {
-                if (!_options.Value.ThrowExceptions)
+                if (!Options.Value.ThrowExceptions)
                 {
                     return result;
                 }
@@ -162,7 +163,7 @@ namespace Sieve.Services
             }
         }
 
-        private IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        protected virtual IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetFiltersParsed() == null)
@@ -216,10 +217,13 @@ namespace Sieve.Services
                                 expression = Expression.Not(expression);
                             }
 
-                            var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
-                            if (filterValueNullCheck != null)
+                            if (expression.NodeType != ExpressionType.NotEqual || Options.Value.IgnoreNullsOnNotEqual)
                             {
-                                expression = Expression.AndAlso(filterValueNullCheck, expression);
+                                var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
+                                if (filterValueNullCheck != null)
+                                {
+                                    expression = Expression.AndAlso(filterValueNullCheck, expression);
+                                }
                             }
 
                             innerExpression = innerExpression == null
@@ -253,8 +257,7 @@ namespace Sieve.Services
                 : result.Where(Expression.Lambda<Func<TEntity, bool>>(outerExpression, parameter));
         }
 
-        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName,
-            bool isFilterTermValueNull)
+        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName, bool isFilterTermValueNull)
         {
             var (propertyValue, nullCheck) = GetPropertyValueAndNullCheckExpression(parameter, fullPropertyName);
 
@@ -336,15 +339,13 @@ namespace Sieve.Services
         }
 
         // Workaround to ensure that the filter value gets passed as a parameter in generated SQL from EF Core
-        // See https://github.com/aspnet/EntityFrameworkCore/issues/3361
-        // Expression.Constant passed the target type to allow Nullable comparison
-        // See http://bradwilson.typepad.com/blog/2008/07/creating-nullab.html
         private static Expression GetClosureOverConstant<T>(T constant, Type targetType)
         {
-            return Expression.Constant(constant, targetType);
+            Expression<Func<T>> hoistedConstant = () => constant;
+            return Expression.Convert(hoistedConstant.Body, targetType);
         }
 
-        private IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        protected virtual IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetSortsParsed() == null)
@@ -373,15 +374,11 @@ namespace Sieve.Services
             return result;
         }
 
-        private IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
+        protected virtual IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
         {
             var page = model?.Page ?? 1;
-            var pageSize = model?.PageSize ?? _options.Value.DefaultPageSize;
-
-            if (_options.Value.MaxPageSize > 0)
-            {
-                pageSize = Math.Min(pageSize, _options.Value.MaxPageSize);
-            }
+            var pageSize = model?.PageSize ?? Options.Value.DefaultPageSize;
+            var maxPageSize = Options.Value.MaxPageSize > 0 ? Options.Value.MaxPageSize : pageSize;
 
             if (pageSize <= 0)
             {
@@ -389,7 +386,7 @@ namespace Sieve.Services
             }
 
             result = result.Skip((page - 1) * pageSize);
-            result = result.Take(pageSize);
+            result = result.Take(Math.Min(pageSize, maxPageSize));
 
             return result;
         }
@@ -403,14 +400,14 @@ namespace Sieve.Services
             string name)
         {
             var property = _mapper.FindProperty<TEntity>(canSortRequired, canFilterRequired, name,
-                _options.Value.CaseSensitive);
+                Options.Value.CaseSensitive);
             if (property.Item1 != null)
             {
                 return property;
             }
 
             var prop = FindPropertyBySieveAttribute<TEntity>(canSortRequired, canFilterRequired, name,
-                _options.Value.CaseSensitive);
+                Options.Value.CaseSensitive);
             return (prop?.Name, prop);
         }
 
@@ -430,7 +427,7 @@ namespace Sieve.Services
         {
             var customMethod = parent?.GetType()
                 .GetMethodExt(name,
-                    _options.Value.CaseSensitive
+                    Options.Value.CaseSensitive
                         ? BindingFlags.Default
                         : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                     typeof(IQueryable<TEntity>));
@@ -441,7 +438,7 @@ namespace Sieve.Services
                 // Find generic methods `public IQueryable<T> Filter<T>(IQueryable<T> source, ...)`
                 var genericCustomMethod = parent?.GetType()
                     .GetMethodExt(name,
-                        _options.Value.CaseSensitive
+                        Options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                         typeof(IQueryable<>));
@@ -485,11 +482,11 @@ namespace Sieve.Services
                 var incompatibleCustomMethods =
                     parent?
                         .GetType()
-                        .GetMethods(_options.Value.CaseSensitive
+                        .GetMethods(Options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance)
                         .Where(method => string.Equals(method.Name, name,
-                            _options.Value.CaseSensitive
+                            Options.Value.CaseSensitive
                                 ? StringComparison.InvariantCulture
                                 : StringComparison.InvariantCultureIgnoreCase))
                         .ToList()

SieveUnitTests/General.cs

@@ -15,16 +15,24 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
+        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<Post> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public General(ITestOutputHelper testOutputHelper)
         {
+            var nullableAccessor = new SieveOptionsAccessor();
+            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
+
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
+            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
+                new SieveCustomSortMethods(),
+                new SieveCustomFilterMethods());
+
             _posts = new List<Post>
             {
                 new Post
@@ -180,10 +188,27 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
+            Assert.True(nullableResult.Count() == 2);
         }
-        
+
+        [Fact]
+        public void CanFilterNullableIntsWithNotEqual()
+        {
+            var model = new SieveModel()
+            {
+                Filters = "CategoryId!=1"
+            };
+
+            var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
+
+            Assert.True(result.Count() == 1);
+            Assert.True(nullableResult.Count() == 2);
+        }
+
         [Theory]
         [InlineData(@"Text@=*\,")]
         [InlineData(@"Text@=*\, ")]
@@ -613,44 +638,61 @@ namespace SieveUnitTests
             Assert.Equal(1,posts[2].Id);
             Assert.Equal(0,posts[3].Id);
         }
-        
-        [Theory]
-        [InlineData(1)]
-        [InlineData(2)]
-        [InlineData(3)]
-        [InlineData(4)]
-        public void Paging_DifferentPages(int page)
+
+        [Fact]
+        public void CanFilter_WithEscapeCharacter()
         {
+            var comments = new List<Comment>
+            {
+                new Comment
+                {
+                    Id = 0,
+                    DateCreated = DateTimeOffset.UtcNow,
+                    Text = "Here is, a comment"
+                },
+                new Comment
+                {
+                    Id = 1,
+                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
+                    Text = "Here is, another comment"
+                },
+            }.AsQueryable();
+
             var model = new SieveModel
             {
-                Page = page,
-                PageSize = 1,
+                Filters = "Text==Here is\\, another comment"
             };
 
-            var result = _processor.Apply(model, _posts);
-            var posts = result.ToList();
-            
-            Assert.Single(posts);
-            var expectedId = page - 1;
-            Assert.Equal(expectedId, posts.First().Id);
+            var result = _processor.Apply(model, comments);
+            Assert.Equal(1, result.Count());
         }
-        
-        [Theory]
-        [InlineData(1)]
-        [InlineData(2)]
-        [InlineData(3)]
-        [InlineData(4)]
-        public void Paging_DifferentPageSizes(int pageSize)
+
+        [Fact]
+        public void OrEscapedPipeValueFilteringWorks()
         {
-            var model = new SieveModel
+            var comments = new List<Comment>
             {
-                Page = 1,
-                PageSize = pageSize,
+                new Comment
+                {
+                    Id = 0,
+                    DateCreated = DateTimeOffset.UtcNow,
+                    Text = "Here is | a comment"
+                },
+                new Comment
+                {
+                    Id = 1,
+                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
+                    Text = "Here is | another comment"
+                },
+            }.AsQueryable();
+
+            var model = new SieveModel()
+            {
+                Filters = "Text==Here is \\| a comment|Here is \\| another comment",
             };
 
-            var result = _processor.Apply(model, _posts);
-
-            Assert.Equal(pageSize, result.Count());
+            var result = _processor.Apply(model, comments);
+            Assert.Equal(2, result.Count());
         }
     }
 }

SieveUnitTests/GeneralWithInterfaces.cs

@@ -16,16 +16,24 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
+        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<IPost> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public GeneralWithInterfaces(ITestOutputHelper testOutputHelper)
         {
+            var nullableAccessor = new SieveOptionsAccessor();
+            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
+
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
+            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
+                new SieveCustomSortMethods(),
+                new SieveCustomFilterMethods());
+
             _posts = new List<IPost>
             {
                 new Post
@@ -181,8 +189,25 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
+            Assert.True(nullableResult.Count() == 2);
+        }
+
+        [Fact]
+        public void CanFilterNullableIntsWithNotEqual()
+        {
+            var model = new SieveModel()
+            {
+                Filters = "CategoryId!=1"
+            };
+
+            var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
+
+            Assert.True(result.Count() == 1);
+            Assert.True(nullableResult.Count() == 2);
         }
 
         [Fact]