Fix broken paging (#136)

* Add unit tests
* Use calculated page size instead of page

Co-authored-by: ITDancer139 <kevinitdancersommer@gmail.com>

README.md

@@ -127,8 +127,7 @@ Then you can add the configuration:
         "CaseSensitive": "boolean: should property names be case-sensitive? Defaults to false",
         "DefaultPageSize": "int number: optional number to fallback to when no page argument is given. Set <=0 to disable paging if no pageSize is specified (default).",
         "MaxPageSize": "int number: maximum allowed page size. Set <=0 to make infinite (default)",
-        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false",
-        "IgnoreNullsOnNotEqual": "boolean: ignore null values when filtering using is not equal operator? Default to true"
+        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false"
     }
 }
 ```

Sieve.Sample/appsettings.Development.json

@@ -1,5 +1,6 @@
 {
   "Logging": {
+    "IncludeScopes": false,
     "LogLevel": {
       "Default": "Debug",
       "System": "Information",

Sieve.Sample/appsettings.json

@@ -4,10 +4,10 @@
   },
   "Sieve": {
     "CaseSensitive": false,
-    "DefaultPageSize": 10,
-    "IgnoreNullsOnNotEqual": true
+    "DefaultPageSize": 10
   },
   "Logging": {
+    "IncludeScopes": false,
     "Debug": {
       "LogLevel": {
         "Default": "Warning"

Sieve/Models/FilterTerm.cs

@@ -9,7 +9,6 @@ namespace Sieve.Models
         public FilterTerm() { }
 
         private const string EscapedPipePattern = @"(?<!($|[^\\])(\\\\)*?\\)\|";
-        private const string PipeToEscape = @"\|";
 
         private static readonly string[] Operators = new string[] {
                     "!@=*",
@@ -37,11 +36,7 @@ namespace Sieve.Models
                 var filterSplits = value.Split(Operators, StringSplitOptions.RemoveEmptyEntries)
                     .Select(t => t.Trim()).ToArray();
                 Names = Regex.Split(filterSplits[0], EscapedPipePattern).Select(t => t.Trim()).ToArray();
-                Values = filterSplits.Length > 1
-                    ? Regex.Split(filterSplits[1], EscapedPipePattern)
-                        .Select(t => t.Replace(PipeToEscape, "|").Trim())
-                        .ToArray()
-                    : null;
+                Values = filterSplits.Length > 1 ? Regex.Split(filterSplits[1], EscapedPipePattern).Select(t => t.Trim()).ToArray() : null;
                 Operator = Array.Find(Operators, o => value.Contains(o)) ?? "==";
                 OperatorParsed = GetOperatorParsed(Operator);
                 OperatorIsCaseInsensitive = Operator.EndsWith("*");
@@ -95,5 +90,6 @@ namespace Sieve.Models
                 && Values.SequenceEqual(other.Values)
                 && Operator == other.Operator;
         }
+
     }
 }

Sieve/Models/SieveModel.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.Linq;

Sieve/Models/SieveOptions.cs

@@ -2,14 +2,27 @@
 {
     public class SieveOptions
     {
+        /// <summary>
+        /// If flag is set, property names have to match including case sensitivity.
+        /// </summary>
         public bool CaseSensitive { get; set; } = false;
 
+        /// <summary>
+        /// Fallback value of no page size is specified in the request.
+        /// </summary>
+        /// <remarks>Values less or equal to 0 disable paging.</remarks>
         public int DefaultPageSize { get; set; } = 0;
 
+        /// <summary>
+        /// Specifies the upper limit of a page size to be requested.
+        /// </summary>
+        /// <remarks>Values less or equal to 0 are ignored.</remarks>
         public int MaxPageSize { get; set; } = 0;
 
-        public bool ThrowExceptions { get; set; } = false;
-
-        public bool IgnoreNullsOnNotEqual { get; set; } = true;
+        /// <summary>
+        /// If flag is set, Sieve throws exception otherwise exceptions are caught and the already processed
+        /// result is returned.
+        /// </summary>
+        public bool ThrowExceptions { get; set; } = true;
     }
 }

Sieve/Services/SieveProcessor.cs

@@ -68,6 +68,7 @@ namespace Sieve.Services
         where TSortTerm : ISortTerm, new()
     {
         private const string NullFilterValue = "null";
+        private readonly IOptions<SieveOptions> _options;
         private readonly ISieveCustomSortMethods _customSortMethods;
         private readonly ISieveCustomFilterMethods _customFilterMethods;
         private readonly SievePropertyMapper _mapper = new SievePropertyMapper();
@@ -77,7 +78,7 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            Options = options;
+            _options = options;
             _customSortMethods = customSortMethods;
             _customFilterMethods = customFilterMethods;
         }
@@ -86,7 +87,7 @@ namespace Sieve.Services
             ISieveCustomSortMethods customSortMethods)
         {
             _mapper = MapProperties(_mapper);
-            Options = options;
+            _options = options;
             _customSortMethods = customSortMethods;
         }
 
@@ -94,18 +95,16 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            Options = options;
+            _options = options;
             _customFilterMethods = customFilterMethods;
         }
 
         public SieveProcessor(IOptions<SieveOptions> options)
         {
             _mapper = MapProperties(_mapper);
-            Options = options;
+            _options = options;
         }
 
-        protected IOptions<SieveOptions> Options { get; }
-
         /// <summary>
         /// Apply filtering, sorting, and pagination parameters found in `model` to `source`
         /// </summary>
@@ -149,7 +148,7 @@ namespace Sieve.Services
             }
             catch (Exception ex)
             {
-                if (!Options.Value.ThrowExceptions)
+                if (!_options.Value.ThrowExceptions)
                 {
                     return result;
                 }
@@ -163,7 +162,7 @@ namespace Sieve.Services
             }
         }
 
-        protected virtual IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        private IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetFiltersParsed() == null)
@@ -217,13 +216,10 @@ namespace Sieve.Services
                                 expression = Expression.Not(expression);
                             }
 
-                            if (expression.NodeType != ExpressionType.NotEqual || Options.Value.IgnoreNullsOnNotEqual)
+                            var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
+                            if (filterValueNullCheck != null)
                             {
-                                var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
-                                if (filterValueNullCheck != null)
-                                {
-                                    expression = Expression.AndAlso(filterValueNullCheck, expression);
-                                }
+                                expression = Expression.AndAlso(filterValueNullCheck, expression);
                             }
 
                             innerExpression = innerExpression == null
@@ -257,7 +253,8 @@ namespace Sieve.Services
                 : result.Where(Expression.Lambda<Func<TEntity, bool>>(outerExpression, parameter));
         }
 
-        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName, bool isFilterTermValueNull)
+        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName,
+            bool isFilterTermValueNull)
         {
             var (propertyValue, nullCheck) = GetPropertyValueAndNullCheckExpression(parameter, fullPropertyName);
 
@@ -339,13 +336,15 @@ namespace Sieve.Services
         }
 
         // Workaround to ensure that the filter value gets passed as a parameter in generated SQL from EF Core
+        // See https://github.com/aspnet/EntityFrameworkCore/issues/3361
+        // Expression.Constant passed the target type to allow Nullable comparison
+        // See http://bradwilson.typepad.com/blog/2008/07/creating-nullab.html
         private static Expression GetClosureOverConstant<T>(T constant, Type targetType)
         {
-            Expression<Func<T>> hoistedConstant = () => constant;
-            return Expression.Convert(hoistedConstant.Body, targetType);
+            return Expression.Constant(constant, targetType);
         }
 
-        protected virtual IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        private IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetSortsParsed() == null)
@@ -374,11 +373,15 @@ namespace Sieve.Services
             return result;
         }
 
-        protected virtual IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
+        private IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
         {
             var page = model?.Page ?? 1;
-            var pageSize = model?.PageSize ?? Options.Value.DefaultPageSize;
-            var maxPageSize = Options.Value.MaxPageSize > 0 ? Options.Value.MaxPageSize : pageSize;
+            var pageSize = model?.PageSize ?? _options.Value.DefaultPageSize;
+
+            if (_options.Value.MaxPageSize > 0)
+            {
+                pageSize = Math.Min(pageSize, _options.Value.MaxPageSize);
+            }
 
             if (pageSize <= 0)
             {
@@ -386,7 +389,7 @@ namespace Sieve.Services
             }
 
             result = result.Skip((page - 1) * pageSize);
-            result = result.Take(Math.Min(pageSize, maxPageSize));
+            result = result.Take(pageSize);
 
             return result;
         }
@@ -400,14 +403,14 @@ namespace Sieve.Services
             string name)
         {
             var property = _mapper.FindProperty<TEntity>(canSortRequired, canFilterRequired, name,
-                Options.Value.CaseSensitive);
+                _options.Value.CaseSensitive);
             if (property.Item1 != null)
             {
                 return property;
             }
 
             var prop = FindPropertyBySieveAttribute<TEntity>(canSortRequired, canFilterRequired, name,
-                Options.Value.CaseSensitive);
+                _options.Value.CaseSensitive);
             return (prop?.Name, prop);
         }
 
@@ -427,7 +430,7 @@ namespace Sieve.Services
         {
             var customMethod = parent?.GetType()
                 .GetMethodExt(name,
-                    Options.Value.CaseSensitive
+                    _options.Value.CaseSensitive
                         ? BindingFlags.Default
                         : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                     typeof(IQueryable<TEntity>));
@@ -438,7 +441,7 @@ namespace Sieve.Services
                 // Find generic methods `public IQueryable<T> Filter<T>(IQueryable<T> source, ...)`
                 var genericCustomMethod = parent?.GetType()
                     .GetMethodExt(name,
-                        Options.Value.CaseSensitive
+                        _options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                         typeof(IQueryable<>));
@@ -482,11 +485,11 @@ namespace Sieve.Services
                 var incompatibleCustomMethods =
                     parent?
                         .GetType()
-                        .GetMethods(Options.Value.CaseSensitive
+                        .GetMethods(_options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance)
                         .Where(method => string.Equals(method.Name, name,
-                            Options.Value.CaseSensitive
+                            _options.Value.CaseSensitive
                                 ? StringComparison.InvariantCulture
                                 : StringComparison.InvariantCultureIgnoreCase))
                         .ToList()

SieveUnitTests/General.cs

@@ -15,24 +15,16 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
-        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<Post> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public General(ITestOutputHelper testOutputHelper)
         {
-            var nullableAccessor = new SieveOptionsAccessor();
-            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
-
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
-            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
-                new SieveCustomSortMethods(),
-                new SieveCustomFilterMethods());
-
             _posts = new List<Post>
             {
                 new Post
@@ -188,27 +180,10 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
-            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
-            Assert.True(nullableResult.Count() == 2);
         }
-
-        [Fact]
-        public void CanFilterNullableIntsWithNotEqual()
-        {
-            var model = new SieveModel()
-            {
-                Filters = "CategoryId!=1"
-            };
-
-            var result = _processor.Apply(model, _posts);
-            var nullableResult = _nullableProcessor.Apply(model, _posts);
-
-            Assert.True(result.Count() == 1);
-            Assert.True(nullableResult.Count() == 2);
-        }
-
+        
         [Theory]
         [InlineData(@"Text@=*\,")]
         [InlineData(@"Text@=*\, ")]
@@ -638,61 +613,44 @@ namespace SieveUnitTests
             Assert.Equal(1,posts[2].Id);
             Assert.Equal(0,posts[3].Id);
         }
-
-        [Fact]
-        public void CanFilter_WithEscapeCharacter()
+        
+        [Theory]
+        [InlineData(1)]
+        [InlineData(2)]
+        [InlineData(3)]
+        [InlineData(4)]
+        public void Paging_DifferentPages(int page)
         {
-            var comments = new List<Comment>
-            {
-                new Comment
-                {
-                    Id = 0,
-                    DateCreated = DateTimeOffset.UtcNow,
-                    Text = "Here is, a comment"
-                },
-                new Comment
-                {
-                    Id = 1,
-                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
-                    Text = "Here is, another comment"
-                },
-            }.AsQueryable();
-
             var model = new SieveModel
             {
-                Filters = "Text==Here is\\, another comment"
+                Page = page,
+                PageSize = 1,
             };
 
-            var result = _processor.Apply(model, comments);
-            Assert.Equal(1, result.Count());
+            var result = _processor.Apply(model, _posts);
+            var posts = result.ToList();
+            
+            Assert.Single(posts);
+            var expectedId = page - 1;
+            Assert.Equal(expectedId, posts.First().Id);
         }
-
-        [Fact]
-        public void OrEscapedPipeValueFilteringWorks()
+        
+        [Theory]
+        [InlineData(1)]
+        [InlineData(2)]
+        [InlineData(3)]
+        [InlineData(4)]
+        public void Paging_DifferentPageSizes(int pageSize)
         {
-            var comments = new List<Comment>
+            var model = new SieveModel
             {
-                new Comment
-                {
-                    Id = 0,
-                    DateCreated = DateTimeOffset.UtcNow,
-                    Text = "Here is | a comment"
-                },
-                new Comment
-                {
-                    Id = 1,
-                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
-                    Text = "Here is | another comment"
-                },
-            }.AsQueryable();
-
-            var model = new SieveModel()
-            {
-                Filters = "Text==Here is \\| a comment|Here is \\| another comment",
+                Page = 1,
+                PageSize = pageSize,
             };
 
-            var result = _processor.Apply(model, comments);
-            Assert.Equal(2, result.Count());
+            var result = _processor.Apply(model, _posts);
+
+            Assert.Equal(pageSize, result.Count());
         }
     }
 }

SieveUnitTests/GeneralWithInterfaces.cs

@@ -16,24 +16,16 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
-        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<IPost> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public GeneralWithInterfaces(ITestOutputHelper testOutputHelper)
         {
-            var nullableAccessor = new SieveOptionsAccessor();
-            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
-
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
-            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
-                new SieveCustomSortMethods(),
-                new SieveCustomFilterMethods());
-
             _posts = new List<IPost>
             {
                 new Post
@@ -189,25 +181,8 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
-            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
-            Assert.True(nullableResult.Count() == 2);
-        }
-
-        [Fact]
-        public void CanFilterNullableIntsWithNotEqual()
-        {
-            var model = new SieveModel()
-            {
-                Filters = "CategoryId!=1"
-            };
-
-            var result = _processor.Apply(model, _posts);
-            var nullableResult = _nullableProcessor.Apply(model, _posts);
-
-            Assert.True(result.Count() == 1);
-            Assert.True(nullableResult.Count() == 2);
         }
 
         [Fact]