Release Sieve 2.5.0 (#151)

* Setup release 2.5.0 with automated build and pre-releases

* #80 added support for escaping pipe control characters (#113)

* #80 added support for escaping comma and pipe control characters

* Update SieveModel.cs

Fix build. Accidentally broken by resolving conflicts.

* Migrate UnitTests to xUnit

Co-authored-by: Clayton Andersen <candersen@restaurant365.com>
Co-authored-by: ITDancer13 <kevin@ksommer.eu>
Co-authored-by: ITDancer139 <kevinitdancersommer@gmail.com>

* SieveProcessor.Options made protected property (#134)

Mapper assignment in constructor is moved to a null-coalescing member pair (a field and a property)
"IncludeScopes" switch is removed from appSettings.{env}.json files

* Revert to _mapper assignment in constructor. (#140)

* reverting fix (#142)

* Revert to _mapper assignment in constructor.

* reverting fix

* pass filter values as parameters (#112)

make GetClosureOverConstant really work

* Make ApplyFiltering, ApplySorting and ApplyPagination protected virtual #139 (#144)

* stop excluding null values when filtering using notEqual (#114)

* stop excluding null values when filtering using notEqual
* add IgnoreNullsOnNotEqual config field, to enable/disable the new feature

Co-authored-by: AnasZakarneh <a.zakarneh@foothillsolutions.com>

Co-authored-by: Clayton Andersen <tunaman65@gmail.com>
Co-authored-by: Clayton Andersen <candersen@restaurant365.com>
Co-authored-by: ITDancer139 <kevinitdancersommer@gmail.com>
Co-authored-by: Hasan Manzak <hasan.manzak@gmail.com>
Co-authored-by: alicak <alicak@users.noreply.github.com>
Co-authored-by: AnasZakarneh <Zakarnehanas1@gmail.com>
Co-authored-by: AnasZakarneh <a.zakarneh@foothillsolutions.com>

.github/workflows/ci.yml

@@ -17,12 +17,10 @@
 name: ci
 
 on:
-  push:
-    branches:
-      - master
   pull_request:
     branches:
       - master
+      - 'releases/*'
 
 jobs:
   ubuntu-latest:

.github/workflows/ci_publish.yml

@@ -0,0 +1,33 @@
+# ------------------------------------------------------------------------------
+# <auto-generated>
+#
+#     This code was generated.
+#
+#     - To turn off auto-generation set:
+#
+#         [GitHubActions (AutoGenerate = false)]
+#
+#     - To trigger manual generation invoke:
+#
+#         nuke --generate-configuration GitHubActions_ci_publish --host GitHubActions
+#
+# </auto-generated>
+# ------------------------------------------------------------------------------
+
+name: ci_publish
+
+on:
+  push:
+    branches:
+      - 'releases/*'
+
+jobs:
+  ubuntu-latest:
+    name: ubuntu-latest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Run './build.cmd CiPublish'
+        run: ./build.cmd CiPublish
+        env:
+          NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}

.nuke/build.schema.json

@@ -43,6 +43,9 @@
           "type": "boolean",
           "description": "Disables displaying the NUKE logo"
         },
+        "NUGET_API_KEY": {
+          "type": "string"
+        },
         "Plan": {
           "type": "boolean",
           "description": "Shows the execution plan (HTML)"
@@ -65,9 +68,11 @@
             "type": "string",
             "enum": [
               "Ci",
+              "CiPublish",
               "Clean",
               "Compile",
               "Package",
+              "Publish",
               "Restore",
               "Test"
             ]
@@ -84,9 +89,11 @@
             "type": "string",
             "enum": [
               "Ci",
+              "CiPublish",
               "Clean",
               "Compile",
               "Package",
+              "Publish",
               "Restore",
               "Test"
             ]

GitVersion.yml

@@ -0,0 +1,3 @@
+branches:
+  release:
+    mode: ContinuousDeployment

README.md

@@ -128,7 +128,8 @@ Then you can add the configuration:
         "CaseSensitive": "boolean: should property names be case-sensitive? Defaults to false",
         "DefaultPageSize": "int number: optional number to fallback to when no page argument is given. Set <=0 to disable paging if no pageSize is specified (default).",
         "MaxPageSize": "int number: maximum allowed page size. Set <=0 to make infinite (default)",
-        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false"
+        "ThrowExceptions": "boolean: should Sieve throw exceptions instead of silently failing? Defaults to false",
+        "IgnoreNullsOnNotEqual": "boolean: ignore null values when filtering using is not equal operator? Default to true"
     }
 }
 ```

Sieve.Sample/appsettings.Development.json

@@ -1,6 +1,5 @@
 {
   "Logging": {
-    "IncludeScopes": false,
     "LogLevel": {
       "Default": "Debug",
       "System": "Information",

Sieve.Sample/appsettings.json

@@ -4,10 +4,10 @@
   },
   "Sieve": {
     "CaseSensitive": false,
-    "DefaultPageSize": 10
+    "DefaultPageSize": 10,
+    "IgnoreNullsOnNotEqual": true
   },
   "Logging": {
-    "IncludeScopes": false,
     "Debug": {
       "LogLevel": {
         "Default": "Warning"

Sieve/Models/FilterTerm.cs

@@ -9,6 +9,7 @@ namespace Sieve.Models
         public FilterTerm() { }
 
         private const string EscapedPipePattern = @"(?<!($|[^\\])(\\\\)*?\\)\|";
+        private const string PipeToEscape = @"\|";
 
         private static readonly string[] Operators = new string[] {
                     "!@=*",
@@ -36,7 +37,11 @@ namespace Sieve.Models
                 var filterSplits = value.Split(Operators, StringSplitOptions.RemoveEmptyEntries)
                     .Select(t => t.Trim()).ToArray();
                 Names = Regex.Split(filterSplits[0], EscapedPipePattern).Select(t => t.Trim()).ToArray();
-                Values = filterSplits.Length > 1 ? Regex.Split(filterSplits[1], EscapedPipePattern).Select(t => t.Trim()).ToArray() : null;
+                Values = filterSplits.Length > 1
+                    ? Regex.Split(filterSplits[1], EscapedPipePattern)
+                        .Select(t => t.Replace(PipeToEscape, "|").Trim())
+                        .ToArray()
+                    : null;
                 Operator = Array.Find(Operators, o => value.Contains(o)) ?? "==";
                 OperatorParsed = GetOperatorParsed(Operator);
                 OperatorIsCaseInsensitive = Operator.EndsWith("*");
@@ -90,6 +95,5 @@ namespace Sieve.Models
                 && Values.SequenceEqual(other.Values)
                 && Operator == other.Operator;
         }
-
     }
 }

Sieve/Models/SieveModel.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.Linq;

Sieve/Models/SieveOptions.cs

@@ -9,5 +9,7 @@
         public int MaxPageSize { get; set; } = 0;
 
         public bool ThrowExceptions { get; set; } = false;
+
+        public bool IgnoreNullsOnNotEqual { get; set; } = true;
     }
-}
+}

Sieve/Services/SieveProcessor.cs

@@ -68,7 +68,6 @@ namespace Sieve.Services
         where TSortTerm : ISortTerm, new()
     {
         private const string NullFilterValue = "null";
-        private readonly IOptions<SieveOptions> _options;
         private readonly ISieveCustomSortMethods _customSortMethods;
         private readonly ISieveCustomFilterMethods _customFilterMethods;
         private readonly SievePropertyMapper _mapper = new SievePropertyMapper();
@@ -78,7 +77,7 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customSortMethods = customSortMethods;
             _customFilterMethods = customFilterMethods;
         }
@@ -87,7 +86,7 @@ namespace Sieve.Services
             ISieveCustomSortMethods customSortMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customSortMethods = customSortMethods;
         }
 
@@ -95,16 +94,18 @@ namespace Sieve.Services
             ISieveCustomFilterMethods customFilterMethods)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
             _customFilterMethods = customFilterMethods;
         }
 
         public SieveProcessor(IOptions<SieveOptions> options)
         {
             _mapper = MapProperties(_mapper);
-            _options = options;
+            Options = options;
         }
 
+        protected IOptions<SieveOptions> Options { get; }
+
         /// <summary>
         /// Apply filtering, sorting, and pagination parameters found in `model` to `source`
         /// </summary>
@@ -148,7 +149,7 @@ namespace Sieve.Services
             }
             catch (Exception ex)
             {
-                if (!_options.Value.ThrowExceptions)
+                if (!Options.Value.ThrowExceptions)
                 {
                     return result;
                 }
@@ -162,7 +163,7 @@ namespace Sieve.Services
             }
         }
 
-        private IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        protected virtual IQueryable<TEntity> ApplyFiltering<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetFiltersParsed() == null)
@@ -216,10 +217,13 @@ namespace Sieve.Services
                                 expression = Expression.Not(expression);
                             }
 
-                            var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
-                            if (filterValueNullCheck != null)
+                            if (expression.NodeType != ExpressionType.NotEqual || Options.Value.IgnoreNullsOnNotEqual)
                             {
-                                expression = Expression.AndAlso(filterValueNullCheck, expression);
+                                var filterValueNullCheck = GetFilterValueNullCheck(parameter, fullPropertyName, isFilterTermValueNull);
+                                if (filterValueNullCheck != null)
+                                {
+                                    expression = Expression.AndAlso(filterValueNullCheck, expression);
+                                }
                             }
 
                             innerExpression = innerExpression == null
@@ -253,8 +257,7 @@ namespace Sieve.Services
                 : result.Where(Expression.Lambda<Func<TEntity, bool>>(outerExpression, parameter));
         }
 
-        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName,
-            bool isFilterTermValueNull)
+        private static Expression GetFilterValueNullCheck(Expression parameter, string fullPropertyName, bool isFilterTermValueNull)
         {
             var (propertyValue, nullCheck) = GetPropertyValueAndNullCheckExpression(parameter, fullPropertyName);
 
@@ -336,15 +339,13 @@ namespace Sieve.Services
         }
 
         // Workaround to ensure that the filter value gets passed as a parameter in generated SQL from EF Core
-        // See https://github.com/aspnet/EntityFrameworkCore/issues/3361
-        // Expression.Constant passed the target type to allow Nullable comparison
-        // See http://bradwilson.typepad.com/blog/2008/07/creating-nullab.html
         private static Expression GetClosureOverConstant<T>(T constant, Type targetType)
         {
-            return Expression.Constant(constant, targetType);
+            Expression<Func<T>> hoistedConstant = () => constant;
+            return Expression.Convert(hoistedConstant.Body, targetType);
         }
 
-        private IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
+        protected virtual IQueryable<TEntity> ApplySorting<TEntity>(TSieveModel model, IQueryable<TEntity> result,
             object[] dataForCustomMethods = null)
         {
             if (model?.GetSortsParsed() == null)
@@ -373,11 +374,11 @@ namespace Sieve.Services
             return result;
         }
 
-        private IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
+        protected virtual IQueryable<TEntity> ApplyPagination<TEntity>(TSieveModel model, IQueryable<TEntity> result)
         {
             var page = model?.Page ?? 1;
-            var pageSize = model?.PageSize ?? _options.Value.DefaultPageSize;
-            var maxPageSize = _options.Value.MaxPageSize > 0 ? _options.Value.MaxPageSize : pageSize;
+            var pageSize = model?.PageSize ?? Options.Value.DefaultPageSize;
+            var maxPageSize = Options.Value.MaxPageSize > 0 ? Options.Value.MaxPageSize : pageSize;
 
             if (pageSize <= 0)
             {
@@ -399,14 +400,14 @@ namespace Sieve.Services
             string name)
         {
             var property = _mapper.FindProperty<TEntity>(canSortRequired, canFilterRequired, name,
-                _options.Value.CaseSensitive);
+                Options.Value.CaseSensitive);
             if (property.Item1 != null)
             {
                 return property;
             }
 
             var prop = FindPropertyBySieveAttribute<TEntity>(canSortRequired, canFilterRequired, name,
-                _options.Value.CaseSensitive);
+                Options.Value.CaseSensitive);
             return (prop?.Name, prop);
         }
 
@@ -426,7 +427,7 @@ namespace Sieve.Services
         {
             var customMethod = parent?.GetType()
                 .GetMethodExt(name,
-                    _options.Value.CaseSensitive
+                    Options.Value.CaseSensitive
                         ? BindingFlags.Default
                         : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                     typeof(IQueryable<TEntity>));
@@ -437,7 +438,7 @@ namespace Sieve.Services
                 // Find generic methods `public IQueryable<T> Filter<T>(IQueryable<T> source, ...)`
                 var genericCustomMethod = parent?.GetType()
                     .GetMethodExt(name,
-                        _options.Value.CaseSensitive
+                        Options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance,
                         typeof(IQueryable<>));
@@ -481,11 +482,11 @@ namespace Sieve.Services
                 var incompatibleCustomMethods =
                     parent?
                         .GetType()
-                        .GetMethods(_options.Value.CaseSensitive
+                        .GetMethods(Options.Value.CaseSensitive
                             ? BindingFlags.Default
                             : BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance)
                         .Where(method => string.Equals(method.Name, name,
-                            _options.Value.CaseSensitive
+                            Options.Value.CaseSensitive
                                 ? StringComparison.InvariantCulture
                                 : StringComparison.InvariantCultureIgnoreCase))
                         .ToList()

SieveUnitTests/General.cs

@@ -15,16 +15,24 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
+        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<Post> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public General(ITestOutputHelper testOutputHelper)
         {
+            var nullableAccessor = new SieveOptionsAccessor();
+            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
+
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
+            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
+                new SieveCustomSortMethods(),
+                new SieveCustomFilterMethods());
+
             _posts = new List<Post>
             {
                 new Post
@@ -180,10 +188,27 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
+            Assert.True(nullableResult.Count() == 2);
         }
-        
+
+        [Fact]
+        public void CanFilterNullableIntsWithNotEqual()
+        {
+            var model = new SieveModel()
+            {
+                Filters = "CategoryId!=1"
+            };
+
+            var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
+
+            Assert.True(result.Count() == 1);
+            Assert.True(nullableResult.Count() == 2);
+        }
+
         [Theory]
         [InlineData(@"Text@=*\,")]
         [InlineData(@"Text@=*\, ")]
@@ -613,5 +638,61 @@ namespace SieveUnitTests
             Assert.Equal(1,posts[2].Id);
             Assert.Equal(0,posts[3].Id);
         }
+
+        [Fact]
+        public void CanFilter_WithEscapeCharacter()
+        {
+            var comments = new List<Comment>
+            {
+                new Comment
+                {
+                    Id = 0,
+                    DateCreated = DateTimeOffset.UtcNow,
+                    Text = "Here is, a comment"
+                },
+                new Comment
+                {
+                    Id = 1,
+                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
+                    Text = "Here is, another comment"
+                },
+            }.AsQueryable();
+
+            var model = new SieveModel
+            {
+                Filters = "Text==Here is\\, another comment"
+            };
+
+            var result = _processor.Apply(model, comments);
+            Assert.Equal(1, result.Count());
+        }
+
+        [Fact]
+        public void OrEscapedPipeValueFilteringWorks()
+        {
+            var comments = new List<Comment>
+            {
+                new Comment
+                {
+                    Id = 0,
+                    DateCreated = DateTimeOffset.UtcNow,
+                    Text = "Here is | a comment"
+                },
+                new Comment
+                {
+                    Id = 1,
+                    DateCreated = DateTimeOffset.UtcNow.AddDays(-1),
+                    Text = "Here is | another comment"
+                },
+            }.AsQueryable();
+
+            var model = new SieveModel()
+            {
+                Filters = "Text==Here is \\| a comment|Here is \\| another comment",
+            };
+
+            var result = _processor.Apply(model, comments);
+            Assert.Equal(2, result.Count());
+        }
     }
 }

SieveUnitTests/GeneralWithInterfaces.cs

@@ -16,16 +16,24 @@ namespace SieveUnitTests
     {
         private readonly ITestOutputHelper _testOutputHelper;
         private readonly SieveProcessor _processor;
+        private readonly SieveProcessor _nullableProcessor;
         private readonly IQueryable<IPost> _posts;
         private readonly IQueryable<Comment> _comments;
 
         public GeneralWithInterfaces(ITestOutputHelper testOutputHelper)
         {
+            var nullableAccessor = new SieveOptionsAccessor();
+            nullableAccessor.Value.IgnoreNullsOnNotEqual = false;
+
             _testOutputHelper = testOutputHelper;
             _processor = new ApplicationSieveProcessor(new SieveOptionsAccessor(),
                 new SieveCustomSortMethods(),
                 new SieveCustomFilterMethods());
 
+            _nullableProcessor = new ApplicationSieveProcessor(nullableAccessor,
+                new SieveCustomSortMethods(),
+                new SieveCustomFilterMethods());
+
             _posts = new List<IPost>
             {
                 new Post
@@ -181,8 +189,25 @@ namespace SieveUnitTests
             };
 
             var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
 
             Assert.True(result.Count() == 2);
+            Assert.True(nullableResult.Count() == 2);
+        }
+
+        [Fact]
+        public void CanFilterNullableIntsWithNotEqual()
+        {
+            var model = new SieveModel()
+            {
+                Filters = "CategoryId!=1"
+            };
+
+            var result = _processor.Apply(model, _posts);
+            var nullableResult = _nullableProcessor.Apply(model, _posts);
+
+            Assert.True(result.Count() == 1);
+            Assert.True(nullableResult.Count() == 2);
         }
 
         [Fact]

build/Build.cs

@@ -1,4 +1,5 @@
 using System.Linq;
+using GlobExpressions;
 using Nuke.Common;
 using Nuke.Common.CI;
 using Nuke.Common.CI.GitHubActions;
@@ -8,17 +9,23 @@ using Nuke.Common.IO;
 using Nuke.Common.ProjectModel;
 using Nuke.Common.Tools.DotNet;
 using Nuke.Common.Tools.GitVersion;
+using Nuke.Common.Utilities.Collections;
 using static Nuke.Common.IO.FileSystemTasks;
 using static Nuke.Common.Tools.DotNet.DotNetTasks;
 
 [CheckBuildProjectConfigurations]
 [ShutdownDotNetAfterServerBuild]
 [GitHubActions("ci", GitHubActionsImage.UbuntuLatest,
-    OnPushBranches = new[] {"master"},
-    OnPullRequestBranches = new[] {"master"},
+    OnPullRequestBranches = new[] {"master", "releases/*"},
     AutoGenerate = true,
     InvokedTargets = new[] {nameof(Ci)},
     CacheKeyFiles = new string[0])]
+[GitHubActions("ci_publish", GitHubActionsImage.UbuntuLatest,
+    OnPushBranches = new[] {"releases/*"},
+    AutoGenerate = true,
+    InvokedTargets = new[] {nameof(CiPublish)},
+    CacheKeyFiles = new string[0],
+    ImportSecrets = new[] {"NUGET_API_KEY"})]
 class Build : NukeBuild
 {
     [Parameter("Configuration to build - Default is 'Debug' (local) or 'Release' (server)")]
@@ -30,6 +37,9 @@ class Build : NukeBuild
 
     [Solution] readonly Solution Solution;
 
+    // ReSharper disable once InconsistentNaming
+    [Parameter] string NUGET_API_KEY;
+
     Project SieveProject => Solution.AllProjects.First(p => p.Name == "Sieve");
 
     AbsolutePath OutputDirectory => RootDirectory / "output";
@@ -83,13 +93,29 @@ class Build : NukeBuild
                 .EnableNoBuild());
         });
 
-    Target Ci => _ => _
-        .DependsOn(Package);
+    Target Publish => _ => _
+        .DependsOn(Package)
+        .Requires(() => IsServerBuild)
+        .Requires(() => NUGET_API_KEY)
+        .Requires(() => Configuration.Equals(Configuration.Release))
+        .Executes(() =>
+        {
+            Glob.Files(OutputDirectory, "*.nupkg")
+                .NotEmpty()
+                .ForEach(x =>
+                {
+                    DotNetNuGetPush(s => s
+                        .SetTargetPath(OutputDirectory / x)
+                        .SetSource("https://api.nuget.org/v3/index.json")
+                        .SetApiKey(NUGET_API_KEY));
+                });
+        });
+
+    Target Ci => _ => _
+        .DependsOn(Test);
+
+    Target CiPublish => _ => _
+        .DependsOn(Publish);
 
-    /// Support plugins are available for:
-    /// - JetBrains ReSharper        https://nuke.build/resharper
-    /// - JetBrains Rider            https://nuke.build/rider
-    /// - Microsoft VisualStudio     https://nuke.build/visualstudio
-    /// - Microsoft VSCode           https://nuke.build/vscode
     public static int Main() => Execute<Build>(x => x.Package);
 }