From b0d3b39311f87d219ba998bddf5148700a44c2ac Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 17 Oct 2024 13:41:30 -0500 Subject: [PATCH 1/6] Adding default field search tokenizer and sample --- docker-compose.yml | 7 +-- .../Extensions/SqlNodeExtensions.cs | 37 ++++++++++++---- .../SqlQueryParser.cs | 5 +++ .../SqlQueryParserConfiguration.cs | 7 +++ .../Visitors/ISqlQueryVisitorContext.cs | 4 +- .../Visitors/SqlQueryVisitorContext.cs | 4 +- .../SampleContext.cs | 15 +++++++ .../SqlQueryParserTests.cs | 44 +++++++++++++++++-- 8 files changed, 105 insertions(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fbb92280..c9db4c86 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,8 +35,11 @@ services: - "1433:1433" # login with sa:P@ssword1 environment: - "ACCEPT_EULA=Y" - - "SA_PASSWORD=P@ssword1" + - "MSSQL_SA_PASSWORD=P@ssword1" - "MSSQL_PID=Developer" + user: root + networks: + - foundatio healthcheck: test: [ @@ -49,8 +52,6 @@ services: ] interval: 1s retries: 20 - networks: - - foundatio ready: image: andrewlock/wait-for-dependencies diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index c1c89c9f..4a73b15a 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -128,26 +128,45 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon return String.Empty; } - for (int index = 0; index < context.DefaultFields.Length; index++) + var fieldTerms = new Dictionary>(); + foreach (string df in context.DefaultFields) + { + var fieldInfo = GetFieldInfo(context.Fields, df); + if (!fieldTerms.TryGetValue(fieldInfo, out var fields)) + { + fields = new List(); + fieldTerms.Add(fieldInfo, fields); + } + + foreach (string token in context.Tokenizer?.Invoke(node.Term) ?? [node.Term]) + fields.Add(token); + } + + var keys = fieldTerms.Keys.ToArray(); + for (int index = 0; index < keys.Length; index++) { builder.Append(index == 0 ? "(" : " OR "); + var fieldInfo = keys[index]; + var terms = fieldTerms[fieldInfo]; - var defaultField = GetFieldInfo(context.Fields, context.DefaultFields[index]); - if (defaultField.IsCollection) + if (fieldInfo.IsCollection) { - int dotIndex = defaultField.Field.LastIndexOf('.'); - string collectionField = defaultField.Field.Substring(0, dotIndex); - string fieldName = defaultField.Field.Substring(dotIndex + 1); + int dotIndex = fieldInfo.Field.LastIndexOf('.'); + string collectionField = fieldInfo.Field.Substring(0, dotIndex); + string fieldName = fieldInfo.Field.Substring(dotIndex + 1); builder.Append(collectionField); builder.Append(".Any("); builder.Append(fieldName); - builder.Append(".Contains(\"").Append(node.Term).Append("\")"); - builder.Append(")"); + builder.Append(" in ("); + builder.Append(String.Join(',', terms.Select(t => "\"" + t + "\""))); + builder.Append("))"); } else { - builder.Append(defaultField.Field).Append(".Contains(\"").Append(node.Term).Append("\")"); + builder.Append(fieldInfo.Field).Append(" in ("); + builder.Append(String.Join(',', terms.Select(t => "\"" + t + "\""))); + builder.Append(")"); } if (index == context.DefaultFields.Length - 1) diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs index 5a029f7b..bc48b229 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -189,5 +189,10 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) if (Configuration.IncludeResolver != null && context.GetIncludeResolver() == null) context.SetIncludeResolver(Configuration.IncludeResolver); } + + if (context is ISqlQueryVisitorContext sqlContext) + { + sqlContext.Tokenizer = Configuration.Tokenizer; + } } } diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs index 2474aebf..1daac952 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -25,6 +25,7 @@ public SqlQueryParserConfiguration() public int MaxFieldDepth { get; private set; } = 10; public QueryFieldResolver FieldResolver { get; private set; } + public Func Tokenizer { get; set; } = static s => [s]; public EntityTypePropertyFilter EntityTypePropertyFilter { get; private set; } = static _ => true; public EntityTypeNavigationFilter EntityTypeNavigationFilter { get; private set; } = static _ => true; public EntityTypeSkipNavigationFilter EntityTypeSkipNavigationFilter { get; private set; } = static _ => true; @@ -48,6 +49,12 @@ public SqlQueryParserConfiguration SetDefaultFields(string[] fields) return this; } + public SqlQueryParserConfiguration SetTokenizer(Func tokenizer) + { + Tokenizer = tokenizer; + return this; + } + public SqlQueryParserConfiguration SetFieldDepth(int maxFieldDepth) { MaxFieldDepth = maxFieldDepth; diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index 2e68df18..3dec670a 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using Foundatio.Parsers.LuceneQueries.Visitors; namespace Foundatio.Parsers.SqlQueries.Visitors; @@ -6,4 +7,5 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } + Func Tokenizer { get; set; } } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index f1ec3e9d..da3c9155 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Diagnostics; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.EntityFrameworkCore.Metadata; @@ -8,6 +9,7 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { public List Fields { get; set; } + public Func Tokenizer { get; set; } = s => [s]; public IEntityType EntityType { get; set; } } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs index 38b372cc..3582f4b8 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using Microsoft.EntityFrameworkCore; namespace Foundatio.Parsers.SqlQueries.Tests; @@ -11,6 +12,7 @@ public SampleContext(DbContextOptions options) : base(options) { public DbSet Companies => Set(); public DbSet DataDefinitions => Set(); public DbSet DataValues => Set(); + public DbSet SearchValues => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -34,17 +36,30 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity().Property(e => e.BooleanValue).IsSparse(); modelBuilder.Entity().Property(e => e.NumberValue).HasColumnType("decimal").HasPrecision(15, 3).IsSparse(); modelBuilder.Entity().HasIndex(e => new { e.StringValue, e.DateValue, e.MoneyValue, e.BooleanValue, e.NumberValue }); + + // Search + modelBuilder.Entity().HasIndex(s => new { s.EmployeeId, s.Term }); } } +public class SearchValue +{ + public int Id { get; set; } + public int EmployeeId { get; set; } + public string Term { get; set; } +} + public class Employee { public int Id { get; set; } public string FullName { get; set; } + public string PhoneNumber { get; set; } public string Title { get; set; } public int Salary { get; set; } public List Companies { get; set; } public List DataValues { get; set; } + + public List SearchValues { get; set; } public DateTime Created { get; set; } = DateTime.Now; } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 03968063..8bce8cb5 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -69,11 +69,39 @@ public async Task CanSearchDefaultFields() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.FullName.Contains("John") || e.Title.Contains("John")).ToQueryString(); - string sqlActual = db.Employees.Where("""FullName.Contains("John") || Title.Contains("John")""").ToQueryString(); + string sqlExpected = db.Employees.Where(e => e.FullName == "John Doe" || e.Title == "John Doe").ToQueryString(); + string sqlActual = db.Employees.Where("""FullName = "John Doe" || Title = "John Doe" """).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); - string sql = await parser.ToDynamicLinqAsync("John", context); + string sql = await parser.ToDynamicLinqAsync("John Doe", context); sqlActual = db.Employees.Where(sql).ToQueryString(); + var results = await db.Employees.Where(sql).ToListAsync(); + Assert.Single(results); + Assert.Equal(sqlExpected, sqlActual); + } + + [Fact] + public async Task CanSearchWithTokenizer() + { + var sp = GetServiceProvider(); + await using var db = await GetSampleContextWithDataAsync(sp); + var parser = sp.GetRequiredService(); + parser.Configuration.SetDefaultFields(["SearchValues.Term"]); + parser.Configuration.SetTokenizer(t => + { + string[] terms = [t.Replace("-", "")]; + return terms.Distinct().ToArray(); + }); + + var context = parser.GetContext(db.Employees.EntityType); + + string sqlExpected = db.Employees.Where(e => e.SearchValues.Any(s => s.Term == "2142222222")).ToQueryString(); + string sqlActual = db.Employees.Where("""SearchValues.Any(Term in ("2142222222"))""").ToQueryString(); + Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("214-222-2222", context); + _logger.LogInformation(sql); + sqlActual = db.Employees.Where(sql).ToQueryString(); + var results = await db.Employees.Where(sql).ToListAsync(); + Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); } @@ -281,9 +309,17 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider { FullName = "John Doe", Title = "Software Developer", + PhoneNumber = "(214) 222-2222", Salary = 80_000, DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 30 }], - Companies = [company] + Companies = [company], + SearchValues = [ + new() { Term = "john" }, + new() { Term = "doe" }, + new() { Term = "software" }, + new() { Term = "developer" }, + new() { Term = "2142222222" } + ] }); db.Employees.Add(new Employee { From fec38522d7a6743e5a1b9d604dcd5a073f524e05 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 17 Oct 2024 14:18:38 -0500 Subject: [PATCH 2/6] More changes --- .../Extensions/SqlNodeExtensions.cs | 19 +++++------ .../SqlQueryParser.cs | 2 +- .../SqlQueryParserConfiguration.cs | 7 ++-- .../Visitors/ISqlQueryVisitorContext.cs | 2 +- .../Visitors/SqlQueryVisitorContext.cs | 29 ++++++++++++++++- .../SqlQueryParserTests.cs | 32 +++++++++++++++++-- 6 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 4a73b15a..66977558 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -128,18 +128,19 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon return String.Empty; } - var fieldTerms = new Dictionary>(); + var fieldTerms = new Dictionary Tokens, SqlSearchOperator Operator)>(); foreach (string df in context.DefaultFields) { var fieldInfo = GetFieldInfo(context.Fields, df); - if (!fieldTerms.TryGetValue(fieldInfo, out var fields)) + if (!fieldTerms.TryGetValue(fieldInfo, out var searchInfo)) { - fields = new List(); - fieldTerms.Add(fieldInfo, fields); + searchInfo = (new List(), SqlSearchOperator.Contains); + fieldTerms.Add(fieldInfo, searchInfo); } - foreach (string token in context.Tokenizer?.Invoke(node.Term) ?? [node.Term]) - fields.Add(token); + var result = context.Tokenizer.Invoke(fieldInfo, node.Term); + searchInfo.Tokens.AddRange(result.Tokens); + searchInfo.Operator = result.Operator; } var keys = fieldTerms.Keys.ToArray(); @@ -147,7 +148,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon { builder.Append(index == 0 ? "(" : " OR "); var fieldInfo = keys[index]; - var terms = fieldTerms[fieldInfo]; + var searchInfo = fieldTerms[fieldInfo]; if (fieldInfo.IsCollection) { @@ -159,13 +160,13 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon builder.Append(".Any("); builder.Append(fieldName); builder.Append(" in ("); - builder.Append(String.Join(',', terms.Select(t => "\"" + t + "\""))); + builder.Append(String.Join(',', searchInfo.Tokens.Select(t => "\"" + t + "\""))); builder.Append("))"); } else { builder.Append(fieldInfo.Field).Append(" in ("); - builder.Append(String.Join(',', terms.Select(t => "\"" + t + "\""))); + builder.Append(String.Join(',', searchInfo.Tokens.Select(t => "\"" + t + "\""))); builder.Append(")"); } diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs index bc48b229..e13cfdc5 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -192,7 +192,7 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) if (context is ISqlQueryVisitorContext sqlContext) { - sqlContext.Tokenizer = Configuration.Tokenizer; + sqlContext.Tokenizer = Configuration.SearchTokenizer; } } } diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs index 1daac952..1c2fdc03 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Visitors; +using Foundatio.Parsers.SqlQueries.Visitors; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -25,7 +26,7 @@ public SqlQueryParserConfiguration() public int MaxFieldDepth { get; private set; } = 10; public QueryFieldResolver FieldResolver { get; private set; } - public Func Tokenizer { get; set; } = static s => [s]; + public Func SearchTokenizer { get; set; } = static (_, t) => new SearchTokenizeResult([t], SqlSearchOperator.Contains); public EntityTypePropertyFilter EntityTypePropertyFilter { get; private set; } = static _ => true; public EntityTypeNavigationFilter EntityTypeNavigationFilter { get; private set; } = static _ => true; public EntityTypeSkipNavigationFilter EntityTypeSkipNavigationFilter { get; private set; } = static _ => true; @@ -49,9 +50,9 @@ public SqlQueryParserConfiguration SetDefaultFields(string[] fields) return this; } - public SqlQueryParserConfiguration SetTokenizer(Func tokenizer) + public SqlQueryParserConfiguration SetSearchTokenizer(Func tokenizer) { - Tokenizer = tokenizer; + SearchTokenizer = tokenizer; return this; } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index 3dec670a..fcce9de9 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -7,5 +7,5 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } - Func Tokenizer { get; set; } + Func Tokenizer { get; set; } } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index da3c9155..5df3a6ec 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -9,13 +9,37 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { public List Fields { get; set; } - public Func Tokenizer { get; set; } = s => [s]; + public Func Tokenizer { get; set; } = static (_, t) => new SearchTokenizeResult([t], SqlSearchOperator.Contains); public IEntityType EntityType { get; set; } } [DebuggerDisplay("{Field} IsNumber: {IsNumber} IsMoney: {IsMoney} IsDate: {IsDate} IsBoolean: {IsBoolean} IsCollection: {IsCollection}")] public class EntityFieldInfo { + protected bool Equals(EntityFieldInfo other) => Field == other.Field; + + public override bool Equals(object obj) + { + if (obj is null) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != GetType()) + { + return false; + } + + return Equals((EntityFieldInfo)obj); + } + + public override int GetHashCode() => (Field != null ? Field.GetHashCode() : 0); + public string Field { get; set; } public bool IsNumber { get; set; } public bool IsMoney { get; set; } @@ -24,3 +48,6 @@ public class EntityFieldInfo public bool IsCollection { get; set; } public IDictionary Data { get; set; } = new Dictionary(); } + +public record SearchTokenizeResult(string[] Tokens, SqlSearchOperator Operator); +public enum SqlSearchOperator { Equals, Contains } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 8bce8cb5..8a9cf900 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -86,10 +86,13 @@ public async Task CanSearchWithTokenizer() await using var db = await GetSampleContextWithDataAsync(sp); var parser = sp.GetRequiredService(); parser.Configuration.SetDefaultFields(["SearchValues.Term"]); - parser.Configuration.SetTokenizer(t => + parser.Configuration.SetSearchTokenizer((f, t) => { + if (f.Field != "SearchValues.Term") + return new SearchTokenizeResult([t], SqlSearchOperator.Contains); + string[] terms = [t.Replace("-", "")]; - return terms.Distinct().ToArray(); + return new SearchTokenizeResult(terms.Distinct().ToArray(), SqlSearchOperator.Equals); }); var context = parser.GetContext(db.Employees.EntityType); @@ -97,12 +100,27 @@ public async Task CanSearchWithTokenizer() string sqlExpected = db.Employees.Where(e => e.SearchValues.Any(s => s.Term == "2142222222")).ToQueryString(); string sqlActual = db.Employees.Where("""SearchValues.Any(Term in ("2142222222"))""").ToQueryString(); Assert.Equal(sqlExpected, sqlActual); + string sql = await parser.ToDynamicLinqAsync("214-222-2222", context); _logger.LogInformation(sql); sqlActual = db.Employees.Where(sql).ToQueryString(); var results = await db.Employees.Where(sql).ToListAsync(); Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); + + sql = await parser.ToDynamicLinqAsync("2142222222", context); + _logger.LogInformation(sql); + sqlActual = db.Employees.Where(sql).ToQueryString(); + results = await db.Employees.Where(sql).ToListAsync(); + Assert.Single(results); + Assert.Equal(sqlExpected, sqlActual); + + sql = await parser.ToDynamicLinqAsync("(214) 222-2222", context); + _logger.LogInformation(sql); + sqlActual = db.Employees.Where(sql).ToQueryString(); + results = await db.Employees.Where(sql).ToListAsync(); + Assert.Single(results); + Assert.Equal(sqlExpected, sqlActual); } [Fact] @@ -325,9 +343,17 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider { FullName = "Jane Doe", Title = "Software Developer", + PhoneNumber = "214-333-1111", Salary = 90_000, DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 23 }], - Companies = [company] + Companies = [company], + SearchValues = [ + new() { Term = "jane" }, + new() { Term = "doe" }, + new() { Term = "software" }, + new() { Term = "developer" }, + new() { Term = "2143331111" } + ] }); await db.SaveChangesAsync(); From 0a1fcc3d4d187530a36b419b56dd56a1100536a1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 17 Oct 2024 17:31:42 -0500 Subject: [PATCH 3/6] A different approach --- .../Extensions/EnumerableExtensions.cs | 38 ++++ .../Extensions/SqlNodeExtensions.cs | 185 ++++++++++-------- .../SqlQueryParserConfiguration.cs | 4 +- .../Visitors/ISqlQueryVisitorContext.cs | 2 +- .../Visitors/SqlQueryVisitorContext.cs | 27 ++- .../Foundatio.Parsers.SqlQueries.Tests.csproj | 1 + .../SampleContext.cs | 14 +- .../SqlQueryParserTests.cs | 49 ++--- 8 files changed, 181 insertions(+), 139 deletions(-) create mode 100644 src/Foundatio.Parsers.SqlQueries/Extensions/EnumerableExtensions.cs diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/EnumerableExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/EnumerableExtensions.cs new file mode 100644 index 00000000..2b61ebeb --- /dev/null +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/EnumerableExtensions.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; + +namespace Foundatio.Parsers.SqlQueries.Extensions; + +internal static class EnumerableExtensions { + public delegate void ElementAction(T element, ElementInfo info); + + public static void ForEach(this IEnumerable elements, ElementAction action) + { + using IEnumerator enumerator = elements.GetEnumerator(); + bool isFirst = true; + bool hasNext = enumerator.MoveNext(); + int index = 0; + + while (hasNext) + { + T current = enumerator.Current; + hasNext = enumerator.MoveNext(); + action(current, new ElementInfo(index, isFirst, !hasNext)); + isFirst = false; + index++; + } + } + + public struct ElementInfo { + public ElementInfo(int index, bool isFirst, bool isLast) + : this() { + Index = index; + IsFirst = isFirst; + IsLast = isLast; + } + + public int Index { get; private set; } + public bool IsFirst { get; private set; } + public bool IsLast { get; private set; } + } +} + diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 66977558..cd4ecfad 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -104,20 +104,15 @@ public static string ToDynamicLinqString(this MissingNode node, ISqlQueryVisitor return builder.ToString(); } - public static EntityFieldInfo GetFieldInfo(List fields, string field) - { - if (fields == null) - return new EntityFieldInfo { Field = field }; - - return fields.FirstOrDefault(f => f.Field.Equals(field, StringComparison.OrdinalIgnoreCase)) ?? - new EntityFieldInfo { Field = field }; - } - public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorContext context) { if (!String.IsNullOrEmpty(node.Prefix)) context.AddValidationError("Prefix is not supported for term range queries."); + // support overriding the generated query + if (node.TryGetQuery(out string query)) + return query; + var builder = new StringBuilder(); if (String.IsNullOrEmpty(node.Field)) @@ -128,59 +123,78 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon return String.Empty; } - var fieldTerms = new Dictionary Tokens, SqlSearchOperator Operator)>(); + var fieldTerms = new Dictionary(); foreach (string df in context.DefaultFields) { var fieldInfo = GetFieldInfo(context.Fields, df); - if (!fieldTerms.TryGetValue(fieldInfo, out var searchInfo)) + if (!fieldTerms.TryGetValue(fieldInfo, out var searchTerm)) { - searchInfo = (new List(), SqlSearchOperator.Contains); - fieldTerms.Add(fieldInfo, searchInfo); + searchTerm = new SearchTerm + { + FieldInfo = fieldInfo, + Tokens = [node.Term], + Operator = SqlSearchOperator.StartsWith + }; + fieldTerms[fieldInfo] = searchTerm; } - var result = context.Tokenizer.Invoke(fieldInfo, node.Term); - searchInfo.Tokens.AddRange(result.Tokens); - searchInfo.Operator = result.Operator; + context.Tokenizer.Invoke(searchTerm); } - var keys = fieldTerms.Keys.ToArray(); - for (int index = 0; index < keys.Length; index++) + fieldTerms.ForEach((kvp, info) => { - builder.Append(index == 0 ? "(" : " OR "); - var fieldInfo = keys[index]; - var searchInfo = fieldTerms[fieldInfo]; + builder.Append(info.IsFirst ? "(" : " OR "); + var searchTerm = kvp.Value; + var tokens = kvp.Value.Tokens ?? [kvp.Value.Term]; - if (fieldInfo.IsCollection) + if (searchTerm.FieldInfo.IsCollection) { - int dotIndex = fieldInfo.Field.LastIndexOf('.'); - string collectionField = fieldInfo.Field.Substring(0, dotIndex); - string fieldName = fieldInfo.Field.Substring(dotIndex + 1); + int dotIndex = searchTerm.FieldInfo.Field.LastIndexOf('.'); + string collectionField = searchTerm.FieldInfo.Field.Substring(0, dotIndex); + string fieldName = searchTerm.FieldInfo.Field.Substring(dotIndex + 1); builder.Append(collectionField); builder.Append(".Any("); builder.Append(fieldName); builder.Append(" in ("); - builder.Append(String.Join(',', searchInfo.Tokens.Select(t => "\"" + t + "\""))); + builder.Append(String.Join(',', tokens.Select(t => "\"" + t + "\""))); builder.Append("))"); } else { - builder.Append(fieldInfo.Field).Append(" in ("); - builder.Append(String.Join(',', searchInfo.Tokens.Select(t => "\"" + t + "\""))); - builder.Append(")"); + if (searchTerm.Operator == SqlSearchOperator.Equals) + { + builder.Append(searchTerm.FieldInfo.Field).Append(" in ("); + builder.Append(String.Join(',', tokens.Select(t => "\"" + t + "\""))); + builder.Append(")"); + } + else if (searchTerm.Operator == SqlSearchOperator.Contains) + { + searchTerm.Tokens.ForEach((token, info) => { + builder.Append(info.IsFirst ? "(" : " OR "); + builder.Append(searchTerm.FieldInfo.Field).Append(".Contains(\"").Append(token).Append("\")"); + if (info.IsLast) + builder.Append(")"); + }); + } + else if (searchTerm.Operator == SqlSearchOperator.StartsWith) + { + searchTerm.Tokens.ForEach((token, info) => { + builder.Append(info.IsFirst ? "(" : " OR "); + builder.Append(searchTerm.FieldInfo.Field).Append(".StartsWith(\"").Append(token).Append("\")"); + if (info.IsLast) + builder.Append(")"); + }); + } } - if (index == context.DefaultFields.Length - 1) + if (info.IsLast) builder.Append(")"); - } + }); return builder.ToString(); } - // support overriding the generated query - if (node.TryGetQuery(out string query)) - return query; - var field = GetFieldInfo(context.Fields, node.Field); if (node.IsNegated.HasValue && node.IsNegated.Value) @@ -219,52 +233,6 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon return builder.ToString(); } - private static void AppendField(StringBuilder builder, EntityFieldInfo field, string term) - { - if (field == null) - return; - - if (field.IsNumber || field.IsBoolean || field.IsMoney) - { - builder.Append(term); - } - else if (field is { IsDate: true }) - { - term = term.Trim(); - if (term.StartsWith("now", StringComparison.OrdinalIgnoreCase)) - { - builder.Append("DateTime.UtcNow"); - - if (term.Length == 3) - return; - - builder.Append("."); - - string method = term[^1..] switch - { - "y" => "AddYears", - "M" => "AddMonths", - "d" => "AddDays", - "h" => "AddHours", - "H" => "AddHours", - "m" => "AddMinutes", - "s" => "AddSeconds", - _ => throw new NotSupportedException("Invalid date operation.") - }; - - bool subtract = term.Substring(3, 1) == "-"; - - builder.Append(method).Append("(").Append(subtract ? "-" : "").Append(term.Substring(4, term.Length - 5)).Append(")"); - } - else - { - builder.Append("DateTime.Parse(\"" + term + "\")"); - } - } - else - builder.Append("\"" + term + "\""); - } - public static string ToDynamicLinqString(this TermRangeNode node, ISqlQueryVisitorContext context) { if (String.IsNullOrEmpty(node.Field)) @@ -326,6 +294,61 @@ public static string ToDynamicLinqString(this IQueryNode node, ISqlQueryVisitorC }; } + public static EntityFieldInfo GetFieldInfo(List fields, string field) + { + if (fields == null) + return new EntityFieldInfo { Field = field }; + + return fields.FirstOrDefault(f => f.Field.Equals(field, StringComparison.OrdinalIgnoreCase)) ?? + new EntityFieldInfo { Field = field }; + } + + private static void AppendField(StringBuilder builder, EntityFieldInfo field, string term) + { + if (field == null) + return; + + if (field.IsNumber || field.IsBoolean || field.IsMoney) + { + builder.Append(term); + } + else if (field is { IsDate: true }) + { + term = term.Trim(); + if (term.StartsWith("now", StringComparison.OrdinalIgnoreCase)) + { + builder.Append("DateTime.UtcNow"); + + if (term.Length == 3) + return; + + builder.Append("."); + + string method = term[^1..] switch + { + "y" => "AddYears", + "M" => "AddMonths", + "d" => "AddDays", + "h" => "AddHours", + "H" => "AddHours", + "m" => "AddMinutes", + "s" => "AddSeconds", + _ => throw new NotSupportedException("Invalid date operation.") + }; + + bool subtract = term.Substring(3, 1) == "-"; + + builder.Append(method).Append("(").Append(subtract ? "-" : "").Append(term.Substring(4, term.Length - 5)).Append(")"); + } + else + { + builder.Append("DateTime.Parse(\"" + term + "\")"); + } + } + else + builder.Append("\"" + term + "\""); + } + private const string QueryKey = "Query"; public static void SetQuery(this IQueryNode node, string query) { diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs index 1c2fdc03..6a295d65 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParserConfiguration.cs @@ -26,7 +26,7 @@ public SqlQueryParserConfiguration() public int MaxFieldDepth { get; private set; } = 10; public QueryFieldResolver FieldResolver { get; private set; } - public Func SearchTokenizer { get; set; } = static (_, t) => new SearchTokenizeResult([t], SqlSearchOperator.Contains); + public Action SearchTokenizer { get; set; } = static _ => { }; public EntityTypePropertyFilter EntityTypePropertyFilter { get; private set; } = static _ => true; public EntityTypeNavigationFilter EntityTypeNavigationFilter { get; private set; } = static _ => true; public EntityTypeSkipNavigationFilter EntityTypeSkipNavigationFilter { get; private set; } = static _ => true; @@ -50,7 +50,7 @@ public SqlQueryParserConfiguration SetDefaultFields(string[] fields) return this; } - public SqlQueryParserConfiguration SetSearchTokenizer(Func tokenizer) + public SqlQueryParserConfiguration SetSearchTokenizer(Action tokenizer) { SearchTokenizer = tokenizer; return this; diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index fcce9de9..c0554c30 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -7,5 +7,5 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } - Func Tokenizer { get; set; } + Action Tokenizer { get; set; } } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index 5df3a6ec..90cf32be 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -9,13 +9,21 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { public List Fields { get; set; } - public Func Tokenizer { get; set; } = static (_, t) => new SearchTokenizeResult([t], SqlSearchOperator.Contains); + public Action Tokenizer { get; set; } = static _ => { }; public IEntityType EntityType { get; set; } } [DebuggerDisplay("{Field} IsNumber: {IsNumber} IsMoney: {IsMoney} IsDate: {IsDate} IsBoolean: {IsBoolean} IsCollection: {IsCollection}")] public class EntityFieldInfo { + public string Field { get; init; } + public bool IsNumber { get; set; } + public bool IsMoney { get; set; } + public bool IsDate { get; set; } + public bool IsBoolean { get; set; } + public bool IsCollection { get; set; } + public IDictionary Data { get; set; } = new Dictionary(); + protected bool Equals(EntityFieldInfo other) => Field == other.Field; public override bool Equals(object obj) @@ -39,15 +47,14 @@ public override bool Equals(object obj) } public override int GetHashCode() => (Field != null ? Field.GetHashCode() : 0); +} - public string Field { get; set; } - public bool IsNumber { get; set; } - public bool IsMoney { get; set; } - public bool IsDate { get; set; } - public bool IsBoolean { get; set; } - public bool IsCollection { get; set; } - public IDictionary Data { get; set; } = new Dictionary(); +public class SearchTerm +{ + public EntityFieldInfo FieldInfo { get; set; } + public string Term { get; set; } + public List Tokens { get; set; } + public SqlSearchOperator Operator { get; set; } = SqlSearchOperator.Contains; } -public record SearchTokenizeResult(string[] Tokens, SqlSearchOperator Operator); -public enum SqlSearchOperator { Equals, Contains } +public enum SqlSearchOperator { Equals, Contains, StartsWith } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj index 56be4a1e..9458c114 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/Foundatio.Parsers.SqlQueries.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs index 3582f4b8..2d106b03 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SampleContext.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; using Microsoft.EntityFrameworkCore; namespace Foundatio.Parsers.SqlQueries.Tests; @@ -12,7 +11,6 @@ public SampleContext(DbContextOptions options) : base(options) { public DbSet Companies => Set(); public DbSet DataDefinitions => Set(); public DbSet DataValues => Set(); - public DbSet SearchValues => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -36,30 +34,20 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity().Property(e => e.BooleanValue).IsSparse(); modelBuilder.Entity().Property(e => e.NumberValue).HasColumnType("decimal").HasPrecision(15, 3).IsSparse(); modelBuilder.Entity().HasIndex(e => new { e.StringValue, e.DateValue, e.MoneyValue, e.BooleanValue, e.NumberValue }); - - // Search - modelBuilder.Entity().HasIndex(s => new { s.EmployeeId, s.Term }); } } -public class SearchValue -{ - public int Id { get; set; } - public int EmployeeId { get; set; } - public string Term { get; set; } -} - public class Employee { public int Id { get; set; } public string FullName { get; set; } public string PhoneNumber { get; set; } + public string NationalPhoneNumber { get; set; } public string Title { get; set; } public int Salary { get; set; } public List Companies { get; set; } public List DataValues { get; set; } - public List SearchValues { get; set; } public DateTime Created { get; set; } = DateTime.Now; } diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 8a9cf900..3edd3011 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Pegasus.Common.Tracing; +using PhoneNumbers; using Xunit; using Xunit.Abstractions; @@ -84,21 +85,22 @@ public async Task CanSearchWithTokenizer() { var sp = GetServiceProvider(); await using var db = await GetSampleContextWithDataAsync(sp); + var phoneNumberUtil = PhoneNumberUtil.GetInstance(); var parser = sp.GetRequiredService(); - parser.Configuration.SetDefaultFields(["SearchValues.Term"]); - parser.Configuration.SetSearchTokenizer((f, t) => + parser.Configuration.SetDefaultFields(["NationalPhoneNumber"]); + parser.Configuration.SetSearchTokenizer(s => { - if (f.Field != "SearchValues.Term") - return new SearchTokenizeResult([t], SqlSearchOperator.Contains); + if (s.FieldInfo.Field != "NationalPhoneNumber") + return; - string[] terms = [t.Replace("-", "")]; - return new SearchTokenizeResult(terms.Distinct().ToArray(), SqlSearchOperator.Equals); + s.Tokens = [phoneNumberUtil.Parse(s.Term, "US").NationalNumber.ToString()]; + s.Operator = SqlSearchOperator.StartsWith; }); var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.SearchValues.Any(s => s.Term == "2142222222")).ToQueryString(); - string sqlActual = db.Employees.Where("""SearchValues.Any(Term in ("2142222222"))""").ToQueryString(); + string sqlExpected = db.Employees.Where(e => e.NationalPhoneNumber.StartsWith("2142222222")).ToQueryString(); + string sqlActual = db.Employees.Where("NationalPhoneNumber.StartsWith(\"2142222222\")").ToQueryString(); Assert.Equal(sqlExpected, sqlActual); string sql = await parser.ToDynamicLinqAsync("214-222-2222", context); @@ -114,13 +116,6 @@ public async Task CanSearchWithTokenizer() results = await db.Employees.Where(sql).ToListAsync(); Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); - - sql = await parser.ToDynamicLinqAsync("(214) 222-2222", context); - _logger.LogInformation(sql); - sqlActual = db.Employees.Where(sql).ToQueryString(); - results = await db.Employees.Where(sql).ToListAsync(); - Assert.Single(results); - Assert.Equal(sqlExpected, sqlActual); } [Fact] @@ -309,6 +304,8 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider var db = sp.GetRequiredService(); var parser = sp.GetRequiredService(); + var phoneNumberUtil = PhoneNumberUtil.GetInstance(); + var dbParser = db.GetService(); Assert.Same(parser, dbParser); var dbSetParser = db.Employees.GetService(); @@ -328,32 +325,20 @@ public async Task GetSampleContextWithDataAsync(IServiceProvider FullName = "John Doe", Title = "Software Developer", PhoneNumber = "(214) 222-2222", + NationalPhoneNumber = phoneNumberUtil.Parse("(214) 222-2222", "US").NationalNumber.ToString(), Salary = 80_000, DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 30 }], - Companies = [company], - SearchValues = [ - new() { Term = "john" }, - new() { Term = "doe" }, - new() { Term = "software" }, - new() { Term = "developer" }, - new() { Term = "2142222222" } - ] + Companies = [company] }); db.Employees.Add(new Employee { FullName = "Jane Doe", Title = "Software Developer", - PhoneNumber = "214-333-1111", + PhoneNumber = "+52 55 1234 5678", // Mexico + NationalPhoneNumber = phoneNumberUtil.Parse("+52 55 1234 5678", "US").NationalNumber.ToString(), Salary = 90_000, DataValues = [new() { Definition = company.DataDefinitions[0], NumberValue = 23 }], - Companies = [company], - SearchValues = [ - new() { Term = "jane" }, - new() { Term = "doe" }, - new() { Term = "software" }, - new() { Term = "developer" }, - new() { Term = "2143331111" } - ] + Companies = [company] }); await db.SaveChangesAsync(); From e36255a05b1b619595686227e9791a048128606f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 21 Oct 2024 13:13:50 -0500 Subject: [PATCH 4/6] Fix tests --- .../Extensions/SqlNodeExtensions.cs | 2 +- .../SqlQueryParserTests.cs | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index cd4ecfad..73eb5e40 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -132,7 +132,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon searchTerm = new SearchTerm { FieldInfo = fieldInfo, - Tokens = [node.Term], + Term = node.Term, Operator = SqlSearchOperator.StartsWith }; fieldTerms[fieldInfo] = searchTerm; diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 3edd3011..6c7178ee 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -70,10 +70,10 @@ public async Task CanSearchDefaultFields() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.FullName == "John Doe" || e.Title == "John Doe").ToQueryString(); - string sqlActual = db.Employees.Where("""FullName = "John Doe" || Title = "John Doe" """).ToQueryString(); + string sqlExpected = db.Employees.Where(e => e.FullName.StartsWith("John") || e.Title.StartsWith("John")).ToQueryString(); + string sqlActual = db.Employees.Where("""FullName.StartsWith("John") || Title.StartsWith("John") """).ToQueryString(); Assert.Equal(sqlExpected, sqlActual); - string sql = await parser.ToDynamicLinqAsync("John Doe", context); + string sql = await parser.ToDynamicLinqAsync("John", context); sqlActual = db.Employees.Where(sql).ToQueryString(); var results = await db.Employees.Where(sql).ToListAsync(); Assert.Single(results); @@ -90,6 +90,9 @@ public async Task CanSearchWithTokenizer() parser.Configuration.SetDefaultFields(["NationalPhoneNumber"]); parser.Configuration.SetSearchTokenizer(s => { + if (String.IsNullOrWhiteSpace(s.Term)) + return; + if (s.FieldInfo.Field != "NationalPhoneNumber") return; @@ -116,6 +119,12 @@ public async Task CanSearchWithTokenizer() results = await db.Employees.Where(sql).ToListAsync(); Assert.Single(results); Assert.Equal(sqlExpected, sqlActual); + + sql = await parser.ToDynamicLinqAsync("21422", context); + _logger.LogInformation(sql); + sqlActual = db.Employees.Where(sql).ToQueryString(); + results = await db.Employees.Where(sql).ToListAsync(); + Assert.Single(results); } [Fact] From 30b560f5f7a53002c013fc173f02723156029518 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 21 Oct 2024 13:34:43 -0500 Subject: [PATCH 5/6] Fix tests again --- .../Extensions/SqlNodeExtensions.cs | 61 ++++++++++++++----- .../SqlQueryParserTests.cs | 4 +- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 73eb5e40..9888e2c1 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -141,9 +141,9 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon context.Tokenizer.Invoke(searchTerm); } - fieldTerms.ForEach((kvp, info) => + fieldTerms.ForEach((kvp, x) => { - builder.Append(info.IsFirst ? "(" : " OR "); + builder.Append(x.IsFirst ? "(" : " OR "); var searchTerm = kvp.Value; var tokens = kvp.Value.Tokens ?? [kvp.Value.Term]; @@ -153,12 +153,43 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon string collectionField = searchTerm.FieldInfo.Field.Substring(0, dotIndex); string fieldName = searchTerm.FieldInfo.Field.Substring(dotIndex + 1); - builder.Append(collectionField); - builder.Append(".Any("); - builder.Append(fieldName); - builder.Append(" in ("); - builder.Append(String.Join(',', tokens.Select(t => "\"" + t + "\""))); - builder.Append("))"); + if (searchTerm.Operator == SqlSearchOperator.Equals) + { + builder.Append(collectionField); + builder.Append(".Any("); + builder.Append(fieldName); + builder.Append(" in ("); + builder.Append(String.Join(',', tokens.Select(t => "\"" + t + "\""))); + builder.Append("))"); + } + else if (searchTerm.Operator == SqlSearchOperator.Contains) + { + tokens.ForEach((token, i) => { + builder.Append(i.IsFirst ? "(" : " OR "); + builder.Append(collectionField); + builder.Append(".Any("); + builder.Append(fieldName); + builder.Append(".Contains(\""); + builder.Append(token); + builder.Append("\"))"); + if (i.IsLast) + builder.Append(")"); + }); + } + else if (searchTerm.Operator == SqlSearchOperator.StartsWith) + { + tokens.ForEach((token, i) => { + builder.Append(i.IsFirst ? "(" : " OR "); + builder.Append(collectionField); + builder.Append(".Any("); + builder.Append(fieldName); + builder.Append(".StartsWith(\""); + builder.Append(token); + builder.Append("\"))"); + if (i.IsLast) + builder.Append(")"); + }); + } } else { @@ -170,25 +201,25 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon } else if (searchTerm.Operator == SqlSearchOperator.Contains) { - searchTerm.Tokens.ForEach((token, info) => { - builder.Append(info.IsFirst ? "(" : " OR "); + tokens.ForEach((token, i) => { + builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(searchTerm.FieldInfo.Field).Append(".Contains(\"").Append(token).Append("\")"); - if (info.IsLast) + if (i.IsLast) builder.Append(")"); }); } else if (searchTerm.Operator == SqlSearchOperator.StartsWith) { - searchTerm.Tokens.ForEach((token, info) => { - builder.Append(info.IsFirst ? "(" : " OR "); + tokens.ForEach((token, i) => { + builder.Append(i.IsFirst ? "(" : " OR "); builder.Append(searchTerm.FieldInfo.Field).Append(".StartsWith(\"").Append(token).Append("\")"); - if (info.IsLast) + if (i.IsLast) builder.Append(")"); }); } } - if (info.IsLast) + if (x.IsLast) builder.Append(")"); }); diff --git a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs index 6c7178ee..23586e5a 100644 --- a/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs +++ b/tests/Foundatio.Parsers.SqlQueries.Tests/SqlQueryParserTests.cs @@ -205,8 +205,8 @@ public async Task CanUseCollectionDefaultFields() var context = parser.GetContext(db.Employees.EntityType); - string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name.Contains("acme"))).ToQueryString(); - string sqlActual = db.Employees.Where("""Companies.Any(Name.Contains("acme"))""").ToQueryString(); + string sqlExpected = db.Employees.Where(e => e.Companies.Any(c => c.Name.StartsWith("acme"))).ToQueryString(); + string sqlActual = db.Employees.Where("""Companies.Any(Name.StartsWith("acme"))""").ToQueryString(); Assert.Equal(sqlExpected, sqlActual); string sql = await parser.ToDynamicLinqAsync("acme", context); sqlActual = db.Employees.Where(sql).ToQueryString(); From 1ce788a749e33cadf9b9fb37885c040a3265b8bf Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 21 Oct 2024 13:40:17 -0500 Subject: [PATCH 6/6] Rename Tokenizer to SearchTokenizer --- .../Extensions/SqlNodeExtensions.cs | 2 +- src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs | 2 +- .../Visitors/ISqlQueryVisitorContext.cs | 2 +- .../Visitors/SqlQueryVisitorContext.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs index 9888e2c1..cf9e9efb 100644 --- a/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs +++ b/src/Foundatio.Parsers.SqlQueries/Extensions/SqlNodeExtensions.cs @@ -138,7 +138,7 @@ public static string ToDynamicLinqString(this TermNode node, ISqlQueryVisitorCon fieldTerms[fieldInfo] = searchTerm; } - context.Tokenizer.Invoke(searchTerm); + context.SearchTokenizer.Invoke(searchTerm); } fieldTerms.ForEach((kvp, x) => diff --git a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs index e13cfdc5..65ed49d0 100644 --- a/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs +++ b/src/Foundatio.Parsers.SqlQueries/SqlQueryParser.cs @@ -192,7 +192,7 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) if (context is ISqlQueryVisitorContext sqlContext) { - sqlContext.Tokenizer = Configuration.SearchTokenizer; + sqlContext.SearchTokenizer = Configuration.SearchTokenizer; } } } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs index c0554c30..c0d549a8 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/ISqlQueryVisitorContext.cs @@ -7,5 +7,5 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public interface ISqlQueryVisitorContext : IQueryVisitorContext { List Fields { get; set; } - Action Tokenizer { get; set; } + Action SearchTokenizer { get; set; } } diff --git a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs index 90cf32be..f2a9be84 100644 --- a/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs +++ b/src/Foundatio.Parsers.SqlQueries/Visitors/SqlQueryVisitorContext.cs @@ -9,7 +9,7 @@ namespace Foundatio.Parsers.SqlQueries.Visitors; public class SqlQueryVisitorContext : QueryVisitorContext, ISqlQueryVisitorContext { public List Fields { get; set; } - public Action Tokenizer { get; set; } = static _ => { }; + public Action SearchTokenizer { get; set; } = static _ => { }; public IEntityType EntityType { get; set; } }