Merge pull request #4402 from sbwalker/dev

search modifications

Oqtane.Server/Controllers/SearchResultsController.cs

@@ -4,7 +4,6 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
-using Oqtane.Documentation;
 using Oqtane.Enums;
 using Oqtane.Infrastructure;
 using Oqtane.Services;
@@ -28,7 +27,7 @@ namespace Oqtane.Controllers
         {
             try
             {
-                return await _searchService.SearchAsync(searchQuery);
+                return await _searchService.GetSearchResultsAsync(searchQuery);
             }
             catch (Exception ex)
             {

Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs

@@ -58,6 +58,8 @@ namespace Oqtane.Infrastructure
 
                 var currentTime = DateTime.UtcNow;
                 var lastIndexedOn = Convert.ToDateTime(siteSettings.GetValue(SearchLastIndexedOnSetting, DateTime.MinValue.ToString()));
+                log += $"Index Date: {lastIndexedOn}<br />";
+
                 var ignorePaths = siteSettings.GetValue(SearchIgnorePathsSetting, "").Split(',');
                 var ignoreEntities = siteSettings.GetValue(SearchIgnoreEntitiesSetting, "").Split(',');
 
@@ -68,7 +70,7 @@ namespace Oqtane.Infrastructure
                 // index pages
                 foreach (var page in pages)
                 {
-                    if (Constants.InternalPagePaths.Contains(page.Path) || ignorePaths.Contains(page.Path))
+                    if (!string.IsNullOrEmpty(page.Path) && (Constants.InternalPagePaths.Contains(page.Path) || ignorePaths.Contains(page.Path)))
                     {
                         continue;
                     }
@@ -169,9 +171,8 @@ namespace Oqtane.Infrastructure
                     }
                 }
 
-                // save search content
+                // save search contents
-                await searchService.SaveSearchContentAsync(searchContents, siteSettings);
+                log += await searchService.SaveSearchContentsAsync(searchContents, siteSettings);
-                log += $"Index Date: {lastIndexedOn}<br />";
                 log += $"Items Indexed: {searchContents.Count}<br />";
 
                 // update last indexed on

Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs

@@ -1,60 +0,0 @@
-using System;
-using System.Linq;
-using Microsoft.Extensions.DependencyInjection;
-using Oqtane.Models;
-using Oqtane.Repository;
-using Oqtane.Security;
-using Oqtane.Services;
-using Oqtane.Shared;
-
-namespace Oqtane.Managers.Search
-{
-    public class ModuleSearchResultManager : ISearchResultManager
-    {
-        public string Name => EntityNames.Module;
-
-        private readonly IServiceProvider _serviceProvider;
-
-        public ModuleSearchResultManager(
-            IServiceProvider serviceProvider)
-        {
-            _serviceProvider = serviceProvider;
-        }
-
-        public string GetUrl(SearchResult searchResult, SearchQuery searchQuery)
-        {
-            var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
-            var pageIdValue = searchResult.SearchContentProperties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
-            if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
-            {
-                var page = pageRepository.GetPage(pageId);
-                if (page != null)
-                {
-                    return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
-                }
-            }
-            
-            return string.Empty;
-        }
-
-        public bool Visible(SearchContent searchResult, SearchQuery searchQuery)
-        {
-            var pageIdValue = searchResult.SearchContentProperties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
-            if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
-            {
-                return CanViewPage(pageId, searchQuery.User);
-            }
-
-            return false;
-        }
-
-        private bool CanViewPage(int pageId, User user)
-        {
-            var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
-            var page = pageRepository.GetPage(pageId);
-
-            return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList)
-                    && (Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList));
-        }
-    }
-}

Oqtane.Server/Providers/DatabaseSearchProvider.cs

@@ -28,105 +28,10 @@ namespace Oqtane.Providers
             _searchContentRepository = searchContentRepository;
         }
 
-        public void Commit()
+        public async Task<List<SearchResult>> GetSearchResultsAsync(SearchQuery searchQuery)
         {
-        }
+            var searchContents = await _searchContentRepository.GetSearchContentsAsync(searchQuery);
-
+            return searchContents.Select(item => ConvertToSearchResult(item, searchQuery)).ToList();
-        public void DeleteSearchContent(string id)
-        {
-            _searchContentRepository.DeleteSearchContent(id);
-        }
-
-        public bool Optimize()
-        {
-            return true;
-        }
-
-        public void ResetIndex()
-        {
-            _searchContentRepository.DeleteAllSearchContent();
-        }
-
-        public void SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings, bool autoCommit = false)
-        {
-            // remove existing search content
-            _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId);
-
-            if (!searchContent.IsDeleted)
-            {
-                // clean the search content to remove html tags
-                CleanSearchContent(searchContent);
-
-                _searchContentRepository.AddSearchContent(searchContent);
-
-                // save the index words
-                AnalyzeSearchContent(searchContent, siteSettings);
-            }
-        }
-
-        public async Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchContent, SearchQuery, bool> validateFunc)
-        {
-            var totalResults = 0;
-
-            var searchContentList = await _searchContentRepository.GetSearchContentsAsync(searchQuery);
-
-            // convert the search content to search results
-            var results = searchContentList
-                .Where(i => validateFunc(i, searchQuery))
-                .Select(i => ConvertToSearchResult(i, searchQuery));
-
-            if (searchQuery.SortDirection == SearchSortDirections.Descending)
-            {
-                switch (searchQuery.SortField)
-                {
-                    case SearchSortFields.Relevance:
-                        results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
-                        break;
-                    case SearchSortFields.Title:
-                        results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
-                        break;
-                    default:
-                        results = results.OrderByDescending(i => i.ContentModifiedOn);
-                        break;
-                }
-            }
-            else
-            {
-                switch (searchQuery.SortField)
-                {
-                    case SearchSortFields.Relevance:
-                        results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
-                        break;
-                    case SearchSortFields.Title:
-                        results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
-                        break;
-                    default:
-                        results = results.OrderBy(i => i.ContentModifiedOn);
-                        break;
-                }
-            }
-
-            // remove duplicated results based on page id for Page and Module types
-            results = results.DistinctBy(i =>
-            {
-                if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module)
-                {
-                    var pageId = i.SearchContentProperties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
-                    return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
-                }
-                else
-                {
-                    return i.UniqueKey;
-                }
-            });
-
-            totalResults = results.Count();
-
-            return new SearchResults
-            {
-                Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(),
-                TotalResults = totalResults
-            };
         }
 
         private SearchResult ConvertToSearchResult(SearchContent searchContent, SearchQuery searchQuery)
@@ -152,17 +57,6 @@ namespace Oqtane.Providers
             return searchResult;
         }
 
-        private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery)
-        {
-            var score = 0f;
-            foreach (var keyword in SearchUtils.GetKeywords(searchQuery.Keywords))
-            {
-                score += searchContent.SearchContentWords.Where(i => i.SearchWord.Word.StartsWith(keyword)).Sum(i => i.Count);
-            }
-
-            return score / 100;
-        }
-
         private string BuildSnippet(SearchContent searchContent, SearchQuery searchQuery)
         {
             var content = $"{searchContent.Title} {searchContent.Description} {searchContent.Body}";
@@ -189,8 +83,6 @@ namespace Oqtane.Providers
 
                     snippet = $"{prefix}{content.Substring(start, length)}{suffix}";
                     break;
-
-
                 }
             }
 
@@ -207,6 +99,36 @@ namespace Oqtane.Providers
             return snippet;
         }
 
+        private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery)
+        {
+            var score = 0f;
+            foreach (var keyword in SearchUtils.GetKeywords(searchQuery.Keywords))
+            {
+                score += searchContent.SearchContentWords.Where(i => i.SearchWord.Word.StartsWith(keyword)).Sum(i => i.Count);
+            }
+
+            return score / 100;
+        }
+
+        public Task SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings)
+        {
+            // remove existing search content
+            _searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId);
+
+            if (!searchContent.IsDeleted)
+            {
+                // clean the search content to remove html tags
+                CleanSearchContent(searchContent);
+
+                _searchContentRepository.AddSearchContent(searchContent);
+
+                // save the index words
+                AnalyzeSearchContent(searchContent, siteSettings);
+            }
+
+            return Task.CompletedTask;
+        }
+
         private void AnalyzeSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings)
         {
             var ignoreWords = IgnoreWords.Split(',');
@@ -324,5 +246,11 @@ namespace Oqtane.Providers
 
             return string.Join(" ", phrases);
         }
+
+        public Task ResetIndex()
+        {
+            _searchContentRepository.DeleteAllSearchContent();
+            return Task.CompletedTask;
+        }
     }
 }

Oqtane.Server/Services/SearchService.cs

@@ -2,10 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
-using Oqtane.Infrastructure;
 using Oqtane.Models;
 using Oqtane.Repository;
 using Oqtane.Security;
@@ -16,49 +14,132 @@ namespace Oqtane.Services
     public class SearchService : ISearchService
     {
         private const string SearchProviderSettingName = "SearchProvider";
-        private const string SearchEnabledSettingName = "SearchEnabled";
 
         private readonly IServiceProvider _serviceProvider;
-        private readonly ITenantManager _tenantManager;
-        private readonly IAliasRepository _aliasRepository;
         private readonly ISettingRepository _settingRepository;
         private readonly IPermissionRepository _permissionRepository;
         private readonly ILogger<SearchService> _logger;
-        private readonly IMemoryCache _cache;
 
         public SearchService(
             IServiceProvider serviceProvider,
-            ITenantManager tenantManager,
-            IAliasRepository aliasRepository,
             ISettingRepository settingRepository,
             IPermissionRepository permissionRepository,
-            ILogger<SearchService> logger,
+            ILogger<SearchService> logger)
-            IMemoryCache cache)
         {
-            _tenantManager = tenantManager;
-            _aliasRepository = aliasRepository;
             _settingRepository = settingRepository;
             _permissionRepository = permissionRepository;
             _serviceProvider = serviceProvider;
             _logger = logger;
-            _cache = cache;
         }
 
-        public async Task<SearchResults> SearchAsync(SearchQuery searchQuery)
+        public async Task<SearchResults> GetSearchResultsAsync(SearchQuery searchQuery)
         {
             var searchProvider = GetSearchProvider(searchQuery.SiteId);
-            var searchResults = await searchProvider.SearchAsync(searchQuery, Visible);
+            var searchResults = await searchProvider.GetSearchResultsAsync(searchQuery);
 
-            //generate the document url if it's not set.
+            var totalResults = 0;
-            foreach (var result in searchResults.Results)
+
+            // trim results based on permissions
+            var results = searchResults.Where(i => IsVisible(i, searchQuery));
+
+            if (searchQuery.SortDirection == SearchSortDirections.Descending)
             {
-                if(string.IsNullOrEmpty(result.Url))
+                switch (searchQuery.SortField)
                 {
-                    result.Url = GetDocumentUrl(result, searchQuery);
+                    case SearchSortFields.Relevance:
+                        results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
+                        break;
+                    case SearchSortFields.Title:
+                        results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
+                        break;
+                    default:
+                        results = results.OrderByDescending(i => i.ContentModifiedOn);
+                        break;
+                }
+            }
+            else
+            {
+                switch (searchQuery.SortField)
+                {
+                    case SearchSortFields.Relevance:
+                        results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
+                        break;
+                    case SearchSortFields.Title:
+                        results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
+                        break;
+                    default:
+                        results = results.OrderBy(i => i.ContentModifiedOn);
+                        break;
                 }
             }
 
-            return searchResults;
+            // remove duplicated results based on page id for Page and Module types
+            results = results.DistinctBy(i =>
+            {
+                if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module)
+                {
+                    var pageId = i.SearchContentProperties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
+                    return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
+                }
+                else
+                {
+                    return i.UniqueKey;
+                }
+            });
+
+            totalResults = results.Count();
+
+            return new SearchResults
+            {
+                Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(),
+                TotalResults = totalResults
+            };
+        }
+
+        private bool IsVisible(SearchContent searchContent, SearchQuery searchQuery)
+        {
+            var visible = true;
+            foreach (var permission in searchContent.Permissions.Split(','))
+            {
+                var entityName = permission.Split(":")[0];
+                var entityId = int.Parse(permission.Split(":")[1]);
+                if (!HasViewPermission(searchQuery.SiteId, searchQuery.User, entityName, entityId))
+                {
+                    visible = false;
+                    break;
+                }
+            }
+            return visible;
+        }
+
+        private bool HasViewPermission(int siteId, User user, string entityName, int entityId)
+        {
+            var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList();
+            return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions);
+        }
+
+        public async Task<string> SaveSearchContentsAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings)
+        {
+            var result = "";
+
+            if (searchContents.Any())
+            {
+                var searchProvider = GetSearchProvider(searchContents.First().SiteId);
+
+                foreach (var searchContent in searchContents)
+                {
+                    try
+                    {
+                        await searchProvider.SaveSearchContent(searchContent, siteSettings);
+                    }
+                    catch (Exception ex)
+                    {
+                        result += $"Error Saving Search Content With UniqueKey {searchContent.UniqueKey} - {ex.Message}<br />";
+                    }
+                }
+            }
+
+            return result;
         }
 
         private ISearchProvider GetSearchProvider(int siteId)
@@ -84,96 +165,5 @@ namespace Oqtane.Services
 
             return Constants.DefaultSearchProviderName;
         }
-
-        private bool SearchEnabled(int siteId)
-        {
-            var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchEnabledSettingName);
-            if (!string.IsNullOrEmpty(setting?.SettingValue))
-            {
-                return bool.TryParse(setting.SettingValue, out bool enabled) && enabled;
-            }
-
-            return true;
-        }
-
-        private void SetTenant(int siteId)
-        {
-            var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId);
-            _tenantManager.SetAlias(alias);
-        }
-
-        private List<ISearchResultManager> GetSearchResultManagers()
-        {
-            var managers = new List<ISearchResultManager>();
-            var managerTypes = AppDomain.CurrentDomain.GetAssemblies()
-                .SelectMany(s => s.GetTypes())
-                .Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract);
-
-            foreach (var type in managerTypes)
-            {
-                var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type);
-                managers.Add(manager);
-            }
-
-            return managers.ToList();
-        }
-
-        public async Task SaveSearchContentAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings)
-        {
-            if(searchContents.Any())
-            {
-                var searchProvider = GetSearchProvider(searchContents.First().SiteId);
-
-                foreach (var searchContent in searchContents)
-                {
-                    try
-                    {
-                        searchProvider.SaveSearchContent(searchContent, siteSettings);
-                    }
-                    catch(Exception ex)
-                    {
-                        _logger.LogError(ex, $"Search: Save search content {searchContent.UniqueKey} failed.");
-                    }
-                }
-
-                await Task.CompletedTask;
-
-                //commit the index changes
-                searchProvider.Commit();
-            }
-        }
-
-        private bool Visible(SearchContent searchContent, SearchQuery searchQuery)
-        {
-            var visible = true;
-            foreach (var permission in searchContent.Permissions.Split(','))
-            {
-                var entityName = permission.Split(":")[0];
-                var entityId = int.Parse(permission.Split(":")[1]);
-                if (!HasViewPermission(searchQuery.SiteId, searchQuery.User, entityName, entityId))
-                {
-                    visible = false;
-                    break;
-                }
-            }
-            return visible;
-        }
-
-        private bool HasViewPermission(int siteId, User user, string entityName, int entityId)
-        {
-            var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList();
-            return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions);
-        }
-
-        private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery)
-        {
-            var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.EntityName);
-            if(searchResultManager != null)
-            {
-                return searchResultManager.GetUrl(result, searchQuery);
-            }
-
-            return string.Empty;
-        }
     }
 }

Oqtane.Shared/Interfaces/ISearchProvider.cs

@@ -11,16 +11,10 @@ namespace Oqtane.Services
     {
         string Name { get; }
 
-        void SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings, bool autoCommit = false);
+        Task<List<SearchResult>> GetSearchResultsAsync(SearchQuery searchQuery);
 
-        void DeleteSearchContent(string id);
+        Task SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings);
         
-        Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchContent, SearchQuery, bool> validateFunc);
+        Task ResetIndex();
-        
-        bool Optimize();
-
-        void Commit();
-
-        void ResetIndex();
     }
 }

Oqtane.Shared/Interfaces/ISearchResultManager.cs

@@ -1,14 +0,0 @@
-using System;
-using Oqtane.Models;
-
-namespace Oqtane.Services
-{
-    public interface ISearchResultManager
-    {
-        string Name { get; }
-
-        bool Visible(SearchContent searchResult, SearchQuery searchQuery);
-
-        string GetUrl(SearchResult searchResult, SearchQuery searchQuery);
-    }
-}

Oqtane.Shared/Interfaces/ISearchService.cs

@@ -6,8 +6,8 @@ namespace Oqtane.Services
 {
     public interface ISearchService
     {
-        Task SaveSearchContentAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings);
+        Task<SearchResults> GetSearchResultsAsync(SearchQuery searchQuery);
 
-        Task<SearchResults> SearchAsync(SearchQuery searchQuery);
+        Task<string> SaveSearchContentsAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings);
     }
 }