Diplomarbeit-Absolventenverein/oqtane.framework
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4402 from sbwalker/dev

Browse Source 
search modifications
This commit is contained in:
Shaun Walker 2024-07-12 10:33:34 -04:00 committed by GitHub
commit 7abc2289de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 144 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Oqtane.Server/Controllers/SearchResultsController.cs
View File

@ -4,7 +4,6 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Oqtane.Documentation;
using Oqtane.Enums;
using Oqtane.Infrastructure;
using Oqtane.Services;
@ -28,7 +27,7 @@ namespace Oqtane.Controllers
{
try
{
return await _searchService.SearchAsync(searchQuery);
return await _searchService.GetSearchResultsAsync(searchQuery);
}
catch (Exception ex)
{

9
Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs
View File

@ -58,6 +58,8 @@ namespace Oqtane.Infrastructure
var currentTime = DateTime.UtcNow;
var lastIndexedOn = Convert.ToDateTime(siteSettings.GetValue(SearchLastIndexedOnSetting, DateTime.MinValue.ToString()));
log += $"Index Date: {lastIndexedOn}<br />";
var ignorePaths = siteSettings.GetValue(SearchIgnorePathsSetting, "").Split(',');
var ignoreEntities = siteSettings.GetValue(SearchIgnoreEntitiesSetting, "").Split(',');
@ -68,7 +70,7 @@ namespace Oqtane.Infrastructure
// index pages
foreach (var page in pages)
{
if (Constants.InternalPagePaths.Contains(page.Path) || ignorePaths.Contains(page.Path))
if (!string.IsNullOrEmpty(page.Path) && (Constants.InternalPagePaths.Contains(page.Path) || ignorePaths.Contains(page.Path)))
{
continue;
}
@ -169,9 +171,8 @@ namespace Oqtane.Infrastructure
}
}
// save search content
await searchService.SaveSearchContentAsync(searchContents, siteSettings);
log += $"Index Date: {lastIndexedOn}<br />";
// save search contents
log += await searchService.SaveSearchContentsAsync(searchContents, siteSettings);
log += $"Items Indexed: {searchContents.Count}<br />";
// update last indexed on

60
Oqtane.Server/Managers/Search/ModuleSearchResultManager.cs
View File

@ -1,60 +0,0 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class ModuleSearchResultManager : ISearchResultManager
{
public string Name => EntityNames.Module;
private readonly IServiceProvider _serviceProvider;
public ModuleSearchResultManager(
IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public string GetUrl(SearchResult searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var pageIdValue = searchResult.SearchContentProperties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
{
var page = pageRepository.GetPage(pageId);
if (page != null)
{
return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
}
}
return string.Empty;
}
public bool Visible(SearchContent searchResult, SearchQuery searchQuery)
{
var pageIdValue = searchResult.SearchContentProperties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
{
return CanViewPage(pageId, searchQuery.User);
}
return false;
}
private bool CanViewPage(int pageId, User user)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(pageId);
return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList)
&& (Utilities.IsEffectiveOrExpired(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList));
}
}
}

150
Oqtane.Server/Providers/DatabaseSearchProvider.cs
View File

@ -28,105 +28,10 @@ namespace Oqtane.Providers
_searchContentRepository = searchContentRepository;
}
public void Commit()
public async Task<List<SearchResult>> GetSearchResultsAsync(SearchQuery searchQuery)
{
}
public void DeleteSearchContent(string id)
{
_searchContentRepository.DeleteSearchContent(id);
}
public bool Optimize()
{
return true;
}
public void ResetIndex()
{
_searchContentRepository.DeleteAllSearchContent();
}
public void SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings, bool autoCommit = false)
{
// remove existing search content
_searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId);
if (!searchContent.IsDeleted)
{
// clean the search content to remove html tags
CleanSearchContent(searchContent);
_searchContentRepository.AddSearchContent(searchContent);
// save the index words
AnalyzeSearchContent(searchContent, siteSettings);
}
}
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchContent, SearchQuery, bool> validateFunc)
{
var totalResults = 0;
var searchContentList = await _searchContentRepository.GetSearchContentsAsync(searchQuery);
// convert the search content to search results
var results = searchContentList
.Where(i => validateFunc(i, searchQuery))
.Select(i => ConvertToSearchResult(i, searchQuery));
if (searchQuery.SortDirection == SearchSortDirections.Descending)
{
switch (searchQuery.SortField)
{
case SearchSortFields.Relevance:
results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
break;
case SearchSortFields.Title:
results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
break;
default:
results = results.OrderByDescending(i => i.ContentModifiedOn);
break;
}
}
else
{
switch (searchQuery.SortField)
{
case SearchSortFields.Relevance:
results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
break;
case SearchSortFields.Title:
results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
break;
default:
results = results.OrderBy(i => i.ContentModifiedOn);
break;
}
}
// remove duplicated results based on page id for Page and Module types
results = results.DistinctBy(i =>
{
if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module)
{
var pageId = i.SearchContentProperties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
}
else
{
return i.UniqueKey;
}
});
totalResults = results.Count();
return new SearchResults
{
Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(),
TotalResults = totalResults
};
var searchContents = await _searchContentRepository.GetSearchContentsAsync(searchQuery);
return searchContents.Select(item => ConvertToSearchResult(item, searchQuery)).ToList();
}
private SearchResult ConvertToSearchResult(SearchContent searchContent, SearchQuery searchQuery)
@ -152,17 +57,6 @@ namespace Oqtane.Providers
return searchResult;
}
private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery)
{
var score = 0f;
foreach (var keyword in SearchUtils.GetKeywords(searchQuery.Keywords))
{
score += searchContent.SearchContentWords.Where(i => i.SearchWord.Word.StartsWith(keyword)).Sum(i => i.Count);
}
return score / 100;
}
private string BuildSnippet(SearchContent searchContent, SearchQuery searchQuery)
{
var content = $"{searchContent.Title} {searchContent.Description} {searchContent.Body}";
@ -189,8 +83,6 @@ namespace Oqtane.Providers
snippet = $"{prefix}{content.Substring(start, length)}{suffix}";
break;
}
}
@ -207,6 +99,36 @@ namespace Oqtane.Providers
return snippet;
}
private float CalculateScore(SearchContent searchContent, SearchQuery searchQuery)
{
var score = 0f;
foreach (var keyword in SearchUtils.GetKeywords(searchQuery.Keywords))
{
score += searchContent.SearchContentWords.Where(i => i.SearchWord.Word.StartsWith(keyword)).Sum(i => i.Count);
}
return score / 100;
}
public Task SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings)
{
// remove existing search content
_searchContentRepository.DeleteSearchContent(searchContent.EntityName, searchContent.EntityId);
if (!searchContent.IsDeleted)
{
// clean the search content to remove html tags
CleanSearchContent(searchContent);
_searchContentRepository.AddSearchContent(searchContent);
// save the index words
AnalyzeSearchContent(searchContent, siteSettings);
}
return Task.CompletedTask;
}
private void AnalyzeSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings)
{
var ignoreWords = IgnoreWords.Split(',');
@ -324,5 +246,11 @@ namespace Oqtane.Providers
return string.Join(" ", phrases);
}
public Task ResetIndex()
{
_searchContentRepository.DeleteAllSearchContent();
return Task.CompletedTask;
}
}
}

212
Oqtane.Server/Services/SearchService.cs
View File

@ -2,10 +2,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Oqtane.Infrastructure;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
@ -16,49 +14,132 @@ namespace Oqtane.Services
public class SearchService : ISearchService
{
private const string SearchProviderSettingName = "SearchProvider";
private const string SearchEnabledSettingName = "SearchEnabled";
private readonly IServiceProvider _serviceProvider;
private readonly ITenantManager _tenantManager;
private readonly IAliasRepository _aliasRepository;
private readonly ISettingRepository _settingRepository;
private readonly IPermissionRepository _permissionRepository;
private readonly ILogger<SearchService> _logger;
private readonly IMemoryCache _cache;
public SearchService(
IServiceProvider serviceProvider,
ITenantManager tenantManager,
IAliasRepository aliasRepository,
ISettingRepository settingRepository,
IPermissionRepository permissionRepository,
ILogger<SearchService> logger,
IMemoryCache cache)
ILogger<SearchService> logger)
{
_tenantManager = tenantManager;
_aliasRepository = aliasRepository;
_settingRepository = settingRepository;
_permissionRepository = permissionRepository;
_serviceProvider = serviceProvider;
_logger = logger;
_cache = cache;
}
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery)
public async Task<SearchResults> GetSearchResultsAsync(SearchQuery searchQuery)
{
var searchProvider = GetSearchProvider(searchQuery.SiteId);
var searchResults = await searchProvider.SearchAsync(searchQuery, Visible);
var searchResults = await searchProvider.GetSearchResultsAsync(searchQuery);
//generate the document url if it's not set.
foreach (var result in searchResults.Results)
var totalResults = 0;
// trim results based on permissions
var results = searchResults.Where(i => IsVisible(i, searchQuery));
if (searchQuery.SortDirection == SearchSortDirections.Descending)
{
if(string.IsNullOrEmpty(result.Url))
switch (searchQuery.SortField)
{
result.Url = GetDocumentUrl(result, searchQuery);
case SearchSortFields.Relevance:
results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
break;
case SearchSortFields.Title:
results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
break;
default:
results = results.OrderByDescending(i => i.ContentModifiedOn);
break;
}
}
else
{
switch (searchQuery.SortField)
{
case SearchSortFields.Relevance:
results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ContentModifiedOn);
break;
case SearchSortFields.Title:
results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ContentModifiedOn);
break;
default:
results = results.OrderBy(i => i.ContentModifiedOn);
break;
}
}
return searchResults;
// remove duplicated results based on page id for Page and Module types
results = results.DistinctBy(i =>
{
if (i.EntityName == EntityNames.Page || i.EntityName == EntityNames.Module)
{
var pageId = i.SearchContentProperties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
}
else
{
return i.UniqueKey;
}
});
totalResults = results.Count();
return new SearchResults
{
Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(),
TotalResults = totalResults
};
}
private bool IsVisible(SearchContent searchContent, SearchQuery searchQuery)
{
var visible = true;
foreach (var permission in searchContent.Permissions.Split(','))
{
var entityName = permission.Split(":")[0];
var entityId = int.Parse(permission.Split(":")[1]);
if (!HasViewPermission(searchQuery.SiteId, searchQuery.User, entityName, entityId))
{
visible = false;
break;
}
}
return visible;
}
private bool HasViewPermission(int siteId, User user, string entityName, int entityId)
{
var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList();
return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions);
}
public async Task<string> SaveSearchContentsAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings)
{
var result = "";
if (searchContents.Any())
{
var searchProvider = GetSearchProvider(searchContents.First().SiteId);
foreach (var searchContent in searchContents)
{
try
{
await searchProvider.SaveSearchContent(searchContent, siteSettings);
}
catch (Exception ex)
{
result += $"Error Saving Search Content With UniqueKey {searchContent.UniqueKey} - {ex.Message}<br />";
}
}
}
return result;
}
private ISearchProvider GetSearchProvider(int siteId)
@ -84,96 +165,5 @@ namespace Oqtane.Services
return Constants.DefaultSearchProviderName;
}
private bool SearchEnabled(int siteId)
{
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, SearchEnabledSettingName);
if (!string.IsNullOrEmpty(setting?.SettingValue))
{
return bool.TryParse(setting.SettingValue, out bool enabled) && enabled;
}
return true;
}
private void SetTenant(int siteId)
{
var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId);
_tenantManager.SetAlias(alias);
}
private List<ISearchResultManager> GetSearchResultManagers()
{
var managers = new List<ISearchResultManager>();
var managerTypes = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(s => s.GetTypes())
.Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract);
foreach (var type in managerTypes)
{
var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type);
managers.Add(manager);
}
return managers.ToList();
}
public async Task SaveSearchContentAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings)
{
if(searchContents.Any())
{
var searchProvider = GetSearchProvider(searchContents.First().SiteId);
foreach (var searchContent in searchContents)
{
try
{
searchProvider.SaveSearchContent(searchContent, siteSettings);
}
catch(Exception ex)
{
_logger.LogError(ex, $"Search: Save search content {searchContent.UniqueKey} failed.");
}
}
await Task.CompletedTask;
//commit the index changes
searchProvider.Commit();
}
}
private bool Visible(SearchContent searchContent, SearchQuery searchQuery)
{
var visible = true;
foreach (var permission in searchContent.Permissions.Split(','))
{
var entityName = permission.Split(":")[0];
var entityId = int.Parse(permission.Split(":")[1]);
if (!HasViewPermission(searchQuery.SiteId, searchQuery.User, entityName, entityId))
{
visible = false;
break;
}
}
return visible;
}
private bool HasViewPermission(int siteId, User user, string entityName, int entityId)
{
var permissions = _permissionRepository.GetPermissions(siteId, entityName, entityId).ToList();
return UserSecurity.IsAuthorized(user, PermissionNames.View, permissions);
}
private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery)
{
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.EntityName);
if(searchResultManager != null)
{
return searchResultManager.GetUrl(result, searchQuery);
}
return string.Empty;
}
}
}

12
Oqtane.Shared/Interfaces/ISearchProvider.cs
View File

@ -11,16 +11,10 @@ namespace Oqtane.Services
{
string Name { get; }
void SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings, bool autoCommit = false);
Task<List<SearchResult>> GetSearchResultsAsync(SearchQuery searchQuery);
void DeleteSearchContent(string id);
Task SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings);
Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchContent, SearchQuery, bool> validateFunc);
bool Optimize();
void Commit();
void ResetIndex();
Task ResetIndex();
}
}

14
Oqtane.Shared/Interfaces/ISearchResultManager.cs
View File

@ -1,14 +0,0 @@
using System;
using Oqtane.Models;
namespace Oqtane.Services
{
public interface ISearchResultManager
{
string Name { get; }
bool Visible(SearchContent searchResult, SearchQuery searchQuery);
string GetUrl(SearchResult searchResult, SearchQuery searchQuery);
}
}

4
Oqtane.Shared/Interfaces/ISearchService.cs
View File

@ -6,8 +6,8 @@ namespace Oqtane.Services
{
public interface ISearchService
{
Task SaveSearchContentAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings);
Task<SearchResults> GetSearchResultsAsync(SearchQuery searchQuery);
Task<SearchResults> SearchAsync(SearchQuery searchQuery);
Task<string> SaveSearchContentsAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings);
}
}