#4303: add search function.

This commit is contained in:
Ben
2024-06-03 21:19:42 +08:00
parent d75e3acdf3
commit 9d85ca07f4
50 changed files with 2478 additions and 4 deletions

View File

@ -21,6 +21,7 @@ using Oqtane.Infrastructure;
using Oqtane.Infrastructure.Interfaces;
using Oqtane.Managers;
using Oqtane.Modules;
using Oqtane.Providers;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Services;
@ -97,6 +98,8 @@ namespace Microsoft.Extensions.DependencyInjection
services.AddScoped<IUrlMappingService, UrlMappingService>();
services.AddScoped<IVisitorService, VisitorService>();
services.AddScoped<ISyncService, SyncService>();
services.AddScoped<ISearchService, SearchService>();
services.AddScoped<ISearchProvider, DatabaseSearchProvider>();
return services;
}
@ -131,6 +134,7 @@ namespace Microsoft.Extensions.DependencyInjection
services.AddTransient<ILanguageRepository, LanguageRepository>();
services.AddTransient<IVisitorRepository, VisitorRepository>();
services.AddTransient<IUrlMappingRepository, UrlMappingRepository>();
services.AddTransient<ISearchDocumentRepository, SearchDocumentRepository>();
// managers
services.AddTransient<IDBContextDependencies, DBContextDependencies>();

View File

@ -0,0 +1,88 @@
using System;
using System.Linq;
using System.Text;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Infrastructure
{
public class SearchIndexJob : HostedServiceBase
{
public SearchIndexJob(IServiceScopeFactory serviceScopeFactory) : base(serviceScopeFactory)
{
Name = "Search Index Job";
Frequency = "m"; // run every minute.
Interval = 1;
IsEnabled = true;
}
public override string ExecuteJob(IServiceProvider provider)
{
// get services
var siteRepository = provider.GetRequiredService<ISiteRepository>();
var settingRepository = provider.GetRequiredService<ISettingRepository>();
var logRepository = provider.GetRequiredService<ILogRepository>();
var searchService = provider.GetRequiredService<ISearchService>();
var sites = siteRepository.GetSites().ToList();
var logs = new StringBuilder();
foreach (var site in sites)
{
var startTime = GetSearchStartTime(site.SiteId, settingRepository);
logs.AppendLine($"Search: Begin index site: {site.Name}<br />");
var currentTime = DateTime.UtcNow;
searchService.IndexContent(site.SiteId, startTime, logNote =>
{
logs.AppendLine(logNote);
}, handleError =>
{
logs.AppendLine(handleError);
});
UpdateSearchStartTime(site.SiteId, currentTime, settingRepository);
logs.AppendLine($"Search: End index site: {site.Name}<br />");
}
return logs.ToString();
}
private DateTime? GetSearchStartTime(int siteId, ISettingRepository settingRepository)
{
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName);
if(setting == null)
{
return null;
}
return Convert.ToDateTime(setting.SettingValue);
}
private void UpdateSearchStartTime(int siteId, DateTime startTime, ISettingRepository settingRepository)
{
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchIndexStartTimeSettingName);
if (setting == null)
{
setting = new Setting
{
EntityName = EntityNames.Site,
EntityId = siteId,
SettingName = Constants.SearchIndexStartTimeSettingName,
SettingValue = Convert.ToString(startTime),
};
settingRepository.AddSetting(setting);
}
else
{
setting.SettingValue = Convert.ToString(startTime);
settingRepository.UpdateSetting(setting);
}
}
}
}

View File

@ -133,6 +133,30 @@ namespace Oqtane.SiteTemplates
}
}
});
_pageTemplates.Add(new PageTemplate
{
Name = "Search Results",
Parent = "",
Order = 7,
Path = "search-results",
Icon = "oi oi-magnifying-glass",
IsNavigation = false,
IsPersonalizable = false,
PermissionList = new List<Permission> {
new Permission(PermissionNames.View, RoleNames.Everyone, true),
new Permission(PermissionNames.View, RoleNames.Admin, true),
new Permission(PermissionNames.Edit, RoleNames.Admin, true)
},
PageTemplateModules = new List<PageTemplateModule> {
new PageTemplateModule { ModuleDefinitionName = "Oqtane.Modules.SearchResults, Oqtane.Client", Title = "Search Results", Pane = PaneNames.Default,
PermissionList = new List<Permission> {
new Permission(PermissionNames.View, RoleNames.Everyone, true),
new Permission(PermissionNames.View, RoleNames.Admin, true),
new Permission(PermissionNames.Edit, RoleNames.Admin, true)
}
}
}
});
if (System.IO.File.Exists(Path.Combine(_environment.WebRootPath, "images", "logo-white.png")))
{

View File

@ -0,0 +1,127 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Oqtane.Interfaces;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class ModuleSearchIndexManager : SearchIndexManagerBase
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<ModuleSearchIndexManager> _logger;
private readonly IPageModuleRepository _pageModuleRepostory;
private readonly IPageRepository _pageRepository;
public ModuleSearchIndexManager(
IServiceProvider serviceProvider,
IPageModuleRepository pageModuleRepostory,
ILogger<ModuleSearchIndexManager> logger,
IPageRepository pageRepository)
: base(serviceProvider)
{
_serviceProvider = serviceProvider;
_logger = logger;
_pageModuleRepostory = pageModuleRepostory;
_pageRepository = pageRepository;
}
public override string Name => Constants.ModuleSearchIndexManagerName;
public override int Priority => Constants.ModuleSearchIndexManagerPriority;
public override int IndexDocuments(int siteId, DateTime? startTime, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError)
{
var pageModules = _pageModuleRepostory.GetPageModules(siteId).DistinctBy(i => i.ModuleId);
var searchDocuments = new List<SearchDocument>();
foreach(var pageModule in pageModules)
{
var module = pageModule.Module;
if (module.ModuleDefinition.ServerManagerType != "")
{
_logger.LogDebug($"Search: Begin index module {module.ModuleId}.");
var type = Type.GetType(module.ModuleDefinition.ServerManagerType);
if (type?.GetInterface("IModuleSearch") != null)
{
try
{
var moduleSearch = (IModuleSearch)ActivatorUtilities.CreateInstance(_serviceProvider, type);
var documents = moduleSearch.GetSearchDocuments(module, startTime.GetValueOrDefault(DateTime.MinValue));
if(documents != null)
{
foreach(var document in documents)
{
SaveModuleMetaData(document, pageModule);
searchDocuments.Add(document);
}
}
}
catch(Exception ex)
{
_logger.LogError(ex, $"Search: Index module {module.ModuleId} failed.");
handleError($"Search: Index module {module.ModuleId} failed: {ex.Message}");
}
}
_logger.LogDebug($"Search: End index module {module.ModuleId}.");
}
}
processSearchDocuments(searchDocuments);
return searchDocuments.Count;
}
private void SaveModuleMetaData(SearchDocument document, PageModule pageModule)
{
document.EntryId = pageModule.ModuleId;
document.IndexerName = Name;
document.SiteId = pageModule.Module.SiteId;
document.LanguageCode = string.Empty;
if(document.ModifiedTime == DateTime.MinValue)
{
document.ModifiedTime = pageModule.ModifiedOn;
}
if (string.IsNullOrEmpty(document.AdditionalContent))
{
document.AdditionalContent = string.Empty;
}
var page = _pageRepository.GetPage(pageModule.PageId);
if (string.IsNullOrEmpty(document.Url) && page != null)
{
document.Url = page.Url;
}
if (string.IsNullOrEmpty(document.Title) && page != null)
{
document.Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name;
}
if (document.Properties == null)
{
document.Properties = new List<SearchDocumentProperty>();
}
if(!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = pageModule.PageId.ToString() });
}
if (!document.Properties.Any(i => i.Name == Constants.SearchModuleIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchModuleIdPropertyName, Value = pageModule.ModuleId.ToString() });
}
}
}
}

View File

@ -0,0 +1,69 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class ModuleSearchResultManager : ISearchResultManager
{
public string Name => Constants.ModuleSearchIndexManagerName;
private readonly IServiceProvider _serviceProvider;
public ModuleSearchResultManager(
IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public string GetUrl(SearchResult searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
if(!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId))
{
var page = pageRepository.GetPage(pageId);
if (page != null)
{
return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
}
}
return string.Empty;
}
public bool Visible(SearchDocument searchResult, SearchQuery searchQuery)
{
var pageIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
var moduleIdValue = searchResult.Properties?.FirstOrDefault(i => i.Name == Constants.SearchModuleIdPropertyName)?.Value ?? string.Empty;
if (!string.IsNullOrEmpty(pageIdValue) && int.TryParse(pageIdValue, out int pageId)
&& !string.IsNullOrEmpty(moduleIdValue) && int.TryParse(moduleIdValue, out int moduleId))
{
return CanViewPage(pageId, searchQuery.User) && CanViewModule(moduleId, searchQuery.User);
}
return false;
}
private bool CanViewModule(int moduleId, User user)
{
var moduleRepository = _serviceProvider.GetRequiredService<IModuleRepository>();
var module = moduleRepository.GetModule(moduleId);
return module != null && !module.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, module.PermissionList);
}
private bool CanViewPage(int pageId, User user)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(pageId);
return page != null && !page.IsDeleted && UserSecurity.IsAuthorized(user, PermissionNames.View, page.PermissionList)
&& (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(user, PermissionNames.Edit, page.PermissionList));
}
}
}

View File

@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Mvc.RazorPages;
using System.Reflection.Metadata;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Oqtane.Interfaces;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class PageSearchIndexManager : SearchIndexManagerBase
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger<ModuleSearchIndexManager> _logger;
private readonly IPageRepository _pageRepository;
public PageSearchIndexManager(
IServiceProvider serviceProvider,
ILogger<ModuleSearchIndexManager> logger,
IPageRepository pageRepository)
: base(serviceProvider)
{
_serviceProvider = serviceProvider;
_logger = logger;
_pageRepository = pageRepository;
}
public override string Name => Constants.PageSearchIndexManagerName;
public override int Priority => Constants.PageSearchIndexManagerPriority;
public override int IndexDocuments(int siteId, DateTime? startTime, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError)
{
var startTimeValue = startTime.GetValueOrDefault(DateTime.MinValue);
var pages = _pageRepository.GetPages(siteId).Where(i => i.ModifiedOn >= startTimeValue);
var searchDocuments = new List<SearchDocument>();
foreach(var page in pages)
{
try
{
if(IsSystemPage(page))
{
continue;
}
var document = new SearchDocument
{
EntryId = page.PageId,
IndexerName = Name,
SiteId = page.SiteId,
LanguageCode = string.Empty,
ModifiedTime = page.ModifiedOn,
AdditionalContent = string.Empty,
Url = page.Url ?? string.Empty,
Title = !string.IsNullOrEmpty(page.Title) ? page.Title : page.Name,
Description = string.Empty,
Body = $"{page.Name} {page.Title}"
};
if (document.Properties == null)
{
document.Properties = new List<SearchDocumentProperty>();
}
if (!document.Properties.Any(i => i.Name == Constants.SearchPageIdPropertyName))
{
document.Properties.Add(new SearchDocumentProperty { Name = Constants.SearchPageIdPropertyName, Value = page.PageId.ToString() });
}
searchDocuments.Add(document);
}
catch(Exception ex)
{
_logger.LogError(ex, $"Search: Index page {page.PageId} failed.");
handleError($"Search: Index page {page.PageId} failed: {ex.Message}");
}
}
processSearchDocuments(searchDocuments);
return searchDocuments.Count;
}
private bool IsSystemPage(Models.Page page)
{
return page.Path.Contains("admin") || page.Path == "login" || page.Path == "register" || page.Path == "profile";
}
}
}

View File

@ -0,0 +1,46 @@
using System;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Security;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public class PageSearchResultManager : ISearchResultManager
{
public string Name => Constants.PageSearchIndexManagerName;
private readonly IServiceProvider _serviceProvider;
public PageSearchResultManager(
IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public string GetUrl(SearchResult searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(searchResult.EntryId);
if (page != null)
{
return $"{searchQuery.Alias.Protocol}{searchQuery.Alias.Name}{(!string.IsNullOrEmpty(page.Path) && !page.Path.StartsWith("/") ? "/" : "")}{page.Path}";
}
return string.Empty;
}
public bool Visible(SearchDocument searchResult, SearchQuery searchQuery)
{
var pageRepository = _serviceProvider.GetRequiredService<IPageRepository>();
var page = pageRepository.GetPage(searchResult.EntryId);
return page != null && !page.IsDeleted
&& UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.View, page.PermissionList)
&& (Utilities.IsPageModuleVisible(page.EffectiveDate, page.ExpiryDate) || UserSecurity.IsAuthorized(searchQuery.User, PermissionNames.Edit, page.PermissionList));
}
}
}

View File

@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Services;
using Oqtane.Shared;
namespace Oqtane.Managers.Search
{
public abstract class SearchIndexManagerBase : ISearchIndexManager
{
private readonly IServiceProvider _serviceProvider;
public SearchIndexManagerBase(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public abstract int Priority { get; }
public abstract string Name { get; }
public abstract int IndexDocuments(int siteId, DateTime? startDate, Action<IList<SearchDocument>> processSearchDocuments, Action<string> handleError);
public virtual bool IsIndexEnabled(int siteId)
{
var settingName = string.Format(Constants.SearchIndexManagerEnabledSettingFormat, Name);
var settingRepository = _serviceProvider.GetRequiredService<ISettingRepository>();
var setting = settingRepository.GetSetting(EntityNames.Site, siteId, settingName);
return setting == null || setting.SettingValue == "true";
}
}
}

View File

@ -0,0 +1,64 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentEntityBuilder : AuditableBaseEntityBuilder<SearchDocumentEntityBuilder>
{
private const string _entityTableName = "SearchDocument";
private readonly PrimaryKey<SearchDocumentEntityBuilder> _primaryKey = new("PK_SearchDocument", x => x.SearchDocumentId);
public SearchDocumentEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
}
protected override SearchDocumentEntityBuilder BuildTable(ColumnsBuilder table)
{
SearchDocumentId = AddAutoIncrementColumn(table, "SearchDocumentId");
EntryId = AddIntegerColumn(table, "EntryId");
IndexerName = AddStringColumn(table, "IndexerName", 50);
SiteId = AddIntegerColumn(table, "SiteId");
Title = AddStringColumn(table, "Title", 255);
Description = AddMaxStringColumn(table, "Description");
Body = AddMaxStringColumn(table, "Body");
Url = AddStringColumn(table, "Url", 255);
ModifiedTime = AddDateTimeColumn(table, "ModifiedTime");
IsActive = AddBooleanColumn(table, "IsActive");
AdditionalContent = AddMaxStringColumn(table, "AdditionalContent");
LanguageCode = AddStringColumn(table, "LanguageCode", 20);
AddAuditableColumns(table);
return this;
}
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> EntryId { get; private set; }
public OperationBuilder<AddColumnOperation> IndexerName { get; private set; }
public OperationBuilder<AddColumnOperation> SiteId { get; private set; }
public OperationBuilder<AddColumnOperation> Title { get; private set; }
public OperationBuilder<AddColumnOperation> Description { get; private set; }
public OperationBuilder<AddColumnOperation> Body { get; private set; }
public OperationBuilder<AddColumnOperation> Url { get; private set; }
public OperationBuilder<AddColumnOperation> ModifiedTime { get; private set; }
public OperationBuilder<AddColumnOperation> IsActive { get; private set; }
public OperationBuilder<AddColumnOperation> AdditionalContent { get; private set; }
public OperationBuilder<AddColumnOperation> LanguageCode { get; private set; }
}
}

View File

@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentPropertyEntityBuilder : BaseEntityBuilder<SearchDocumentPropertyEntityBuilder>
{
private const string _entityTableName = "SearchDocumentProperty";
private readonly PrimaryKey<SearchDocumentPropertyEntityBuilder> _primaryKey = new("PK_SearchDocumentProperty", x => x.PropertyId);
private readonly ForeignKey<SearchDocumentPropertyEntityBuilder> _searchDocumentForeignKey = new("FK_SearchDocumentProperty_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade);
public SearchDocumentPropertyEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchDocumentForeignKey);
}
protected override SearchDocumentPropertyEntityBuilder BuildTable(ColumnsBuilder table)
{
PropertyId = AddAutoIncrementColumn(table, "PropertyId");
SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId");
Name = AddStringColumn(table, "Name", 50);
Value = AddStringColumn(table, "Value", 50);
return this;
}
public OperationBuilder<AddColumnOperation> PropertyId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> Name { get; private set; }
public OperationBuilder<AddColumnOperation> Value { get; private set; }
}
}

View File

@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using Microsoft.EntityFrameworkCore.Migrations.Operations.Builders;
using Oqtane.Databases.Interfaces;
namespace Oqtane.Migrations.EntityBuilders
{
public class SearchDocumentTagEntityBuilder : BaseEntityBuilder<SearchDocumentTagEntityBuilder>
{
private const string _entityTableName = "SearchDocumentTag";
private readonly PrimaryKey<SearchDocumentTagEntityBuilder> _primaryKey = new("PK_SearchDocumentTag", x => x.TagId);
private readonly ForeignKey<SearchDocumentTagEntityBuilder> _searchDocumentForeignKey = new("FK_SearchDocumentTag_SearchDocument", x => x.SearchDocumentId, "SearchDocument", "SearchDocumentId", ReferentialAction.Cascade);
public SearchDocumentTagEntityBuilder(MigrationBuilder migrationBuilder, IDatabase database) : base(migrationBuilder, database)
{
EntityTableName = _entityTableName;
PrimaryKey = _primaryKey;
ForeignKeys.Add(_searchDocumentForeignKey);
}
protected override SearchDocumentTagEntityBuilder BuildTable(ColumnsBuilder table)
{
TagId = AddAutoIncrementColumn(table, "TagId");
SearchDocumentId = AddIntegerColumn(table, "SearchDocumentId");
Tag = AddStringColumn(table, "Tag", 50);
return this;
}
public OperationBuilder<AddColumnOperation> TagId { get; private set; }
public OperationBuilder<AddColumnOperation> SearchDocumentId { get; private set; }
public OperationBuilder<AddColumnOperation> Tag { get; private set; }
}
}

View File

@ -0,0 +1,42 @@
using System;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Oqtane.Databases.Interfaces;
using Oqtane.Migrations.EntityBuilders;
using Oqtane.Repository;
namespace Oqtane.Migrations.Tenant
{
[DbContext(typeof(TenantDBContext))]
[Migration("Tenant.05.02.00.01")]
public class AddSearchTables : MultiDatabaseMigration
{
public AddSearchTables(IDatabase database) : base(database)
{
}
protected override void Up(MigrationBuilder migrationBuilder)
{
var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentEntityBuilder.Create();
var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentPropertyEntityBuilder.Create();
var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentTagEntityBuilder.Create();
}
protected override void Down(MigrationBuilder migrationBuilder)
{
var searchDocumentPropertyEntityBuilder = new SearchDocumentPropertyEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentPropertyEntityBuilder.Drop();
var searchDocumentTagEntityBuilder = new SearchDocumentTagEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentTagEntityBuilder.Drop();
var searchDocumentEntityBuilder = new SearchDocumentEntityBuilder(migrationBuilder, ActiveDatabase);
searchDocumentEntityBuilder.Drop();
}
}
}

View File

@ -8,20 +8,29 @@ using Oqtane.Shared;
using Oqtane.Migrations.Framework;
using Oqtane.Documentation;
using System.Linq;
using Oqtane.Interfaces;
using System.Collections.Generic;
using System;
// ReSharper disable ConvertToUsingDeclaration
namespace Oqtane.Modules.HtmlText.Manager
{
[PrivateApi("Mark HtmlText classes as private, since it's not very useful in the public docs")]
public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable
public class HtmlTextManager : MigratableModuleBase, IInstallable, IPortable, IModuleSearch
{
private readonly IServiceProvider _serviceProvider;
private readonly IHtmlTextRepository _htmlText;
private readonly IDBContextDependencies _DBContextDependencies;
private readonly ISqlRepository _sqlRepository;
public HtmlTextManager(IHtmlTextRepository htmlText, IDBContextDependencies DBContextDependencies, ISqlRepository sqlRepository)
public HtmlTextManager(
IServiceProvider serviceProvider,
IHtmlTextRepository htmlText,
IDBContextDependencies DBContextDependencies,
ISqlRepository sqlRepository)
{
_serviceProvider = serviceProvider;
_htmlText = htmlText;
_DBContextDependencies = DBContextDependencies;
_sqlRepository = sqlRepository;
@ -39,6 +48,27 @@ namespace Oqtane.Modules.HtmlText.Manager
return content;
}
public IList<SearchDocument> GetSearchDocuments(Module module, DateTime startDate)
{
var searchDocuments = new List<SearchDocument>();
var htmltexts = _htmlText.GetHtmlTexts(module.ModuleId);
if (htmltexts != null && htmltexts.Any(i => i.CreatedOn >= startDate))
{
var htmltext = htmltexts.OrderByDescending(item => item.CreatedOn).First();
searchDocuments.Add(new SearchDocument
{
Title = module.Title,
Description = string.Empty,
Body = SearchUtils.Clean(htmltext.Content, true),
ModifiedTime = htmltext.ModifiedOn
});
}
return searchDocuments;
}
public void ImportModule(Module module, string content, string version)
{
content = WebUtility.HtmlDecode(content);

View File

@ -0,0 +1,43 @@
using System;
using System.Net;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Oqtane.Controllers;
using Oqtane.Documentation;
using Oqtane.Enums;
using Oqtane.Infrastructure;
using Oqtane.Modules.SearchResults.Services;
using Oqtane.Shared;
namespace Oqtane.Modules.SearchResults.Controllers
{
[Route(ControllerRoutes.ApiRoute)]
[PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")]
public class SearchResultsController : ModuleControllerBase
{
private readonly ISearchResultsService _searchResultsService;
public SearchResultsController(ISearchResultsService searchResultsService, ILogManager logger, IHttpContextAccessor accessor) : base(logger, accessor)
{
_searchResultsService = searchResultsService;
}
[HttpPost]
[Authorize(Policy = PolicyNames.ViewModule)]
public async Task<Models.SearchResults> Post([FromBody] Models.SearchQuery searchQuery)
{
try
{
return await _searchResultsService.SearchAsync(AuthEntityId(EntityNames.Module), searchQuery);
}
catch(Exception ex)
{
_logger.Log(LogLevel.Error, this, LogFunction.Other, ex, "Fetch search results failed.", searchQuery);
HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest;
return null;
}
}
}
}

View File

@ -0,0 +1,36 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Oqtane.Documentation;
using Oqtane.Infrastructure;
using Oqtane.Models;
using Oqtane.Services;
namespace Oqtane.Modules.SearchResults.Services
{
[PrivateApi("Mark SearchResults classes as private, since it's not very useful in the public docs")]
public class ServerSearchResultsService : ISearchResultsService, ITransientService
{
private readonly ILogManager _logger;
private readonly IHttpContextAccessor _accessor;
private readonly Alias _alias;
private readonly ISearchService _searchService;
public ServerSearchResultsService(
ITenantManager tenantManager,
ILogManager logger,
IHttpContextAccessor accessor,
ISearchService searchService)
{
_logger = logger;
_accessor = accessor;
_alias = tenantManager.GetAlias();
_searchService = searchService;
}
public async Task<Models.SearchResults> SearchAsync(int moduleId, SearchQuery searchQuery)
{
var results = await _searchService.SearchAsync(searchQuery);
return results;
}
}
}

View File

@ -0,0 +1,24 @@
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
using Oqtane.Infrastructure;
using Oqtane.Modules.SearchResults.Services;
namespace Oqtane.Modules.SearchResults.Startup
{
public class ServerStartup : IServerStartup
{
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
}
public void ConfigureMvc(IMvcBuilder mvcBuilder)
{
}
public void ConfigureServices(IServiceCollection services)
{
services.AddTransient<ISearchResultsService, ServerSearchResultsService>();
}
}
}

View File

@ -0,0 +1,202 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Services;
using Oqtane.Shared;
using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource;
namespace Oqtane.Providers
{
public class DatabaseSearchProvider : ISearchProvider
{
private readonly ISearchDocumentRepository _searchDocumentRepository;
private const float TitleBoost = 100f;
private const float DescriptionBoost = 10f;
private const float BodyBoost = 10f;
private const float AdditionalContentBoost = 5f;
public string Name => Constants.DefaultSearchProviderName;
public DatabaseSearchProvider(ISearchDocumentRepository searchDocumentRepository)
{
_searchDocumentRepository = searchDocumentRepository;
}
public void Commit()
{
}
public void DeleteDocument(string id)
{
_searchDocumentRepository.DeleteSearchDocument(id);
}
public bool Optimize()
{
return true;
}
public void ResetIndex()
{
_searchDocumentRepository.DeleteAllSearchDocuments();
}
public void SaveDocument(SearchDocument document, bool autoCommit = false)
{
//remove exist document
_searchDocumentRepository.DeleteSearchDocument(document.IndexerName, document.EntryId);
_searchDocumentRepository.AddSearchDocument(document);
}
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery, Func<SearchDocument, SearchQuery, bool> validateFunc)
{
var totalResults = 0;
var documents = await _searchDocumentRepository.GetSearchDocumentsAsync(searchQuery);
//convert the search documents to search results.
var results = documents
.Where(i => validateFunc(i, searchQuery))
.Select(i => ConvertToSearchResult(i, searchQuery));
if (searchQuery.SortDirection == SearchSortDirections.Descending)
{
switch (searchQuery.SortField)
{
case SearchSortFields.Relevance:
results = results.OrderByDescending(i => i.Score).ThenByDescending(i => i.ModifiedTime);
break;
case SearchSortFields.Title:
results = results.OrderByDescending(i => i.Title).ThenByDescending(i => i.ModifiedTime);
break;
default:
results = results.OrderByDescending(i => i.ModifiedTime);
break;
}
}
else
{
switch (searchQuery.SortField)
{
case SearchSortFields.Relevance:
results = results.OrderBy(i => i.Score).ThenByDescending(i => i.ModifiedTime);
break;
case SearchSortFields.Title:
results = results.OrderBy(i => i.Title).ThenByDescending(i => i.ModifiedTime);
break;
default:
results = results.OrderBy(i => i.ModifiedTime);
break;
}
}
//remove duplicated results based on page id for Page and Module types
results = results.DistinctBy(i =>
{
if (i.IndexerName == Constants.PageSearchIndexManagerName || i.IndexerName == Constants.ModuleSearchIndexManagerName)
{
var pageId = i.Properties.FirstOrDefault(p => p.Name == Constants.SearchPageIdPropertyName)?.Value ?? string.Empty;
return !string.IsNullOrEmpty(pageId) ? pageId : i.UniqueKey;
}
else
{
return i.UniqueKey;
}
});
totalResults = results.Count();
return new SearchResults
{
Results = results.Skip(searchQuery.PageIndex * searchQuery.PageSize).Take(searchQuery.PageSize).ToList(),
TotalResults = totalResults
};
}
private SearchResult ConvertToSearchResult(SearchDocument searchDocument, SearchQuery searchQuery)
{
var searchResult = new SearchResult()
{
SearchDocumentId = searchDocument.SearchDocumentId,
SiteId = searchDocument.SiteId,
IndexerName = searchDocument.IndexerName,
EntryId = searchDocument.EntryId,
Title = searchDocument.Title,
Description = searchDocument.Description,
Body = searchDocument.Body,
Url = searchDocument.Url,
ModifiedTime = searchDocument.ModifiedTime,
Tags = searchDocument.Tags,
Properties = searchDocument.Properties,
Snippet = BuildSnippet(searchDocument, searchQuery),
Score = CalculateScore(searchDocument, searchQuery)
};
return searchResult;
}
private float CalculateScore(SearchDocument searchDocument, SearchQuery searchQuery)
{
var score = 0f;
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
score += Regex.Matches(searchDocument.Title, keyword, RegexOptions.IgnoreCase).Count * TitleBoost;
score += Regex.Matches(searchDocument.Description, keyword, RegexOptions.IgnoreCase).Count * DescriptionBoost;
score += Regex.Matches(searchDocument.Body, keyword, RegexOptions.IgnoreCase).Count * BodyBoost;
score += Regex.Matches(searchDocument.AdditionalContent, keyword, RegexOptions.IgnoreCase).Count * AdditionalContentBoost;
}
return score / 100;
}
private string BuildSnippet(SearchDocument searchDocument, SearchQuery searchQuery)
{
var content = $"{searchDocument.Title} {searchDocument.Description} {searchDocument.Body}";
var snippet = string.Empty;
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
if (!string.IsNullOrWhiteSpace(keyword) && content.Contains(keyword, StringComparison.OrdinalIgnoreCase))
{
var start = content.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) - 20;
var prefix = "...";
var suffix = "...";
if (start <= 0)
{
start = 0;
prefix = string.Empty;
}
var length = searchQuery.BodySnippetLength;
if (start + length >= content.Length)
{
length = content.Length - start;
suffix = string.Empty;
}
snippet = $"{prefix}{content.Substring(start, length)}{suffix}";
break;
}
}
if (string.IsNullOrEmpty(snippet))
{
snippet = content.Substring(0, searchQuery.BodySnippetLength);
}
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
snippet = Regex.Replace(snippet, $"({keyword})", $"<b>$1</b>", RegexOptions.IgnoreCase);
}
return snippet;
}
}
}

View File

@ -29,5 +29,8 @@ namespace Oqtane.Repository
public virtual DbSet<Language> Language { get; set; }
public virtual DbSet<Visitor> Visitor { get; set; }
public virtual DbSet<UrlMapping> UrlMapping { get; set; }
public virtual DbSet<SearchDocument> SearchDocument { get; set; }
public virtual DbSet<SearchDocumentProperty> SearchDocumentProperty { get; set; }
public virtual DbSet<SearchDocumentTag> SearchDocumentTag { get; set; }
}
}

View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Oqtane.Models;
namespace Oqtane.Repository
{
public interface ISearchDocumentRepository
{
Task<IEnumerable<SearchDocument>> GetSearchDocumentsAsync(SearchQuery searchQuery);
SearchDocument AddSearchDocument(SearchDocument searchDocument);
void DeleteSearchDocument(int searchDocumentId);
void DeleteSearchDocument(string indexerName, int entryId);
void DeleteSearchDocument(string uniqueKey);
void DeleteAllSearchDocuments();
}
}

View File

@ -0,0 +1,136 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Oqtane.Models;
using Oqtane.Shared;
namespace Oqtane.Repository
{
public class SearchDocumentRepository : ISearchDocumentRepository
{
private readonly IDbContextFactory<TenantDBContext> _dbContextFactory;
public SearchDocumentRepository(IDbContextFactory<TenantDBContext> dbContextFactory)
{
_dbContextFactory = dbContextFactory;
}
public async Task<IEnumerable<SearchDocument>> GetSearchDocumentsAsync(SearchQuery searchQuery)
{
using var db = _dbContextFactory.CreateDbContext();
var documents = db.SearchDocument.AsNoTracking()
.Include(i => i.Properties)
.Include(i => i.Tags)
.Where(i => i.SiteId == searchQuery.SiteId);
if (searchQuery.Sources != null && searchQuery.Sources.Any())
{
documents = documents.Where(i => searchQuery.Sources.Contains(i.IndexerName));
}
if (searchQuery.BeginModifiedTimeUtc != DateTime.MinValue)
{
documents = documents.Where(i => i.ModifiedTime >= searchQuery.BeginModifiedTimeUtc);
}
if (searchQuery.EndModifiedTimeUtc != DateTime.MinValue)
{
documents = documents.Where(i => i.ModifiedTime <= searchQuery.EndModifiedTimeUtc);
}
if (searchQuery.Tags != null && searchQuery.Tags.Any())
{
foreach (var tag in searchQuery.Tags)
{
documents = documents.Where(i => i.Tags.Any(t => t.Tag == tag));
}
}
if (searchQuery.Properties != null && searchQuery.Properties.Any())
{
foreach (var property in searchQuery.Properties)
{
documents = documents.Where(i => i.Properties.Any(p => p.Name == property.Key && p.Value == property.Value));
}
}
var filteredDocuments = new List<SearchDocument>();
if (!string.IsNullOrEmpty(searchQuery.Keywords))
{
foreach (var keyword in SearchUtils.GetKeywordsList(searchQuery.Keywords))
{
filteredDocuments.AddRange(await documents.Where(i => i.Title.Contains(keyword) || i.Description.Contains(keyword) || i.Body.Contains(keyword)).ToListAsync());
}
}
return filteredDocuments.DistinctBy(i => i.UniqueKey);
}
public SearchDocument AddSearchDocument(SearchDocument searchDocument)
{
using var context = _dbContextFactory.CreateDbContext();
context.SearchDocument.Add(searchDocument);
if(searchDocument.Properties != null && searchDocument.Properties.Any())
{
foreach(var property in searchDocument.Properties)
{
property.SearchDocumentId = searchDocument.SearchDocumentId;
context.SearchDocumentProperty.Add(property);
}
}
if (searchDocument.Tags != null && searchDocument.Tags.Any())
{
foreach (var tag in searchDocument.Tags)
{
tag.SearchDocumentId = searchDocument.SearchDocumentId;
context.SearchDocumentTag.Add(tag);
}
}
context.SaveChanges();
return searchDocument;
}
public void DeleteSearchDocument(int searchDocumentId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.Find(searchDocumentId);
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
public void DeleteSearchDocument(string indexerName, int entryId)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.FirstOrDefault(i => i.IndexerName == indexerName && i.EntryId == entryId);
if(searchDocument != null)
{
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
}
public void DeleteSearchDocument(string uniqueKey)
{
using var db = _dbContextFactory.CreateDbContext();
var searchDocument = db.SearchDocument.FirstOrDefault(i => (i.IndexerName + ":" + i.EntryId) == uniqueKey);
if (searchDocument != null)
{
db.SearchDocument.Remove(searchDocument);
db.SaveChanges();
}
}
public void DeleteAllSearchDocuments()
{
using var db = _dbContextFactory.CreateDbContext();
db.SearchDocument.RemoveRange(db.SearchDocument);
db.SaveChanges();
}
}
}

View File

@ -0,0 +1,214 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Oqtane.Infrastructure;
using Oqtane.Models;
using Oqtane.Repository;
using Oqtane.Shared;
namespace Oqtane.Services
{
public class SearchService : ISearchService
{
private readonly IServiceProvider _serviceProvider;
private readonly ITenantManager _tenantManager;
private readonly IAliasRepository _aliasRepository;
private readonly ISettingRepository _settingRepository;
private readonly ILogger<SearchService> _logger;
private readonly IMemoryCache _cache;
public SearchService(
IServiceProvider serviceProvider,
ITenantManager tenantManager,
IAliasRepository aliasRepository,
ISettingRepository settingRepository,
ILogger<SearchService> logger,
IMemoryCache cache)
{
_tenantManager = tenantManager;
_aliasRepository = aliasRepository;
_settingRepository = settingRepository;
_serviceProvider = serviceProvider;
_logger = logger;
_cache = cache;
}
public void IndexContent(int siteId, DateTime? startTime, Action<string> logNote, Action<string> handleError)
{
var searchEnabled = SearchEnabled(siteId);
if(!searchEnabled)
{
logNote($"Search: Search is disabled on site {siteId}.<br />");
return;
}
_logger.LogDebug($"Search: Start Index Content of {siteId}, Start Time: {startTime.GetValueOrDefault(DateTime.MinValue)}");
var searchProvider = GetSearchProvider(siteId);
SetTenant(siteId);
if (startTime == null)
{
searchProvider.ResetIndex();
}
var searchIndexManagers = GetSearchIndexManagers(m => { });
foreach (var searchIndexManager in searchIndexManagers)
{
if (!searchIndexManager.IsIndexEnabled(siteId))
{
logNote($"Search: Ignore indexer {searchIndexManager.Name} because it's disabled.<br />");
}
else
{
_logger.LogDebug($"Search: Begin Index {searchIndexManager.Name}");
var count = searchIndexManager.IndexDocuments(siteId, startTime, SaveIndexDocuments, handleError);
logNote($"Search: Indexer {searchIndexManager.Name} processed {count} documents.<br />");
_logger.LogDebug($"Search: End Index {searchIndexManager.Name}");
}
}
}
public async Task<SearchResults> SearchAsync(SearchQuery searchQuery)
{
var searchProvider = GetSearchProvider(searchQuery.SiteId);
var searchResults = await searchProvider.SearchAsync(searchQuery, HasViewPermission);
//generate the document url if it's not set.
foreach (var result in searchResults.Results)
{
if(string.IsNullOrEmpty(result.Url))
{
result.Url = GetDocumentUrl(result, searchQuery);
}
}
return searchResults;
}
private ISearchProvider GetSearchProvider(int siteId)
{
var providerName = GetSearchProviderSetting(siteId);
var searchProviders = _serviceProvider.GetServices<ISearchProvider>();
var provider = searchProviders.FirstOrDefault(i => i.Name == providerName);
if(provider == null)
{
provider = searchProviders.FirstOrDefault(i => i.Name == Constants.DefaultSearchProviderName);
}
return provider;
}
private string GetSearchProviderSetting(int siteId)
{
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchProviderSettingName);
if(!string.IsNullOrEmpty(setting?.SettingValue))
{
return setting.SettingValue;
}
return Constants.DefaultSearchProviderName;
}
private bool SearchEnabled(int siteId)
{
var setting = _settingRepository.GetSetting(EntityNames.Site, siteId, Constants.SearchEnabledSettingName);
if (!string.IsNullOrEmpty(setting?.SettingValue))
{
return bool.TryParse(setting.SettingValue, out bool enabled) && enabled;
}
return true;
}
private void SetTenant(int siteId)
{
var alias = _aliasRepository.GetAliases().OrderBy(i => i.SiteId).ThenByDescending(i => i.IsDefault).FirstOrDefault(i => i.SiteId == siteId);
_tenantManager.SetAlias(alias);
}
private IList<ISearchIndexManager> GetSearchIndexManagers(Action<ISearchIndexManager> initManager)
{
var managers = new List<ISearchIndexManager>();
var managerTypes = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(s => s.GetTypes())
.Where(p => typeof(ISearchIndexManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract);
foreach (var type in managerTypes)
{
var manager = (ISearchIndexManager)ActivatorUtilities.CreateInstance(_serviceProvider, type);
initManager(manager);
managers.Add(manager);
}
return managers.OrderBy(i => i.Priority).ToList();
}
private IList<ISearchResultManager> GetSearchResultManagers()
{
var managers = new List<ISearchResultManager>();
var managerTypes = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(s => s.GetTypes())
.Where(p => typeof(ISearchResultManager).IsAssignableFrom(p) && !p.IsInterface && !p.IsAbstract);
foreach (var type in managerTypes)
{
var manager = (ISearchResultManager)ActivatorUtilities.CreateInstance(_serviceProvider, type);
managers.Add(manager);
}
return managers.ToList();
}
private void SaveIndexDocuments(IList<SearchDocument> searchDocuments)
{
if(searchDocuments.Any())
{
var searchProvider = GetSearchProvider(searchDocuments.First().SiteId);
foreach (var searchDocument in searchDocuments)
{
try
{
searchProvider.SaveDocument(searchDocument);
}
catch(Exception ex)
{
_logger.LogError(ex, $"Search: Save search document {searchDocument.UniqueKey} failed.");
}
}
//commit the index changes
searchProvider.Commit();
}
}
private bool HasViewPermission(SearchDocument searchDocument, SearchQuery searchQuery)
{
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == searchDocument.IndexerName);
if (searchResultManager != null)
{
return searchResultManager.Visible(searchDocument, searchQuery);
}
return true;
}
private string GetDocumentUrl(SearchResult result, SearchQuery searchQuery)
{
var searchResultManager = GetSearchResultManagers().FirstOrDefault(i => i.Name == result.IndexerName);
if(searchResultManager != null)
{
return searchResultManager.GetUrl(result, searchQuery);
}
return string.Empty;
}
}
}

View File

@ -0,0 +1,3 @@
.search-result-container ul.pagination li label, .search-result-container ul.dropdown-menu li label {
cursor: pointer;
}

View File

@ -79,6 +79,10 @@ body {
top: -2px;
}
.app-search input{
width: auto;
}
.navbar-toggler {
background-color: rgba(255, 255, 255, 0.1);
margin: .5rem;

View File

@ -235,3 +235,17 @@ app {
.app-form-inline {
display: inline-block;
}
.app-search{
display: inline-block;
position: relative;
}
.app-search input + button{
background: none;
border: none;
position: absolute;
right: 0;
top: 0;
}
.app-search input + button .oi{
top: 0;
}