Merge pull request #4809 from sbwalker/dev

Added defensive logic to File Indexer for scenarios where file does not exist on disk. Added ability to reset the search index prior to reindexing.

Oqtane.Client/Modules/Admin/Search/Index.razor

@@ -106,9 +106,7 @@
         try
         {
             _lastIndexedOn = DateTime.MinValue.ToString();
-            var settings = await SettingService.GetSiteSettingsAsync(PageState.Site.SiteId);
-            settings = SettingService.SetSetting(settings, "Search_LastIndexedOn", _lastIndexedOn, true);
-            await SettingService.UpdateSiteSettingsAsync(settings, PageState.Site.SiteId);
+            await Save();
             AddModuleMessage(Localizer["Message.Reindex"], MessageType.Success);
         }
         catch (Exception ex)

Oqtane.Server/Infrastructure/Jobs/SearchIndexJob.cs

@@ -59,6 +59,13 @@ namespace Oqtane.Infrastructure
                 var currentTime = DateTime.UtcNow;
                 var lastIndexedOn = Convert.ToDateTime(siteSettings.GetValue(SearchLastIndexedOnSetting, DateTime.MinValue.ToString()));
 
+                if (lastIndexedOn == DateTime.MinValue)
+                {
+                    // reset index
+                    log += $"*Site Index Reset*<br />";
+                    await searchService.DeleteSearchContentsAsync(site.SiteId);
+                }
+
                 var ignorePages = siteSettings.GetValue(SearchIgnorePagesSetting, "").Split(',');
                 var ignoreEntities = siteSettings.GetValue(SearchIgnoreEntitiesSetting, "").Split(',');
 

Oqtane.Server/Modules/Admin/Files/Manager/FileManager.cs

@@ -45,11 +45,26 @@ namespace Oqtane.Modules.Admin.Files.Manager
                         var path = folder.Path + file.Name;
 
                         var body = "";
+                        if (System.IO.File.Exists(_fileRepository.GetFilePath(file)))
+                        {
+                            // only non-binary files can be indexed
                             if (DocumentExtensions.Contains(Path.GetExtension(file.Name)))
                             {
                                 // get the contents of the file
+                                try
+                                {
                                     body = System.IO.File.ReadAllText(_fileRepository.GetFilePath(file));
                                 }
+                                catch
+                                {
+                                    // could not read the file
+                                }
+                            }
+                        }
+                        else
+                        {
+                            removed = true; // file does not exist on disk
+                        }
 
                         var searchContent = new SearchContent
                         {

Oqtane.Server/Providers/DatabaseSearchProvider.cs

@@ -235,9 +235,9 @@ namespace Oqtane.Providers
             return text;
         }
 
-        public Task ResetIndex()
+        public Task DeleteSearchContent(int siteId)
         {
-            _searchContentRepository.DeleteAllSearchContent();
+            _searchContentRepository.DeleteAllSearchContent(siteId);
             return Task.CompletedTask;
         }
     }

Oqtane.Server/Repository/Interfaces/ISearchContentRepository.cs

@@ -12,7 +12,7 @@ namespace Oqtane.Repository
         void DeleteSearchContent(int searchContentId);
         void DeleteSearchContent(string entityName, string entryId);
         void DeleteSearchContent(string uniqueKey);
-        void DeleteAllSearchContent();
+        void DeleteAllSearchContent(int siteId);
 
         SearchWord GetSearchWord(string word);
         SearchWord AddSearchWord(SearchWord searchWord);

Oqtane.Server/Repository/SearchContentRepository.cs

@@ -152,11 +152,17 @@ namespace Oqtane.Repository
             }
         }
 
-        public void DeleteAllSearchContent()
+        public void DeleteAllSearchContent(int siteId)
         {
             using var db = _dbContextFactory.CreateDbContext();
-            db.SearchContent.RemoveRange(db.SearchContent);
+            // delete in batches of 100 records
+            var searchContents = db.SearchContent.Where(item => item.SiteId == siteId).Take(100).ToList();
+            while (searchContents.Count > 0)
+            {
+                db.SearchContent.RemoveRange(searchContents);
                 db.SaveChanges();
+                searchContents = db.SearchContent.Where(item => item.SiteId == siteId).Take(100).ToList();
+            }
         }
 
         public SearchWord GetSearchWord(string word)

Oqtane.Server/Services/SearchService.cs

@@ -149,6 +149,12 @@ namespace Oqtane.Services
             return result;
         }
 
+        public async Task DeleteSearchContentsAsync(int siteId)
+        {
+            var searchProvider = GetSearchProvider(siteId);
+            await searchProvider.DeleteSearchContent(siteId);
+        }
+
         private ISearchProvider GetSearchProvider(int siteId)
         {
             var providerName = GetSearchProviderSetting(siteId);

Oqtane.Shared/Interfaces/ISearchProvider.cs

@@ -12,6 +12,6 @@ namespace Oqtane.Services
 
         Task SaveSearchContent(SearchContent searchContent, Dictionary<string, string> siteSettings);
 
-        Task ResetIndex();
+        Task DeleteSearchContent(int siteId);
     }
 }

Oqtane.Shared/Interfaces/ISearchService.cs

@@ -9,5 +9,7 @@ namespace Oqtane.Services
         Task<SearchResults> GetSearchResultsAsync(SearchQuery searchQuery);
 
         Task<string> SaveSearchContentsAsync(List<SearchContent> searchContents, Dictionary<string, string> siteSettings);
+
+        Task DeleteSearchContentsAsync(int siteId);
     }
 }