From 70f55bff9086eeb7b2ac132ef350ae817a65f911 Mon Sep 17 00:00:00 2001 From: kavics Date: Wed, 4 Jul 2018 09:22:45 +0200 Subject: [PATCH] Fix/step populateindex (#403) * Add new events for tracing population progress/errors. * Ensure PopulateIndex step testability. Write progress and errors to the console. * FIX: Refresh index documents of all versions in the subtree. --- .../Packaging/Steps/PopulateIndex.cs | 46 ++++++- .../Search/Indexing/DocumentPopulator.cs | 77 +++++++---- .../Search/Indexing/NullPopulator.cs | 2 + src/Storage/Search/IIndexPopulator.cs | 11 ++ src/Storage/Search/NodeIndexedEventArgs.cs | 21 ++- .../Search/NodeIndexingErrorEventArgs.cs | 39 ++++++ src/Storage/SenseNet.Storage.csproj | 1 + .../SenseNet.Packaging.Tests.csproj | 5 + .../StepTests/PopulateIndexTests.cs | 127 ++++++++++++++++++ 9 files changed, 298 insertions(+), 31 deletions(-) create mode 100644 src/Storage/Search/NodeIndexingErrorEventArgs.cs create mode 100644 src/Tests/SenseNet.Packaging.Tests/StepTests/PopulateIndexTests.cs diff --git a/src/ContentRepository/Packaging/Steps/PopulateIndex.cs b/src/ContentRepository/Packaging/Steps/PopulateIndex.cs index cbf9d4980..29f7bbf76 100644 --- a/src/ContentRepository/Packaging/Steps/PopulateIndex.cs +++ b/src/ContentRepository/Packaging/Steps/PopulateIndex.cs @@ -4,6 +4,7 @@ using SenseNet.ContentRepository.Search; using SenseNet.ContentRepository.Search.Indexing; using SenseNet.ContentRepository.Storage.Data; +using SenseNet.Diagnostics; namespace SenseNet.Packaging.Steps { @@ -17,12 +18,20 @@ public class PopulateIndex : Step public string Level { get; set; } private long _count; + private long _docCount; private long _versionCount; - private long _factor; private ExecutionContext _context; public override void Execute(ExecutionContext context) + { + ExecuteInternal(context); + } + // Method for indexing tests. + internal void ExecuteInternal(ExecutionContext context, + EventHandler refreshed = null, + EventHandler indexed = null, + EventHandler error = null) { context.AssertRepositoryStarted(); @@ -33,22 +42,29 @@ public override void Execute(ExecutionContext context) { rebuildLevel = string.IsNullOrEmpty(level) ? IndexRebuildLevel.IndexOnly - : (IndexRebuildLevel) Enum.Parse(typeof(IndexRebuildLevel), level, true); + : (IndexRebuildLevel)Enum.Parse(typeof(IndexRebuildLevel), level, true); } - catch(Exception e) + catch (Exception e) { throw new PackagingException(SR.Errors.InvalidParameter + ": Level", e, PackagingExceptionType.InvalidStepParameter); } _context = context; _versionCount = DataProvider.GetVersionCount(path); - _factor = Math.Max(_versionCount / 60, 1); var savedMode = RepositoryEnvironment.WorkingMode.Populating; RepositoryEnvironment.WorkingMode.Populating = true; var populator = SearchManager.GetIndexPopulator(); populator.NodeIndexed += Populator_NodeIndexed; + populator.IndexDocumentRefreshed += Populator_IndexDocumentRefreshed; + populator.IndexingError += Populator_IndexingError; + if (refreshed != null) + populator.IndexDocumentRefreshed += refreshed; + if (indexed != null) + populator.NodeIndexed += indexed; + if (error != null) + populator.IndexingError += error; try { @@ -70,17 +86,33 @@ public override void Execute(ExecutionContext context) populator.NodeIndexed -= Populator_NodeIndexed; RepositoryEnvironment.WorkingMode.Populating = savedMode; + context.Console.Write(" \r"); context.Console.WriteLine(); } Logger.LogMessage("...finished: " + _count + " items indexed."); } + private DateTime _lastWriteTime = DateTime.MinValue; + + private void Populator_IndexDocumentRefreshed(object sender, NodeIndexedEventArgs e) + { + Interlocked.Increment(ref _docCount); + if (DateTime.Now.AddSeconds(-1) < _lastWriteTime) + return; + _context.Console.Write($" Document refreshing progress: {_docCount}/{_versionCount} {_docCount * 100 / _versionCount}% \r"); + _lastWriteTime = DateTime.Now; + } private void Populator_NodeIndexed(object sender, NodeIndexedEventArgs e) { Interlocked.Increment(ref _count); - - if (_count % _factor == 0) - _context.Console.Write("|"); + if (DateTime.Now.AddSeconds(-1) < _lastWriteTime) + return; + _context.Console.Write($" Indexing progress: {_count}/{_versionCount} {_count * 100 / _versionCount}% \r"); + _lastWriteTime = DateTime.Now; + } + private void Populator_IndexingError(object sender, NodeIndexingErrorEventArgs e) + { + Logger.LogException(e.Exception, $"Indexing error: NodeId: {e.NodeId}, VersionId: {e.VersionId}, Path: {e.Path}"); } } } diff --git a/src/ContentRepository/Search/Indexing/DocumentPopulator.cs b/src/ContentRepository/Search/Indexing/DocumentPopulator.cs index a4e4511ed..62043a132 100644 --- a/src/ContentRepository/Search/Indexing/DocumentPopulator.cs +++ b/src/ContentRepository/Search/Indexing/DocumentPopulator.cs @@ -10,6 +10,7 @@ using SenseNet.ContentRepository.Storage.Security; using SenseNet.Diagnostics; using SenseNet.Search; +using SenseNet.Search.Indexing; using SenseNet.Search.Querying; namespace SenseNet.ContentRepository.Search.Indexing @@ -39,15 +40,10 @@ public void ClearAndPopulateAll(TextWriter consoleWriter = null) IndexManager.ClearIndex(); consoleWriter?.WriteLine("ok"); - IndexManager.AddDocuments( - SearchManager.LoadIndexDocumentsByPath("/Root", IndexManager.GetNotIndexedNodeTypes()) - .Select(d => - { - var indexDoc = IndexManager.CompleteIndexDocument(d); - OnNodeIndexed(d.Path); - return indexDoc; - })); + IndexManager.AddDocuments(LoadIndexDocumentsByPath("/Root")); + // delete progress characters + consoleWriter?.Write(" \n"); consoleWriter?.Write(" Commiting ... "); IndexManager.Commit(); // explicit commit consoleWriter?.WriteLine("ok"); @@ -67,11 +63,21 @@ public void RebuildIndexDirectly(string path, IndexRebuildLevel level = IndexReb { using (new SystemAccount()) { - var node = Node.LoadNode(path); - DataBackingStore.SaveIndexDocument(node, false, false, out _); - - Parallel.ForEach(NodeQuery.QueryNodesByPath(node.Path, true).Nodes, - n => { DataBackingStore.SaveIndexDocument(n, false, false, out _); }); + foreach (var node in Node.LoadNode(path).LoadVersions()) + { + DataBackingStore.SaveIndexDocument(node, false, false, out _); + OnIndexDocumentRefreshed(node.Path, node.Id, node.VersionId, node.Version.ToString()); + } + + Parallel.ForEach(NodeQuery.QueryNodesByPath(path, true).Nodes, + n => + { + foreach (var node in n.LoadVersions()) + { + DataBackingStore.SaveIndexDocument(node, false, false, out _); + OnIndexDocumentRefreshed(node.Path, node.Id, node.VersionId, node.Version.ToString()); + } + }); } op2.Successful = true; } @@ -82,13 +88,7 @@ public void RebuildIndexDirectly(string path, IndexRebuildLevel level = IndexReb IndexManager.IndexingEngine.WriteIndex( new[] {new SnTerm(IndexFieldName.InTree, path)}, null, - SearchManager.LoadIndexDocumentsByPath(path, IndexManager.GetNotIndexedNodeTypes()) - .Select(d => - { - var indexDoc = IndexManager.CompleteIndexDocument(d); - OnNodeIndexed(d.Path); - return indexDoc; - })); + LoadIndexDocumentsByPath(path)); op.Successful = true; } } @@ -148,7 +148,8 @@ public void CommitPopulateNode(object data, IndexDocumentData indexDocument = nu UpdateVersion(state, versioningInfo, indexDocument); } - OnNodeIndexed(state.Node.Path); + var node = state.Node; + OnNodeIndexed(node.Path, node.Id, node.VersionId, node.Version.ToString()); op.Successful = true; } @@ -266,14 +267,44 @@ private void RebuildIndex_Recursive(Node node, bool databaseAndIndex) public event EventHandler NodeIndexed; - protected void OnNodeIndexed(string path) + protected void OnNodeIndexed(string path, int nodeId = 0, int versionId = 0, string version = null) { - NodeIndexed?.Invoke(null, new NodeIndexedEventArgs(path)); + NodeIndexed?.Invoke(null, new NodeIndexedEventArgs(path, nodeId, versionId, version)); + } + public event EventHandler IndexDocumentRefreshed; + protected void OnIndexDocumentRefreshed(string path, int nodeId = 0, int versionId = 0, string version = null) + { + IndexDocumentRefreshed?.Invoke(null, new NodeIndexedEventArgs(path, nodeId, versionId, version)); } + public event EventHandler IndexingError; + protected void OnIndexingError(IndexDocumentData doc, Exception exception) + { + IndexingError?.Invoke(null, new NodeIndexingErrorEventArgs(doc.NodeId, doc.VersionId, doc.Path, exception)); + } /*================================================================================================================================*/ + private IEnumerable LoadIndexDocumentsByPath(string path) + { + return SearchManager.LoadIndexDocumentsByPath(path, IndexManager.GetNotIndexedNodeTypes()) + .Select(d => + { + try + { + var indexDoc = IndexManager.CompleteIndexDocument(d); + OnNodeIndexed(d.Path, d.NodeId, d.VersionId, indexDoc.Version); + return indexDoc; + } + catch (Exception e) + { + OnIndexingError(d, e); + return null; + } + }) + .Where(d => d != null); + } + // caller: CommitPopulateNode private static void CreateBrandNewNode(Node node, VersioningInfo versioningInfo, IndexDocumentData indexDocumentData) { diff --git a/src/ContentRepository/Search/Indexing/NullPopulator.cs b/src/ContentRepository/Search/Indexing/NullPopulator.cs index ea65294c1..a82f735a4 100644 --- a/src/ContentRepository/Search/Indexing/NullPopulator.cs +++ b/src/ContentRepository/Search/Indexing/NullPopulator.cs @@ -23,6 +23,8 @@ public void DeleteTree(string path, int nodeId) { } #pragma warning disable 0067 // suppressed because it is not used but the interface declares. public event EventHandler NodeIndexed; + public event EventHandler IndexDocumentRefreshed; + public event EventHandler IndexingError; #pragma warning restore 0067 public void DeleteForest(IEnumerable idSet) { } public void DeleteForest(IEnumerable pathSet) { } diff --git a/src/Storage/Search/IIndexPopulator.cs b/src/Storage/Search/IIndexPopulator.cs index be24ee722..de9d49305 100644 --- a/src/Storage/Search/IIndexPopulator.cs +++ b/src/Storage/Search/IIndexPopulator.cs @@ -93,9 +93,20 @@ public interface IIndexPopulator /// IndexRebuildLevel option. void RebuildIndex(Node node, bool recursive = false, IndexRebuildLevel rebuildLevel = IndexRebuildLevel.IndexOnly); + /// + /// Defines an event that occurs when an index document is refreshed. + /// + event EventHandler IndexDocumentRefreshed; + /// /// Defines an event that occurs when a node has just been indexed. /// event EventHandler NodeIndexed; + + /// + /// Defines an event that occurs when a node indexing causes an error. + /// + event EventHandler IndexingError; + } } diff --git a/src/Storage/Search/NodeIndexedEventArgs.cs b/src/Storage/Search/NodeIndexedEventArgs.cs index 293bf1ef6..56ef26fce 100644 --- a/src/Storage/Search/NodeIndexedEventArgs.cs +++ b/src/Storage/Search/NodeIndexedEventArgs.cs @@ -1,4 +1,5 @@ using System; +using SenseNet.ContentRepository.Storage; // ReSharper disable once CheckNamespace namespace SenseNet.ContentRepository.Search.Indexing @@ -12,10 +13,28 @@ public class NodeIndexedEventArgs : EventArgs /// Gets the path of the currently indexed node. /// public string Path { get; } + /// + /// Gets the id of the currently indexed node. + /// + public int NodeId { get; } + /// + /// Gets the version id of the currently indexed node instance. + /// + public int VersionId { get; } + /// + /// Gets the version of the currently indexed node instance. + /// + public string Version { get; } /// /// Initializes a new NodeIndexedEventArgs instance. /// - public NodeIndexedEventArgs(string path) { Path = path; } + public NodeIndexedEventArgs(string path, int nodeId = 0, int versionId = 0, string version = null) + { + Path = path; + NodeId = nodeId; + VersionId = versionId; + Version = version; + } } } diff --git a/src/Storage/Search/NodeIndexingErrorEventArgs.cs b/src/Storage/Search/NodeIndexingErrorEventArgs.cs new file mode 100644 index 000000000..aa46d9ee4 --- /dev/null +++ b/src/Storage/Search/NodeIndexingErrorEventArgs.cs @@ -0,0 +1,39 @@ +using System; + +// ReSharper disable once CheckNamespace +namespace SenseNet.ContentRepository.Search.Indexing +{ + /// + /// Defines an event argument containing the identifiers of the node that could not be indexed. + /// + public class NodeIndexingErrorEventArgs : EventArgs + { + /// + /// Gets the id of the currently indexed node. + /// + public int NodeId { get; } + /// + /// Gets the id of the currently indexed version. + /// + public int VersionId { get; } + /// + /// Gets the path of the currently indexed node. + /// + public string Path { get; } + /// + /// Gets the indexing error. + /// + public Exception Exception { get; } + + /// + /// Initializes a new NodeIndexedEventArgs instance. + /// + public NodeIndexingErrorEventArgs(int nodeId, int versionId, string path, Exception exception) + { + NodeId = nodeId; + VersionId = versionId; + Path = path; + Exception = exception; + } + } +} diff --git a/src/Storage/SenseNet.Storage.csproj b/src/Storage/SenseNet.Storage.csproj index 81bcb778e..2af566640 100644 --- a/src/Storage/SenseNet.Storage.csproj +++ b/src/Storage/SenseNet.Storage.csproj @@ -257,6 +257,7 @@ + diff --git a/src/Tests/SenseNet.Packaging.Tests/SenseNet.Packaging.Tests.csproj b/src/Tests/SenseNet.Packaging.Tests/SenseNet.Packaging.Tests.csproj index 46a0967e4..f32742332 100644 --- a/src/Tests/SenseNet.Packaging.Tests/SenseNet.Packaging.Tests.csproj +++ b/src/Tests/SenseNet.Packaging.Tests/SenseNet.Packaging.Tests.csproj @@ -91,6 +91,7 @@ + @@ -107,6 +108,10 @@ {786e6165-ca02-45a9-bf58-207a45d7d6df} SenseNet.ContentRepository + + {0279705B-779D-485D-86B9-F7AB3DD1F2C3} + SenseNet.Search + {5db4ddba-81f6-4d81-943a-18f3178b3355} SenseNet.Storage diff --git a/src/Tests/SenseNet.Packaging.Tests/StepTests/PopulateIndexTests.cs b/src/Tests/SenseNet.Packaging.Tests/StepTests/PopulateIndexTests.cs new file mode 100644 index 000000000..0b072a88b --- /dev/null +++ b/src/Tests/SenseNet.Packaging.Tests/StepTests/PopulateIndexTests.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SenseNet.Configuration; +using SenseNet.ContentRepository; +using SenseNet.ContentRepository.Search; +using SenseNet.ContentRepository.Search.Indexing; +using SenseNet.ContentRepository.Versioning; +using SenseNet.ContentRepository.Workspaces; +using SenseNet.Diagnostics; +using SenseNet.Packaging.Steps; +using SenseNet.Packaging.Tests.Implementations; +using SenseNet.Tests; +using SenseNet.Tests.Implementations; + +namespace SenseNet.Packaging.Tests.StepTests +{ + [TestClass] + public class PopulateIndexTests : TestBase + { + private static StringBuilder _log; + [TestInitialize] + public void PrepareTest() + { + // preparing logger + _log = new StringBuilder(); + var loggers = new[] { new PackagingTestLogger(_log) }; + var loggerAcc = new PrivateType(typeof(Logger)); + loggerAcc.SetStaticField("_loggers", loggers); + } + [TestCleanup] + public void AfterTest() + { + } + + [TestMethod] + public void Packaging_PopulateIndex_HardReindexWorksOnAllVersions() + { + Test(() => + { + // arrange + InstallCarContentType(); + var root = new SystemFolder(Repository.Root) {Name = "TestRoot"}; + root.Save(); + + var workspace = new Workspace(root) + { + Name = "Workspace-1", + InheritableVersioningMode = InheritableVersioningType.MajorAndMinor + }; + workspace.AllowChildType("Car"); + workspace.Save(); + + var car1 = Content.CreateNew("Car", workspace, "Car1"); + car1.Save(); + var id = car1.Id; + + // create some versions + car1 = Content.Load(id); car1.Publish(); + car1 = Content.Load(id); car1.CheckOut(); + car1 = Content.Load(id); car1.Index++; car1.Save(); + car1.CheckIn(); + car1 = Content.Load(id); car1.Index++; car1.Save(); + car1 = Content.Load(id); car1.Index++; car1.Save(); + car1 = Content.Load(id); car1.Publish(); + car1 = Content.Load(id); car1.Index++; car1.Save(); + car1 = Content.Load(id); car1.Index++; car1.Save(); + car1 = Content.Load(id); car1.Publish(); + + var versions = car1.Versions.Select(n => n.Version.ToString()).ToArray(); + + var step = new PopulateIndex {Path = car1.Path, Level = "DatabaseAndIndex"}; + + var refreshedVersions = new List(); + var indexedVersions = new List(); + var refreshed = new EventHandler((sender, args) => + { + refreshedVersions.Add(args.Version.ToUpper()); + }); + var indexed = new EventHandler((sender, args) => + { + indexedVersions.Add(args.Version.ToUpper()); + }); + + // action + step.ExecuteInternal(GetExecutionContext(), refreshed, indexed); + + // assert + Assert.AreEqual(6, versions.Length); + + refreshedVersions.Sort(); + Assert.AreEqual(string.Join(",", versions), string.Join(",", refreshedVersions)); + + indexedVersions.Sort(); + Assert.AreEqual(string.Join(",", versions), string.Join(",", indexedVersions)); + + + }); + } + + private ExecutionContext GetExecutionContext() + { + var manifestXml = new XmlDocument(); + manifestXml.LoadXml(@" + + MyCompany.MyComponent + 2017-01-01 + 1.0 + + Package is running. + + "); + + var phase = 0; + var console = new StringWriter(); + var manifest = Manifest.Parse(manifestXml, phase, true, new PackageParameter[0]); + var executionContext = ExecutionContext.CreateForTest("packagePath", "targetPath", new string[0], "sandboxPath", manifest, phase, manifest.CountOfPhases, null, console); + executionContext.RepositoryStarted = true; + return executionContext; + } + } +}