Merge branch 'develop' of https://github.com/SenseNet/sensenet into d…

…evelop

src/ContentRepository/Packaging/Steps/PopulateIndex.cs

@@ -4,6 +4,7 @@
 using SenseNet.ContentRepository.Search;
 using SenseNet.ContentRepository.Search.Indexing;
 using SenseNet.ContentRepository.Storage.Data;
+using SenseNet.Diagnostics;
 
 namespace SenseNet.Packaging.Steps
 {
@@ -17,12 +18,20 @@ public class PopulateIndex : Step
         public string Level { get; set; }
 
         private long _count;
+        private long _docCount;
         private long _versionCount;
-        private long _factor;
 
         private ExecutionContext _context;
 
         public override void Execute(ExecutionContext context)
+        {
+            ExecuteInternal(context);
+        }
+        // Method for indexing tests.
+        internal void ExecuteInternal(ExecutionContext context,
+            EventHandler<NodeIndexedEventArgs> refreshed = null,
+            EventHandler<NodeIndexedEventArgs> indexed = null,
+            EventHandler<NodeIndexingErrorEventArgs> error = null)
         {
             context.AssertRepositoryStarted();
 
@@ -33,22 +42,29 @@ public override void Execute(ExecutionContext context)
             {
                 rebuildLevel = string.IsNullOrEmpty(level)
                     ? IndexRebuildLevel.IndexOnly
-                    : (IndexRebuildLevel) Enum.Parse(typeof(IndexRebuildLevel), level, true);
+                    : (IndexRebuildLevel)Enum.Parse(typeof(IndexRebuildLevel), level, true);
             }
-            catch(Exception e)
+            catch (Exception e)
             {
                 throw new PackagingException(SR.Errors.InvalidParameter + ": Level", e, PackagingExceptionType.InvalidStepParameter);
             }
 
             _context = context;
             _versionCount = DataProvider.GetVersionCount(path);
-            _factor = Math.Max(_versionCount / 60, 1);
 
             var savedMode = RepositoryEnvironment.WorkingMode.Populating;
             RepositoryEnvironment.WorkingMode.Populating = true;
 
             var populator = SearchManager.GetIndexPopulator();
             populator.NodeIndexed += Populator_NodeIndexed;
+            populator.IndexDocumentRefreshed += Populator_IndexDocumentRefreshed;
+            populator.IndexingError += Populator_IndexingError;
+            if (refreshed != null)
+                populator.IndexDocumentRefreshed += refreshed;
+            if (indexed != null)
+                populator.NodeIndexed += indexed;
+            if (error != null)
+                populator.IndexingError += error;
 
             try
             {
@@ -70,17 +86,33 @@ public override void Execute(ExecutionContext context)
                 populator.NodeIndexed -= Populator_NodeIndexed;
                 RepositoryEnvironment.WorkingMode.Populating = savedMode;
 
+                context.Console.Write("                                                      \r");
                 context.Console.WriteLine();
             }
 
             Logger.LogMessage("...finished: " + _count + " items indexed.");
         }
+        private DateTime _lastWriteTime = DateTime.MinValue;
+
+        private void Populator_IndexDocumentRefreshed(object sender, NodeIndexedEventArgs e)
+        {
+            Interlocked.Increment(ref _docCount);
+            if (DateTime.Now.AddSeconds(-1) < _lastWriteTime)
+                return;
+            _context.Console.Write($"  Document refreshing progress: {_docCount}/{_versionCount} {_docCount * 100 / _versionCount}%  \r");
+            _lastWriteTime = DateTime.Now;
+        }
         private void Populator_NodeIndexed(object sender, NodeIndexedEventArgs e)
         {
             Interlocked.Increment(ref _count);
-
-            if (_count % _factor == 0)
-                _context.Console.Write("|");
+            if (DateTime.Now.AddSeconds(-1) < _lastWriteTime)
+                return;
+            _context.Console.Write($"  Indexing progress: {_count}/{_versionCount} {_count * 100 / _versionCount}%               \r");
+            _lastWriteTime = DateTime.Now;
+        }
+        private void Populator_IndexingError(object sender, NodeIndexingErrorEventArgs e)
+        {
+            Logger.LogException(e.Exception, $"Indexing error: NodeId: {e.NodeId}, VersionId: {e.VersionId}, Path: {e.Path}");
         }
     }
 }

src/ContentRepository/Search/Indexing/DocumentPopulator.cs

@@ -10,6 +10,7 @@
 using SenseNet.ContentRepository.Storage.Security;
 using SenseNet.Diagnostics;
 using SenseNet.Search;
+using SenseNet.Search.Indexing;
 using SenseNet.Search.Querying;
 
 namespace SenseNet.ContentRepository.Search.Indexing
@@ -39,15 +40,10 @@ public void ClearAndPopulateAll(TextWriter consoleWriter = null)
                 IndexManager.ClearIndex();
                 consoleWriter?.WriteLine("ok");
 
-                IndexManager.AddDocuments(
-                    SearchManager.LoadIndexDocumentsByPath("/Root", IndexManager.GetNotIndexedNodeTypes())
-                        .Select(d =>
-                        {
-                            var indexDoc = IndexManager.CompleteIndexDocument(d);
-                            OnNodeIndexed(d.Path);
-                            return indexDoc;
-                        }));
+                IndexManager.AddDocuments(LoadIndexDocumentsByPath("/Root"));
 
+                // delete progress characters
+                consoleWriter?.Write("                                             \n");
                 consoleWriter?.Write("  Commiting ... ");
                 IndexManager.Commit(); // explicit commit
                 consoleWriter?.WriteLine("ok");
@@ -67,11 +63,21 @@ public void RebuildIndexDirectly(string path, IndexRebuildLevel level = IndexReb
                 {
                     using (new SystemAccount())
                     {
-                        var node = Node.LoadNode(path);
-                        DataBackingStore.SaveIndexDocument(node, false, false, out _);
-
-                        Parallel.ForEach(NodeQuery.QueryNodesByPath(node.Path, true).Nodes,
-                            n => { DataBackingStore.SaveIndexDocument(n, false, false, out _); });
+                        foreach (var node in Node.LoadNode(path).LoadVersions())
+                        {
+                            DataBackingStore.SaveIndexDocument(node, false, false, out _);
+                            OnIndexDocumentRefreshed(node.Path, node.Id, node.VersionId, node.Version.ToString());
+                        }
+
+                        Parallel.ForEach(NodeQuery.QueryNodesByPath(path, true).Nodes,
+                            n =>
+                            {
+                                foreach (var node in n.LoadVersions())
+                                {
+                                    DataBackingStore.SaveIndexDocument(node, false, false, out _);
+                                    OnIndexDocumentRefreshed(node.Path, node.Id, node.VersionId, node.Version.ToString());
+                                }
+                            });
                     }
                     op2.Successful = true;
                 }
@@ -82,13 +88,7 @@ public void RebuildIndexDirectly(string path, IndexRebuildLevel level = IndexReb
                 IndexManager.IndexingEngine.WriteIndex(
                     new[] {new SnTerm(IndexFieldName.InTree, path)},
                     null,
-                    SearchManager.LoadIndexDocumentsByPath(path, IndexManager.GetNotIndexedNodeTypes())
-                        .Select(d =>
-                        {
-                            var indexDoc = IndexManager.CompleteIndexDocument(d);
-                            OnNodeIndexed(d.Path);
-                            return indexDoc;
-                        }));
+                    LoadIndexDocumentsByPath(path));
                 op.Successful = true;
             }
         }
@@ -148,7 +148,8 @@ public void CommitPopulateNode(object data, IndexDocumentData indexDocument = nu
                     UpdateVersion(state, versioningInfo, indexDocument);
                 }
 
-                OnNodeIndexed(state.Node.Path);
+                var node = state.Node;
+                OnNodeIndexed(node.Path, node.Id, node.VersionId, node.Version.ToString());
 
                 op.Successful = true;
             }
@@ -266,14 +267,44 @@ private void RebuildIndex_Recursive(Node node, bool databaseAndIndex)
 
 
         public event EventHandler<NodeIndexedEventArgs> NodeIndexed;
-        protected void OnNodeIndexed(string path)
+        protected void OnNodeIndexed(string path, int nodeId = 0, int versionId = 0, string version = null)
         {
-            NodeIndexed?.Invoke(null, new NodeIndexedEventArgs(path));
+            NodeIndexed?.Invoke(null, new NodeIndexedEventArgs(path, nodeId, versionId, version));
+        }
+        public event EventHandler<NodeIndexedEventArgs> IndexDocumentRefreshed;
+        protected void OnIndexDocumentRefreshed(string path, int nodeId = 0, int versionId = 0, string version = null)
+        {
+            IndexDocumentRefreshed?.Invoke(null, new NodeIndexedEventArgs(path, nodeId, versionId, version));
         }
 
+        public event EventHandler<NodeIndexingErrorEventArgs> IndexingError;
+        protected void OnIndexingError(IndexDocumentData doc, Exception exception)
+        {
+            IndexingError?.Invoke(null, new NodeIndexingErrorEventArgs(doc.NodeId, doc.VersionId, doc.Path, exception));
+        }
 
         /*================================================================================================================================*/
 
+        private IEnumerable<IndexDocument> LoadIndexDocumentsByPath(string path)
+        {
+            return SearchManager.LoadIndexDocumentsByPath(path, IndexManager.GetNotIndexedNodeTypes())
+                .Select(d =>
+                {
+                    try
+                    {
+                        var indexDoc = IndexManager.CompleteIndexDocument(d);
+                        OnNodeIndexed(d.Path, d.NodeId, d.VersionId, indexDoc.Version);
+                        return indexDoc;
+                    }
+                    catch (Exception e)
+                    {
+                        OnIndexingError(d, e);
+                        return null;
+                    }
+                })
+                .Where(d => d != null);
+        }
+
         // caller: CommitPopulateNode
         private static void CreateBrandNewNode(Node node, VersioningInfo versioningInfo, IndexDocumentData indexDocumentData)
         {

src/ContentRepository/Search/Indexing/NullPopulator.cs

@@ -23,6 +23,8 @@ public void DeleteTree(string path, int nodeId) { }
 #pragma warning disable 0067
         // suppressed because it is not used but the interface declares.
         public event EventHandler<NodeIndexedEventArgs> NodeIndexed;
+        public event EventHandler<NodeIndexedEventArgs> IndexDocumentRefreshed;
+        public event EventHandler<NodeIndexingErrorEventArgs> IndexingError;
 #pragma warning restore 0067
         public void DeleteForest(IEnumerable<int> idSet) { }
         public void DeleteForest(IEnumerable<string> pathSet) { }

src/Storage/Search/IIndexPopulator.cs

@@ -93,9 +93,20 @@ public interface IIndexPopulator
         /// <param name="rebuildLevel">IndexRebuildLevel option.</param>
         void RebuildIndex(Node node, bool recursive = false, IndexRebuildLevel rebuildLevel = IndexRebuildLevel.IndexOnly);
 
+        /// <summary>
+        /// Defines an event that occurs when an index document is refreshed.
+        /// </summary>
+        event EventHandler<NodeIndexedEventArgs> IndexDocumentRefreshed;
+
         /// <summary>
         /// Defines an event that occurs when a node has just been indexed.
         /// </summary>
         event EventHandler<NodeIndexedEventArgs> NodeIndexed;
+
+        /// <summary>
+        /// Defines an event that occurs when a node indexing causes an error.
+        /// </summary>
+        event EventHandler<NodeIndexingErrorEventArgs> IndexingError;
+
     }
 }

src/Storage/Search/NodeIndexedEventArgs.cs

@@ -1,4 +1,5 @@
 using System;
+using SenseNet.ContentRepository.Storage;
 
 // ReSharper disable once CheckNamespace
 namespace SenseNet.ContentRepository.Search.Indexing
@@ -12,10 +13,28 @@ public class NodeIndexedEventArgs : EventArgs
         /// Gets the path of the currently indexed node.
         /// </summary>
         public string Path { get; }
+        /// <summary>
+        /// Gets the id of the currently indexed node.
+        /// </summary>
+        public int NodeId { get; }
+        /// <summary>
+        /// Gets the version id of the currently indexed node instance.
+        /// </summary>
+        public int VersionId { get; }
+        /// <summary>
+        /// Gets the version of the currently indexed node instance.
+        /// </summary>
+        public string Version { get; }
 
         /// <summary>
         /// Initializes a new NodeIndexedEventArgs instance.
         /// </summary>
-        public NodeIndexedEventArgs(string path) { Path = path; }
+        public NodeIndexedEventArgs(string path, int nodeId = 0, int versionId = 0, string version = null)
+        {
+            Path = path;
+            NodeId = nodeId;
+            VersionId = versionId;
+            Version = version;
+        }
     }
 }

src/Storage/Search/NodeIndexingErrorEventArgs.cs

@@ -0,0 +1,39 @@
+using System;
+
+// ReSharper disable once CheckNamespace
+namespace SenseNet.ContentRepository.Search.Indexing
+{
+    /// <summary>
+    /// Defines an event argument containing the identifiers of the node that could not be indexed.
+    /// </summary>
+    public class NodeIndexingErrorEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Gets the id of the currently indexed node.
+        /// </summary>
+        public int NodeId { get; }
+        /// <summary>
+        /// Gets the id of the currently indexed version.
+        /// </summary>
+        public int VersionId { get; }
+        /// <summary>
+        /// Gets the path of the currently indexed node.
+        /// </summary>
+        public string Path { get; }
+        /// <summary>
+        /// Gets the indexing error.
+        /// </summary>
+        public Exception Exception { get; }
+
+        /// <summary>
+        /// Initializes a new NodeIndexedEventArgs instance.
+        /// </summary>
+        public NodeIndexingErrorEventArgs(int nodeId, int versionId, string path, Exception exception)
+        {
+            NodeId = nodeId;
+            VersionId = versionId;
+            Path = path;
+            Exception = exception;
+        }
+    }
+}

src/Storage/SenseNet.Storage.csproj

@@ -257,6 +257,7 @@
     <Compile Include="NodeSaveSettings.cs" />
     <Compile Include="Package.cs" />
     <Compile Include="PreviewProvider.cs" />
+    <Compile Include="Search\NodeIndexingErrorEventArgs.cs" />
     <Compile Include="Search\QueryResult.cs" />
     <Compile Include="Retrier.cs" />
     <Compile Include="Search\SearchManager.cs" />

src/Tests/SenseNet.Packaging.Tests/SenseNet.Packaging.Tests.csproj

@@ -91,6 +91,7 @@
     <Compile Include="StepTests\DeleteContentTypeTests.cs" />
     <Compile Include="StepTests\EditAllowedChildTypesTests.cs" />
     <Compile Include="StepTests\EditConfigurationTests.cs" />
+    <Compile Include="StepTests\PopulateIndexTests.cs" />
     <Compile Include="StepTests\UpgradeProviderConfigurationTests.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -107,6 +108,10 @@
       <Project>{786e6165-ca02-45a9-bf58-207a45d7d6df}</Project>
       <Name>SenseNet.ContentRepository</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\Search\SenseNet.Search.csproj">
+      <Project>{0279705B-779D-485D-86B9-F7AB3DD1F2C3}</Project>
+      <Name>SenseNet.Search</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\Storage\SenseNet.Storage.csproj">
       <Project>{5db4ddba-81f6-4d81-943a-18f3178b3355}</Project>
       <Name>SenseNet.Storage</Name>