Merge branch 'master' of https://github.com/DynamoDS/Dynamo

.github/workflows/Issues_workflow.yaml

@@ -27,9 +27,11 @@ jobs:
           replace-with: '-'
           flags: g    
       - name: Check Information
+        env:
+          ISSUE_TITLE_PARSED: ${{steps.remove_quotations.outputs.replaced}}
         id: check-info
         run: |
-          echo "content_analysis_response=$(pwsh .\\.github\\scripts\\title_analyzer.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )" >> $GITHUB_ENV
+          echo "content_analysis_response=$(pwsh .\\.github\\scripts\\title_analyzer.ps1 "${{ env.ISSUE_TITLE_PARSED }}" )" >> $GITHUB_ENV
       - name: Label issue
         if: env.content_analysis_response != 'Valid'
         #Uses DYNAMOBOTTOKEN to allow interaction between repos
@@ -99,9 +101,11 @@ jobs:
 
       #Checks for missing information inside the issue content
       - name: Check Information
+        env:
+          ISSUE_TITLE_PARSED: ${{steps.remove_quotations.outputs.replaced}}
         id: check-info
         run: |
-          echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ steps.remove_quotations.outputs.replaced }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV
+          echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ env.ISSUE_TITLE_PARSED }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV
       
       #Closes the issue if the analysis response is "Empty"
       - name: Close issue

.github/workflows/disabled/cherry-picking.yaml

@@ -33,9 +33,11 @@ jobs:
 
     #Checks the message looking for a cherry-pick request and extracts the target branch name
     - name: Check Information
+      env:
+          ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}}
       id: check-info
       run: |
-        echo "destination_branch=$(pwsh .\\.github\\scripts\\cherry_pick_check.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )" >> $GITHUB_ENV
+        echo "destination_branch=$(pwsh .\\.github\\scripts\\cherry_pick_check.ps1 "${{ env.ISSUE_BODY_PARSED }}" )" >> $GITHUB_ENV
     
       #If a target branch was found will run the action
     - if: env.destination_branch != 'invalid'
@@ -50,6 +52,7 @@ jobs:
       env: 
         #Token used for the pull request. Corresponds to the DynamoBot account
         GITHUB_TOKEN: ${{secrets.DYNAMOBOTTOKEN}}
+        ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}}
         #This represents the title and description of the pr in Markdown format
         #Everything before the first blank line will be the title
         #Everything after will be included in the description
@@ -60,4 +63,4 @@ jobs:
                   [Commit](https://github.com/DynamoDS/Dynamo/commit/${{github.event.after}})
                   
                   ### Pull request:
-                  ${{ steps.remove_quotations.outputs.replaced }}
+                  ${{ env.ISSUE_BODY_PARSED }}

.github/workflows/issue_type_predicter.yaml

@@ -36,17 +36,21 @@ jobs:
 
        #Checks for missing information inside the issue content
       - name: Check Information
+        env:
+          ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}}
         id: check-info
         run: |
           ls -la
-          echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ steps.remove_quotations.outputs.replaced }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV
+          echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ env.ISSUE_BODY_PARSED }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV
       
       #Remove sections in the issue body like "Dynamo version", "Stack Trace" because won't be used to predict the issue type
       - name: Clean Issue Body
+        env:
+          ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}}
         if: env.analysis_response == 'Valid'
         id: clean-issue-body
         run: |
-          echo "parsed_issue_body="$(pwsh  .\\.github\\scripts\\issue_body_cleaner.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )"" >> $GITHUB_ENV
+          echo "parsed_issue_body="$(pwsh  .\\.github\\scripts\\issue_body_cleaner.ps1 "${{ env.ISSUE_BODY_PARSED }}" )"" >> $GITHUB_ENV
 
       #The IssuesTypePredicter program receives as a parameter a json string with the issue content, then It's creating the json string in this section based in the issue body
       - name: Create Issue JSON String

.github/workflows/move_issue.yaml

@@ -23,10 +23,11 @@ jobs:
       #Adds a comment to the issue before moving it
       - name: Add comment
         if: (github.event.label.name == matrix.label)
-        uses: ben-z/[email protected]
+        uses: peter-evans/create-or-update-comment@v3
         with:
-          message: ${{ env.bot_comment }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          issue-number: ${{github.event.issue.number}}
+          body: ${{ env.bot_comment }}
+          token: ${{ secrets.DYNAMOBOTTOKEN }}
 
       #Move the issue depending on its labels
       - name: (Label ${{ matrix.label }}) Move to ${{ matrix.repoName }}

src/DynamoCore/Utilities/LuceneSearchUtility.cs

@@ -21,6 +21,16 @@ internal class LuceneSearchUtility
         internal Lucene.Net.Store.Directory indexDir;
         internal IndexWriter writer;
         internal string directory;
+        internal LuceneStorage currentStorageType;
+
+        public enum LuceneStorage
+        {
+            //Lucene Storage will be located in RAM and all the info indexed will be lost when Dynamo app is closed
+            RAM,
+
+            //Lucene Storage will be located in the local File System and the files will remain in ...AppData\Roaming\Dynamo\Dynamo Core\2.19\Index folder
+            FILE_SYSTEM
+        }
 
         // Used for creating the StandardAnalyzer
         internal Analyzer Analyzer;
@@ -36,23 +46,34 @@ internal LuceneSearchUtility(DynamoModel model)
         /// <summary>
         /// Initialize Lucene config file writer.
         /// </summary>
-        internal void InitializeLuceneConfig(string dirName)
+        internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM)
         {
             addedFields = new List<string>();
 
-            DirectoryInfo webBrowserUserDataFolder;
+            DirectoryInfo luceneUserDataFolder;
             var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory);
-            webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null;
+            luceneUserDataFolder = userDataDir.Exists ? userDataDir : null;
 
             directory = dirName;
-            string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName);
-            indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath);
+            string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName);
+
+            currentStorageType = storageType;
+
+            if (storageType == LuceneStorage.RAM)
+            {
+                indexDir = new RAMDirectory();
+            }
+            else
+            {
+                indexDir = FSDirectory.Open(indexPath);
+            }
+
 
             // Create an analyzer to process the text
             Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion);
 
-            // Initialize Lucene index writer, unless in test mode.
-            if (!DynamoModel.IsTestMode)
+            // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info. 
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 // Create an index writer
                 IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer)
@@ -77,7 +98,7 @@ internal void InitializeLuceneConfig(string dirName)
         /// <returns></returns>
         internal Document InitializeIndexDocumentForNodes()
         {
-            if (DynamoModel.IsTestMode) return null;
+            if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null;
 
             var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES);
             var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES);
@@ -153,7 +174,7 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo
                 ((StringField)doc.GetField(field)).SetStringValue(value);
             }
 
-            if (isLast && indexedFields.Any())
+            if (isLast && indexedFields != null && indexedFields.Any())
             {
                 List<string> diff = indexedFields.Except(addedFields).ToList();
                 foreach (var d in diff)
@@ -248,7 +269,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm)
         internal void DisposeWriter()
         {
             //We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location
-            if (!DynamoModel.IsTestMode)
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 writer?.Dispose();
                 writer = null;
@@ -257,7 +278,7 @@ internal void DisposeWriter()
 
         internal void CommitWriterChanges()
         {
-            if (!DynamoModel.IsTestMode)
+            if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM)
             {
                 //Commit the info indexed
                 writer?.Commit();

src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs

@@ -483,10 +483,18 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi
         {
             PackageManagerClientViewModel = client;
             HostFilter = InitializeHostFilter();
-            LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model);
+            InitializeLuceneForPackageManager();
+        }
+
+        internal void InitializeLuceneForPackageManager()
+        {
+            if(LuceneSearchUtility == null)
+            {
+                LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model);
+            }          
             LuceneSearchUtility.InitializeLuceneConfig(LuceneConfig.PackagesIndexingDirectory);
         }
-        
+
         /// <summary>
         /// Sort the default package results in the view based on the sorting key and sorting direction.
         /// </summary>

src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs

@@ -17,6 +17,9 @@
 using Dynamo.Utilities;
 using Dynamo.Wpf.ViewModels;
 using Greg;
+using Lucene.Net.Documents;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
 using Newtonsoft.Json;
 using ProtoCore.AST.AssociativeAST;
 using ProtoCore.Mirror;
@@ -38,6 +41,9 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel
         private bool displayLowConfidence;
         private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete";
 
+        // Lucene search utility to perform indexing operations just for NodeAutocomplete.
+        internal LuceneSearchUtility LuceneSearchUtilityNodeAutocomplete { get; set; }
+
         /// <summary>
         /// The Node AutoComplete ML service version, this could be empty if user has not used ML way
         /// </summary>
@@ -601,6 +607,61 @@ private NodeSearchElementViewModel GetViewModelForNodeSearchElement(NodeSearchEl
             return null;
         }
 
+
+        /// <summary>
+        ///     Performs a search using the given string as query and subset, if provided.
+        /// </summary>
+        /// <returns> Returns a list with a maximum MaxNumSearchResults elements.</returns>
+        /// <param name="search"> The search query </param>
+        /// <param name="useLucene"> Temporary flag that will be used for searching using Lucene.NET </param>
+        internal IEnumerable<NodeSearchElementViewModel> SearchNodeAutocomplete(string search, bool useLucene)
+        {
+            if (useLucene)
+            {
+                //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
+                LuceneSearchUtilityNodeAutocomplete.dirReader = LuceneSearchUtilityNodeAutocomplete.writer?.GetReader(applyAllDeletes: true);
+                if (LuceneSearchUtilityNodeAutocomplete.dirReader == null) return null;
+
+                LuceneSearchUtilityNodeAutocomplete.Searcher = new IndexSearcher(LuceneSearchUtilityNodeAutocomplete.dirReader);
+
+                string searchTerm = search.Trim();
+                var candidates = new List<NodeSearchElementViewModel>();
+                var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtilityNodeAutocomplete.Analyzer)
+                {
+                    AllowLeadingWildcard = true,
+                    DefaultOperator = LuceneConfig.DefaultOperator,
+                    FuzzyMinSim = LuceneConfig.MinimumSimilarity
+                };
+
+                Query query = parser.Parse(LuceneSearchUtilityNodeAutocomplete.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm));
+                TopDocs topDocs = LuceneSearchUtilityNodeAutocomplete.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount);
+
+                for (int i = 0; i < topDocs.ScoreDocs.Length; i++)
+                {
+                    // read back a Lucene doc from results
+                    Document resultDoc = LuceneSearchUtilityNodeAutocomplete.Searcher.Doc(topDocs.ScoreDocs[i].Doc);
+
+                    string name = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Name));
+                    string docName = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.DocName));
+                    string cat = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName));
+                    string parameters = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Parameters));
+
+
+                    var foundNode = FindViewModelForNodeNameAndCategory(name, cat, parameters);
+                    if (foundNode != null)
+                    {
+                        candidates.Add(foundNode);
+                    }
+                }
+
+                return candidates;
+            }
+            else
+            {
+                return Search(search);
+            }
+        }
+
         /// <summary>
         /// Filters the matching node search elements based on user input in the search field. 
         /// </summary>
@@ -616,9 +677,25 @@ internal void SearchAutoCompleteCandidates(string input)
                 }
                 else
                 {
-                    // Providing the saved search results to limit the scope of the query search.
-                    // Then add back the ML info on filterted nodes as the Search function accepts elements of type NodeSearchElement
-                    var foundNodes = Search(input, searchElementsCache.Select(x => x.Model));
+                    LuceneSearchUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model);
+
+                    //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created
+                    LuceneSearchUtilityNodeAutocomplete.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
+
+                    //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search)
+                    foreach (var node in searchElementsCache.Select(x => x.Model))
+                    {
+                        var doc = LuceneSearchUtilityNodeAutocomplete.InitializeIndexDocumentForNodes();
+                        AddNodeTypeToSearchIndex(node, doc);
+                    }
+
+                    //Write the Lucene documents to memory
+                    LuceneSearchUtilityNodeAutocomplete.CommitWriterChanges();
+
+                    var luceneResults = SearchNodeAutocomplete(input, true);
+                    var foundNodesModels = luceneResults.Select(x => x.Model);
+                    var foundNodes = foundNodesModels.Select(MakeNodeSearchElementVM);
+
                     var filteredSearchElements = new List<NodeSearchElementViewModel>();
 
                     foreach (var node in foundNodes)
@@ -634,10 +711,30 @@ internal void SearchAutoCompleteCandidates(string input)
                         }
                     }
                     FilteredResults = new List<NodeSearchElementViewModel>(filteredSearchElements).OrderBy(x => x.Name).ThenBy(x => x.Description);
+
+                    LuceneSearchUtilityNodeAutocomplete.DisposeWriter();
                 }
             }
         }
 
+        /// <summary>
+        /// Add node information to Lucene index
+        /// </summary>
+        /// <param name="node">node info that will be indexed</param>
+        /// <param name="doc">Lucene document in which the node info will be indexed</param>
+        private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
+        {
+            if (LuceneSearchUtilityNodeAutocomplete.addedFields == null) return;
+
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
+            if (node.SearchKeywords.Count > 0) LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
+            LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);
+
+            LuceneSearchUtilityNodeAutocomplete.writer?.AddDocument(doc);
+        }
+
         /// <summary>
         /// Returns a collection of node search elements for nodes
         /// that output a type compatible with the port type if it's an input port.

src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs

@@ -996,7 +996,7 @@ internal IEnumerable<NodeSearchElementViewModel> Search(string search, bool useL
         /// <param name="nodeCategory">Full Category of the node</param>
         /// <param name="parameters">Node input parameters</param>
         /// <returns></returns>
-        private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
+        internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters)
         {
             var result = Model.SearchEntries.Where(e => {
                 if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory))
@@ -1035,7 +1035,7 @@ private static IEnumerable<NodeSearchElementViewModel> GetVisibleSearchResults(N
             }
         }
 
-        private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
+        internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry)
         {
             var element = entry as CustomNodeSearchElement;
             var elementVM = element != null

src/DynamoCoreWpf/Views/Core/DynamoView.xaml.cs

@@ -1444,6 +1444,10 @@ private void DynamoViewModelRequestShowPackageManagerSearch(object s, EventArgs
             {
                 _pkgSearchVM = new PackageManagerSearchViewModel(dynamoViewModel.PackageManagerClientViewModel);
             }
+            else
+            {
+                _pkgSearchVM.InitializeLuceneForPackageManager();
+            }
 
             if (_searchPkgsView == null)
             {