diff --git a/.github/workflows/Issues_workflow.yaml b/.github/workflows/Issues_workflow.yaml index 1af3fd6c588..285d228f20b 100644 --- a/.github/workflows/Issues_workflow.yaml +++ b/.github/workflows/Issues_workflow.yaml @@ -27,9 +27,11 @@ jobs: replace-with: '-' flags: g - name: Check Information + env: + ISSUE_TITLE_PARSED: ${{steps.remove_quotations.outputs.replaced}} id: check-info run: | - echo "content_analysis_response=$(pwsh .\\.github\\scripts\\title_analyzer.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )" >> $GITHUB_ENV + echo "content_analysis_response=$(pwsh .\\.github\\scripts\\title_analyzer.ps1 "${{ env.ISSUE_TITLE_PARSED }}" )" >> $GITHUB_ENV - name: Label issue if: env.content_analysis_response != 'Valid' #Uses DYNAMOBOTTOKEN to allow interaction between repos @@ -99,9 +101,11 @@ jobs: #Checks for missing information inside the issue content - name: Check Information + env: + ISSUE_TITLE_PARSED: ${{steps.remove_quotations.outputs.replaced}} id: check-info run: | - echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ steps.remove_quotations.outputs.replaced }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV + echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ env.ISSUE_TITLE_PARSED }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV #Closes the issue if the analysis response is "Empty" - name: Close issue diff --git a/.github/workflows/disabled/cherry-picking.yaml b/.github/workflows/disabled/cherry-picking.yaml index 43f98ea222b..3c5d35da6c7 100644 --- a/.github/workflows/disabled/cherry-picking.yaml +++ b/.github/workflows/disabled/cherry-picking.yaml @@ -33,9 +33,11 @@ jobs: #Checks the message looking for a cherry-pick request and extracts the target branch name - name: Check Information + env: + ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}} id: check-info run: | - echo "destination_branch=$(pwsh .\\.github\\scripts\\cherry_pick_check.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )" >> $GITHUB_ENV + echo "destination_branch=$(pwsh .\\.github\\scripts\\cherry_pick_check.ps1 "${{ env.ISSUE_BODY_PARSED }}" )" >> $GITHUB_ENV #If a target branch was found will run the action - if: env.destination_branch != 'invalid' @@ -50,6 +52,7 @@ jobs: env: #Token used for the pull request. Corresponds to the DynamoBot account GITHUB_TOKEN: ${{secrets.DYNAMOBOTTOKEN}} + ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}} #This represents the title and description of the pr in Markdown format #Everything before the first blank line will be the title #Everything after will be included in the description @@ -60,4 +63,4 @@ jobs: [Commit](https://github.com/DynamoDS/Dynamo/commit/${{github.event.after}}) ### Pull request: - ${{ steps.remove_quotations.outputs.replaced }} + ${{ env.ISSUE_BODY_PARSED }} diff --git a/.github/workflows/issue_type_predicter.yaml b/.github/workflows/issue_type_predicter.yaml index 5f9221a67ae..55a98a6c830 100644 --- a/.github/workflows/issue_type_predicter.yaml +++ b/.github/workflows/issue_type_predicter.yaml @@ -36,17 +36,21 @@ jobs: #Checks for missing information inside the issue content - name: Check Information + env: + ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}} id: check-info run: | ls -la - echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ steps.remove_quotations.outputs.replaced }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV + echo "analysis_response=$(pwsh .\\.github\\scripts\\issue_analyzer.ps1 "${{ env.template }}" "${{ env.ISSUE_BODY_PARSED }}" "${{ env.acceptable_missing_info }}" )" >> $GITHUB_ENV #Remove sections in the issue body like "Dynamo version", "Stack Trace" because won't be used to predict the issue type - name: Clean Issue Body + env: + ISSUE_BODY_PARSED: ${{steps.remove_quotations.outputs.replaced}} if: env.analysis_response == 'Valid' id: clean-issue-body run: | - echo "parsed_issue_body="$(pwsh .\\.github\\scripts\\issue_body_cleaner.ps1 "${{ steps.remove_quotations.outputs.replaced }}" )"" >> $GITHUB_ENV + echo "parsed_issue_body="$(pwsh .\\.github\\scripts\\issue_body_cleaner.ps1 "${{ env.ISSUE_BODY_PARSED }}" )"" >> $GITHUB_ENV #The IssuesTypePredicter program receives as a parameter a json string with the issue content, then It's creating the json string in this section based in the issue body - name: Create Issue JSON String diff --git a/.github/workflows/move_issue.yaml b/.github/workflows/move_issue.yaml index d6140238a0a..a385146c114 100644 --- a/.github/workflows/move_issue.yaml +++ b/.github/workflows/move_issue.yaml @@ -23,10 +23,11 @@ jobs: #Adds a comment to the issue before moving it - name: Add comment if: (github.event.label.name == matrix.label) - uses: ben-z/actions-comment-on-issue@1.0.2 + uses: peter-evans/create-or-update-comment@v3 with: - message: ${{ env.bot_comment }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + issue-number: ${{github.event.issue.number}} + body: ${{ env.bot_comment }} + token: ${{ secrets.DYNAMOBOTTOKEN }} #Move the issue depending on its labels - name: (Label ${{ matrix.label }}) Move to ${{ matrix.repoName }} diff --git a/extern/legacy_remove_me/bin/CSharpAnalytics.Net45.dll b/extern/legacy_remove_me/bin/CSharpAnalytics.Net45.dll new file mode 100644 index 00000000000..bc270afb872 Binary files /dev/null and b/extern/legacy_remove_me/bin/CSharpAnalytics.Net45.dll differ diff --git a/extern/legacy_remove_me/bin/NDesk.Options.dll b/extern/legacy_remove_me/bin/NDesk.Options.dll new file mode 100644 index 00000000000..df458789570 Binary files /dev/null and b/extern/legacy_remove_me/bin/NDesk.Options.dll differ diff --git a/src/DynamoCore/Utilities/LuceneSearchUtility.cs b/src/DynamoCore/Utilities/LuceneSearchUtility.cs index d40097b681b..982e1d0c508 100644 --- a/src/DynamoCore/Utilities/LuceneSearchUtility.cs +++ b/src/DynamoCore/Utilities/LuceneSearchUtility.cs @@ -21,6 +21,16 @@ internal class LuceneSearchUtility internal Lucene.Net.Store.Directory indexDir; internal IndexWriter writer; internal string directory; + internal LuceneStorage currentStorageType; + + public enum LuceneStorage + { + //Lucene Storage will be located in RAM and all the info indexed will be lost when Dynamo app is closed + RAM, + + //Lucene Storage will be located in the local File System and the files will remain in ...AppData\Roaming\Dynamo\Dynamo Core\2.19\Index folder + FILE_SYSTEM + } // Used for creating the StandardAnalyzer internal Analyzer Analyzer; @@ -36,23 +46,34 @@ internal LuceneSearchUtility(DynamoModel model) /// /// Initialize Lucene config file writer. /// - internal void InitializeLuceneConfig(string dirName) + internal void InitializeLuceneConfig(string dirName, LuceneStorage storageType = LuceneStorage.FILE_SYSTEM) { addedFields = new List(); - DirectoryInfo webBrowserUserDataFolder; + DirectoryInfo luceneUserDataFolder; var userDataDir = new DirectoryInfo(dynamoModel.PathManager.UserDataDirectory); - webBrowserUserDataFolder = userDataDir.Exists ? userDataDir : null; + luceneUserDataFolder = userDataDir.Exists ? userDataDir : null; directory = dirName; - string indexPath = Path.Combine(webBrowserUserDataFolder.FullName, LuceneConfig.Index, dirName); - indexDir = Lucene.Net.Store.FSDirectory.Open(indexPath); + string indexPath = Path.Combine(luceneUserDataFolder.FullName, LuceneConfig.Index, dirName); + + currentStorageType = storageType; + + if (storageType == LuceneStorage.RAM) + { + indexDir = new RAMDirectory(); + } + else + { + indexDir = FSDirectory.Open(indexPath); + } + // Create an analyzer to process the text Analyzer = new StandardAnalyzer(LuceneConfig.LuceneNetVersion); - // Initialize Lucene index writer, unless in test mode. - if (!DynamoModel.IsTestMode) + // Initialize Lucene index writer, unless in test mode or we are using RAMDirectory for indexing info. + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { // Create an index writer IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneConfig.LuceneNetVersion, Analyzer) @@ -77,7 +98,7 @@ internal void InitializeLuceneConfig(string dirName) /// internal Document InitializeIndexDocumentForNodes() { - if (DynamoModel.IsTestMode) return null; + if (DynamoModel.IsTestMode && currentStorageType == LuceneStorage.FILE_SYSTEM) return null; var name = new TextField(nameof(LuceneConfig.NodeFieldsEnum.Name), string.Empty, Field.Store.YES); var fullCategory = new TextField(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), string.Empty, Field.Store.YES); @@ -153,7 +174,7 @@ internal void SetDocumentFieldValue(Document doc, string field, string value, bo ((StringField)doc.GetField(field)).SetStringValue(value); } - if (isLast && indexedFields.Any()) + if (isLast && indexedFields != null && indexedFields.Any()) { List diff = indexedFields.Except(addedFields).ToList(); foreach (var d in diff) @@ -248,7 +269,7 @@ internal string CreateSearchQuery(string[] fields, string SearchTerm) internal void DisposeWriter() { //We need to check if we are not running Dynamo tests because otherwise parallel test start to fail when trying to write in the same Lucene directory location - if (!DynamoModel.IsTestMode) + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { writer?.Dispose(); writer = null; @@ -257,7 +278,7 @@ internal void DisposeWriter() internal void CommitWriterChanges() { - if (!DynamoModel.IsTestMode) + if (!DynamoModel.IsTestMode || currentStorageType == LuceneStorage.RAM) { //Commit the info indexed writer?.Commit(); diff --git a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs index 41625a42a5f..af4870a6fbe 100644 --- a/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/PackageManager/PackageManagerSearchViewModel.cs @@ -483,10 +483,18 @@ public PackageManagerSearchViewModel(PackageManagerClientViewModel client) : thi { PackageManagerClientViewModel = client; HostFilter = InitializeHostFilter(); - LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model); + InitializeLuceneForPackageManager(); + } + + internal void InitializeLuceneForPackageManager() + { + if(LuceneSearchUtility == null) + { + LuceneSearchUtility = new LuceneSearchUtility(PackageManagerClientViewModel.DynamoViewModel.Model); + } LuceneSearchUtility.InitializeLuceneConfig(LuceneConfig.PackagesIndexingDirectory); } - + /// /// Sort the default package results in the view based on the sorting key and sorting direction. /// diff --git a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs index e2b4dd37302..b07581cd01f 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/NodeAutoCompleteSearchViewModel.cs @@ -17,6 +17,9 @@ using Dynamo.Utilities; using Dynamo.Wpf.ViewModels; using Greg; +using Lucene.Net.Documents; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; using Newtonsoft.Json; using ProtoCore.AST.AssociativeAST; using ProtoCore.Mirror; @@ -38,6 +41,9 @@ public class NodeAutoCompleteSearchViewModel : SearchViewModel private bool displayLowConfidence; private const string nodeAutocompleteMLEndpoint = "MLNodeAutocomplete"; + // Lucene search utility to perform indexing operations just for NodeAutocomplete. + internal LuceneSearchUtility LuceneSearchUtilityNodeAutocomplete { get; set; } + /// /// The Node AutoComplete ML service version, this could be empty if user has not used ML way /// @@ -601,6 +607,61 @@ private NodeSearchElementViewModel GetViewModelForNodeSearchElement(NodeSearchEl return null; } + + /// + /// Performs a search using the given string as query and subset, if provided. + /// + /// Returns a list with a maximum MaxNumSearchResults elements. + /// The search query + /// Temporary flag that will be used for searching using Lucene.NET + internal IEnumerable SearchNodeAutocomplete(string search, bool useLucene) + { + if (useLucene) + { + //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected + LuceneSearchUtilityNodeAutocomplete.dirReader = LuceneSearchUtilityNodeAutocomplete.writer?.GetReader(applyAllDeletes: true); + if (LuceneSearchUtilityNodeAutocomplete.dirReader == null) return null; + + LuceneSearchUtilityNodeAutocomplete.Searcher = new IndexSearcher(LuceneSearchUtilityNodeAutocomplete.dirReader); + + string searchTerm = search.Trim(); + var candidates = new List(); + var parser = new MultiFieldQueryParser(LuceneConfig.LuceneNetVersion, LuceneConfig.NodeIndexFields, LuceneSearchUtilityNodeAutocomplete.Analyzer) + { + AllowLeadingWildcard = true, + DefaultOperator = LuceneConfig.DefaultOperator, + FuzzyMinSim = LuceneConfig.MinimumSimilarity + }; + + Query query = parser.Parse(LuceneSearchUtilityNodeAutocomplete.CreateSearchQuery(LuceneConfig.NodeIndexFields, searchTerm)); + TopDocs topDocs = LuceneSearchUtilityNodeAutocomplete.Searcher.Search(query, n: LuceneConfig.DefaultResultsCount); + + for (int i = 0; i < topDocs.ScoreDocs.Length; i++) + { + // read back a Lucene doc from results + Document resultDoc = LuceneSearchUtilityNodeAutocomplete.Searcher.Doc(topDocs.ScoreDocs[i].Doc); + + string name = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Name)); + string docName = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.DocName)); + string cat = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName)); + string parameters = resultDoc.Get(nameof(LuceneConfig.NodeFieldsEnum.Parameters)); + + + var foundNode = FindViewModelForNodeNameAndCategory(name, cat, parameters); + if (foundNode != null) + { + candidates.Add(foundNode); + } + } + + return candidates; + } + else + { + return Search(search); + } + } + /// /// Filters the matching node search elements based on user input in the search field. /// @@ -616,9 +677,25 @@ internal void SearchAutoCompleteCandidates(string input) } else { - // Providing the saved search results to limit the scope of the query search. - // Then add back the ML info on filterted nodes as the Search function accepts elements of type NodeSearchElement - var foundNodes = Search(input, searchElementsCache.Select(x => x.Model)); + LuceneSearchUtilityNodeAutocomplete = new LuceneSearchUtility(dynamoViewModel.Model); + + //The dirName parameter doesn't matter because we are using RAMDirectory indexing and no files are created + LuceneSearchUtilityNodeAutocomplete.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM); + + //Memory indexing process for Node Autocomplete (indexing just the nodes returned by the NodeAutocomplete service so we limit the scope of the query search) + foreach (var node in searchElementsCache.Select(x => x.Model)) + { + var doc = LuceneSearchUtilityNodeAutocomplete.InitializeIndexDocumentForNodes(); + AddNodeTypeToSearchIndex(node, doc); + } + + //Write the Lucene documents to memory + LuceneSearchUtilityNodeAutocomplete.CommitWriterChanges(); + + var luceneResults = SearchNodeAutocomplete(input, true); + var foundNodesModels = luceneResults.Select(x => x.Model); + var foundNodes = foundNodesModels.Select(MakeNodeSearchElementVM); + var filteredSearchElements = new List(); foreach (var node in foundNodes) @@ -634,10 +711,30 @@ internal void SearchAutoCompleteCandidates(string input) } } FilteredResults = new List(filteredSearchElements).OrderBy(x => x.Name).ThenBy(x => x.Description); + + LuceneSearchUtilityNodeAutocomplete.DisposeWriter(); } } } + /// + /// Add node information to Lucene index + /// + /// node info that will be indexed + /// Lucene document in which the node info will be indexed + private void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc) + { + if (LuceneSearchUtilityNodeAutocomplete.addedFields == null) return; + + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description); + if (node.SearchKeywords.Count > 0) LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true); + LuceneSearchUtilityNodeAutocomplete.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty); + + LuceneSearchUtilityNodeAutocomplete.writer?.AddDocument(doc); + } + /// /// Returns a collection of node search elements for nodes /// that output a type compatible with the port type if it's an input port. diff --git a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs index 9369672e85c..bc513bca3f6 100644 --- a/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Search/SearchViewModel.cs @@ -996,7 +996,7 @@ internal IEnumerable Search(string search, bool useL /// Full Category of the node /// Node input parameters /// - private NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters) + internal NodeSearchElementViewModel FindViewModelForNodeNameAndCategory(string nodeName, string nodeCategory, string parameters) { var result = Model.SearchEntries.Where(e => { if (e.Name.Equals(nodeName) && e.FullCategoryName.Equals(nodeCategory)) @@ -1035,7 +1035,7 @@ private static IEnumerable GetVisibleSearchResults(N } } - private NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry) + internal NodeSearchElementViewModel MakeNodeSearchElementVM(NodeSearchElement entry) { var element = entry as CustomNodeSearchElement; var elementVM = element != null diff --git a/src/DynamoCoreWpf/Views/Core/DynamoView.xaml.cs b/src/DynamoCoreWpf/Views/Core/DynamoView.xaml.cs index b164c6601e6..7b01c2b8e85 100644 --- a/src/DynamoCoreWpf/Views/Core/DynamoView.xaml.cs +++ b/src/DynamoCoreWpf/Views/Core/DynamoView.xaml.cs @@ -1444,6 +1444,10 @@ private void DynamoViewModelRequestShowPackageManagerSearch(object s, EventArgs { _pkgSearchVM = new PackageManagerSearchViewModel(dynamoViewModel.PackageManagerClientViewModel); } + else + { + _pkgSearchVM.InitializeLuceneForPackageManager(); + } if (_searchPkgsView == null) { diff --git a/tools/NuGet/template-artifactory/DynamoVisualProgramming.SignDynamo.nuspec b/tools/NuGet/template-artifactory/DynamoVisualProgramming.SignDynamo.nuspec index 4bb98f33805..f37be820b43 100644 --- a/tools/NuGet/template-artifactory/DynamoVisualProgramming.SignDynamo.nuspec +++ b/tools/NuGet/template-artifactory/DynamoVisualProgramming.SignDynamo.nuspec @@ -19,7 +19,7 @@ - +