Optimize the merge operation of the large number of contributor records.

Signed-off-by: EdmondFrank <[email protected]>
oss-compass · Feb 22, 2024 · 234b142 · 234b142
1 parent 2e6417c
commit 234b142
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 20 deletions.
diff --git a/app/api/openapi/shared_params/export.rb b/app/api/openapi/shared_params/export.rb
@@ -36,8 +36,8 @@ def extract_params!(params)
 
       def refresh_download_path(state)
         blob = state[:blob_id] ? ActiveStorage::Attachment.find_by(blob_id: state[:blob_id], name: 'exports') : nil
-        state.merge!(download_path: Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)) if blob
-        state.merge!(downdload_path: Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)) if blob
+        state[:download_path] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true) if blob
+        state[:downdload_path] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true) if blob
         state
       end
 

diff --git a/app/controllers/concerns/common.rb b/app/controllers/concerns/common.rb
@@ -59,7 +59,7 @@ def generate_yml_url(agent, branch, path)
 
   def each_patch_with_action(diff_url, &action)
     req = { method: :get, url: diff_url }
-    req.merge!(proxy: PROXY) unless extract_domain(diff_url).start_with?('gitee')
+    req[:proxy] = PROXY unless extract_domain(diff_url).start_with?('gitee')
     diff = RestClient::Request.new(req).execute.body
     patches = GitDiffParser.parse(diff)
     patches.each do |patch|
@@ -85,7 +85,7 @@ def analyze_or_submit_yaml_file(analyzer, user_agent, branch, path, extra={})
         base_config.merge(yaml_url: yaml_url)
       else
         req = { method: :get, url: yaml_url }
-        req.merge!(proxy: PROXY) unless extract_domain(yaml_url).start_with?('gitee')
+        req[:proxy] = PROXY unless extract_domain(yaml_url).start_with?('gitee')
         yaml = YAML.load(RestClient::Request.new(req).execute.body)
         base_config.merge(repo_url: yaml['resource_types']['repo_urls'], developers: yaml['developers'] || {})
       end

diff --git a/app/models/concerns/contributor_enrich.rb b/app/models/concerns/contributor_enrich.rb
@@ -2,8 +2,8 @@
 module ContributorEnrich
   extend ActiveSupport::Concern
 
-  MAX_DEPTH = 10
-  MAX_PER_PAGE = 10000
+  MAX_DEPTH = 100
+  MAX_PER_PAGE = 8000
 
   class_methods do
     def fetch_contributors_list(repo_urls, begin_date, end_date, label: nil, level: nil)
@@ -13,7 +13,7 @@ def fetch_contributors_list(repo_urls, begin_date, end_date, label: nil, level:
         mileage_step = 0
         mileage_types = ['core', 'regular', 'guest']
         depth = 0
-        contributors_list = []
+        contributors_map = {}
 
         query =
           self
@@ -31,26 +31,18 @@ def fetch_contributors_list(repo_urls, begin_date, end_date, label: nil, level:
             .raw_response
             .dig('hits', 'hits')
             .map do |hit|
-            contributors_list << hit['_source'].slice(*Types::Meta::ContributorDetailType.fields.keys.map(&:underscore))
+            row = hit['_source'].slice(*Types::Meta::ContributorDetailType.fields.keys.map(&:underscore))
+            key = row['contributor']
+            contributors_map[key] = contributors_map[key] ? merge_contributor(contributors_map[key], row) : row
+            contribution_count += row['contribution'].to_i
           end
           query = query.scroll(id: query.scroll_id, timeout: '1m')
           depth += 1
           break if (query.last_page? || depth > MAX_DEPTH)
         end
 
         contributors_list =
-          contributors_list
-            .reduce({}) do |map, row|
-          key = row['contributor']
-          if !['openharmony_ci'].include?(key)
-            map[key] = map[key] ? merge_contributor(map[key], row) : row
-            contribution_count += row['contribution'].to_i
-          end
-          map
-        end
-
-        contributors_list =
-          contributors_list
+          contributors_map
             .sort_by { |_, row| -row['contribution_without_observe'].to_i }
             .map do |_, row|
           row['mileage_type'] = (mileage_step < 2 && row['contribution_without_observe'] > 0) ?