feat: add lazy join decorator to improve performance

packages/forest_admin_agent/lib/forest_admin_agent/routes/charts/charts.rb

@@ -99,7 +99,7 @@ def make_pie
 
           result = @collection.aggregate(@caller, @filter, aggregation)
 
-          PieChart.new(result.map { |row| { key: row[:group][group_field], value: row[:value] } }).serialize
+          PieChart.new(result.map { |row| { key: row['group'][group_field], value: row['value'] } }).serialize
         end
 
         def make_line
@@ -124,7 +124,7 @@ def make_line
           )
 
           values = {}
-          rows.each { |row| values[row[:group][group_by_field_name]] = row[:value] }
+          rows.each { |row| values[row['group'][group_by_field_name]] = row['value'] }
           dates = values.keys.sort
           current = dates[0]
           last = dates.last
@@ -189,8 +189,8 @@ def make_leaderboard
 
             result = rows.map do |row|
               {
-                key: row[:group][aggregation.groups[0][:field]],
-                value: row[:value]
+                key: row['group'][aggregation.groups[0][:field]],
+                value: row['value']
               }
             end
 
@@ -206,7 +206,7 @@ def compute_value(filter)
                                         field: @args[:params][:aggregateFieldName])
           result = @collection.aggregate(@caller, filter, aggregation)
 
-          result[0][:value] || 0
+          result[0]['value'] || 0
         end
       end
     end

packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/charts/charts_spec.rb

@@ -148,7 +148,7 @@ module Charts
                                                   type: 'Value',
                                                   timezone: 'Europe/Paris'
                                                 })
-            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ value: 10, group: [] }])
+            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ 'value' => 10, 'group' => [] }])
             result = chart.handle_request(args)
 
             expect(result).to match(
@@ -175,8 +175,8 @@ module Charts
                                                 })
             @datasource.get_collection('book')
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
-              [{ value: 10, group: [] }], # first call
-              [{ value: 5, group: [] }] # second call
+              [{ 'value' => 10, 'group' => [] }], # first call
+              [{ 'value' => 5, 'group' => [] }] # second call
             )
             result = chart.handle_request(args)
 
@@ -203,7 +203,7 @@ module Charts
                                                   type: 'Objective',
                                                   timezone: 'Europe/Paris'
                                                 })
-            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ value: 10, group: [] }])
+            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ 'value' => 10, 'group' => [] }])
             result = chart.handle_request(args)
 
             expect(result).to match(
@@ -231,8 +231,8 @@ module Charts
                                                 })
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
               [
-                { value: 100, group: { 'year' => 2021 } },
-                { value: 150, group: { 'year' => 2022 } }
+                { 'value' => 100, 'group' => { 'year' => 2021 } },
+                { 'value' => 150, 'group' => { 'year' => 2022 } }
               ]
             )
             result = chart.handle_request(args)
@@ -263,8 +263,8 @@ module Charts
                                                 })
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'date' => Time.parse('2022-01-03 00:00:00') } },
-                { value: 15, group: { 'date' => Time.parse('2022-01-07 00:00:00') } }
+                { 'value' => 10, 'group' => { 'date' => Time.parse('2022-01-03 00:00:00') } },
+                { 'value' => 15, 'group' => { 'date' => Time.parse('2022-01-07 00:00:00') } }
               ]
             )
             result = chart.handle_request(args)
@@ -299,8 +299,8 @@ module Charts
                                                 })
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'date' => Time.parse('2022-01-03 00:00:00') } },
-                { value: 15, group: { 'date' => Time.parse('2022-01-10 00:00:00') } }
+                { 'value' => 10, 'group' => { 'date' => Time.parse('2022-01-03 00:00:00') } },
+                { 'value' => 15, 'group' => { 'date' => Time.parse('2022-01-10 00:00:00') } }
               ]
             )
             result = chart.handle_request(args)
@@ -332,8 +332,8 @@ module Charts
                                                 })
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'date' => Time.parse('2022-01-01 00:00:00') } },
-                { value: 15, group: { 'date' => Time.parse('2022-02-01 00:00:00') } }
+                { 'value' => 10, 'group' => { 'date' => Time.parse('2022-01-01 00:00:00') } },
+                { 'value' => 15, 'group' => { 'date' => Time.parse('2022-02-01 00:00:00') } }
               ]
             )
             result = chart.handle_request(args)
@@ -365,8 +365,8 @@ module Charts
                                                 })
             allow(@datasource.get_collection('book')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'date' => Time.parse('2022-01-01 00:00:00') } },
-                { value: 15, group: { 'date' => Time.parse('2023-01-01 00:00:00') } }
+                { 'value' => 10, 'group' => { 'date' => Time.parse('2022-01-01 00:00:00') } },
+                { 'value' => 15, 'group' => { 'date' => Time.parse('2023-01-01 00:00:00') } }
               ]
             )
             result = chart.handle_request(args)
@@ -402,8 +402,8 @@ module Charts
             allow(@datasource.get_collection('book')).to receive(:datasource).and_return(@datasource)
             allow(@datasource.get_collection('review')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'author' => 'Isaac Asimov' } },
-                { value: 15, group: { 'author' => 'Jules Verne' } }
+                { 'value' => 10, 'group' => { 'author' => 'Isaac Asimov' } },
+                { 'value' => 15, 'group' => { 'author' => 'Jules Verne' } }
               ]
             )
             result = chart.handle_request(args)
@@ -434,8 +434,8 @@ module Charts
             allow(@datasource.get_collection('book')).to receive(:datasource).and_return(@datasource)
             allow(@datasource.get_collection('book_review')).to receive(:aggregate).and_return(
               [
-                { value: 10, group: { 'book:year' => 2022 } },
-                { value: 15, group: { 'book:year' => 2023 } }
+                { 'value' => 10, 'group' => { 'book:year' => 2022 } },
+                { 'value' => 15, 'group' => { 'book:year' => 2023 } }
               ]
             )
             result = chart.handle_request(args)
@@ -489,7 +489,7 @@ module Charts
                 timezone: 'Europe/Paris'
               }
             )
-            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ value: 10, group: [] }])
+            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ 'value' => 10, 'group' => [] }])
             chart.handle_request(args)
 
             expect(chart.filter).to have_attributes(
@@ -510,7 +510,7 @@ module Charts
                                                   aggregator: 'Count',
                                                   timezone: 'Europe/Paris'
                                                 })
-            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ value: 10, group: [] }])
+            allow(@datasource.get_collection('book')).to receive(:aggregate).and_return([{ 'value' => 10, 'group' => [] }])
             chart.handle_request(args)
 
             expect(chart.filter).to have_attributes(

packages/forest_admin_datasource_active_record/lib/forest_admin_datasource_active_record/utils/query.rb

@@ -109,11 +109,8 @@ def build_select
       end
 
       def apply_select
-        unless @projection.nil?
-          @query = @query.select(@select.join(', '))
-
-          @query = @query.includes(format_relation_projection(@projection))
-        end
+        @query = @query.select(@select.join(', ')) if @select
+        @query = @query.includes(format_relation_projection(@projection)) unless @projection.nil?
 
         @query
       end

packages/forest_admin_datasource_active_record/lib/forest_admin_datasource_active_record/utils/query_aggregate.rb

@@ -44,6 +44,12 @@ def compute_result_aggregate(rows)
           }
         end
       end
+
+      def add_join_relation(relation_name)
+        @query = @query.left_joins(relation_name.to_sym)
+
+        @query
+      end
     end
   end
 end

packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/decorators_stack.rb

@@ -5,7 +5,7 @@ class DecoratorsStack
 
       attr_reader :datasource, :schema, :search, :early_computed, :late_computed, :action, :relation, :late_op_emulate,
                   :early_op_emulate, :validation, :sort, :rename_field, :publication, :write, :chart, :hook, :segment,
-                  :binary, :override
+                  :binary, :override, :lazy_join
 
       def initialize(datasource)
         @customizations = []
@@ -19,6 +19,8 @@ def initialize(datasource)
         last = @early_op_emulate = DatasourceDecorator.new(last, OperatorsEmulate::OperatorsEmulateCollectionDecorator)
         last = DatasourceDecorator.new(last, OperatorsEquivalence::OperatorsEquivalenceCollectionDecorator)
         last = @relation = DatasourceDecorator.new(last, Relation::RelationCollectionDecorator)
+        # lazy join is just before relation, to avoid relations to do useless stuff
+        last = @lazy_join = DatasourceDecorator.new(last, LazyJoin::LazyJoinCollectionDecorator)
         last = @late_computed = DatasourceDecorator.new(last, Computed::ComputeCollectionDecorator)
         last = @late_op_emulate = DatasourceDecorator.new(last, OperatorsEmulate::OperatorsEmulateCollectionDecorator)
         last = DatasourceDecorator.new(last, OperatorsEquivalence::OperatorsEquivalenceCollectionDecorator)

packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/lazy_join/lazy_join_collection_decorator.rb

@@ -0,0 +1,129 @@
+module ForestAdminDatasourceCustomizer
+  module Decorators
+    module LazyJoin
+      class LazyJoinCollectionDecorator < ForestAdminDatasourceToolkit::Decorators::CollectionDecorator
+        include ForestAdminDatasourceToolkit::Decorators
+        include ForestAdminDatasourceToolkit::Components::Query
+        include ForestAdminDatasourceToolkit::Components::Query::ConditionTree
+
+        def list(caller, filter, projection)
+          simplified_projection = get_projection_without_useless_joins(projection)
+          refined_filter = refine_filter(caller, filter)
+          records = child_collection.list(caller, refined_filter, simplified_projection)
+
+          apply_joins_on_records(projection, simplified_projection, records)
+        end
+
+        def aggregate(caller, filter, aggregation, limit = nil)
+          refined_filter = refine_filter(caller, filter)
+          replaced = {}
+          refined_aggregation = aggregation.replace_fields do |field_name|
+            if useless_join?(field_name.split(':')[0], aggregation.projection)
+              new_field_name = get_foreign_key_for_projection(field_name)
+              replaced[new_field_name] = field_name
+
+              new_field_name
+            else
+              field_name
+            end
+          end
+
+          results = child_collection.aggregate(caller, refined_filter, refined_aggregation, limit)
+
+          apply_joins_on_aggregate_result(aggregation, refined_aggregation, results, replaced)
+        end
+
+        def refine_filter(_caller, filter = nil)
+          filter&.override(
+            condition_tree: filter.condition_tree&.replace_leafs do |leaf|
+              if useless_join?(leaf.field.split(':')[0], filter.condition_tree.projection)
+                leaf.override(field: get_foreign_key_for_projection(leaf.field))
+              else
+                leaf
+              end
+            end
+          )
+        end
+
+        private
+
+        def get_foreign_key_for_projection(field_name)
+          relation_name = field_name.split(':')[0]
+          relation_schema = schema[:fields][relation_name]
+
+          relation_schema.foreign_key
+        end
+
+        def useless_join?(relation_name, projection)
+          relation_schema = schema[:fields][relation_name]
+          sub_projection = projection.relations[relation_name]
+
+          relation_schema.type == 'ManyToOne' &&
+            sub_projection.size == 1 &&
+            sub_projection[0] == relation_schema.foreign_key_target
+        end
+
+        def get_projection_without_useless_joins(projection)
+          new_projection = Projection.new(projection)
+
+          projection.relations.each do |relation_name, relation_projection|
+            next unless useless_join?(relation_name, projection)
+
+            # remove foreign key target from projection
+            new_projection.delete("#{relation_name}:#{relation_projection[0]}")
+
+            # add foreign keys to projection
+            fk_field = get_foreign_key_for_projection("#{relation_name}:#{relation_projection[0]}")
+            new_projection << fk_field
+          end
+
+          new_projection
+        end
+
+        def apply_joins_on_records(initial_projection, requested_projection, records)
+          return records if initial_projection == requested_projection
+
+          projections_to_add = Projection.new(initial_projection.reject do |field|
+            requested_projection.include?(field)
+          end)
+          projections_to_rm = Projection.new(requested_projection.reject { |field| initial_projection.include?(field) })
+
+          records.each do |record|
+            # add to record relation:id
+            projections_to_add.relations.each do |relation_name, relation_projection|
+              relation_schema = schema[:fields][relation_name]
+
+              if relation_schema && relation_schema.type == 'ManyToOne'
+                fk_value = record[get_foreign_key_for_projection("#{relation_name}:#{relation_projection[0]}")]
+                record[relation_name] = fk_value.nil? ? nil : { relation_projection[0] => fk_value }
+              end
+            end
+
+            # remove foreign keys
+            projections_to_rm.each { |field| record.delete(field) }
+          end
+
+          records
+        end
+
+        def apply_joins_on_aggregate_result(initial_aggregation, requested_aggregation, results, fields_to_replace)
+          return result if initial_aggregation == requested_aggregation
+
+          results.each do |result|
+            group = {}
+            result['group'].each do |field, value|
+              if fields_to_replace.include?(field)
+                group[fields_to_replace[field]] = value
+              else
+                group[field] = value
+              end
+            end
+            result['group'] = group
+          end
+
+          results
+        end
+      end
+    end
+  end
+end