Merge pull request #17453 from lpichler/multi_region_chargeback

Report chargeback from all regions

app/models/chargeback.rb

@@ -32,22 +32,26 @@ def self.build_results_for_report_chargeback(options)
 
     data = {}
     rates = RatesCache.new
-    ConsumptionHistory.for_report(self, options) do |consumption|
-      rates_to_apply = rates.get(consumption)
 
-      key = report_row_key(consumption)
-      data[key] ||= new(options, consumption)
+    MiqRegion.all.each do |region|
+      ConsumptionHistory.for_report(self, options, region.region) do |consumption|
+        rates_to_apply = rates.get(consumption)
 
-      chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
-      data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')
+        key = report_row_key(consumption)
+        data[key] ||= new(options, consumption, region.region)
 
-      # we are getting hash with metrics and costs for metrics defined for chargeback
-      if Settings[:new_chargeback]
-        data[key].new_chargeback_calculate_costs(consumption, rates_to_apply)
-      else
-        data[key].calculate_costs(consumption, rates_to_apply)
+        chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
+        data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')
+
+        # we are getting hash with metrics and costs for metrics defined for chargeback
+        if Settings[:new_chargeback]
+          data[key].new_chargeback_calculate_costs(consumption, rates_to_apply)
+        else
+          data[key].calculate_costs(consumption, rates_to_apply)
+        end
       end
     end
+
     _log.info("Calculating chargeback costs...Complete")
 
     [data.values]
@@ -77,7 +81,7 @@ def self.groupby_label_value(consumption, groupby_label)
     nil
   end
 
-  def initialize(options, consumption)
+  def initialize(options, consumption, region)
     @options = options
     super()
     if @options[:groupby_tag].present?
@@ -87,7 +91,7 @@ def initialize(options, consumption)
       label_value = self.class.groupby_label_value(consumption, options[:groupby_label])
       self.label_name = label_value.present? ? label_value : _('<Empty>')
     else
-      init_extra_fields(consumption)
+      init_extra_fields(consumption, region)
     end
     self.start_date, self.end_date, self.display_range = options.report_step_range(consumption.timestamp)
     self.interval_name = options.interval

app/models/chargeback/consumption_history.rb

@@ -1,11 +1,11 @@
 class Chargeback
   class ConsumptionHistory
-    def self.for_report(cb_class, options)
-      base_rollup = base_rollup_scope
+    def self.for_report(cb_class, options, region)
+      base_rollup = base_rollup_scope.in_region(region)
       timerange = options.report_time_range
       interval_duration = options.duration_of_report_step
 
-      extra_resources = cb_class.try(:extra_resources_without_rollups) || []
+      extra_resources = cb_class.try(:extra_resources_without_rollups, region) || []
       timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time|
         extra_resources.each do |resource|
           consumption = ConsumptionWithoutRollups.new(resource, query_start_time, query_end_time)
@@ -15,7 +15,7 @@ def self.for_report(cb_class, options)
         next unless options.include_metrics?
 
         records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
-        records = cb_class.where_clause(records, options)
+        records = cb_class.where_clause(records, options, region)
         records = Metric::Helper.remove_duplicate_timestamps(records)
         next if records.empty?
         _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")

app/models/chargeback_container_image.rb

@@ -83,8 +83,8 @@ def self.project(consumption)
     @data_index.fetch_path(:container_project, :by_container_id, consumption.resource_id) || @unknown_project
   end
 
-  def self.where_clause(records, _options)
-    records.where(:resource_type => Container.name, :resource_id => @containers.pluck(:id))
+  def self.where_clause(records, _options, region)
+    records.where(:resource_type => Container.name, :resource_id => @containers.in_region(region).pluck(:id))
   end
 
   def self.report_static_cols
@@ -115,7 +115,7 @@ def self.report_col_options
 
   private
 
-  def init_extra_fields(consumption)
+  def init_extra_fields(consumption, _region)
     self.project_name  = self.class.project(consumption).name
     self.image_name    = self.class.image(consumption).try(:full_name)
     self.project_uid   = self.class.project(consumption).ems_ref

app/models/chargeback_container_project.rb

@@ -46,8 +46,8 @@ def self.build_results_for_report_ChargebackContainerProject(options)
     @projects = nil
   end
 
-  def self.where_clause(records, _options)
-    records.where(:resource_type => ContainerProject.name, :resource_id => @projects.select(:id))
+  def self.where_clause(records, _options, region)
+    records.where(:resource_type => ContainerProject.name, :resource_id => @projects.in_region(region).select(:id))
   end
 
   def self.report_static_cols
@@ -74,7 +74,7 @@ def self.report_col_options
 
   private
 
-  def init_extra_fields(consumption)
+  def init_extra_fields(consumption, _region)
     self.project_name  = consumption.resource_name
     self.project_uid   = consumption.resource.ems_ref
     self.provider_name = consumption.parent_ems.try(:name)

app/models/chargeback_vm.rb

@@ -80,25 +80,25 @@ def self.build_results_for_report_ChargebackVm(options)
     build_results_for_report_chargeback(options)
   end
 
-  def self.where_clause(records, options)
+  def self.where_clause(records, options, region)
     scope = records.where(:resource_type => "VmOrTemplate")
     if options[:tag] && (@report_user.nil? || !@report_user.self_service?)
       scope.for_tag_names(options[:tag].split("/")[2..-1])
     else
-      scope.where(:resource => vms)
+      scope.where(:resource => vms(region))
     end
   end
 
-  def self.extra_resources_without_rollups
+  def self.extra_resources_without_rollups(region)
     # support hyper-v for which we do not collect metrics yet (also when we are including metrics in calculations)
-    scope = @options.include_metrics? ? ManageIQ::Providers::Microsoft::InfraManager::Vm : vms
+    scope = @options.include_metrics? ? ManageIQ::Providers::Microsoft::InfraManager::Vm : vms(region)
     scope = scope.eager_load(:hardware, :taggings, :tags, :host, :ems_cluster, :storage, :ext_management_system,
                              :tenant)
 
     if @options[:tag] && (@report_user.nil? || !@report_user.self_service?)
       scope.find_tagged_with(:any => @options[:tag], :ns => '*')
     else
-      scope.where(:id => vms)
+      scope.where(:id => vms(region))
     end
   end
 
@@ -143,13 +143,14 @@ def self.report_col_options
     }.merge(sub_metric_columns)
   end
 
-  def self.vm_owner(consumption)
-    @vm_owners ||= vms.each_with_object({}) { |vm, res| res[vm.id] = vm.evm_owner_name }
+  def self.vm_owner(consumption, region)
+    @vm_owners ||= vms(region).each_with_object({}) { |vm, res| res[vm.id] = vm.evm_owner_name }
     @vm_owners[consumption.resource_id] ||= consumption.resource.try(:evm_owner_name)
   end
 
-  def self.vms
-    @vms ||=
+  def self.vms(region)
+    @vms ||= {}
+    @vms[region] ||=
       begin
         # Find Vms by user or by tag
         if @options[:entity_id]
@@ -167,6 +168,7 @@ def self.vms
           vms
         elsif @options[:tenant_id]
           tenant = Tenant.find(@options[:tenant_id])
+          tenant = Tenant.in_region(region).find_by(:name => tenant.name)
           if tenant.nil?
             _log.error("Unable to find tenant '#{@options[:tenant_id]}'. Calculating chargeback costs aborted.")
             raise MiqException::Error, "Unable to find tenant '#{@options[:tenant_id]}'"
@@ -187,12 +189,12 @@ def self.vms
 
   private
 
-  def init_extra_fields(consumption)
+  def init_extra_fields(consumption, region)
     self.vm_id         = consumption.resource_id
     self.vm_name       = consumption.resource_name
     self.vm_uid        = consumption.resource.try(:ems_ref)
     self.vm_guid       = consumption.resource.try(:guid)
-    self.owner_name    = self.class.vm_owner(consumption)
+    self.owner_name    = self.class.vm_owner(consumption, region)
     self.provider_name = consumption.parent_ems.try(:name)
     self.provider_uid  = consumption.parent_ems.try(:guid)
   end

spec/models/chargeback_vm_spec.rb

@@ -856,7 +856,7 @@ def result_row_for_vm(vm)
           {'vm_name' => @vm1.name, 'owner_name' => admin.name, 'vm_uid' => 'ems_ref', 'vm_guid' => @vm1.guid,
            'vm_id' => @vm1.id}
         end
-        subject { ChargebackVm.new(report_options, consumption).attributes }
+        subject { ChargebackVm.new(report_options, consumption, MiqRegion.my_region_number).attributes }
 
         before do
           ChargebackVm.instance_variable_set(:@vm_owners, vm_owners)
@@ -933,6 +933,116 @@ def result_row_for_vm(vm)
 
         subject { ChargebackVm.build_results_for_report_ChargebackVm(options).first.first }
 
+        context "with global and remote regions" do
+          let(:options_tenant)  { base_options.merge(:interval => 'monthly', :tenant_id => tenant_1.id).tap { |t| t.delete(:tag) } }
+          let(:vm_global)       { FactoryGirl.create(:vm_vmware) }
+          let!(:region_1) { FactoryGirl.create(:miq_region) }
+
+          def region_id_for(klass, region)
+            klass.id_in_region(klass.count + 1_000_000, region)
+          end
+
+          def find_result_by_vm_name_and_region(chargeback_result, vm_name, region)
+            first_region_id, last_region_id = MiqRegion.region_to_array(region)
+
+            chargeback_result.detect do |result|
+              result.vm_name == vm_name && result.vm_id.between?(first_region_id, last_region_id)
+            end
+          end
+
+          let(:tenant_name_1) { "T1" }
+          let(:tenant_name_2) { "T2" }
+          let(:tenant_name_3) { "T3" }
+
+          let(:vm_name_1) { "VM 1 T1" }
+
+          # BUILD tenants and VMs structure for default region
+          #
+          # T1(vm_1, vm_2) ->
+          #   T2(vm_1, vm_2)
+          #   T3(vm_1, vm_2)
+          let!(:tenant_1) { FactoryGirl.create(:tenant, :parent => Tenant.root_tenant, :name => tenant_name_1, :description => tenant_name_1) }
+          let(:vm_1_t_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1, :name => vm_name_1) }
+          let(:vm_2_t_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1) }
+
+          let(:tenant_2) { FactoryGirl.create(:tenant, :name => tenant_name_2, :parent => tenant_1, :description => tenant_name_2) }
+          let(:vm_1_t_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2) }
+          let(:vm_2_t_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2) }
+
+          let(:tenant_3) { FactoryGirl.create(:tenant, :name => tenant_name_3, :parent => tenant_1, :description => tenant_name_3) }
+          let(:vm_1_t_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3) }
+          let(:vm_2_t_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3) }
+
+          # BUILD tenants and VMs structure for region_1
+          #
+          # T1(vm_1, vm_2) ->
+          #   T2(vm_1, vm_2)
+          #   T3(vm_1, vm_2)
+          #
+          let!(:root_tenant_region_1) do
+            tenant = FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region))
+            tenant.parent = nil
+            tenant.save(:validate => false) # skip validate to set parent = nil
+            tenant
+          end
+
+          let!(:tenant_1_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_1, :parent => root_tenant_region_1, :description => tenant_name_1) }
+          let(:vm_1_region_1_t_1) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1, :name => vm_name_1) }
+          let(:vm_2_region_1_t_1) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) }
+
+          let!(:tenant_2_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_2, :parent => tenant_1_region_1, :description => tenant_name_2) }
+          let(:vm_1_region_1_t_2) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) }
+          let(:vm_2_region_1_t_2) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) }
+
+          let!(:tenant_3_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_3, :parent => tenant_1_region_1, :description => tenant_name_3) }
+          let(:vm_1_region_1_t_3) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_3_region_1) }
+          let(:vm_2_region_1_t_3) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_3_region_1) }
+
+          before do
+            # default region
+            add_metric_rollups_for(vm_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+            add_metric_rollups_for(vm_2_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+            add_metric_rollups_for(vm_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+            add_metric_rollups_for(vm_2_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+            add_metric_rollups_for(vm_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+            add_metric_rollups_for(vm_2_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data)
+
+            # region 1
+            add_metric_rollups_for(vm_1_region_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+            add_metric_rollups_for(vm_2_region_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+            add_metric_rollups_for(vm_1_region_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+            add_metric_rollups_for(vm_2_region_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+            add_metric_rollups_for(vm_1_region_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+            add_metric_rollups_for(vm_2_region_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region)
+          end
+
+          subject! { ChargebackVm.build_results_for_report_ChargebackVm(options_tenant).first }
+
+          it "report from all regions and only for tenant_1" do
+            # report only VMs from tenant 1
+            vm_ids = subject.map(&:vm_id)
+            vm_ids_from_tenant = [tenant_1, tenant_1_region_1].map { |t| t.subtree.map(&:vms).map(&:ids) }.flatten
+            expect(vm_ids).to match_array(vm_ids_from_tenant)
+
+            # default region subject
+            default_region_chargeback = find_result_by_vm_name_and_region(subject, vm_name_1, MiqRegion.my_region_number)
+            used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, vm_1_t_1)
+            expect(default_region_chargeback.cpu_used_metric).to be_within(0.01).of(used_metric)
+            expect(default_region_chargeback.cpu_used_cost).to be_within(0.01).of(used_metric * hourly_rate * hours_in_month)
+            expect(default_region_chargeback.cpu_allocated_cost).to be_within(0.01).of(cpu_count * count_hourly_rate * hours_in_month)
+            expect(default_region_chargeback.cpu_allocated_metric).to eq(cpu_count)
+
+            # region 1
+            region_1_chargeback = find_result_by_vm_name_and_region(subject, vm_name_1, region_1.region)
+            used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, vm_1_region_1_t_1)
+            expect(region_1_chargeback.cpu_used_metric).to be_within(0.01).of(used_metric)
+            expect(region_1_chargeback.cpu_used_cost).to be_within(0.01).of(used_metric * hourly_rate * hours_in_month)
+            expect(region_1_chargeback.cpu_allocated_cost).to be_within(0.01).of(cpu_count * count_hourly_rate * hours_in_month)
+
+            expect(region_1_chargeback.vm_id).to eq(vm_1_region_1_t_1.id)
+          end
+        end
+
         it "cpu" do
           expect(subject.cpu_allocated_metric).to eq(cpu_count)
           used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, @vm1)

spec/support/chargeback_helper.rb

@@ -12,9 +12,10 @@ def used_average_for(metric, hours_in_interval, resource)
         resource.metric_rollups.sum(&metric) / hours_in_interval
       end
 
-      def add_metric_rollups_for(resources, range, step, metric_rollup_params, trait = :with_data)
+      def add_metric_rollups_for(resources, range, step, metric_rollup_params, trait = :with_data, region = nil)
         range.step_value(step).each do |time|
           Array(resources).each do |resource|
+            metric_rollup_params[:id] = region_id_for(MetricRollup, region) if region
             metric_rollup_params[:timestamp]     = time
             metric_rollup_params[:resource_id]   = resource.id
             metric_rollup_params[:resource_name] = resource.name