diff --git a/Gemfile b/Gemfile index e20b1260..dbd7ba62 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,14 @@ gem 'rails', '~> 5.2.3' gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false +gem 'rack-mini-profiler' +gem 'bullet' +gem 'ruby-prof' +gem 'stackprof' +gem 'memory_profiler' +gem 'oj' +gem "pghero" +gem "pg_query", ">= 2" group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console @@ -17,6 +25,7 @@ group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'meta_request' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..aead7a62 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,145 +1,188 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (5.2.8.1) + actionpack (= 5.2.8.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailer (5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) - rack (~> 2.0) + actionpack (5.2.8.1) + actionview (= 5.2.8.1) + activesupport (= 5.2.8.1) + rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + actionview (5.2.8.1) + activesupport (= 5.2.8.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + activejob (5.2.8.1) + activesupport (= 5.2.8.1) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) + activemodel (5.2.8.1) + activesupport (= 5.2.8.1) + activerecord (5.2.8.1) + activemodel (= 5.2.8.1) + activesupport (= 5.2.8.1) arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) - marcel (~> 0.3.1) - activesupport (5.2.3) + activestorage (5.2.8.1) + actionpack (= 5.2.8.1) + activerecord (= 5.2.8.1) + marcel (~> 1.0.0) + activesupport (5.2.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) - msgpack (~> 1.0) - builder (3.2.3) - byebug (11.0.1) - concurrent-ruby (1.1.5) - crass (1.0.4) - erubi (1.8.0) - ffi (1.10.0) - globalid (0.4.2) - activesupport (>= 4.2.0) - i18n (1.6.0) + bindex (0.8.1) + bootsnap (1.16.0) + msgpack (~> 1.2) + builder (3.2.4) + bullet (7.0.7) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) + byebug (11.1.3) + concurrent-ruby (1.2.2) + crass (1.0.6) + date (3.3.3) + erubi (1.12.0) + ffi (1.15.5) + globalid (1.1.0) + activesupport (>= 5.0) + google-protobuf (3.22.2) + i18n (1.12.0) concurrent-ruby (~> 1.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) - loofah (2.2.3) + loofah (2.19.1) crass (~> 1.0.2) nokogiri (>= 1.5.9) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) - method_source (0.9.2) - mimemagic (0.3.3) - mini_mime (1.0.1) - mini_portile2 (2.4.0) - minitest (5.11.3) - msgpack (1.2.9) - nio4r (2.3.1) - nokogiri (1.10.2) - mini_portile2 (~> 2.4.0) - pg (1.1.4) - puma (3.12.1) - rack (2.0.6) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) + net-imap + net-pop + net-smtp + marcel (1.0.2) + memory_profiler (1.0.1) + meta_request (0.7.4) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7.1) + method_source (1.0.0) + mini_mime (1.1.2) + mini_portile2 (2.8.1) + minitest (5.18.0) + msgpack (1.6.1) + net-imap (0.3.4) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.1) + timeout + net-smtp (0.3.3) + net-protocol + nio4r (2.5.8) + nokogiri (1.13.10) + mini_portile2 (~> 2.8.0) + racc (~> 1.4) + oj (3.14.2) + pg (1.4.6) + pg_query (4.2.0) + google-protobuf (>= 3.19.2) + pghero (2.8.3) + activerecord (>= 5) + puma (3.12.6) + racc (1.6.2) + rack (2.2.6.4) + rack-contrib (2.3.0) + rack (~> 2.0) + rack-mini-profiler (3.0.0) + rack (>= 1.2.0) + rack-test (2.1.0) + rack (>= 1.3) + rails (5.2.8.1) + actioncable (= 5.2.8.1) + actionmailer (= 5.2.8.1) + actionpack (= 5.2.8.1) + actionview (= 5.2.8.1) + activejob (= 5.2.8.1) + activemodel (= 5.2.8.1) + activerecord (= 5.2.8.1) + activestorage (= 5.2.8.1) + activesupport (= 5.2.8.1) bundler (>= 1.3.0) - railties (= 5.2.3) + railties (= 5.2.8.1) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (1.5.0) + loofah (~> 2.19, >= 2.19.1) + railties (5.2.8.1) + actionpack (= 5.2.8.1) + activesupport (= 5.2.8.1) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (12.3.2) - rb-fsevent (0.10.3) - rb-inotify (0.10.0) + rake (13.0.6) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) ffi (~> 1.0) + ruby-prof (1.4.3) ruby_dep (1.5.0) - sprockets (3.7.2) + sprockets (4.2.0) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.1) - actionpack (>= 4.0) - activesupport (>= 4.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - thor (0.20.3) + stackprof (0.2.24) + thor (1.2.1) thread_safe (0.3.6) - tzinfo (1.2.5) + timeout (0.3.2) + tzinfo (1.2.11) thread_safe (~> 0.1) + uniform_notifier (1.16.0) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) bindex (>= 0.4.0) railties (>= 5.0) - websocket-driver (0.7.0) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.5) PLATFORMS ruby DEPENDENCIES bootsnap (>= 1.1.0) + bullet byebug listen (>= 3.0.5, < 3.2) + memory_profiler + meta_request + oj pg (>= 0.18, < 2.0) + pg_query (>= 2) + pghero puma (~> 3.11) + rack-mini-profiler rails (~> 5.2.3) + ruby-prof + stackprof tzinfo-data web-console (>= 3.3.0) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..1431925c 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.includes(bus: :services).where(from: @from, to: @to).order(:start_time) end end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad0..00000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c0..00000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639fc..00000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • - diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb deleted file mode 100644 index fa1de9aa..00000000 --- a/app/views/trips/_trip.html.erb +++ /dev/null @@ -1,5 +0,0 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..4e26ce11 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -1,16 +1,27 @@ -

    - <%= "Автобусы #{@from.name} – #{@to.name}" %> -

    -

    - <%= "В расписании #{@trips.count} рейсов" %> -

    +<%= cache([@from.name, @to.name]) do %> +

    + <%= "Автобусы #{@from.name} – #{@to.name}" %> +

    +

    + <%= "В расписании #{@trips.count} рейсов" %> +

    -<% @trips.each do |trip| %> - - <%= render "delimiter" %> -<% end %> + <% @trips.each do |trip| %> + + ==================================================== + <% end %> +<% end %> \ No newline at end of file diff --git a/bin/setup b/bin/setup index f294207b..5534a287 100755 --- a/bin/setup +++ b/bin/setup @@ -28,8 +28,8 @@ chdir APP_ROOT do puts "\n== Preparing database ==" system! 'bin/rails db:setup' - puts "\n== Loading data from fixtures/small.json ==" - system! 'bin/rake reload_json[fixtures/small.json]' + puts "\n== Loading data from fixtures/large.json ==" + system! 'bin/rake reload_json[fixtures/large.json]' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..6577fee0 --- /dev/null +++ b/case-study.md @@ -0,0 +1,24 @@ +# Case-study оптимизации + +## Оптимизация импорта +Изначально время импорта файла large.json занимало ~450s +Я переписал импорт сразу с потоковой записью в `Postgres`. После рефакторинга время обарботки файла large.json стало ~13s + +## Оптимизация рендеринга страницы +Изначальное время рендеринга страницы с данными из файла large.json было ~34000ms. + +### Правка №1 +bullet и rack mini profiler указывали на проблему n+1 +Добавил includes к формированию списка поездок: +@trips = Trip.includes(bus: :services).where(from: @from, to: @to).order(:start_time) +После рефакторинга время рендера страницы изменилось с ~34000ms до ~24000ms + +### Правка №2 +Rails panel указвыл, что практически все время тратиться на рендеринг шаблонов. Решил убрать все шаблоны и формировать html сразу на одной странице, тк он достаточно простой. +После рефакторинга время рендера страницы изменилось с ~24000ms до ~6000ms + +### Правка №3 +Решил установить pg_hero и проанализировать запросы. pg_hero не предлавагал добавлять никакие индексы и все было в норме. Но я добавил индыксы на имя города, и составной индекс для трипов по from_id, to_id. Какой-то прибавки к скорости рендера это не дало, все осталось в пределах погрешности. + +### Правка №4 +Принял решение добавить кэширование. Закэшировал сразу всю страницу index. Время рендера закешированной версии страницы изменилось с ~6000ms до ~250ms diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..bc8c0849 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -58,4 +58,14 @@ # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + + config.after_initialize do + Bullet.enable = true + Bullet.bullet_logger = true + Bullet.console = true + Bullet.rails_logger = true + Bullet.add_footer = true + Bullet.skip_html_injection = false + Bullet.stacktrace_includes = [ 'your_gem', 'your_middleware' ] + end end diff --git a/config/routes.rb b/config/routes.rb index a2da6a7b..84053141 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,7 @@ Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html + mount PgHero::Engine, at: "pghero" + get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" end diff --git a/db/migrate/20230327052906_create_pghero_query_stats.rb b/db/migrate/20230327052906_create_pghero_query_stats.rb new file mode 100644 index 00000000..fbf41263 --- /dev/null +++ b/db/migrate/20230327052906_create_pghero_query_stats.rb @@ -0,0 +1,15 @@ +class CreatePgheroQueryStats < ActiveRecord::Migration[5.2] + def change + create_table :pghero_query_stats do |t| + t.text :database + t.text :user + t.text :query + t.integer :query_hash, limit: 8 + t.float :total_time + t.integer :calls, limit: 8 + t.timestamp :captured_at + end + + add_index :pghero_query_stats, [:database, :captured_at] + end +end diff --git a/db/migrate/20230327070032_add_indexes.rb b/db/migrate/20230327070032_add_indexes.rb new file mode 100644 index 00000000..9f22feab --- /dev/null +++ b/db/migrate/20230327070032_add_indexes.rb @@ -0,0 +1,6 @@ +class AddIndexes < ActiveRecord::Migration[5.2] + def change + add_index :cities, :name + add_index :trips, [:from_id, :to_id] + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..07f86208 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,10 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2023_03_27_070032) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| @@ -27,6 +28,18 @@ create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name" + end + + create_table "pghero_query_stats", force: :cascade do |t| + t.text "database" + t.text "user" + t.text "query" + t.bigint "query_hash" + t.float "total_time" + t.bigint "calls" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" end create_table "services", force: :cascade do |t| @@ -40,6 +53,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/lib/data_import.rb b/lib/data_import.rb new file mode 100644 index 00000000..5df75dfc --- /dev/null +++ b/lib/data_import.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +class DataImport + def initialize(file_name) + @file_name = file_name + @cities = {} + @buses = {} + @buses_services = {} + end + + def call + time = Benchmark.realtime do + @cities = {} + @buses = {} + @buses_services = {} + + ActiveRecord::Base.transaction do + connection = ActiveRecord::Base.connection.raw_connection + trips_command = "copy trips (from_id, to_id, start_time, duration_minutes, price_cents, bus_id) from stdin with csv delimiter ';'" + connection.copy_data trips_command do + File.open(@file_name) do |ff| + nesting = 0 + str = +"" + + while !ff.eof? + ch = ff.read(1) # читаем по одному символу + case + when ch == '{' # начинается объект, повышается вложенность + nesting += 1 + str << ch + when ch == '}' # заканчивается объект, понижается вложенность + nesting -= 1 + str << ch + if nesting == 0 # если закончился объкет уровня trip, парсим и импортируем его + trip = Oj.load(str) + import_trip(trip, connection) + str = +"" + end + when nesting >= 1 + str << ch + end + end + end + end + + services_command = "copy services (name) from stdin with csv delimiter ';'" + connection.copy_data services_command do + Service::SERVICES.each do |service| + connection.put_copy_data("#{service}\n") + end + end + + cities_command = "copy cities (name) from stdin with csv delimiter ';'" + connection.copy_data cities_command do + @cities.keys.each do |city_name| + connection.put_copy_data("#{city_name}\n") + end + end + + buses_command = "copy buses (model, number) from stdin with csv delimiter ';'" + connection.copy_data buses_command do + @buses.keys.each do |bus_key| + connection.put_copy_data("#{bus_key}\n") + end + end + + buses_services_command = "copy buses_services (bus_id, service_id) from stdin with csv delimiter ';'" + connection.copy_data buses_services_command do + @buses_services.each do |bus_id, service_ids| + service_ids.each do |service_id| + connection.put_copy_data("#{bus_id};#{service_id}\n") + end + end + end + end + end + puts "Finish in #{time.round(2)}" + end + + + private + + + def import_trip(trip, connection) + from_id = @cities[trip['from']] + if !from_id + from_id = @cities.size + 1 + @cities[trip['from']] = from_id + end + + to_id = @cities[trip['to']] + if !to_id + to_id = @cities.size + 1 + @cities[trip['to']] = to_id + end + + bus = trip['bus'] + bus_key = "#{bus['model']};#{bus['number']}" + bus_id = @buses[bus_key] + if !bus_id + bus_id = @buses.size + 1 + @buses[bus_key] = bus_id + end + service_ids = @buses_services[bus_id] + if !service_ids + @buses_services[bus_id] = [] + bus['services'].each do |service| + service_id = Service::SERVICES.index(service) + 1 + @buses_services[bus_id] << service_id + end + end + + # стримим подготовленный чанк данных в postgres + connection.put_copy_data("#{from_id};#{to_id};#{trip['start_time']};#{trip['duration_minutes']};#{trip['price_cents']};#{bus_id}\n") + end +end \ No newline at end of file diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..0bb084d6 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,7 @@ -# Наивная загрузка данных из json-файла в БД +require 'data_import' + # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end - end + DataImport.new(args.file_name).call end +