diff --git a/.ruby-version b/.ruby-version index ec1cf33c..351227fc 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.6.3 +3.2.4 diff --git a/Gemfile b/Gemfile index e20b1260..61ebaeaf 100644 --- a/Gemfile +++ b/Gemfile @@ -1,26 +1,36 @@ +# frozen_string_literal: true + source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.6.3' +ruby '3.2.4' -gem 'rails', '~> 5.2.3' +gem 'activerecord-import' +gem 'bootsnap', '>= 1.1.0', require: false gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' -gem 'bootsnap', '>= 1.1.0', require: false +gem 'rails', '~> 6.1' +gem 'stackprof' +gem 'vernier', '~> 1.0' +# gem 'rack-mini-profiler' +gem 'bullet' +gem 'memory_profiler' +gem "pghero" +gem "pg_query", ">= 2" group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console - gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + gem 'byebug', platforms: %i[mri mingw x64_mingw] end group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. - gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'web-console', '>= 3.3.0' end group :test do end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem -gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] diff --git a/Gemfile.lock b/Gemfile.lock index fccf6f5f..7fc33310 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,150 +1,206 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailbox (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) + mail (>= 2.7.1) + actionmailer (6.1.7.7) + actionpack (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activesupport (= 6.1.7.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) - rack (~> 2.0) + actionpack (6.1.7.7) + actionview (= 6.1.7.7) + activesupport (= 6.1.7.7) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.1.7.7) + actionpack (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) + nokogiri (>= 1.8.5) + actionview (6.1.7.7) + activesupport (= 6.1.7.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.1.7.7) + activesupport (= 6.1.7.7) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) - arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) - marcel (~> 0.3.1) - activesupport (5.2.3) + activemodel (6.1.7.7) + activesupport (= 6.1.7.7) + activerecord (6.1.7.7) + activemodel (= 6.1.7.7) + activesupport (= 6.1.7.7) + activerecord-import (1.6.0) + activerecord (>= 4.2) + activestorage (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activesupport (= 6.1.7.7) + marcel (~> 1.0) + mini_mime (>= 1.1.0) + activesupport (6.1.7.7) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) - msgpack (~> 1.0) - builder (3.2.3) - byebug (11.0.1) - concurrent-ruby (1.1.5) - crass (1.0.4) - erubi (1.8.0) - ffi (1.10.0) - globalid (0.4.2) - activesupport (>= 4.2.0) - i18n (1.6.0) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + bindex (0.8.1) + bootsnap (1.18.3) + msgpack (~> 1.2) + builder (3.2.4) + bullet (7.1.6) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) + byebug (11.1.3) + concurrent-ruby (1.2.3) + crass (1.0.6) + date (3.3.4) + erubi (1.12.0) + ffi (1.16.3) + globalid (1.2.1) + activesupport (>= 6.1) + google-protobuf (4.26.1-arm64-darwin) + rake (>= 13) + i18n (1.14.5) concurrent-ruby (~> 1.0) - listen (3.1.5) + listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - loofah (2.2.3) + loofah (2.22.0) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) + nokogiri (>= 1.12.0) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) - method_source (0.9.2) - mimemagic (0.3.3) - mini_mime (1.0.1) - mini_portile2 (2.4.0) - minitest (5.11.3) - msgpack (1.2.9) - nio4r (2.3.1) - nokogiri (1.10.2) - mini_portile2 (~> 2.4.0) - pg (1.1.4) - puma (3.12.1) - rack (2.0.6) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) - bundler (>= 1.3.0) - railties (= 5.2.3) + net-imap + net-pop + net-smtp + marcel (1.0.4) + memory_profiler (1.0.1) + method_source (1.1.0) + mini_mime (1.1.5) + minitest (5.22.3) + msgpack (1.7.2) + net-imap (0.4.11) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.5.0) + net-protocol + nio4r (2.7.3) + nokogiri (1.16.4-arm64-darwin) + racc (~> 1.4) + pg (1.5.6) + pg_query (5.1.0) + google-protobuf (>= 3.22.3) + pghero (3.4.1) + activerecord (>= 6) + puma (3.12.6) + racc (1.7.3) + rack (2.2.9) + rack-mini-profiler (3.3.1) + rack (>= 1.2.0) + rack-test (2.1.0) + rack (>= 1.3) + rails (6.1.7.7) + actioncable (= 6.1.7.7) + actionmailbox (= 6.1.7.7) + actionmailer (= 6.1.7.7) + actionpack (= 6.1.7.7) + actiontext (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activemodel (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) + bundler (>= 1.15.0) + railties (= 6.1.7.7) sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) method_source - rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) - rake (12.3.2) - rb-fsevent (0.10.3) - rb-inotify (0.10.0) + rake (>= 12.2) + thor (~> 1.0) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) ffi (~> 1.0) - ruby_dep (1.5.0) - sprockets (3.7.2) + sprockets (4.2.1) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.2.1) - actionpack (>= 4.0) - activesupport (>= 4.0) + rack (>= 2.2.4, < 4) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - thor (0.20.3) - thread_safe (0.3.6) - tzinfo (1.2.5) - thread_safe (~> 0.1) + stackprof (0.2.26) + thor (1.3.1) + timeout (0.4.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + uniform_notifier (1.16.0) + vernier (1.0.1) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) bindex (>= 0.4.0) railties (>= 5.0) - websocket-driver (0.7.0) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.5) + zeitwerk (2.6.13) PLATFORMS - ruby + arm64-darwin-23 DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) + bullet byebug listen (>= 3.0.5, < 3.2) + memory_profiler pg (>= 0.18, < 2.0) + pg_query (>= 2) + pghero puma (~> 3.11) - rails (~> 5.2.3) + rack-mini-profiler + rails (~> 6.1) + stackprof tzinfo-data + vernier (~> 1.0) web-console (>= 3.3.0) RUBY VERSION - ruby 2.6.3p62 + ruby 3.2.4p170 BUNDLED WITH - 2.0.2 + 2.4.19 diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..1431925c 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.includes(bus: :services).where(from: @from, to: @to).order(:start_time) end end diff --git a/app/models/buses_service.rb b/app/models/buses_service.rb new file mode 100644 index 00000000..976a0c10 --- /dev/null +++ b/app/models/buses_service.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class BusesService < ApplicationRecord + belongs_to :bus + belongs_to :service +end diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..c941e529 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -7,10 +7,19 @@ <% @trips.each do |trip| %> - <%= render "delimiter" %> + ==================================================== <% end %> diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..c22828e4 --- /dev/null +++ b/case-study.md @@ -0,0 +1,33 @@ +# Case-study Report + +## Актуальная проблема + +Нужно оптимизировать импорт данных и отображение рассписаний: +- `rake reload_json[fixtures/large.json]` должнен выполняться в пределах минуты; +- загрузка страницы `автобусы/Самара/Москва` должна укалывать в 200 мс; + + +## Формирование метрики +Для того, чтобы понимать, дают ли мои изменения положительный эффект на быстродействие программы измеряется время выполнения импорта данных для файла large и скорость закрузки стратрицы. + + +## Главные точки роста +Для того, чтобы найти "точки роста" для оптимизации я воспользовался инструментами rack-mini-profiler, memory-profiler, pghero и bullet. + + +### Hаходки +- Использовал gem activerecord-import и следовал рекомендациям по импорту данных: создал вспомогательные справочники для `cities`, `buses` и использовал `on_duplicate_key_ignore` в связке с `CONSTRAINT for_upsert UNIQUE (bus_id, service_id)` + * Время выполнения импорта файла `fixtures/large.json` составило 24 секунды +- Профилирование показало, что значительное время уходит на `ActiveModel::AttributeMethods#method_missing` при формировании данных для импорта. Для ускорения данные для импорта формировались с id объекта из справочтика, а не с объектом из справочника. + * ![img.png](img.png) + * Время выполнения импорта файла `fixtures/large.json` составило 18 секунд вместо 24 секунд + +- Гем bullet показал наличие n+1 при загрузки страницы `автобусы/Самара/Москва` + * Первым шагом было добалвение загрузки связанных объектов Bus с помощью `includes` для Trip. Время загрузки значительно не изменилось: было 2095.1ms стало 2290ms + * Вторым шагом было добавлние загрузки связанных объектов Services с помощью `includes` для Trip и Bus. Время загрузки уменьшилось до 548ms +- Логи показали много обращений к шаблон для рендера + * Создание общего шаблона для рендера без вложенных шаблонов уменьшило время загрузки страницы `автобусы/Самара/Москва` до 178ms +- Добавление индексов согласно рекомендациям PG Hero сократило время загрузки до 156ms + +## Результаты +В результате проделанной оптимизации удалось обработать большой файл с данными за целевое время и значительно ускорить загрузку страницы diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e4..8468e80a 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,4 +1,13 @@ Rails.application.configure do + # config.after_initialize do + # Bullet.enable = true + # Bullet.alert = true + # Bullet.bullet_logger = true + # Bullet.console = true + # Bullet.rails_logger = true + # Bullet.add_footer = true + # end + # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on diff --git a/config/routes.rb b/config/routes.rb index a2da6a7b..4526e018 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,6 @@ Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html - get "/" => "statistics#index" + # get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" + mount PgHero::Engine, at: "pghero" end diff --git a/db/schema.rb b/db/schema.rb index f6921e45..d96f7b5c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,17 +2,18 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2024_05_11_124734) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| @@ -23,6 +24,7 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id"], name: "index_buses_services_on_bus_id" end create_table "cities", force: :cascade do |t| @@ -40,6 +42,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/img.png b/img.png new file mode 100644 index 00000000..b1b7a9c9 Binary files /dev/null and b/img.png differ diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe871..623a97e9 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,66 @@ +# frozen_string_literal: true + # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] + task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) + # Vernier.run(out: "time_profile.json") do + # profile = StackProf.run(mode: :wall, raw: true) do + # Rails.logger.level = Logger::DEBUG + # ActiveRecord::Base.logger = Logger.new STDOUT - ActiveRecord::Base.transaction do City.delete_all Bus.delete_all Service.delete_all Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) + json = JSON.parse(File.read(args.file_name)) + + ActiveRecord::Base.connection.execute <<-SQL + ALTER TABLE buses_services DROP CONSTRAINT IF EXISTS for_upsert; + ALTER TABLE buses_services + ADD CONSTRAINT for_upsert UNIQUE (bus_id, service_id); + SQL + + ActiveRecord::Base.transaction do + cities = {} + ac_services = {} + ac_buses = {} + trips = [] services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + + json.each do |trip| + cities[trip['from']] ||= City.create!(name: trip['from']) + cities[trip['to']] ||= City.create!(name: trip['to']) + ac_buses[trip['bus']['number']] ||= Bus.create(number: trip['bus']['number'], model: trip['bus']['model']) + + trip['bus']['services'].each do |service| + ac_services[service] ||= Service.create!(name: service) + s = [ac_buses[trip['bus']['number']].id, ac_services[service].id] + services << s + end + + trips << [cities[trip['from']].id, cities[trip['to']].id, trip['start_time'], trip['duration_minutes'], + trip['price_cents'], ac_buses[trip['bus']['number']].id] + + next unless trips.length == 1000 + + BusesService.import %i[bus_id service_id], services, validate: false, on_duplicate_key_ignore: true + Trip.import %i[from_id to_id start_time duration_minutes price_cents bus_id], trips, validate: false + trips = [] + services = [] end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) + + BusesService.import %i[bus_id service_id], services, validate: false, on_duplicate_key_ignore: true + Trip.import %i[from_id to_id start_time duration_minutes price_cents bus_id], trips, validate: false + # puts format('MEMORY USAGE: %d MB', (`ps -o rss= -p #{Process.pid}`.to_i / 1024)) end - end + # end + + ActiveRecord::Base.connection.execute <<-SQL + ALTER TABLE buses_services DROP CONSTRAINT IF EXISTS for_upsert; + SQL + File.write('stackprof.json', JSON.generate(profile)) + # end end