From 9331f067843236c3f084ea4fc696697fb796efdf Mon Sep 17 00:00:00 2001 From: rubic0n Date: Wed, 3 Feb 2021 00:48:19 -0600 Subject: [PATCH 1/7] Add rack-mini-profiler, to diagnose perf issues --- Gemfile | 1 + Gemfile.lock | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index 6d3c4281..82f43fbb 100644 --- a/Gemfile +++ b/Gemfile @@ -141,6 +141,7 @@ group :development do gem 'capistrano-yarn', '~> 2.0' gem 'derailed_benchmarks' + gem 'rack-mini-profiler' gem 'stackprof' end diff --git a/Gemfile.lock b/Gemfile.lock index 35b7fe0e..60cd1d74 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -455,6 +455,8 @@ GEM rack (>= 1.0, < 3) rack-cors (1.1.1) rack (>= 2.0.0) + rack-mini-profiler (2.3.1) + rack (>= 1.2.0) rack-proxy (0.6.5) rack rack-test (1.1.0) @@ -753,6 +755,7 @@ DEPENDENCIES pundit (~> 2.0) rack-attack (~> 6.0) rack-cors (~> 1.0) + rack-mini-profiler rails (= 6.0.3.4) rails-controller-testing (~> 1.0) rails-i18n (~> 6.0.0) From 1179957a32372093b60d8331c547af512c1075ad Mon Sep 17 00:00:00 2001 From: rubic0n Date: Wed, 3 Feb 2021 00:48:40 -0600 Subject: [PATCH 2/7] Start redis server on VM boot Otherwise you have to start it yourself after vagrant halt / vagrant up --- Vagrantfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 4ca75ecf..11cddd77 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -48,8 +48,8 @@ sudo apt-get install \ libpam0g-dev \ -y -# Start Redis Server -sudo service redis-server start +# Start Redis Server on boot (and right now) +sudo systemctl enable --now redis-server # Install rvm read RUBY_VERSION < .ruby-version From 6221581cf064d7b1a931088b286ab6510c7afec2 Mon Sep 17 00:00:00 2001 From: rubic0n Date: Wed, 3 Feb 2021 01:06:42 -0600 Subject: [PATCH 3/7] Create script to generate dummy data, for perf investigation --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- lib/tasks/dummy_data.rake | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 lib/tasks/dummy_data.rake diff --git a/Gemfile b/Gemfile index 82f43fbb..80b5e867 100644 --- a/Gemfile +++ b/Gemfile @@ -111,7 +111,6 @@ end group :test do gem 'capybara', '~> 3.22' gem 'climate_control', '~> 0.2' - gem 'faker', '~> 1.9' gem 'microformats', '~> 4.1' gem 'rails-controller-testing', '~> 1.0' gem 'rspec-sidekiq', '~> 3.0' @@ -126,6 +125,7 @@ group :development do gem 'better_errors', '~> 2.5' gem 'binding_of_caller', '~> 0.7' gem 'bullet', '~> 6.0' + gem 'faker', '~> 2.15' gem 'listen' gem 'letter_opener', '~> 1.7' gem 'letter_opener_web', '~> 1.3' diff --git a/Gemfile.lock b/Gemfile.lock index 60cd1d74..12f89d88 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -227,8 +227,8 @@ GEM tzinfo excon (0.78.1) fabrication (2.21.1) - faker (1.9.6) - i18n (>= 0.7) + faker (2.15.1) + i18n (>= 1.6, < 2) faraday (1.3.0) faraday-net_http (~> 1.0) multipart-post (>= 1.2, < 3) @@ -296,7 +296,7 @@ GEM httplog (1.4.3) rack (>= 1.0) rainbow (>= 2.0.0) - i18n (1.8.7) + i18n (1.8.8) concurrent-ruby (~> 1.0) i18n-tasks (0.9.33) activesupport (>= 4.0.2) @@ -703,7 +703,7 @@ DEPENDENCIES dotenv-rails (~> 2.7) elastic-apm (~> 3.13) fabrication (~> 2.20) - faker (~> 1.9) + faker (~> 2.15) fast_blank (~> 1.0) fastimage fog-core (<= 2.1.0) diff --git a/lib/tasks/dummy_data.rake b/lib/tasks/dummy_data.rake new file mode 100644 index 00000000..41b5911d --- /dev/null +++ b/lib/tasks/dummy_data.rake @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +desc 'Import dummy data into the database, to create a more realistic development experience' +task dummy_data: :environment do + raise 'Only run this in development' unless Rails.env.development? + + domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain + + admin = Account.find_by!(username: 'admin') + + accounts = 1.upto(1_000).map do |num| + username = "#{Faker::Internet.username(separators: %w[_])}#{num}" + password = Faker::Internet.password + + Account.create!(username: username).tap do |account| + account.create_user!({ + email: "#{username}@#{domain}", + password: password, + password_confirmation: password, + agreement: true, + approved: true + }) + end + end + + accounts.each do |acct| + FollowService.new.call(admin, acct) + end + + statuses = (accounts + [admin]).map do |account| + PostStatusService.new.call(account, text: Faker::Lorem.paragraph(sentence_count: 20)) + end + + accounts.sample(200).zip(statuses.sample(200)).each do |account, status| + ReblogService.new.call(account, status) + end +end From 70c44fc68dde4281c28498e8b299279d74bd8d94 Mon Sep 17 00:00:00 2001 From: rubic0n Date: Sat, 13 Feb 2021 22:49:29 -0600 Subject: [PATCH 4/7] Fix vagrant now that Rails 6 replicas are being used --- .env.vagrant | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.vagrant b/.env.vagrant index 28c347c4..8c5a21f4 100644 --- a/.env.vagrant +++ b/.env.vagrant @@ -1,3 +1,6 @@ VAGRANT=true LOCAL_DOMAIN=gabsocial.local DB_HOST=/var/run/postgresql/ +DB_MASTER_URL=postgres:///gabsocial_development +DB_SLAVE1_URL=postgres:///gabsocial_development +DB_SLAVE2_URL=postgres:///gabsocial_development From 903b6c79692cd6f9c50b2aa3bd01ccd71b6854d4 Mon Sep 17 00:00:00 2001 From: rubic0n Date: Sat, 13 Feb 2021 22:51:23 -0600 Subject: [PATCH 5/7] Simplify database.yml by merging in default options Anything in the &default section can be merged in using <<: *default In this case, the default already specifies that the adapter is postgresql, the timeout is 5000, the encoding is unicode. It also already specified the db pool, but the default actually checked 2 different ENVs and provided a fallback if it's not provided. Lastly, I was able to move the prepared_statements section to the defaults, because the .env.vagrant file doesn't specify ENV['PREPARED_STATEMENTS'], development still just sets it to 'false'. --- config/database.yml | 38 +++++++------------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/config/database.yml b/config/database.yml index 8c483cd3..e60018bd 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,26 +4,18 @@ default: &default timeout: 5000 encoding: unicode sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> + prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'false' %> development: master: - adapter: postgresql + <<: *default url: <%= ENV['DB_MASTER_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode slave1: - adapter: postgresql + <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode slave2: - adapter: postgresql + <<: *default url: <%= ENV['DB_SLAVE2_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -46,27 +38,11 @@ test: # prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %> production: master: - adapter: postgresql - sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> - prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'false' %> + <<: *default url: <%= ENV['DB_MASTER_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode slave1: - adapter: postgresql - sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> - prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'false' %> + <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode slave2: - adapter: postgresql - sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> - prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'false' %> + <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> - pool: <%= ENV['DB_POOL'] %> - timeout: 5000 - encoding: unicode - From f472a6154e84d41e94c0fe68b82df7198247b863 Mon Sep 17 00:00:00 2001 From: rubic0n Date: Sat, 13 Feb 2021 22:59:02 -0600 Subject: [PATCH 6/7] Mark the replicas as replicas Rails won't even attempt to write to a replica. Without this option, a write attempt would actually run against Postgres, and it would be up to Postgres to throw a readonly error. Additionally, marking it as a replica teaches ActiveRecord that it should not attempt to run migrations against this connection. https://api.rubyonrails.org/classes/ActiveRecord/DatabaseConfigurations.html#method-i-configs_for I'm actually pretty sure that the lack of `replica: true` is why there's currently a db/slave1_schema.rb and db/slave2_schema.rb --- config/database.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/database.yml b/config/database.yml index e60018bd..03878e8d 100644 --- a/config/database.yml +++ b/config/database.yml @@ -13,9 +13,11 @@ development: slave1: <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> + replica: true slave2: <<: *default url: <%= ENV['DB_SLAVE2_URL'] %> + replica: true # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -43,6 +45,8 @@ production: slave1: <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> + replica: true slave2: <<: *default url: <%= ENV['DB_SLAVE1_URL'] %> + replica: true From 73c0e509360e1f96bd01c7d3c4d1519371d3e079 Mon Sep 17 00:00:00 2001 From: rubic0n Date: Sat, 13 Feb 2021 23:02:44 -0600 Subject: [PATCH 7/7] Prevent constant ElasticAPM warnings in development by disabling it No changes are needed in prod, since the `enabled` setting defaults to "true", and prod obviously isn't setting this ENV. --- .env.vagrant | 1 + config/elastic_apm.yml | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.env.vagrant b/.env.vagrant index 8c5a21f4..36fe6493 100644 --- a/.env.vagrant +++ b/.env.vagrant @@ -4,3 +4,4 @@ DB_HOST=/var/run/postgresql/ DB_MASTER_URL=postgres:///gabsocial_development DB_SLAVE1_URL=postgres:///gabsocial_development DB_SLAVE2_URL=postgres:///gabsocial_development +ELASTIC_APM_ENABLED=false diff --git a/config/elastic_apm.yml b/config/elastic_apm.yml index 1a35ed53..d359457b 100644 --- a/config/elastic_apm.yml +++ b/config/elastic_apm.yml @@ -2,10 +2,12 @@ # Set service name - allowed characters: a-z, A-Z, 0-9, -, _ and space # Defaults to the name of your Rails app -service_name: <%= ENV['ELASTIC_APM_SERVICE_NAME'] || 'gabsocial' %> +service_name: <%= ENV['ELASTIC_APM_SERVICE_NAME'] || 'gabsocial' %> # Use if APM Server requires a token -secret_token: <%= ENV['ELASTIC_APM_SECRET_TOKEN'] || '' %> +secret_token: <%= ENV['ELASTIC_APM_SECRET_TOKEN'] || '' %> # Set custom APM Server URL (default: http://localhost:8200) -server_url: <%= ENV['ELASTIC_APM_URL'] || 'http://localhost:8200'%> +server_url: <%= ENV['ELASTIC_APM_URL'] || 'http://localhost:8200'%> + +enabled: <%= ENV['ELASTIC_APM_ENABLED'] || 'true' %>