random-mirrors/gab-social
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Gab Social. All are welcome.

Browse Source
This commit is contained in:
robcolbert
2019-07-02 03:10:25 -04:00
commit bd0b5afc92
5366 changed files with 222812 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

487
lib/gabsocial/accounts_cli.rb Normal file
View File

@@ -0,0 +1,487 @@
# frozen_string_literal: true
require 'set'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class AccountsCLI < Thor
def self.exit_on_failure?
true
end
option :all, type: :boolean
desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
long_desc <<-LONG_DESC
Generate and broadcast new RSA keys as part of security
maintenance.
With the --all option, all local accounts will be subject
to the rotation. Otherwise, and by default, only a single
account specified by the USERNAME argument will be
processed.
LONG_DESC
def rotate(username = nil)
if options[:all]
processed = 0
delay = 0
Account.local.without_suspended.find_in_batches do |accounts|
accounts.each do |account|
rotate_keys_for_account(account, delay)
processed += 1
say('.', :green, false)
end
delay += 5.minutes
end
say
say("OK, rotated keys for #{processed} accounts", :green)
elsif username.present?
rotate_keys_for_account(Account.find_local(username))
say('OK', :green)
else
say('No account(s) given', :red)
exit(1)
end
end
option :email, required: true
option :confirmed, type: :boolean
option :role, default: 'user'
option :reattach, type: :boolean
option :force, type: :boolean
desc 'create USERNAME', 'Create a new user'
long_desc <<-LONG_DESC
Create a new user account with a given USERNAME and an
e-mail address provided with --email.
With the --confirmed option, the confirmation e-mail will
be skipped and the account will be active straight away.
With the --role option one of "user", "admin" or "moderator"
can be supplied. Defaults to "user"
With the --reattach option, the new user will be reattached
to a given existing username of an old account. If the old
account is still in use by someone else, you can supply
the --force option to delete the old record and reattach the
username to the new account anyway.
LONG_DESC
def create(username)
account = Account.new(username: username)
password = SecureRandom.hex
user = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil)
if options[:reattach]
account = Account.find_local(username) || Account.new(username: username)
if account.user.present? && !options[:force]
say('The chosen username is currently in use', :red)
say('Use --force to reattach it anyway and delete the other user')
return
elsif account.user.present?
account.user.destroy!
end
end
account.suspended_at = nil
user.account = account
if user.save
if options[:confirmed]
user.confirmed_at = nil
user.confirm!
end
say('OK', :green)
say("New password: #{password}")
else
user.errors.to_h.each do |key, error|
say('Failure/Error: ', :red)
say(key)
say(' ' + error, :red)
end
exit(1)
end
end
option :role
option :email
option :confirm, type: :boolean
option :enable, type: :boolean
option :disable, type: :boolean
option :disable_2fa, type: :boolean
option :approve, type: :boolean
desc 'modify USERNAME', 'Modify a user'
long_desc <<-LONG_DESC
Modify a user account.
With the --role option, update the user's role to one of "user",
"moderator" or "admin".
With the --email option, update the user's e-mail address. With
the --confirm option, mark the user's e-mail as confirmed.
With the --disable option, lock the user out of their account. The
--enable option is the opposite.
With the --approve option, the account will be approved, if it was
previously not due to not having open registrations.
With the --disable-2fa option, the two-factor authentication
requirement for the user can be removed.
LONG_DESC
def modify(username)
user = Account.find_local(username)&.user
if user.nil?
say('No user with such username', :red)
exit(1)
end
if options[:role]
user.admin = options[:role] == 'admin'
user.moderator = options[:role] == 'moderator'
end
user.email = options[:email] if options[:email]
user.disabled = false if options[:enable]
user.disabled = true if options[:disable]
user.approved = true if options[:approve]
user.otp_required_for_login = false if options[:disable_2fa]
user.confirm if options[:confirm]
if user.save
say('OK', :green)
else
user.errors.to_h.each do |key, error|
say('Failure/Error: ', :red)
say(key)
say(' ' + error, :red)
end
exit(1)
end
end
desc 'delete USERNAME', 'Delete a user'
long_desc <<-LONG_DESC
Remove a user account with a given USERNAME.
LONG_DESC
def delete(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
SuspendAccountService.new.call(account, including_user: true)
say('OK', :green)
end
desc 'backup USERNAME', 'Request a backup for a user'
long_desc <<-LONG_DESC
Request a new backup for an account with a given USERNAME.
The backup will be created in Sidekiq asynchronously, and
the user will receive an e-mail with a link to it once
it's done.
LONG_DESC
def backup(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
backup = account.user.backups.create!
BackupWorker.perform_async(backup.id)
say('OK', :green)
end
option :dry_run, type: :boolean
desc 'cull', 'Remove remote accounts that no longer exist'
long_desc <<-LONG_DESC
Query every single remote account in the database to determine
if it still exists on the origin server, and if it doesn't,
remove it from the database.
Accounts that have had confirmed activity within the last week
are excluded from the checks.
Domains that are unreachable are not checked.
With the --dry-run option, no deletes will actually be carried
out.
LONG_DESC
def cull
skip_threshold = 7.days.ago
culled = 0
dry_run_culled = []
skip_domains = Set.new
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Account.remote.where(protocol: :activitypub).partitioned.find_each do |account|
next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold)
code = 0
unless skip_domains.include?(account.domain)
begin
code = Request.new(:head, account.uri).perform(&:code)
rescue HTTP::ConnectionError
skip_domains << account.domain
rescue StandardError
next
end
end
if [404, 410].include?(code)
if options[:dry_run]
dry_run_culled << account.acct
else
SuspendAccountService.new.call(account, destroy: true)
end
culled += 1
say('+', :green, false)
else
account.touch # Touch account even during dry run to avoid getting the account into the window again
say('.', nil, false)
end
end
say
say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow)
unless skip_domains.empty?
say('The following servers were not available during the check:', :yellow)
skip_domains.each { |domain| say(' ' + domain) }
end
unless dry_run_culled.empty?
say('The following accounts would have been deleted:', :green)
dry_run_culled.each { |account| say(' ' + account) }
end
end
option :all, type: :boolean
option :domain
desc 'refresh [USERNAME]', 'Fetch remote user data and files'
long_desc <<-LONG_DESC
Fetch remote user data and files for one or multiple accounts.
With the --all option, all remote accounts will be processed.
Through the --domain option, this can be narrowed down to a
specific domain only. Otherwise, a single remote account must
be specified with USERNAME.
All processing is done in the background through Sidekiq.
LONG_DESC
def refresh(username = nil)
if options[:domain] || options[:all]
queued = 0
scope = Account.remote
scope = scope.where(domain: options[:domain]) if options[:domain]
scope.select(:id).reorder(nil).find_in_batches do |accounts|
Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts.map(&:id))
queued += accounts.size
end
say("Scheduled refreshment of #{queued} accounts", :green, true)
elsif username.present?
username, domain = username.split('@')
account = Account.find_remote(username, domain)
if account.nil?
say('No such account', :red)
exit(1)
end
Maintenance::RedownloadAccountMediaWorker.perform_async(account.id)
say('OK', :green)
else
say('No account(s) given', :red)
exit(1)
end
end
desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT'
long_desc <<-LONG_DESC
Make all local accounts follow an account specified by ACCT. ACCT can be
a simple username, in case of a local user. It can also be in the format
username@domain, in case of a remote user.
LONG_DESC
def follow(acct)
target_account = ResolveAccountService.new.call(acct)
processed = 0
failed = 0
if target_account.nil?
say("Target account (#{acct}) could not be resolved", :red)
exit(1)
end
Account.local.without_suspended.find_each do |account|
begin
FollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, followed target from #{processed} accounts, skipped #{failed}", :green)
end
desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT'
long_desc <<-LONG_DESC
Make all local accounts unfollow an account specified by ACCT. ACCT can be
a simple username, in case of a local user. It can also be in the format
username@domain, in case of a remote user.
LONG_DESC
def unfollow(acct)
target_account = Account.find_remote(*acct.split('@'))
processed = 0
failed = 0
if target_account.nil?
say("Target account (#{acct}) was not found", :red)
exit(1)
end
target_account.followers.local.find_each do |account|
begin
UnfollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, unfollowed target from #{processed} accounts, skipped #{failed}", :green)
end
option :follows, type: :boolean, default: false
option :followers, type: :boolean, default: false
desc 'reset-relationships USERNAME', 'Reset all follows and/or followers for a user'
long_desc <<-LONG_DESC
Reset all follows and/or followers for a user specified by USERNAME.
With the --follows option, the command unfollows everyone that the account follows,
and then re-follows the users that would be followed by a brand new account.
With the --followers option, the command removes all followers of the account.
LONG_DESC
def reset_relationships(username)
unless options[:follows] || options[:followers]
say('Please specify either --follows or --followers, or both', :red)
exit(1)
end
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
if options[:follows]
processed = 0
failed = 0
say("Unfollowing #{account.username}'s followees, this might take a while...")
Account.where(id: ::Follow.where(account: account).select(:target_account_id)).find_each do |target_account|
begin
UnfollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
BootstrapTimelineWorker.perform_async(account.id)
say("OK, unfollowed #{processed} followees, skipped #{failed}", :green)
end
if options[:followers]
processed = 0
failed = 0
say("Removing #{account.username}'s followers, this might take a while...")
Account.where(id: ::Follow.where(target_account: account).select(:account_id)).find_each do |target_account|
begin
UnfollowService.new.call(target_account, account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, removed #{processed} followers, skipped #{failed}", :green)
end
end
option :number, type: :numeric, aliases: [:n]
option :all, type: :boolean
desc 'approve [USERNAME]', 'Approve pending accounts'
long_desc <<~LONG_DESC
When registrations require review from staff, approve pending accounts,
either all of them with the --all option, or a specific number of them
specified with the --number (-n) option, or only a single specific
account identified by its username.
LONG_DESC
def approve(username = nil)
if options[:all]
User.pending.find_each(&:approve!)
say('OK', :green)
elsif options[:number]
User.pending.limit(options[:number]).each(&:approve!)
say('OK', :green)
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
account.user&.approve!
say('OK', :green)
else
exit(1)
end
end
private
def rotate_keys_for_account(account, delay = 0)
if account.nil?
say('No such account', :red)
exit(1)
end
old_key = account.private_key
new_key = OpenSSL::PKey::RSA.new(2048)
account.update(private_key: new_key.to_pem, public_key: new_key.public_key.to_pem)
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, sign_with: old_key)
end
end
end

19
lib/gabsocial/cache_cli.rb Normal file
View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class CacheCLI < Thor
def self.exit_on_failure?
true
end
desc 'clear', 'Clear the cache storage'
def clear
Rails.cache.clear
say('OK', :green)
end
end
end

9
lib/gabsocial/cli_helper.rb Normal file
View File

@@ -0,0 +1,9 @@
# frozen_string_literal: true
dev_null = Logger.new('/dev/null')
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
ActiveJob::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
Paperclip.options[:log] = false

152
lib/gabsocial/domains_cli.rb Normal file
View File

@@ -0,0 +1,152 @@
# frozen_string_literal: true
require 'concurrent'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class DomainsCLI < Thor
def self.exit_on_failure?
true
end
option :dry_run, type: :boolean
desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
long_desc <<-LONG_DESC
Remove all accounts from a given DOMAIN without leaving behind any
records. Unlike a suspension, if the DOMAIN still exists in the wild,
it means the accounts could return if they are resolved again.
LONG_DESC
def purge(domain)
removed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Account.where(domain: domain).find_each do |account|
SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
removed += 1
say('.', :green, false)
end
DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
say
say("Removed #{removed} accounts#{dry_run}", :green)
custom_emojis = CustomEmoji.where(domain: domain)
custom_emojis_count = custom_emojis.count
custom_emojis.destroy_all unless options[:dry_run]
say("Removed #{custom_emojis_count} custom emojis", :green)
end
option :concurrency, type: :numeric, default: 50, aliases: [:c]
option :silent, type: :boolean, default: false, aliases: [:s]
option :format, type: :string, default: 'summary', aliases: [:f]
desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
long_desc <<-LONG_DESC
Crawl the fediverse by using the Gab Social REST API endpoints that expose
all known peers, and collect statistics from those peers, as long as those
peers support those API endpoints. When no START is given, the command uses
this server's own database of known peers to seed the crawl.
The --concurrency (-c) option controls the number of threads performing HTTP
requests at the same time. More threads means the crawl may complete faster.
The --silent (-s) option controls progress output.
The --format (-f) option controls how the data is displayed at the end. By
default (`summary`), a summary of the statistics is returned. The other options
are `domains`, which returns a newline-delimited list of all discovered peers,
and `json`, which dumps all the aggregated data raw.
LONG_DESC
def crawl(start = nil)
stats = Concurrent::Hash.new
processed = Concurrent::AtomicFixnum.new(0)
failed = Concurrent::AtomicFixnum.new(0)
start_at = Time.now.to_f
seed = start ? [start] : Account.remote.domains
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
work_unit = ->(domain) do
next if stats.key?(domain)
stats[domain] = nil
processed.increment
begin
Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res|
next unless res.code == 200
stats[domain] = Oj.load(res.to_s)
end
Request.new(:get, "https://#{domain}/api/v1/instance/peers").perform do |res|
next unless res.code == 200
Oj.load(res.to_s).reject { |peer| stats.key?(peer) }.each do |peer|
pool.post(peer, &work_unit)
end
end
Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res|
next unless res.code == 200
stats[domain]['activity'] = Oj.load(res.to_s)
end
say('.', :green, false) unless options[:silent]
rescue StandardError
failed.increment
say('.', :red, false) unless options[:silent]
end
end
seed.each do |domain|
pool.post(domain, &work_unit)
end
sleep 20
sleep 20 until pool.queue_length.zero?
pool.shutdown
pool.wait_for_termination(20)
ensure
pool.shutdown
say unless options[:silent]
case options[:format]
when 'summary'
stats_to_summary(stats, processed, failed, start_at)
when 'domains'
stats_to_domains(stats)
when 'json'
stats_to_json(stats)
end
end
private
def stats_to_summary(stats, processed, failed, start_at)
stats.compact!
total_domains = stats.size
total_users = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['stats'].is_a?(Hash) ? sum + val['stats']['user_count'].to_i : sum }
total_active = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['logins'].to_i : sum }
total_joined = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['registrations'].to_i : sum }
say("Visited #{processed.value} domains, #{failed.value} failed (#{(Time.now.to_f - start_at).round}s elapsed)", :green)
say("Total servers: #{total_domains}", :green)
say("Total registered: #{total_users}", :green)
say("Total active last week: #{total_active}", :green)
say("Total joined last week: #{total_joined}", :green)
end
def stats_to_domains(stats)
say(stats.keys.join("\n"))
end
def stats_to_json(stats)
stats.compact!
say(Oj.dump(stats))
end
end
end

87
lib/gabsocial/emoji_cli.rb Normal file
View File

@@ -0,0 +1,87 @@
# frozen_string_literal: true
require 'rubygems/package'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class EmojiCLI < Thor
def self.exit_on_failure?
true
end
option :prefix
option :suffix
option :overwrite, type: :boolean
option :unlisted, type: :boolean
desc 'import PATH', 'Import emoji from a TAR archive at PATH'
long_desc <<-LONG_DESC
Imports custom emoji from a TAR archive specified by PATH.
Existing emoji will be skipped unless the --overwrite option
is provided, in which case they will be overwritten.
With the --prefix option, a prefix can be added to all
generated shortcodes. Likewise, the --suffix option controls
the suffix of all shortcodes.
With the --unlisted option, the processed emoji will not be
visible in the emoji picker (but still usable via other means)
LONG_DESC
def import(path)
imported = 0
skipped = 0
failed = 0
Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
tar.each do |entry|
next unless entry.file? && entry.full_name.end_with?('.png')
shortcode = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
if custom_emoji && !options[:overwrite]
skipped += 1
next
end
custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
custom_emoji.image = StringIO.new(entry.read)
custom_emoji.image_file_name = File.basename(entry.full_name)
custom_emoji.visible_in_picker = !options[:unlisted]
if custom_emoji.save
imported += 1
else
failed += 1
say('Failure/Error: ', :red)
say(entry.full_name)
say(' ' + custom_emoji.errors[:image].join(', '), :red)
end
end
end
puts
say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
end
desc 'purge', 'Remove all custom emoji'
def purge
CustomEmoji.in_batches.destroy_all
say('OK', :green)
end
private
def color(green, _yellow, red)
if !green.zero? && red.zero?
:green
elsif red.zero?
:yellow
else
:red
end
end
end
end

90
lib/gabsocial/feeds_cli.rb Normal file
View File

@@ -0,0 +1,90 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class FeedsCLI < Thor
def self.exit_on_failure?
true
end
option :all, type: :boolean, default: false
option :background, type: :boolean, default: false
option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
long_desc <<-LONG_DESC
Build home and list feeds that are stored in Redis from the database.
With the --all option, all active users will be processed.
Otherwise, a single user specified by USERNAME.
With the --background option, regeneration will be queued into Sidekiq,
and the command will exit as soon as possible.
With the --dry-run option, no work will be done.
With the --verbose option, when accounts are processed sequentially in the
foreground, the IDs of the accounts will be printed.
LONG_DESC
def build(username = nil)
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:all] || username.nil?
processed = 0
queued = 0
User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users|
if options[:background]
RegenerationWorker.push_bulk(users.map(&:account_id)) unless options[:dry_run]
queued += users.size
else
users.each do |user|
RegenerationWorker.new.perform(user.account_id) unless options[:dry_run]
options[:verbose] ? say(user.account_id) : say('.', :green, false)
processed += 1
end
end
end
if options[:background]
say("Scheduled feed regeneration for #{queued} accounts #{dry_run}", :green, true)
else
say
say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
end
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
if options[:background]
RegenerationWorker.perform_async(account.id) unless options[:dry_run]
else
RegenerationWorker.new.perform(account.id) unless options[:dry_run]
end
say("OK #{dry_run}", :green, true)
else
say('No account(s) given', :red)
exit(1)
end
end
desc 'clear', 'Remove all home and list feeds from Redis'
def clear
keys = Redis.current.keys('feed:*')
Redis.current.pipelined do
keys.each { |key| Redis.current.del(key) }
end
say('OK', :green)
end
end
end

70
lib/gabsocial/media_cli.rb Normal file
View File

@@ -0,0 +1,70 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class MediaCLI < Thor
include ActionView::Helpers::NumberHelper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 7
option :background, type: :boolean, default: false
option :verbose, type: :boolean, default: false
option :dry_run, type: :boolean, default: false
desc 'remove', 'Remove remote media files'
long_desc <<-DESC
Removes locally cached copies of media attachments from other servers.
The --days option specifies how old media attachments have to be before
they are removed. It defaults to 7 days.
With the --background option, instead of deleting the files sequentially,
they will be queued into Sidekiq and the command will exit as soon as
possible. In Sidekiq they will be processed with higher concurrency, but
it may impact other operations of the Gab Social server, and it may overload
the underlying file storage.
With the --dry-run option, no work will be done.
With the --verbose option, when media attachments are processed sequentially in the
foreground, the IDs of the media attachments will be printed.
DESC
def remove
time_ago = options[:days].days.ago
queued = 0
processed = 0
size = 0
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:background]
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id, :file_file_size).reorder(nil).find_in_batches do |media_attachments|
queued += media_attachments.size
size += media_attachments.reduce(0) { |sum, m| sum + (m.file_file_size || 0) }
Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) unless options[:dry_run]
end
else
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).reorder(nil).find_in_batches do |media_attachments|
media_attachments.each do |m|
size += m.file_file_size || 0
Maintenance::UncacheMediaWorker.new.perform(m) unless options[:dry_run]
options[:verbose] ? say(m.id) : say('.', :green, false)
processed += 1
end
end
end
say
if options[:background]
say("Scheduled the deletion of #{queued} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
else
say("Removed #{processed} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
end
end
end
end

1002
lib/gabsocial/migration_helpers.rb Normal file
View File

File diff suppressed because it is too large Load Diff

23
lib/gabsocial/premailer_webpack_strategy.rb Normal file
View File

@@ -0,0 +1,23 @@
# frozen_string_literal: true
module PremailerWebpackStrategy
def load(url)
asset_host = ENV['CDN_HOST'] || ENV['WEB_DOMAIN'] || ENV['LOCAL_DOMAIN']
if Webpacker.dev_server.running?
asset_host = "#{Webpacker.dev_server.protocol}://#{Webpacker.dev_server.host_with_port}"
url = File.join(asset_host, url)
end
css = if url.start_with?('http')
HTTP.get(url).to_s
else
url = url[1..-1] if url.start_with?('/')
File.read(Rails.root.join('public', url))
end
css.gsub(/url\(\//, "url(#{asset_host}/")
end
module_function :load
end

30
lib/gabsocial/redis_config.rb Normal file
View File

@@ -0,0 +1,30 @@
# frozen_string_literal: true
def setup_redis_env_url(prefix = nil, defaults = true)
prefix = prefix.to_s.upcase + '_' unless prefix.nil?
prefix = '' if prefix.nil?
return if ENV[prefix + 'REDIS_URL'].present?
password = ENV.fetch(prefix + 'REDIS_PASSWORD') { '' if defaults }
host = ENV.fetch(prefix + 'REDIS_HOST') { 'localhost' if defaults }
port = ENV.fetch(prefix + 'REDIS_PORT') { 6379 if defaults }
db = ENV.fetch(prefix + 'REDIS_DB') { 0 if defaults }
ENV[prefix + 'REDIS_URL'] = if [password, host, port, db].all?(&:nil?)
ENV['REDIS_URL']
else
"redis://#{password.blank? ? '' : ":#{password}@"}#{host}:#{port}/#{db}"
end
end
setup_redis_env_url
setup_redis_env_url(:cache, false)
namespace = ENV.fetch('REDIS_NAMESPACE') { nil }
cache_namespace = namespace ? namespace + '_cache' : 'cache'
REDIS_CACHE_PARAMS = {
expires_in: 10.minutes,
namespace: cache_namespace,
}.freeze

22
lib/gabsocial/search_cli.rb Normal file
View File

@@ -0,0 +1,22 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class SearchCLI < Thor
desc 'deploy', 'Create or update an ElasticSearch index and populate it'
long_desc <<~LONG_DESC
If ElasticSearch is empty, this command will create the necessary indices
and then import data from the database into those indices.
This command will also upgrade indices if the underlying schema has been
changed since the last run.
LONG_DESC
def deploy
processed = Chewy::RakeHelper.upgrade
Chewy::RakeHelper.sync(except: processed)
end
end
end

30
lib/gabsocial/settings_cli.rb Normal file
View File

@@ -0,0 +1,30 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class RegistrationsCLI < Thor
def self.exit_on_failure?
true
end
desc 'open', 'Open registrations'
def open
Setting.registrations_mode = 'open'
say('OK', :green)
end
desc 'close', 'Close registrations'
def close
Setting.registrations_mode = 'none'
say('OK', :green)
end
end
class SettingsCLI < Thor
desc 'registrations SUBCOMMAND ...ARGS', 'Manage state of registrations'
subcommand 'registrations', RegistrationsCLI
end
end

162
lib/gabsocial/snowflake.rb Normal file
View File

@@ -0,0 +1,162 @@
# frozen_string_literal: true
module GabSocial::Snowflake
DEFAULT_REGEX = /timestamp_id\('(?<seq_prefix>\w+)'/
class Callbacks
def self.around_create(record)
now = Time.now.utc
if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps
yield
else
record.id = GabSocial::Snowflake.id_at(record.created_at)
tries = 0
begin
yield
rescue ActiveRecord::RecordNotUnique
raise if tries > 100
tries += 1
record.id += rand(100)
retry
end
end
end
end
class << self
# Our ID will be composed of the following:
# 6 bytes (48 bits) of millisecond-level timestamp
# 2 bytes (16 bits) of sequence data
#
# The 'sequence data' is intended to be unique within a
# given millisecond, yet obscure the 'serial number' of
# this row.
#
# To do this, we hash the following data:
# * Table name (if provided, skipped if not)
# * Secret salt (should not be guessable)
# * Timestamp (again, millisecond-level granularity)
#
# We then take the first two bytes of that value, and add
# the lowest two bytes of the table ID sequence number
# (`table_name`_id_seq). This means that even if we insert
# two rows at the same millisecond, they will have
# distinct 'sequence data' portions.
#
# If this happens, and an attacker can see both such IDs,
# they can determine which of the two entries was inserted
# first, but not the total number of entries in the table
# (even mod 2**16).
#
# The table name is included in the hash to ensure that
# different tables derive separate sequence bases so rows
# inserted in the same millisecond in different tables do
# not reveal the table ID sequence number for one another.
#
# The secret salt is included in the hash to ensure that
# external users cannot derive the sequence base given the
# timestamp and table name, which would allow them to
# compute the table ID sequence number.
def define_timestamp_id
return if already_defined?
connection.execute(<<~SQL)
CREATE OR REPLACE FUNCTION timestamp_id(table_name text)
RETURNS bigint AS
$$
DECLARE
time_part bigint;
sequence_base bigint;
tail bigint;
BEGIN
time_part := (
-- Get the time in milliseconds
((date_part('epoch', now()) * 1000))::bigint
-- And shift it over two bytes
<< 16);
sequence_base := (
'x' ||
-- Take the first two bytes (four hex characters)
substr(
-- Of the MD5 hash of the data we documented
md5(table_name ||
'#{SecureRandom.hex(16)}' ||
time_part::text
),
1, 4
)
-- And turn it into a bigint
)::bit(16)::bigint;
-- Finally, add our sequence number to our base, and chop
-- it to the last two bytes
tail := (
(sequence_base + nextval(table_name || '_id_seq'))
& 65535);
-- Return the time part and the sequence part. OR appears
-- faster here than addition, but they're equivalent:
-- time_part has no trailing two bytes, and tail is only
-- the last two bytes.
RETURN time_part | tail;
END
$$ LANGUAGE plpgsql VOLATILE;
SQL
end
def ensure_id_sequences_exist
# Find tables using timestamp IDs.
connection.tables.each do |table|
# We're only concerned with "id" columns.
next unless (id_col = connection.columns(table).find { |col| col.name == 'id' })
# And only those that are using timestamp_id.
next unless (data = DEFAULT_REGEX.match(id_col.default_function))
seq_name = data[:seq_prefix] + '_id_seq'
# If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF
# NOT EXISTS, but we can't depend on that. Instead, catch the
# possible exception and ignore it.
# Note that seq_name isn't a column name, but it's a
# relation, like a column, and follows the same quoting rules
# in Postgres.
connection.execute(<<~SQL)
DO $$
BEGIN
CREATE SEQUENCE #{connection.quote_column_name(seq_name)};
EXCEPTION WHEN duplicate_table THEN
-- Do nothing, we have the sequence already.
END
$$ LANGUAGE plpgsql;
SQL
end
end
def id_at(timestamp)
id = timestamp.to_i * 1000 + rand(1000)
id = id << 16
id += rand(2**16)
id
end
private
def already_defined?
connection.execute(<<~SQL).values.first.first
SELECT EXISTS(
SELECT * FROM pg_proc WHERE proname = 'timestamp_id'
);
SQL
end
def connection
ActiveRecord::Base.connection
end
end
end

62
lib/gabsocial/statuses_cli.rb Normal file
View File

@@ -0,0 +1,62 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class StatusesCLI < Thor
include ActionView::Helpers::NumberHelper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 90
desc 'remove', 'Remove unreferenced statuses'
long_desc <<~LONG_DESC
Remove statuses that are not referenced by local user activity, such as
ones that came from relays, or belonging to users that were once followed
by someone locally but no longer are.
This is a computationally heavy procedure that creates extra database
indicides before commencing, and removes them afterward.
LONG_DESC
def remove
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local)
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id)
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url)
max_id = GabSocial::Snowflake.id_at(options[:days].days.ago)
start_at = Time.now.to_f
say('Beginning removal... This might take a while...')
Status.remote
.where('id < ?', max_id)
.where(reblog_of_id: nil) # Skip reblogs
.where(in_reply_to_id: nil) # Skip replies
.where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') # Skip statuses that are pinned on profiles
.where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') # Skip statuses that mention local accounts
.where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') # Skip statuses favorited by local accounts
.where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND statuses1.account_id IN (SELECT accounts.id FROM accounts WHERE accounts.domain IS NULL))') # Skip statuses reblogged by local accounts
.where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') # Skip accounts followed by local accounts
.in_batches
.delete_all
say('Beginning removal of now-orphaned media attachments to free up disk space...')
Scheduler::MediaCleanupScheduler.new.perform
say("Done after #{Time.now.to_f - start_at}s", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local) if ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id) if ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id)
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url) if ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url)
end
end
end

60
lib/gabsocial/version.rb Normal file
View File

@@ -0,0 +1,60 @@
# frozen_string_literal: true
module GabSocial
module Version
module_function
def major
2
end
def minor
8
end
def patch
4
end
def pre
nil
end
def flags
''
end
def to_a
[major, minor, patch, pre].compact
end
def to_s
[to_a.join('.'), flags].join
end
def repository
ENV.fetch('GITHUB_REPOSITORY') { 'gab-ai-inc/gab-social' }
end
def source_base_url
ENV.fetch('SOURCE_BASE_URL') { "https://github.com/#{repository}" }
end
# specify git tag or commit hash here
def source_tag
ENV.fetch('SOURCE_TAG') { nil }
end
def source_url
if source_tag
"#{source_base_url}/tree/#{source_tag}"
else
source_base_url
end
end
def user_agent
@user_agent ||= "#{HTTP::Request::USER_AGENT} (GabSocial/#{Version}; +http#{Rails.configuration.x.use_https ? 's' : ''}://#{Rails.configuration.x.web_domain}/)"
end
end
end