random-mirrors/gab-social
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Gab Social. All are welcome.

Browse Source
This commit is contained in:
robcolbert
2019-07-02 03:10:25 -04:00
commit bd0b5afc92
5366 changed files with 222812 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lib/assets/.keep Normal file
View File

128
lib/cli.rb Normal file
View File

@@ -0,0 +1,128 @@
# frozen_string_literal: true
require 'thor'
require_relative 'gabsocial/media_cli'
require_relative 'gabsocial/emoji_cli'
require_relative 'gabsocial/accounts_cli'
require_relative 'gabsocial/feeds_cli'
require_relative 'gabsocial/search_cli'
require_relative 'gabsocial/settings_cli'
require_relative 'gabsocial/statuses_cli'
require_relative 'gabsocial/domains_cli'
require_relative 'gabsocial/cache_cli'
require_relative 'gabsocial/version'
module GabSocial
class CLI < Thor
def self.exit_on_failure?
true
end
desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
subcommand 'media', GabSocial::MediaCLI
desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji'
subcommand 'emoji', GabSocial::EmojiCLI
desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts'
subcommand 'accounts', GabSocial::AccountsCLI
desc 'feeds SUBCOMMAND ...ARGS', 'Manage feeds'
subcommand 'feeds', GabSocial::FeedsCLI
desc 'search SUBCOMMAND ...ARGS', 'Manage the search engine'
subcommand 'search', GabSocial::SearchCLI
desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings'
subcommand 'settings', GabSocial::SettingsCLI
desc 'statuses SUBCOMMAND ...ARGS', 'Manage statuses'
subcommand 'statuses', GabSocial::StatusesCLI
desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
subcommand 'domains', GabSocial::DomainsCLI
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', GabSocial::CacheCLI
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC
Erase the server from the federation by broadcasting account delete
activities to all known other servers. This allows a "clean exit" from
running a Gab Social server, as it leaves next to no cache behind on
other servers.
This command is always interactive and requires confirmation twice.
No local data is actually deleted, because emptying the
database or removing files is much faster through other, external
means, such as e.g. deleting the entire VPS. However, because other
servers will delete data about local users, but no local data will be
updated (such as e.g. followers), there will be a state mismatch
that will lead to glitches and issues if you then continue to run and use
the server.
So either you know exactly what you are doing, or you are starting
from a blank slate afterwards by manually clearing out all the local
data!
LONG_DESC
def self_destruct
require 'tty-prompt'
prompt = TTY::Prompt.new
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
exit(1) if prompt.no?('Are you sure you want to proceed?')
inboxes = Account.inboxes
processed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if inboxes.empty?
prompt.ok('It seems like your server has not federated with anything')
prompt.ok('You can shut it down and delete it any time')
return
end
prompt.warn('Do NOT interrupt this process...')
Account.local.without_suspended.find_each do |account|
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,
adapter: ActivityPub::Adapter
).as_json
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
unless options[:dry_run]
ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
[json, account.id, inbox_url]
end
account.suspend!
end
processed += 1
end
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
rescue TTY::Reader::InputInterrupt
exit(1)
end
map %w(--version -v) => :version
desc 'version', 'Show version'
def version
say(GabSocial::Version.to_s)
end
end
end

55
lib/devise/ldap_authenticatable.rb Normal file
View File

@@ -0,0 +1,55 @@
# frozen_string_literal: true
require 'net/ldap'
require 'devise/strategies/authenticatable'
module Devise
module Strategies
class LdapAuthenticatable < Authenticatable
def authenticate!
if params[:user]
ldap = Net::LDAP.new(
host: Devise.ldap_host,
port: Devise.ldap_port,
base: Devise.ldap_base,
encryption: {
method: Devise.ldap_method,
tls_options: tls_options,
},
auth: {
method: :simple,
username: Devise.ldap_bind_dn,
password: Devise.ldap_password,
},
connect_timeout: 10
)
filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: email)
if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: password))
user = User.ldap_get_user(user_info.first)
success!(user)
else
return fail(:invalid)
end
end
end
def email
params[:user][:email]
end
def password
params[:user][:password]
end
def tls_options
OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.tap do |options|
options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if Devise.ldap_tls_no_verify
end
end
end
end
end
Warden::Strategies.add(:ldap_authenticatable, Devise::Strategies::LdapAuthenticatable)

487
lib/gabsocial/accounts_cli.rb Normal file
View File

@@ -0,0 +1,487 @@
# frozen_string_literal: true
require 'set'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class AccountsCLI < Thor
def self.exit_on_failure?
true
end
option :all, type: :boolean
desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
long_desc <<-LONG_DESC
Generate and broadcast new RSA keys as part of security
maintenance.
With the --all option, all local accounts will be subject
to the rotation. Otherwise, and by default, only a single
account specified by the USERNAME argument will be
processed.
LONG_DESC
def rotate(username = nil)
if options[:all]
processed = 0
delay = 0
Account.local.without_suspended.find_in_batches do |accounts|
accounts.each do |account|
rotate_keys_for_account(account, delay)
processed += 1
say('.', :green, false)
end
delay += 5.minutes
end
say
say("OK, rotated keys for #{processed} accounts", :green)
elsif username.present?
rotate_keys_for_account(Account.find_local(username))
say('OK', :green)
else
say('No account(s) given', :red)
exit(1)
end
end
option :email, required: true
option :confirmed, type: :boolean
option :role, default: 'user'
option :reattach, type: :boolean
option :force, type: :boolean
desc 'create USERNAME', 'Create a new user'
long_desc <<-LONG_DESC
Create a new user account with a given USERNAME and an
e-mail address provided with --email.
With the --confirmed option, the confirmation e-mail will
be skipped and the account will be active straight away.
With the --role option one of "user", "admin" or "moderator"
can be supplied. Defaults to "user"
With the --reattach option, the new user will be reattached
to a given existing username of an old account. If the old
account is still in use by someone else, you can supply
the --force option to delete the old record and reattach the
username to the new account anyway.
LONG_DESC
def create(username)
account = Account.new(username: username)
password = SecureRandom.hex
user = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil)
if options[:reattach]
account = Account.find_local(username) || Account.new(username: username)
if account.user.present? && !options[:force]
say('The chosen username is currently in use', :red)
say('Use --force to reattach it anyway and delete the other user')
return
elsif account.user.present?
account.user.destroy!
end
end
account.suspended_at = nil
user.account = account
if user.save
if options[:confirmed]
user.confirmed_at = nil
user.confirm!
end
say('OK', :green)
say("New password: #{password}")
else
user.errors.to_h.each do |key, error|
say('Failure/Error: ', :red)
say(key)
say(' ' + error, :red)
end
exit(1)
end
end
option :role
option :email
option :confirm, type: :boolean
option :enable, type: :boolean
option :disable, type: :boolean
option :disable_2fa, type: :boolean
option :approve, type: :boolean
desc 'modify USERNAME', 'Modify a user'
long_desc <<-LONG_DESC
Modify a user account.
With the --role option, update the user's role to one of "user",
"moderator" or "admin".
With the --email option, update the user's e-mail address. With
the --confirm option, mark the user's e-mail as confirmed.
With the --disable option, lock the user out of their account. The
--enable option is the opposite.
With the --approve option, the account will be approved, if it was
previously not due to not having open registrations.
With the --disable-2fa option, the two-factor authentication
requirement for the user can be removed.
LONG_DESC
def modify(username)
user = Account.find_local(username)&.user
if user.nil?
say('No user with such username', :red)
exit(1)
end
if options[:role]
user.admin = options[:role] == 'admin'
user.moderator = options[:role] == 'moderator'
end
user.email = options[:email] if options[:email]
user.disabled = false if options[:enable]
user.disabled = true if options[:disable]
user.approved = true if options[:approve]
user.otp_required_for_login = false if options[:disable_2fa]
user.confirm if options[:confirm]
if user.save
say('OK', :green)
else
user.errors.to_h.each do |key, error|
say('Failure/Error: ', :red)
say(key)
say(' ' + error, :red)
end
exit(1)
end
end
desc 'delete USERNAME', 'Delete a user'
long_desc <<-LONG_DESC
Remove a user account with a given USERNAME.
LONG_DESC
def delete(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
SuspendAccountService.new.call(account, including_user: true)
say('OK', :green)
end
desc 'backup USERNAME', 'Request a backup for a user'
long_desc <<-LONG_DESC
Request a new backup for an account with a given USERNAME.
The backup will be created in Sidekiq asynchronously, and
the user will receive an e-mail with a link to it once
it's done.
LONG_DESC
def backup(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
backup = account.user.backups.create!
BackupWorker.perform_async(backup.id)
say('OK', :green)
end
option :dry_run, type: :boolean
desc 'cull', 'Remove remote accounts that no longer exist'
long_desc <<-LONG_DESC
Query every single remote account in the database to determine
if it still exists on the origin server, and if it doesn't,
remove it from the database.
Accounts that have had confirmed activity within the last week
are excluded from the checks.
Domains that are unreachable are not checked.
With the --dry-run option, no deletes will actually be carried
out.
LONG_DESC
def cull
skip_threshold = 7.days.ago
culled = 0
dry_run_culled = []
skip_domains = Set.new
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Account.remote.where(protocol: :activitypub).partitioned.find_each do |account|
next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold)
code = 0
unless skip_domains.include?(account.domain)
begin
code = Request.new(:head, account.uri).perform(&:code)
rescue HTTP::ConnectionError
skip_domains << account.domain
rescue StandardError
next
end
end
if [404, 410].include?(code)
if options[:dry_run]
dry_run_culled << account.acct
else
SuspendAccountService.new.call(account, destroy: true)
end
culled += 1
say('+', :green, false)
else
account.touch # Touch account even during dry run to avoid getting the account into the window again
say('.', nil, false)
end
end
say
say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow)
unless skip_domains.empty?
say('The following servers were not available during the check:', :yellow)
skip_domains.each { |domain| say(' ' + domain) }
end
unless dry_run_culled.empty?
say('The following accounts would have been deleted:', :green)
dry_run_culled.each { |account| say(' ' + account) }
end
end
option :all, type: :boolean
option :domain
desc 'refresh [USERNAME]', 'Fetch remote user data and files'
long_desc <<-LONG_DESC
Fetch remote user data and files for one or multiple accounts.
With the --all option, all remote accounts will be processed.
Through the --domain option, this can be narrowed down to a
specific domain only. Otherwise, a single remote account must
be specified with USERNAME.
All processing is done in the background through Sidekiq.
LONG_DESC
def refresh(username = nil)
if options[:domain] || options[:all]
queued = 0
scope = Account.remote
scope = scope.where(domain: options[:domain]) if options[:domain]
scope.select(:id).reorder(nil).find_in_batches do |accounts|
Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts.map(&:id))
queued += accounts.size
end
say("Scheduled refreshment of #{queued} accounts", :green, true)
elsif username.present?
username, domain = username.split('@')
account = Account.find_remote(username, domain)
if account.nil?
say('No such account', :red)
exit(1)
end
Maintenance::RedownloadAccountMediaWorker.perform_async(account.id)
say('OK', :green)
else
say('No account(s) given', :red)
exit(1)
end
end
desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT'
long_desc <<-LONG_DESC
Make all local accounts follow an account specified by ACCT. ACCT can be
a simple username, in case of a local user. It can also be in the format
username@domain, in case of a remote user.
LONG_DESC
def follow(acct)
target_account = ResolveAccountService.new.call(acct)
processed = 0
failed = 0
if target_account.nil?
say("Target account (#{acct}) could not be resolved", :red)
exit(1)
end
Account.local.without_suspended.find_each do |account|
begin
FollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, followed target from #{processed} accounts, skipped #{failed}", :green)
end
desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT'
long_desc <<-LONG_DESC
Make all local accounts unfollow an account specified by ACCT. ACCT can be
a simple username, in case of a local user. It can also be in the format
username@domain, in case of a remote user.
LONG_DESC
def unfollow(acct)
target_account = Account.find_remote(*acct.split('@'))
processed = 0
failed = 0
if target_account.nil?
say("Target account (#{acct}) was not found", :red)
exit(1)
end
target_account.followers.local.find_each do |account|
begin
UnfollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, unfollowed target from #{processed} accounts, skipped #{failed}", :green)
end
option :follows, type: :boolean, default: false
option :followers, type: :boolean, default: false
desc 'reset-relationships USERNAME', 'Reset all follows and/or followers for a user'
long_desc <<-LONG_DESC
Reset all follows and/or followers for a user specified by USERNAME.
With the --follows option, the command unfollows everyone that the account follows,
and then re-follows the users that would be followed by a brand new account.
With the --followers option, the command removes all followers of the account.
LONG_DESC
def reset_relationships(username)
unless options[:follows] || options[:followers]
say('Please specify either --follows or --followers, or both', :red)
exit(1)
end
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
if options[:follows]
processed = 0
failed = 0
say("Unfollowing #{account.username}'s followees, this might take a while...")
Account.where(id: ::Follow.where(account: account).select(:target_account_id)).find_each do |target_account|
begin
UnfollowService.new.call(account, target_account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
BootstrapTimelineWorker.perform_async(account.id)
say("OK, unfollowed #{processed} followees, skipped #{failed}", :green)
end
if options[:followers]
processed = 0
failed = 0
say("Removing #{account.username}'s followers, this might take a while...")
Account.where(id: ::Follow.where(target_account: account).select(:account_id)).find_each do |target_account|
begin
UnfollowService.new.call(target_account, account)
processed += 1
say('.', :green, false)
rescue StandardError
failed += 1
say('.', :red, false)
end
end
say("OK, removed #{processed} followers, skipped #{failed}", :green)
end
end
option :number, type: :numeric, aliases: [:n]
option :all, type: :boolean
desc 'approve [USERNAME]', 'Approve pending accounts'
long_desc <<~LONG_DESC
When registrations require review from staff, approve pending accounts,
either all of them with the --all option, or a specific number of them
specified with the --number (-n) option, or only a single specific
account identified by its username.
LONG_DESC
def approve(username = nil)
if options[:all]
User.pending.find_each(&:approve!)
say('OK', :green)
elsif options[:number]
User.pending.limit(options[:number]).each(&:approve!)
say('OK', :green)
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
account.user&.approve!
say('OK', :green)
else
exit(1)
end
end
private
def rotate_keys_for_account(account, delay = 0)
if account.nil?
say('No such account', :red)
exit(1)
end
old_key = account.private_key
new_key = OpenSSL::PKey::RSA.new(2048)
account.update(private_key: new_key.to_pem, public_key: new_key.public_key.to_pem)
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, sign_with: old_key)
end
end
end

19
lib/gabsocial/cache_cli.rb Normal file
View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class CacheCLI < Thor
def self.exit_on_failure?
true
end
desc 'clear', 'Clear the cache storage'
def clear
Rails.cache.clear
say('OK', :green)
end
end
end

9
lib/gabsocial/cli_helper.rb Normal file
View File

@@ -0,0 +1,9 @@
# frozen_string_literal: true
dev_null = Logger.new('/dev/null')
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
ActiveJob::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
Paperclip.options[:log] = false

152
lib/gabsocial/domains_cli.rb Normal file
View File

@@ -0,0 +1,152 @@
# frozen_string_literal: true
require 'concurrent'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class DomainsCLI < Thor
def self.exit_on_failure?
true
end
option :dry_run, type: :boolean
desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
long_desc <<-LONG_DESC
Remove all accounts from a given DOMAIN without leaving behind any
records. Unlike a suspension, if the DOMAIN still exists in the wild,
it means the accounts could return if they are resolved again.
LONG_DESC
def purge(domain)
removed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Account.where(domain: domain).find_each do |account|
SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
removed += 1
say('.', :green, false)
end
DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
say
say("Removed #{removed} accounts#{dry_run}", :green)
custom_emojis = CustomEmoji.where(domain: domain)
custom_emojis_count = custom_emojis.count
custom_emojis.destroy_all unless options[:dry_run]
say("Removed #{custom_emojis_count} custom emojis", :green)
end
option :concurrency, type: :numeric, default: 50, aliases: [:c]
option :silent, type: :boolean, default: false, aliases: [:s]
option :format, type: :string, default: 'summary', aliases: [:f]
desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
long_desc <<-LONG_DESC
Crawl the fediverse by using the Gab Social REST API endpoints that expose
all known peers, and collect statistics from those peers, as long as those
peers support those API endpoints. When no START is given, the command uses
this server's own database of known peers to seed the crawl.
The --concurrency (-c) option controls the number of threads performing HTTP
requests at the same time. More threads means the crawl may complete faster.
The --silent (-s) option controls progress output.
The --format (-f) option controls how the data is displayed at the end. By
default (`summary`), a summary of the statistics is returned. The other options
are `domains`, which returns a newline-delimited list of all discovered peers,
and `json`, which dumps all the aggregated data raw.
LONG_DESC
def crawl(start = nil)
stats = Concurrent::Hash.new
processed = Concurrent::AtomicFixnum.new(0)
failed = Concurrent::AtomicFixnum.new(0)
start_at = Time.now.to_f
seed = start ? [start] : Account.remote.domains
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
work_unit = ->(domain) do
next if stats.key?(domain)
stats[domain] = nil
processed.increment
begin
Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res|
next unless res.code == 200
stats[domain] = Oj.load(res.to_s)
end
Request.new(:get, "https://#{domain}/api/v1/instance/peers").perform do |res|
next unless res.code == 200
Oj.load(res.to_s).reject { |peer| stats.key?(peer) }.each do |peer|
pool.post(peer, &work_unit)
end
end
Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res|
next unless res.code == 200
stats[domain]['activity'] = Oj.load(res.to_s)
end
say('.', :green, false) unless options[:silent]
rescue StandardError
failed.increment
say('.', :red, false) unless options[:silent]
end
end
seed.each do |domain|
pool.post(domain, &work_unit)
end
sleep 20
sleep 20 until pool.queue_length.zero?
pool.shutdown
pool.wait_for_termination(20)
ensure
pool.shutdown
say unless options[:silent]
case options[:format]
when 'summary'
stats_to_summary(stats, processed, failed, start_at)
when 'domains'
stats_to_domains(stats)
when 'json'
stats_to_json(stats)
end
end
private
def stats_to_summary(stats, processed, failed, start_at)
stats.compact!
total_domains = stats.size
total_users = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['stats'].is_a?(Hash) ? sum + val['stats']['user_count'].to_i : sum }
total_active = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['logins'].to_i : sum }
total_joined = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['registrations'].to_i : sum }
say("Visited #{processed.value} domains, #{failed.value} failed (#{(Time.now.to_f - start_at).round}s elapsed)", :green)
say("Total servers: #{total_domains}", :green)
say("Total registered: #{total_users}", :green)
say("Total active last week: #{total_active}", :green)
say("Total joined last week: #{total_joined}", :green)
end
def stats_to_domains(stats)
say(stats.keys.join("\n"))
end
def stats_to_json(stats)
stats.compact!
say(Oj.dump(stats))
end
end
end

87
lib/gabsocial/emoji_cli.rb Normal file
View File

@@ -0,0 +1,87 @@
# frozen_string_literal: true
require 'rubygems/package'
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class EmojiCLI < Thor
def self.exit_on_failure?
true
end
option :prefix
option :suffix
option :overwrite, type: :boolean
option :unlisted, type: :boolean
desc 'import PATH', 'Import emoji from a TAR archive at PATH'
long_desc <<-LONG_DESC
Imports custom emoji from a TAR archive specified by PATH.
Existing emoji will be skipped unless the --overwrite option
is provided, in which case they will be overwritten.
With the --prefix option, a prefix can be added to all
generated shortcodes. Likewise, the --suffix option controls
the suffix of all shortcodes.
With the --unlisted option, the processed emoji will not be
visible in the emoji picker (but still usable via other means)
LONG_DESC
def import(path)
imported = 0
skipped = 0
failed = 0
Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
tar.each do |entry|
next unless entry.file? && entry.full_name.end_with?('.png')
shortcode = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
if custom_emoji && !options[:overwrite]
skipped += 1
next
end
custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
custom_emoji.image = StringIO.new(entry.read)
custom_emoji.image_file_name = File.basename(entry.full_name)
custom_emoji.visible_in_picker = !options[:unlisted]
if custom_emoji.save
imported += 1
else
failed += 1
say('Failure/Error: ', :red)
say(entry.full_name)
say(' ' + custom_emoji.errors[:image].join(', '), :red)
end
end
end
puts
say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
end
desc 'purge', 'Remove all custom emoji'
def purge
CustomEmoji.in_batches.destroy_all
say('OK', :green)
end
private
def color(green, _yellow, red)
if !green.zero? && red.zero?
:green
elsif red.zero?
:yellow
else
:red
end
end
end
end

90
lib/gabsocial/feeds_cli.rb Normal file
View File

@@ -0,0 +1,90 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class FeedsCLI < Thor
def self.exit_on_failure?
true
end
option :all, type: :boolean, default: false
option :background, type: :boolean, default: false
option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
long_desc <<-LONG_DESC
Build home and list feeds that are stored in Redis from the database.
With the --all option, all active users will be processed.
Otherwise, a single user specified by USERNAME.
With the --background option, regeneration will be queued into Sidekiq,
and the command will exit as soon as possible.
With the --dry-run option, no work will be done.
With the --verbose option, when accounts are processed sequentially in the
foreground, the IDs of the accounts will be printed.
LONG_DESC
def build(username = nil)
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:all] || username.nil?
processed = 0
queued = 0
User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users|
if options[:background]
RegenerationWorker.push_bulk(users.map(&:account_id)) unless options[:dry_run]
queued += users.size
else
users.each do |user|
RegenerationWorker.new.perform(user.account_id) unless options[:dry_run]
options[:verbose] ? say(user.account_id) : say('.', :green, false)
processed += 1
end
end
end
if options[:background]
say("Scheduled feed regeneration for #{queued} accounts #{dry_run}", :green, true)
else
say
say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
end
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
if options[:background]
RegenerationWorker.perform_async(account.id) unless options[:dry_run]
else
RegenerationWorker.new.perform(account.id) unless options[:dry_run]
end
say("OK #{dry_run}", :green, true)
else
say('No account(s) given', :red)
exit(1)
end
end
desc 'clear', 'Remove all home and list feeds from Redis'
def clear
keys = Redis.current.keys('feed:*')
Redis.current.pipelined do
keys.each { |key| Redis.current.del(key) }
end
say('OK', :green)
end
end
end

70
lib/gabsocial/media_cli.rb Normal file
View File

@@ -0,0 +1,70 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class MediaCLI < Thor
include ActionView::Helpers::NumberHelper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 7
option :background, type: :boolean, default: false
option :verbose, type: :boolean, default: false
option :dry_run, type: :boolean, default: false
desc 'remove', 'Remove remote media files'
long_desc <<-DESC
Removes locally cached copies of media attachments from other servers.
The --days option specifies how old media attachments have to be before
they are removed. It defaults to 7 days.
With the --background option, instead of deleting the files sequentially,
they will be queued into Sidekiq and the command will exit as soon as
possible. In Sidekiq they will be processed with higher concurrency, but
it may impact other operations of the Gab Social server, and it may overload
the underlying file storage.
With the --dry-run option, no work will be done.
With the --verbose option, when media attachments are processed sequentially in the
foreground, the IDs of the media attachments will be printed.
DESC
def remove
time_ago = options[:days].days.ago
queued = 0
processed = 0
size = 0
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:background]
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id, :file_file_size).reorder(nil).find_in_batches do |media_attachments|
queued += media_attachments.size
size += media_attachments.reduce(0) { |sum, m| sum + (m.file_file_size || 0) }
Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) unless options[:dry_run]
end
else
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).reorder(nil).find_in_batches do |media_attachments|
media_attachments.each do |m|
size += m.file_file_size || 0
Maintenance::UncacheMediaWorker.new.perform(m) unless options[:dry_run]
options[:verbose] ? say(m.id) : say('.', :green, false)
processed += 1
end
end
end
say
if options[:background]
say("Scheduled the deletion of #{queued} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
else
say("Removed #{processed} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
end
end
end
end

1002
lib/gabsocial/migration_helpers.rb Normal file
View File

File diff suppressed because it is too large Load Diff

23
lib/gabsocial/premailer_webpack_strategy.rb Normal file
View File

@@ -0,0 +1,23 @@
# frozen_string_literal: true
module PremailerWebpackStrategy
def load(url)
asset_host = ENV['CDN_HOST'] || ENV['WEB_DOMAIN'] || ENV['LOCAL_DOMAIN']
if Webpacker.dev_server.running?
asset_host = "#{Webpacker.dev_server.protocol}://#{Webpacker.dev_server.host_with_port}"
url = File.join(asset_host, url)
end
css = if url.start_with?('http')
HTTP.get(url).to_s
else
url = url[1..-1] if url.start_with?('/')
File.read(Rails.root.join('public', url))
end
css.gsub(/url\(\//, "url(#{asset_host}/")
end
module_function :load
end

30
lib/gabsocial/redis_config.rb Normal file
View File

@@ -0,0 +1,30 @@
# frozen_string_literal: true
def setup_redis_env_url(prefix = nil, defaults = true)
prefix = prefix.to_s.upcase + '_' unless prefix.nil?
prefix = '' if prefix.nil?
return if ENV[prefix + 'REDIS_URL'].present?
password = ENV.fetch(prefix + 'REDIS_PASSWORD') { '' if defaults }
host = ENV.fetch(prefix + 'REDIS_HOST') { 'localhost' if defaults }
port = ENV.fetch(prefix + 'REDIS_PORT') { 6379 if defaults }
db = ENV.fetch(prefix + 'REDIS_DB') { 0 if defaults }
ENV[prefix + 'REDIS_URL'] = if [password, host, port, db].all?(&:nil?)
ENV['REDIS_URL']
else
"redis://#{password.blank? ? '' : ":#{password}@"}#{host}:#{port}/#{db}"
end
end
setup_redis_env_url
setup_redis_env_url(:cache, false)
namespace = ENV.fetch('REDIS_NAMESPACE') { nil }
cache_namespace = namespace ? namespace + '_cache' : 'cache'
REDIS_CACHE_PARAMS = {
expires_in: 10.minutes,
namespace: cache_namespace,
}.freeze

22
lib/gabsocial/search_cli.rb Normal file
View File

@@ -0,0 +1,22 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class SearchCLI < Thor
desc 'deploy', 'Create or update an ElasticSearch index and populate it'
long_desc <<~LONG_DESC
If ElasticSearch is empty, this command will create the necessary indices
and then import data from the database into those indices.
This command will also upgrade indices if the underlying schema has been
changed since the last run.
LONG_DESC
def deploy
processed = Chewy::RakeHelper.upgrade
Chewy::RakeHelper.sync(except: processed)
end
end
end

30
lib/gabsocial/settings_cli.rb Normal file
View File

@@ -0,0 +1,30 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class RegistrationsCLI < Thor
def self.exit_on_failure?
true
end
desc 'open', 'Open registrations'
def open
Setting.registrations_mode = 'open'
say('OK', :green)
end
desc 'close', 'Close registrations'
def close
Setting.registrations_mode = 'none'
say('OK', :green)
end
end
class SettingsCLI < Thor
desc 'registrations SUBCOMMAND ...ARGS', 'Manage state of registrations'
subcommand 'registrations', RegistrationsCLI
end
end

162
lib/gabsocial/snowflake.rb Normal file
View File

@@ -0,0 +1,162 @@
# frozen_string_literal: true
module GabSocial::Snowflake
DEFAULT_REGEX = /timestamp_id\('(?<seq_prefix>\w+)'/
class Callbacks
def self.around_create(record)
now = Time.now.utc
if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps
yield
else
record.id = GabSocial::Snowflake.id_at(record.created_at)
tries = 0
begin
yield
rescue ActiveRecord::RecordNotUnique
raise if tries > 100
tries += 1
record.id += rand(100)
retry
end
end
end
end
class << self
# Our ID will be composed of the following:
# 6 bytes (48 bits) of millisecond-level timestamp
# 2 bytes (16 bits) of sequence data
#
# The 'sequence data' is intended to be unique within a
# given millisecond, yet obscure the 'serial number' of
# this row.
#
# To do this, we hash the following data:
# * Table name (if provided, skipped if not)
# * Secret salt (should not be guessable)
# * Timestamp (again, millisecond-level granularity)
#
# We then take the first two bytes of that value, and add
# the lowest two bytes of the table ID sequence number
# (`table_name`_id_seq). This means that even if we insert
# two rows at the same millisecond, they will have
# distinct 'sequence data' portions.
#
# If this happens, and an attacker can see both such IDs,
# they can determine which of the two entries was inserted
# first, but not the total number of entries in the table
# (even mod 2**16).
#
# The table name is included in the hash to ensure that
# different tables derive separate sequence bases so rows
# inserted in the same millisecond in different tables do
# not reveal the table ID sequence number for one another.
#
# The secret salt is included in the hash to ensure that
# external users cannot derive the sequence base given the
# timestamp and table name, which would allow them to
# compute the table ID sequence number.
def define_timestamp_id
return if already_defined?
connection.execute(<<~SQL)
CREATE OR REPLACE FUNCTION timestamp_id(table_name text)
RETURNS bigint AS
$$
DECLARE
time_part bigint;
sequence_base bigint;
tail bigint;
BEGIN
time_part := (
-- Get the time in milliseconds
((date_part('epoch', now()) * 1000))::bigint
-- And shift it over two bytes
<< 16);
sequence_base := (
'x' ||
-- Take the first two bytes (four hex characters)
substr(
-- Of the MD5 hash of the data we documented
md5(table_name ||
'#{SecureRandom.hex(16)}' ||
time_part::text
),
1, 4
)
-- And turn it into a bigint
)::bit(16)::bigint;
-- Finally, add our sequence number to our base, and chop
-- it to the last two bytes
tail := (
(sequence_base + nextval(table_name || '_id_seq'))
& 65535);
-- Return the time part and the sequence part. OR appears
-- faster here than addition, but they're equivalent:
-- time_part has no trailing two bytes, and tail is only
-- the last two bytes.
RETURN time_part | tail;
END
$$ LANGUAGE plpgsql VOLATILE;
SQL
end
def ensure_id_sequences_exist
# Find tables using timestamp IDs.
connection.tables.each do |table|
# We're only concerned with "id" columns.
next unless (id_col = connection.columns(table).find { |col| col.name == 'id' })
# And only those that are using timestamp_id.
next unless (data = DEFAULT_REGEX.match(id_col.default_function))
seq_name = data[:seq_prefix] + '_id_seq'
# If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF
# NOT EXISTS, but we can't depend on that. Instead, catch the
# possible exception and ignore it.
# Note that seq_name isn't a column name, but it's a
# relation, like a column, and follows the same quoting rules
# in Postgres.
connection.execute(<<~SQL)
DO $$
BEGIN
CREATE SEQUENCE #{connection.quote_column_name(seq_name)};
EXCEPTION WHEN duplicate_table THEN
-- Do nothing, we have the sequence already.
END
$$ LANGUAGE plpgsql;
SQL
end
end
def id_at(timestamp)
id = timestamp.to_i * 1000 + rand(1000)
id = id << 16
id += rand(2**16)
id
end
private
def already_defined?
connection.execute(<<~SQL).values.first.first
SELECT EXISTS(
SELECT * FROM pg_proc WHERE proname = 'timestamp_id'
);
SQL
end
def connection
ActiveRecord::Base.connection
end
end
end

62
lib/gabsocial/statuses_cli.rb Normal file
View File

@@ -0,0 +1,62 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module GabSocial
class StatusesCLI < Thor
include ActionView::Helpers::NumberHelper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 90
desc 'remove', 'Remove unreferenced statuses'
long_desc <<~LONG_DESC
Remove statuses that are not referenced by local user activity, such as
ones that came from relays, or belonging to users that were once followed
by someone locally but no longer are.
This is a computationally heavy procedure that creates extra database
indicides before commencing, and removes them afterward.
LONG_DESC
def remove
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local)
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id)
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url)
max_id = GabSocial::Snowflake.id_at(options[:days].days.ago)
start_at = Time.now.to_f
say('Beginning removal... This might take a while...')
Status.remote
.where('id < ?', max_id)
.where(reblog_of_id: nil) # Skip reblogs
.where(in_reply_to_id: nil) # Skip replies
.where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') # Skip statuses that are pinned on profiles
.where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') # Skip statuses that mention local accounts
.where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') # Skip statuses favorited by local accounts
.where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND statuses1.account_id IN (SELECT accounts.id FROM accounts WHERE accounts.domain IS NULL))') # Skip statuses reblogged by local accounts
.where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') # Skip accounts followed by local accounts
.in_batches
.delete_all
say('Beginning removal of now-orphaned media attachments to free up disk space...')
Scheduler::MediaCleanupScheduler.new.perform
say("Done after #{Time.now.to_f - start_at}s", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local) if ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id) if ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id)
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url) if ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url)
end
end
end

60
lib/gabsocial/version.rb Normal file
View File

@@ -0,0 +1,60 @@
# frozen_string_literal: true
module GabSocial
module Version
module_function
def major
2
end
def minor
8
end
def patch
4
end
def pre
nil
end
def flags
''
end
def to_a
[major, minor, patch, pre].compact
end
def to_s
[to_a.join('.'), flags].join
end
def repository
ENV.fetch('GITHUB_REPOSITORY') { 'gab-ai-inc/gab-social' }
end
def source_base_url
ENV.fetch('SOURCE_BASE_URL') { "https://github.com/#{repository}" }
end
# specify git tag or commit hash here
def source_tag
ENV.fetch('SOURCE_TAG') { nil }
end
def source_url
if source_tag
"#{source_base_url}/tree/#{source_tag}"
else
source_base_url
end
end
def user_agent
@user_agent ||= "#{HTTP::Request::USER_AGENT} (GabSocial/#{Version}; +http#{Rails.configuration.x.use_https ? 's' : ''}://#{Rails.configuration.x.web_domain}/)"
end
end
end

17
lib/generators/post_deployment_migration_generator.rb Normal file
View File

@@ -0,0 +1,17 @@
# frozen_string_literal: true
require 'rails/generators'
module Rails
class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase
def create_migration_file
timestamp = Time.zone.now.strftime('%Y%m%d%H%M%S')
template 'migration.rb', "db/post_migrate/#{timestamp}_#{file_name}.rb"
end
def migration_class_name
file_name.camelize
end
end
end

50
lib/json_ld/security.rb Normal file
View File

@@ -0,0 +1,50 @@
# -*- encoding: utf-8 -*-
# frozen_string_literal: true
# This file generated automatically from https://w3id.org/security/v1
require 'json/ld'
class JSON::LD::Context
add_preloaded("https://w3id.org/security/v1") do
new(processingMode: "json-ld-1.0", term_definitions: {
"CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true),
"EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true),
"EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true),
"GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true),
"LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true),
"LinkedDataSignature2016" => TermDefinition.new("LinkedDataSignature2016", id: "https://w3id.org/security#LinkedDataSignature2016", simple: true),
"authenticationTag" => TermDefinition.new("authenticationTag", id: "https://w3id.org/security#authenticationTag", simple: true),
"canonicalizationAlgorithm" => TermDefinition.new("canonicalizationAlgorithm", id: "https://w3id.org/security#canonicalizationAlgorithm", simple: true),
"cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true),
"cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true),
"cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true),
"created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
"creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"),
"dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true),
"digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true),
"digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true),
"domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true),
"encryptionKey" => TermDefinition.new("encryptionKey", id: "https://w3id.org/security#encryptionKey", simple: true),
"expiration" => TermDefinition.new("expiration", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
"expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
"id" => TermDefinition.new("id", id: "@id", simple: true),
"initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true),
"iterationCount" => TermDefinition.new("iterationCount", id: "https://w3id.org/security#iterationCount", simple: true),
"nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true),
"normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true),
"owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"),
"password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true),
"privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"),
"privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true),
"publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"),
"publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true),
"publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"),
"revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
"salt" => TermDefinition.new("salt", id: "https://w3id.org/security#salt", simple: true),
"sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true),
"signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true),
"signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signingAlgorithm", simple: true),
"signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true),
"type" => TermDefinition.new("type", id: "@type", simple: true),
"xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true)
})
end
end

16
lib/paperclip/blurhash_transcoder.rb Normal file
View File

@@ -0,0 +1,16 @@
# frozen_string_literal: true
module Paperclip
class BlurhashTranscoder < Paperclip::Processor
def make
return @file unless options[:style] == :small
pixels = convert(':source RGB:-', source: File.expand_path(@file.path)).unpack('C*')
geometry = options.fetch(:file_geometry_parser).from_file(@file)
attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, options[:blurhash] || {})
@file
end
end
end

26
lib/paperclip/gif_transcoder.rb Normal file
View File

@@ -0,0 +1,26 @@
# frozen_string_literal: true
module Paperclip
# This transcoder is only to be used for the MediaAttachment model
# to convert animated gifs to webm
class GifTranscoder < Paperclip::Processor
def make
return File.open(@file.path) unless needs_convert?
final_file = Paperclip::Transcoder.make(file, options, attachment)
attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.mp4'
attachment.instance.file_content_type = 'video/mp4'
attachment.instance.type = MediaAttachment.types[:gifv]
final_file
end
private
def needs_convert?
num_frames = identify('-format %n :file', file: file.path).to_i
options[:style] == :original && num_frames > 1
end
end
end

39
lib/paperclip/lazy_thumbnail.rb Normal file
View File

@@ -0,0 +1,39 @@
# frozen_string_literal: true
module Paperclip
class LazyThumbnail < Paperclip::Thumbnail
def make
return File.open(@file.path) unless needs_convert?
if options[:geometry]
min_side = [@current_geometry.width, @current_geometry.height].min.to_i
options[:geometry] = "#{min_side}x#{min_side}#" if @target_geometry.square? && min_side < @target_geometry.width
elsif options[:pixels]
width = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height.to_f)).round.to_i
height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width.to_f)).round.to_i
options[:geometry] = "#{width}x#{height}>"
end
Paperclip::Thumbnail.make(file, options, attachment)
end
private
def needs_convert?
needs_different_geometry? || needs_different_format? || needs_metadata_stripping?
end
def needs_different_geometry?
(options[:geometry] && @current_geometry.width != @target_geometry.width && @current_geometry.height != @target_geometry.height) ||
(options[:pixels] && @current_geometry.width * @current_geometry.height > options[:pixels])
end
def needs_different_format?
@format.present? && @current_format != @format
end
def needs_metadata_stripping?
@attachment.instance.respond_to?(:local?) && @attachment.instance.local?
end
end
end

14
lib/paperclip/video_transcoder.rb Normal file
View File

@@ -0,0 +1,14 @@
# frozen_string_literal: true
module Paperclip
# This transcoder is only to be used for the MediaAttachment model
# to check when uploaded videos are actually gifv's
class VideoTranscoder < Paperclip::Processor
def make
meta = ::Av.cli.identify(@file.path)
attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode]
Paperclip::Transcoder.make(file, options, attachment)
end
end
end

20
lib/tasks/assets.rake Normal file
View File

@@ -0,0 +1,20 @@
# frozen_string_literal: true
def render_static_page(action, dest:, **opts)
html = ApplicationController.render(action, opts)
File.write(dest, html)
end
namespace :assets do
desc 'Generate static pages'
task generate_static_pages: :environment do
render_static_page 'errors/500', layout: 'error', dest: Rails.root.join('public', 'assets', '500.html')
end
end
if Rake::Task.task_defined?('assets:precompile')
Rake::Task['assets:precompile'].enhance do
Webpacker.manifest.refresh
Rake::Task['assets:generate_static_pages'].invoke
end
end

45
lib/tasks/auto_annotate_models.rake Normal file
View File

@@ -0,0 +1,45 @@
# frozen_string_literal: true
if Rails.env.development?
task :set_annotation_options do
Annotate.set_defaults(
'routes' => 'false',
'position_in_routes' => 'before',
'position_in_class' => 'before',
'position_in_test' => 'before',
'position_in_fixture' => 'before',
'position_in_factory' => 'before',
'position_in_serializer' => 'before',
'show_foreign_keys' => 'false',
'show_indexes' => 'false',
'simple_indexes' => 'false',
'model_dir' => 'app/models',
'root_dir' => '',
'include_version' => 'false',
'require' => '',
'exclude_tests' => 'true',
'exclude_fixtures' => 'true',
'exclude_factories' => 'true',
'exclude_serializers' => 'true',
'exclude_scaffolds' => 'true',
'exclude_controllers' => 'true',
'exclude_helpers' => 'true',
'ignore_model_sub_dir' => 'false',
'ignore_columns' => nil,
'ignore_routes' => nil,
'ignore_unknown_models' => 'false',
'hide_limit_column_types' => 'integer,boolean',
'skip_on_db_migrate' => 'false',
'format_bare' => 'true',
'format_rdoc' => 'false',
'format_markdown' => 'false',
'sort' => 'false',
'force' => 'false',
'trace' => 'false',
'wrapper_open' => nil,
'wrapper_close' => nil
)
end
Annotate.load_tasks
end

75
lib/tasks/db.rake Normal file
View File

@@ -0,0 +1,75 @@
# frozen_string_literal: true
require_relative '../gabsocial/snowflake'
def each_schema_load_environment
# If we're in development, also run this for the test environment.
# This is a somewhat hacky way to do this, so here's why:
# 1. We have to define this before we load the schema, or we won't
# have a timestamp_id function when we get to it in the schema.
# 2. db:setup calls db:schema:load_if_ruby, which calls
# db:schema:load, which we define above as having a prerequisite
# of this task.
# 3. db:schema:load ends up running
# ActiveRecord::Tasks::DatabaseTasks.load_schema_current, which
# calls a private method `each_current_configuration`, which
# explicitly also does the loading for the `test` environment
# if the current environment is `development`, so we end up
# needing to do the same, and we can't even use the same method
# to do it.
if Rails.env.development?
test_conf = ActiveRecord::Base.configurations['test']
if test_conf['database']&.present?
ActiveRecord::Base.establish_connection(:test)
yield
ActiveRecord::Base.establish_connection(Rails.env.to_sym)
end
end
yield
end
namespace :db do
namespace :migrate do
desc 'Setup the db or migrate depending on state of db'
task setup: :environment do
begin
if ActiveRecord::Migrator.current_version.zero?
Rake::Task['db:migrate'].invoke
Rake::Task['db:seed'].invoke
end
rescue ActiveRecord::NoDatabaseError
Rake::Task['db:setup'].invoke
else
Rake::Task['db:migrate'].invoke
end
end
end
# Before we load the schema, define the timestamp_id function.
# Idiomatically, we might do this in a migration, but then it
# wouldn't end up in schema.rb, so we'd need to figure out a way to
# get it in before doing db:setup as well. This is simpler, and
# ensures it's always in place.
Rake::Task['db:schema:load'].enhance ['db:define_timestamp_id']
# After we load the schema, make sure we have sequences for each
# table using timestamp IDs.
Rake::Task['db:schema:load'].enhance do
Rake::Task['db:ensure_id_sequences_exist'].invoke
end
task :define_timestamp_id do
each_schema_load_environment do
GabSocial::Snowflake.define_timestamp_id
end
end
task :ensure_id_sequences_exist do
each_schema_load_environment do
GabSocial::Snowflake.ensure_id_sequences_exist
end
end
end

58
lib/tasks/emojis.rake Normal file
View File

@@ -0,0 +1,58 @@
# frozen_string_literal: true
def codepoints_to_filename(codepoints)
codepoints.downcase.gsub(/\A[0]+/, '').tr(' ', '-')
end
def codepoints_to_unicode(codepoints)
if codepoints.include?(' ')
codepoints.split(' ').map(&:hex).pack('U*')
else
[codepoints.hex].pack('U')
end
end
namespace :emojis do
desc 'Generate a unicode to filename mapping'
task :generate do
source = 'http://www.unicode.org/Public/emoji/11.0/emoji-test.txt'
codes = []
dest = Rails.root.join('app', 'javascript', 'gabsocial', 'features', 'emoji', 'emoji_map.json')
puts "Downloading emojos from source... (#{source})"
HTTP.get(source).to_s.split("\n").each do |line|
next if line.start_with? '#'
parts = line.split(';').map(&:strip)
next if parts.size < 2
codes << [parts[0], parts[1].start_with?('fully-qualified')]
end
grouped_codes = codes.reduce([]) do |agg, current|
if current[1]
agg << [current[0]]
else
agg.last << current[0]
agg
end
end
existence_maps = grouped_codes.map { |c| c.map { |cc| [cc, File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg'))] }.to_h }
map = {}
existence_maps.each do |group|
existing_one = group.key(true)
next if existing_one.nil?
group.each_key do |key|
map[codepoints_to_unicode(key)] = codepoints_to_filename(existing_one)
end
end
map = map.sort { |a, b| a[0].size <=> b[0].size }.to_h
File.write(dest, Oj.dump(map))
puts "Wrote emojo to destination! (#{dest})"
end
end

408
lib/tasks/gabsocial.rake Normal file
View File

@@ -0,0 +1,408 @@
# frozen_string_literal: true
require 'tty-command'
require 'tty-prompt'
namespace :gabsocial do
desc 'Configure the instance for production use'
task :setup do
prompt = TTY::Prompt.new
env = {}
begin
prompt.say('Your instance is identified by its domain name. Changing it afterward will break things.')
env['LOCAL_DOMAIN'] = prompt.ask('Domain name:') do |q|
q.required true
q.modify :strip
q.validate(/\A[a-z0-9\.\-]+\z/i)
q.messages[:valid?] = 'Invalid domain. If you intend to use unicode characters, enter punycode here'
end
prompt.say "\n"
prompt.say('Single user mode disables registrations and redirects the landing page to your public profile.')
env['SINGLE_USER_MODE'] = prompt.yes?('Do you want to enable single user mode?', default: false)
%w(SECRET_KEY_BASE OTP_SECRET).each do |key|
env[key] = SecureRandom.hex(64)
end
vapid_key = Webpush.generate_key
env['VAPID_PRIVATE_KEY'] = vapid_key.private_key
env['VAPID_PUBLIC_KEY'] = vapid_key.public_key
prompt.say "\n"
using_docker = prompt.yes?('Are you using Docker to run Gab Social?')
db_connection_works = false
prompt.say "\n"
loop do
env['DB_HOST'] = prompt.ask('PostgreSQL host:') do |q|
q.required true
q.default using_docker ? 'db' : '/var/run/postgresql'
q.modify :strip
end
env['DB_PORT'] = prompt.ask('PostgreSQL port:') do |q|
q.required true
q.default 5432
q.convert :int
end
env['DB_NAME'] = prompt.ask('Name of PostgreSQL database:') do |q|
q.required true
q.default using_docker ? 'postgres' : 'gabsocial_production'
q.modify :strip
end
env['DB_USER'] = prompt.ask('Name of PostgreSQL user:') do |q|
q.required true
q.default using_docker ? 'postgres' : 'gabsocial'
q.modify :strip
end
env['DB_PASS'] = prompt.ask('Password of PostgreSQL user:') do |q|
q.echo false
end
# The chosen database may not exist yet. Connect to default database
# to avoid "database does not exist" error.
db_options = {
adapter: :postgresql,
database: 'postgres',
host: env['DB_HOST'],
port: env['DB_PORT'],
user: env['DB_USER'],
password: env['DB_PASS'],
}
begin
ActiveRecord::Base.establish_connection(db_options)
ActiveRecord::Base.connection
prompt.ok 'Database configuration works! 🎆'
db_connection_works = true
break
rescue StandardError => e
prompt.error 'Database connection could not be established with this configuration, try again.'
prompt.error e.message
break unless prompt.yes?('Try again?')
end
end
prompt.say "\n"
loop do
env['REDIS_HOST'] = prompt.ask('Redis host:') do |q|
q.required true
q.default using_docker ? 'redis' : 'localhost'
q.modify :strip
end
env['REDIS_PORT'] = prompt.ask('Redis port:') do |q|
q.required true
q.default 6379
q.convert :int
end
env['REDIS_PASSWORD'] = prompt.ask('Redis password:') do |q|
q.required false
q.default nil
q.modify :strip
end
redis_options = {
host: env['REDIS_HOST'],
port: env['REDIS_PORT'],
password: env['REDIS_PASSWORD'],
driver: :hiredis,
}
begin
redis = Redis.new(redis_options)
redis.ping
prompt.ok 'Redis configuration works! 🎆'
break
rescue StandardError => e
prompt.error 'Redis connection could not be established with this configuration, try again.'
prompt.error e.message
break unless prompt.yes?('Try again?')
end
end
prompt.say "\n"
if prompt.yes?('Do you want to store uploaded files on the cloud?', default: false)
case prompt.select('Provider', ['Amazon S3', 'Wasabi', 'Minio'])
when 'Amazon S3'
env['S3_ENABLED'] = 'true'
env['S3_PROTOCOL'] = 'https'
env['S3_BUCKET'] = prompt.ask('S3 bucket name:') do |q|
q.required true
q.default "files.#{env['LOCAL_DOMAIN']}"
q.modify :strip
end
env['S3_REGION'] = prompt.ask('S3 region:') do |q|
q.required true
q.default 'us-east-1'
q.modify :strip
end
env['S3_HOSTNAME'] = prompt.ask('S3 hostname:') do |q|
q.required true
q.default 's3-us-east-1.amazonaws.com'
q.modify :strip
end
env['AWS_ACCESS_KEY_ID'] = prompt.ask('S3 access key:') do |q|
q.required true
q.modify :strip
end
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('S3 secret key:') do |q|
q.required true
q.modify :strip
end
when 'Wasabi'
env['S3_ENABLED'] = 'true'
env['S3_PROTOCOL'] = 'https'
env['S3_REGION'] = 'us-east-1'
env['S3_HOSTNAME'] = 's3.wasabisys.com'
env['S3_ENDPOINT'] = 'https://s3.wasabisys.com/'
env['S3_BUCKET'] = prompt.ask('Wasabi bucket name:') do |q|
q.required true
q.default "files.#{env['LOCAL_DOMAIN']}"
q.modify :strip
end
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Wasabi access key:') do |q|
q.required true
q.modify :strip
end
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Wasabi secret key:') do |q|
q.required true
q.modify :strip
end
when 'Minio'
env['S3_ENABLED'] = 'true'
env['S3_PROTOCOL'] = 'https'
env['S3_REGION'] = 'us-east-1'
env['S3_ENDPOINT'] = prompt.ask('Minio endpoint URL:') do |q|
q.required true
q.modify :strip
end
env['S3_PROTOCOL'] = env['S3_ENDPOINT'].start_with?('https') ? 'https' : 'http'
env['S3_HOSTNAME'] = env['S3_ENDPOINT'].gsub(/\Ahttps?:\/\//, '')
env['S3_BUCKET'] = prompt.ask('Minio bucket name:') do |q|
q.required true
q.default "files.#{env['LOCAL_DOMAIN']}"
q.modify :strip
end
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Minio access key:') do |q|
q.required true
q.modify :strip
end
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Minio secret key:') do |q|
q.required true
q.modify :strip
end
end
if prompt.yes?('Do you want to access the uploaded files from your own domain?')
env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
q.required true
q.default "files.#{env['LOCAL_DOMAIN']}"
q.modify :strip
end
end
end
prompt.say "\n"
loop do
if prompt.yes?('Do you want to send e-mails from localhost?', default: false)
env['SMTP_SERVER'] = 'localhost'
env['SMTP_PORT'] = 25
env['SMTP_AUTH_METHOD'] = 'none'
env['SMTP_OPENSSL_VERIFY_MODE'] = 'none'
else
env['SMTP_SERVER'] = prompt.ask('SMTP server:') do |q|
q.required true
q.default 'smtp.mailgun.org'
q.modify :strip
end
env['SMTP_PORT'] = prompt.ask('SMTP port:') do |q|
q.required true
q.default 587
q.convert :int
end
env['SMTP_LOGIN'] = prompt.ask('SMTP username:') do |q|
q.modify :strip
end
env['SMTP_PASSWORD'] = prompt.ask('SMTP password:') do |q|
q.echo false
end
env['SMTP_AUTH_METHOD'] = prompt.ask('SMTP authentication:') do |q|
q.required
q.default 'plain'
q.modify :strip
end
env['SMTP_OPENSSL_VERIFY_MODE'] = prompt.select('SMTP OpenSSL verify mode:', %w(none peer client_once fail_if_no_peer_cert))
end
env['SMTP_FROM_ADDRESS'] = prompt.ask('E-mail address to send e-mails "from":') do |q|
q.required true
q.default "Gab Social <noreply@#{env['LOCAL_DOMAIN']}>"
q.modify :strip
end
break unless prompt.yes?('Send a test e-mail with this configuration right now?')
send_to = prompt.ask('Send test e-mail to:', required: true)
begin
ActionMailer::Base.smtp_settings = {
port: env['SMTP_PORT'],
address: env['SMTP_SERVER'],
user_name: env['SMTP_LOGIN'].presence,
password: env['SMTP_PASSWORD'].presence,
domain: env['LOCAL_DOMAIN'],
authentication: env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain,
openssl_verify_mode: env['SMTP_OPENSSL_VERIFY_MODE'],
enable_starttls_auto: true,
}
ActionMailer::Base.default_options = {
from: env['SMTP_FROM_ADDRESS'],
}
mail = ActionMailer::Base.new.mail to: send_to, subject: 'Test', body: 'Gab Social SMTP configuration works!'
mail.deliver
break
rescue StandardError => e
prompt.error 'E-mail could not be sent with this configuration, try again.'
prompt.error e.message
break unless prompt.yes?('Try again?')
end
end
prompt.say "\n"
prompt.say 'This configuration will be written to .env.production'
if prompt.yes?('Save configuration?')
cmd = TTY::Command.new(printer: :quiet)
File.write(Rails.root.join('.env.production'), "# Generated with gabsocial:setup on #{Time.now.utc}\n\n" + env.each_pair.map { |key, value| "#{key}=#{value}" }.join("\n") + "\n")
if using_docker
prompt.ok 'Below is your configuration, save it to an .env.production file outside Docker:'
prompt.say "\n"
prompt.say File.read(Rails.root.join('.env.production'))
prompt.say "\n"
prompt.ok 'It is also saved within this container so you can proceed with this wizard.'
end
prompt.say "\n"
prompt.say 'Now that configuration is saved, the database schema must be loaded.'
prompt.warn 'If the database already exists, this will erase its contents.'
if prompt.yes?('Prepare the database now?')
prompt.say 'Running `RAILS_ENV=production rails db:setup` ...'
prompt.say "\n\n"
if cmd.run!({ RAILS_ENV: 'production', SAFETY_ASSURED: 1 }, :rails, 'db:setup').failure?
prompt.error 'That failed! Perhaps your configuration is not right'
else
prompt.ok 'Done!'
end
end
prompt.say "\n"
prompt.say 'The final step is compiling CSS/JS assets.'
prompt.say 'This may take a while and consume a lot of RAM.'
if prompt.yes?('Compile the assets now?')
prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...'
prompt.say "\n\n"
if cmd.run!({ RAILS_ENV: 'production' }, :rails, 'assets:precompile').failure?
prompt.error 'That failed! Maybe you need swap space?'
else
prompt.say 'Done!'
end
end
prompt.say "\n"
prompt.ok 'All done! You can now power on the Gab Social server.'
prompt.say "\n"
if db_connection_works && prompt.yes?('Do you want to create an admin user straight away?')
env.each_pair do |key, value|
ENV[key] = value.to_s
end
require_relative '../../config/environment'
disable_log_stdout!
username = prompt.ask('Username:') do |q|
q.required true
q.default 'admin'
q.validate(/\A[a-z0-9_]+\z/i)
q.modify :strip
end
email = prompt.ask('E-mail:') do |q|
q.required true
q.modify :strip
end
password = SecureRandom.hex(16)
user = User.new(admin: true, email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username })
user.save(validate: false)
prompt.ok "You can login with the password: #{password}"
prompt.warn 'You can change your password once you login.'
end
else
prompt.warn 'Nothing saved. Bye!'
end
rescue TTY::Reader::InputInterrupt
prompt.ok 'Aborting. Bye!'
end
end
namespace :webpush do
desc 'Generate VAPID key'
task generate_vapid_key: :environment do
vapid_key = Webpush.generate_key
puts "VAPID_PRIVATE_KEY=#{vapid_key.private_key}"
puts "VAPID_PUBLIC_KEY=#{vapid_key.public_key}"
end
end
end
def disable_log_stdout!
dev_null = Logger.new('/dev/null')
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
Paperclip.options[:log] = false
end

57
lib/tasks/repo.rake Normal file
View File

@@ -0,0 +1,57 @@
# frozen_string_literal: true
namespace :repo do
desc 'Generate the AUTHORS.md file'
task :authors do
# Gab AI, Inc., does not disclose information about contributors to the Gab
# Social project in this way. Please refer to our public git repository or
# contact us directly at legal@gab.com with any questions about the
# contributors to Gab Social.
end
desc 'Replace pull requests with authors in the CHANGELOG.md file'
task :changelog do
path = Rails.root.join('CHANGELOG.md')
tmp = Tempfile.new
HttpLog.config.compact_log = true
begin
File.open(path, 'r') do |file|
file.each_line do |line|
if line.start_with?('-')
new_line = line.gsub(/#([[:digit:]]+)*/) do |pull_request_reference|
pull_request_number = pull_request_reference[1..-1]
response = nil
loop do
response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/gab-ai-inc/gab-social/pulls/#{pull_request_number}")
if response.code == 403
sleep_for = (response.headers['X-RateLimit-Reset'].to_i - Time.now.to_i).abs
puts "Sleeping for #{sleep_for} seconds to get over rate limit"
sleep sleep_for
else
break
end
end
pull_request = Oj.load(response.to_s)
"[#{pull_request['user']['login']}](#{pull_request['html_url']})"
end
tmp.puts new_line
else
tmp.puts line
end
end
end
tmp.close
FileUtils.mv(tmp.path, path)
ensure
tmp.close
tmp.unlink
end
end
end

19
lib/tasks/statistics.rake Normal file
View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true
task stats: 'gabsocial:stats'
namespace :gabsocial do
desc 'Report code statistics (KLOCs, etc)'
task :stats do
require 'rails/code_statistics'
[
%w(App\ Libraries app/lib),
%w(Presenters app/presenters),
%w(Services app/services),
%w(Validators app/validators),
%w(Workers app/workers),
].each do |name, dir|
::STATS_DIRECTORIES << [name, Rails.root.join(dir)]
end
end
end

10
lib/templates/haml/scaffold/_form.html.haml Normal file
View File

@@ -0,0 +1,10 @@
= simple_form_for(@<%= singular_table_name %>) do |f|
= f.error_notification
.form-inputs
<%- attributes.each do |attribute| -%>
= f.<%= attribute.reference? ? :association : :input %> :<%= attribute.name %>
<%- end -%>
.form-actions
= f.button :submit

8
lib/templates/rails/post_deployment_migration/migration.rb Normal file
View File

@@ -0,0 +1,8 @@
# frozen_string_literal: true
class <%= migration_class_name %> < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
def change
end
end