diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 9962bbde6..b3aa9f45c 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -12,6 +12,7 @@ on: - Dockerfile permissions: contents: read + packages: write jobs: build-image: @@ -26,15 +27,28 @@ jobs: - uses: hadolint/hadolint-action@v3.1.0 - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - if: github.event_name != 'pull_request' + if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' + + - name: Log in to the Github Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' + - uses: docker/metadata-action@v4 id: meta with: - images: tootsuite/mastodon + images: | + tootsuite/mastodon + ghcr.io/mastodon/mastodon flavor: | latest=auto tags: | @@ -42,13 +56,15 @@ jobs: type=pep440,pattern={{raw}} type=pep440,pattern=v{{major}}.{{minor}} type=ref,event=pr + - uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64,linux/arm64 provenance: false builder: ${{ steps.buildx.outputs.name }} - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a6998e8c..7416d2e6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,57 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.1] - 2023-03-16 + +### Added + +- Add redirection from paths with url-encoded `@` to their decoded form ([thijskh](https://github.com/mastodon/mastodon/pull/23593)) +- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749)) +- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597)) +- Add support for refreshing many accounts at once with `tootctl accounts refresh` ([9p4](https://github.com/mastodon/mastodon/pull/23304)) +- Add confirmation modal when clicking to edit a post with a non-empty compose form ([PauloVilarinho](https://github.com/mastodon/mastodon/pull/23936)) +- Add support for the HAproxy PROXY protocol through the `PROXY_PROTO_V1` environment variable ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24064)) +- Add `SENDFILE_HEADER` environment variable ([Gargron](https://github.com/mastodon/mastodon/pull/24123)) +- Add cache headers to static files served through Rails ([Gargron](https://github.com/mastodon/mastodon/pull/24120)) + +### Changed + +- Increase contrast of upload progress bar background ([toolmantim](https://github.com/mastodon/mastodon/pull/23836)) +- Change post auto-deletion throttling constants to better scale with server size ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23320)) +- Change order of bookmark and favourite sidebar entries in single-column UI for consistency ([TerryGarcia](https://github.com/mastodon/mastodon/pull/23701)) +- Change `ActivityPub::DeliveryWorker` retries to be spread out more ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21956)) + +### Fixed + +- Fix “Remove all followers from the selected domains” also removing follows and notifications ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805)) +- Fix streaming metrics format ([emilweth](https://github.com/mastodon/mastodon/pull/23519), [emilweth](https://github.com/mastodon/mastodon/pull/23520)) +- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526)) +- Fix focus point of already-attached media not saving after edit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23566)) +- Fix sidebar behavior in settings/admin UI on mobile ([wxt2005](https://github.com/mastodon/mastodon/pull/23764)) +- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801)) +- Fix duplicate “Publish” button on mobile ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23804)) +- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787)) +- Fix server error when attempting to display the edit history of a trendable post in the admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23574)) +- Fix `tootctl accounts migrate` crashing because of a typo ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23567)) +- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957)) +- Fix the “Back” button in column headers sometimes leaving Mastodon ([c960657](https://github.com/mastodon/mastodon/pull/23953)) +- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958)) +- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803)) +- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988)) +- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029)) +- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046)) +- Fix tags being unnecessarily stripped from plain-text short site description ([c960657](https://github.com/mastodon/mastodon/pull/23975)) +- Fix HTML entities not being un-escaped in extracted plain-text from remote posts ([c960657](https://github.com/mastodon/mastodon/pull/24019)) +- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751)) +- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611)) +- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568)) +- Fix duplicate mails being sent when the SMTP server is too slow to close the connection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23750)) + +### Security + +- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136)) +- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137)) + ## [4.1.0] - 2023-02-10 ### Added diff --git a/README.md b/README.md index 1b5db92a8..abf16d674 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,11 @@ [![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci] [![Code Climate](https://img.shields.io/codeclimate/maintainability/mastodon/mastodon.svg)][code_climate] [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin] -[![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker] [releases]: https://github.com/mastodon/mastodon/releases [circleci]: https://circleci.com/gh/mastodon/mastodon [code_climate]: https://codeclimate.com/github/mastodon/mastodon [crowdin]: https://crowdin.com/project/mastodon -[docker]: https://hub.docker.com/r/tootsuite/mastodon/ Mastodon is a **free, open-source social network server** based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub!) @@ -31,6 +29,7 @@ Click below to **learn more** in a video: - [View sponsors](https://joinmastodon.org/sponsors) - [Blog](https://blog.joinmastodon.org) - [Documentation](https://docs.joinmastodon.org) +- [Official Docker image](https://github.com/mastodon/mastodon/pkgs/container/mastodon) - [Browse Mastodon servers](https://joinmastodon.org/communities) - [Browse Mastodon apps](https://joinmastodon.org/apps) diff --git a/app/controllers/backups_controller.rb b/app/controllers/backups_controller.rb new file mode 100644 index 000000000..2f4b400b8 --- /dev/null +++ b/app/controllers/backups_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackupsController < ApplicationController + include RoutingHelper + + skip_before_action :require_functional! + + before_action :authenticate_user! + before_action :set_backup + + def download + case Paperclip::Attachment.default_options[:storage] + when :s3 + redirect_to @backup.dump.expiring_url(10) + when :fog + redirect_to @backup.dump.expiring_url(Time.now.utc + 10) + when :filesystem + redirect_to full_asset_url(@backup.dump.url) + end + end + + private + + def set_backup + @backup = current_user.backups.find(params[:id]) + end +end diff --git a/app/controllers/relationships_controller.rb b/app/controllers/relationships_controller.rb index 96cce55e9..de5dc5879 100644 --- a/app/controllers/relationships_controller.rb +++ b/app/controllers/relationships_controller.rb @@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController @form.save rescue ActionController::ParameterMissing # Do nothing + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound + flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow' ensure redirect_to relationships_path(filter_params) end @@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController 'unfollow' elsif params[:remove_from_followers] 'remove_from_followers' - elsif params[:block_domains] - 'block_domains' + elsif params[:block_domains] || params[:remove_domains_from_followers] + 'remove_domains_from_followers' end end diff --git a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb index a50d30f06..8435155dd 100644 --- a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb +++ b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb @@ -52,7 +52,7 @@ module Settings end else flash[:error] = I18n.t('webauthn_credentials.create.error') - status = :internal_server_error + status = :unprocessable_entity end else flash[:error] = t('webauthn_credentials.create.error') diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js index 72e592935..3756a975b 100644 --- a/app/javascript/mastodon/actions/compose.js +++ b/app/javascript/mastodon/actions/compose.js @@ -165,11 +165,19 @@ export function submitCompose(routerHistory) { // API call. let media_attributes; if (statusId !== null) { - media_attributes = media.map(item => ({ - id: item.get('id'), - description: item.get('description'), - focus: item.get('focus'), - })); + media_attributes = media.map(item => { + let focus; + + if (item.getIn(['meta', 'focus'])) { + focus = `${item.getIn(['meta', 'focus', 'x']).toFixed(2)},${item.getIn(['meta', 'focus', 'y']).toFixed(2)}`; + } + + return { + id: item.get('id'), + description: item.get('description'), + focus, + }; + }); } api(getState).request({ diff --git a/app/javascript/mastodon/components/column_back_button.js b/app/javascript/mastodon/components/column_back_button.js index 5bbf11652..5c5226b7e 100644 --- a/app/javascript/mastodon/components/column_back_button.js +++ b/app/javascript/mastodon/components/column_back_button.js @@ -15,10 +15,10 @@ export default class ColumnBackButton extends React.PureComponent { }; handleClick = () => { - if (window.history && window.history.length === 1) { - this.context.router.history.push('/'); - } else { + if (window.history && window.history.state) { this.context.router.history.goBack(); + } else { + this.context.router.history.push('/'); } }; diff --git a/app/javascript/mastodon/components/column_header.js b/app/javascript/mastodon/components/column_header.js index 38f6ad60f..9ba783d90 100644 --- a/app/javascript/mastodon/components/column_header.js +++ b/app/javascript/mastodon/components/column_header.js @@ -43,14 +43,6 @@ class ColumnHeader extends React.PureComponent { animating: false, }; - historyBack = () => { - if (window.history && window.history.length === 1) { - this.context.router.history.push('/'); - } else { - this.context.router.history.goBack(); - } - }; - handleToggleClick = (e) => { e.stopPropagation(); this.setState({ collapsed: !this.state.collapsed, animating: true }); @@ -69,7 +61,11 @@ class ColumnHeader extends React.PureComponent { }; handleBackClick = () => { - this.historyBack(); + if (window.history && window.history.state) { + this.context.router.history.goBack(); + } else { + this.context.router.history.push('/'); + } }; handleTransitionEnd = () => { diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js index 294105f25..580f409e9 100644 --- a/app/javascript/mastodon/containers/status_container.js +++ b/app/javascript/mastodon/containers/status_container.js @@ -56,6 +56,8 @@ const messages = defineMessages({ redraftMessage: { id: 'confirmations.redraft.message', defaultMessage: 'Are you sure you want to delete this status and re-draft it? Favourites and boosts will be lost, and replies to the original post will be orphaned.' }, replyConfirm: { id: 'confirmations.reply.confirm', defaultMessage: 'Reply' }, replyMessage: { id: 'confirmations.reply.message', defaultMessage: 'Replying now will overwrite the message you are currently composing. Are you sure you want to proceed?' }, + editConfirm: { id: 'confirmations.edit.confirm', defaultMessage: 'Edit' }, + editMessage: { id: 'confirmations.edit.message', defaultMessage: 'Editing now will overwrite the message you are currently composing. Are you sure you want to proceed?' }, blockDomainConfirm: { id: 'confirmations.domain_block.confirm', defaultMessage: 'Hide entire domain' }, }); @@ -149,7 +151,18 @@ const mapDispatchToProps = (dispatch, { intl, contextType }) => ({ }, onEdit (status, history) { - dispatch(editStatus(status.get('id'), history)); + dispatch((_, getState) => { + let state = getState(); + if (state.getIn(['compose', 'text']).trim().length !== 0) { + dispatch(openModal('CONFIRM', { + message: intl.formatMessage(messages.editMessage), + confirm: intl.formatMessage(messages.editConfirm), + onConfirm: () => dispatch(editStatus(status.get('id'), history)), + })); + } else { + dispatch(editStatus(status.get('id'), history)); + } + }); }, onTranslate (status) { diff --git a/app/javascript/mastodon/features/compose/components/language_dropdown.js b/app/javascript/mastodon/features/compose/components/language_dropdown.js index d96d39f23..82547e079 100644 --- a/app/javascript/mastodon/features/compose/components/language_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/language_dropdown.js @@ -210,7 +210,7 @@ class LanguageDropdownMenu extends React.PureComponent { return (
- {lang[2]} ({lang[1]}) + {lang[2]} ({lang[1]})
); }; diff --git a/app/javascript/mastodon/features/ui/components/header.js b/app/javascript/mastodon/features/ui/components/header.js index 1384bebda..92adc47a9 100644 --- a/app/javascript/mastodon/features/ui/components/header.js +++ b/app/javascript/mastodon/features/ui/components/header.js @@ -22,8 +22,8 @@ const mapDispatchToProps = (dispatch) => ({ }, }); -export default @connect(null, mapDispatchToProps) -@withRouter +export default @withRouter +@connect(null, mapDispatchToProps) class Header extends React.PureComponent { static contextTypes = { diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js index 9a9309be0..755b19349 100644 --- a/app/javascript/mastodon/features/ui/components/navigation_panel.js +++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js @@ -82,8 +82,8 @@ class NavigationPanel extends React.Component { {signedIn && ( - + diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js index 4f0ea0450..2dd59f95d 100644 --- a/app/javascript/mastodon/features/ui/index.js +++ b/app/javascript/mastodon/features/ui/index.js @@ -474,10 +474,10 @@ class UI extends React.PureComponent { }; handleHotkeyBack = () => { - if (window.history && window.history.length === 1) { - this.context.router.history.push('/'); - } else { + if (window.history && window.history.state) { this.context.router.history.goBack(); + } else { + this.context.router.history.push('/'); } }; diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json index ae7722635..f4d051940 100644 --- a/app/javascript/mastodon/locales/en.json +++ b/app/javascript/mastodon/locales/en.json @@ -162,6 +162,8 @@ "confirmations.discard_edit_media.message": "You have unsaved changes to the media description or preview, discard them anyway?", "confirmations.domain_block.confirm": "Block entire domain", "confirmations.domain_block.message": "Are you really, really sure you want to block the entire {domain}? In most cases a few targeted blocks or mutes are sufficient and preferable. You will not see content from that domain in any public timelines or your notifications. Your followers from that domain will be removed.", + "confirmations.edit.confirm": "Edit", + "confirmations.edit.message": "Editing now will overwrite the message you are currently composing. Are you sure you want to proceed?", "confirmations.logout.confirm": "Log out", "confirmations.logout.message": "Are you sure you want to log out?", "confirmations.mute.confirm": "Mute", diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index 783d748ae..842b7af51 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -186,11 +186,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => { }; const sortHashtagsByUse = (state, tags) => { - const personalHistory = state.get('tagHistory'); + const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase()); - return tags.sort((a, b) => { - const usedA = personalHistory.includes(a.name); - const usedB = personalHistory.includes(b.name); + const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() })); + const sorted = tagsWithLowercase.sort((a, b) => { + const usedA = personalHistory.includes(a.lowerName); + const usedB = personalHistory.includes(b.lowerName); if (usedA === usedB) { return 0; @@ -200,6 +201,8 @@ const sortHashtagsByUse = (state, tags) => { return 1; } }); + sorted.forEach(tag => delete tag.lowerName); + return sorted; }; const insertEmoji = (state, position, emojiData, needsSpace) => { diff --git a/app/javascript/styles/mastodon-light/diff.scss b/app/javascript/styles/mastodon-light/diff.scss index c37100a28..db564b227 100644 --- a/app/javascript/styles/mastodon-light/diff.scss +++ b/app/javascript/styles/mastodon-light/diff.scss @@ -254,6 +254,10 @@ html { border-color: $ui-base-color; } +.upload-progress__backdrop { + background: $ui-base-color; +} + // Change the background colors of statuses .focusable:focus { background: $ui-base-color; diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss index 674fafbe9..505f59a46 100644 --- a/app/javascript/styles/mastodon/admin.scss +++ b/app/javascript/styles/mastodon/admin.scss @@ -384,7 +384,7 @@ $content-width: 840px; position: fixed; z-index: 10; width: 100%; - height: calc(100vh - 56px); + height: calc(100% - 56px); left: 0; bottom: 0; overflow-y: auto; diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 493efea30..e4616c110 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -4482,6 +4482,7 @@ a.status-card.compact:hover { display: flex; align-items: center; justify-content: center; + text-align: center; color: $secondary-text-color; font-size: 18px; font-weight: 500; @@ -4516,7 +4517,7 @@ a.status-card.compact:hover { width: 100%; height: 6px; border-radius: 6px; - background: $ui-base-lighter-color; + background: darken($simple-background-color, 8%); position: relative; margin-top: 5px; } diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb index f512635ab..89dfcef9f 100644 --- a/app/lib/admin/system_check.rb +++ b/app/lib/admin/system_check.rb @@ -2,6 +2,7 @@ class Admin::SystemCheck ACTIVE_CHECKS = [ + Admin::SystemCheck::MediaPrivacyCheck, Admin::SystemCheck::DatabaseSchemaCheck, Admin::SystemCheck::SidekiqProcessCheck, Admin::SystemCheck::RulesCheck, diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 5b4c12399..0b55be350 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -31,7 +31,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def running_version @running_version ||= begin Chewy.client.info['version']['number'] - rescue Faraday::ConnectionFailed + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error nil end end diff --git a/app/lib/admin/system_check/media_privacy_check.rb b/app/lib/admin/system_check/media_privacy_check.rb new file mode 100644 index 000000000..1df05b120 --- /dev/null +++ b/app/lib/admin/system_check/media_privacy_check.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck + include RoutingHelper + + def skip? + !current_user.can?(:view_devops) + end + + def pass? + check_media_uploads! + @failure_message.nil? + end + + def message + Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true) + end + + private + + def check_media_uploads! + if Rails.configuration.x.use_s3 + check_media_listing_inaccessible_s3! + else + check_media_listing_inaccessible! + end + end + + def check_media_listing_inaccessible! + full_url = full_asset_url(media_attachment.file.url(:original, false)) + + # Check if we can list the uploaded file. If true, that's an error + directory_url = Addressable::URI.parse(full_url) + directory_url.query = nil + filename = directory_url.path.gsub(%r{.*/}, '') + directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/') + Request.new(:get, directory_url, allow_local: true).perform do |res| + if res.truncated_body&.include?(filename) + @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS' + end + end + rescue + nil + end + + def check_media_listing_inaccessible_s3! + urls_to_check = [] + paperclip_options = Paperclip::Attachment.default_options + s3_protocol = paperclip_options[:s3_protocol] + s3_host_alias = paperclip_options[:s3_host_alias] + s3_host_name = paperclip_options[:s3_host_name] + bucket_name = paperclip_options.dig(:s3_credentials, :bucket) + + urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present? + urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/" + urls_to_check.uniq.each do |full_url| + check_s3_listing!(full_url) + break if @failure_message.present? + end + rescue + nil + end + + def check_s3_listing!(full_url) + bucket_url = Addressable::URI.parse(full_url) + bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original)) + bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}" + Request.new(:get, bucket_url, allow_local: true).perform do |res| + if res.truncated_body&.include?('ListBucketResult') + @failure_message = :upload_check_privacy_error_object_storage + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3' + end + end + end + + def media_attachment + @media_attachment ||= begin + attachment = Account.representative.media_attachments.first + if attachment.present? + attachment.touch # rubocop:disable Rails/SkipsModelValidations + attachment + else + create_test_attachment! + end + end + end + + def create_test_attachment! + Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file| + tmp_file.write( + Base64.decode64( + '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \ + 'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \ + 'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \ + 'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \ + 'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \ + 'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q==' + ) + ) + tmp_file.flush + Account.representative.media_attachments.create!(file: tmp_file) + end + end +end diff --git a/app/lib/admin/system_check/message.rb b/app/lib/admin/system_check/message.rb index bfcad3bf3..ad8d4b607 100644 --- a/app/lib/admin/system_check/message.rb +++ b/app/lib/admin/system_check/message.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true class Admin::SystemCheck::Message - attr_reader :key, :value, :action + attr_reader :key, :value, :action, :critical - def initialize(key, value = nil, action = nil) - @key = key - @value = value - @action = action + def initialize(key, value = nil, action = nil, critical = false) + @key = key + @value = value + @action = action + @critical = critical end end diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb index 08aa29696..6fa2bc5d2 100644 --- a/app/lib/plain_text_formatter.rb +++ b/app/lib/plain_text_formatter.rb @@ -18,7 +18,7 @@ class PlainTextFormatter if local? text else - strip_tags(insert_newlines).chomp + html_entities.decode(strip_tags(insert_newlines)).chomp end end @@ -27,4 +27,8 @@ class PlainTextFormatter def insert_newlines text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" } end + + def html_entities + HTMLEntities.new + end end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index a37682eca..4edcb75f3 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base helper :instance helper :formatting + after_action :set_autoreply_headers! + protected def locale_for_account(account) @@ -14,4 +16,10 @@ class ApplicationMailer < ActionMailer::Base yield end end + + def set_autoreply_headers! + headers['Precedence'] = 'list' + headers['X-Auto-Response-Suppress'] = 'All' + headers['Auto-Submitted'] = 'auto-generated' + end end diff --git a/app/models/account.rb b/app/models/account.rb index 262285a09..28bd828b0 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -107,7 +107,7 @@ class Account < ApplicationRecord scope :bots, -> { where(actor_type: %w(Application Service)) } scope :groups, -> { where(actor_type: 'Group') } scope :alphabetic, -> { order(domain: :asc, username: :asc) } - scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) } + scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") } scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) } diff --git a/app/models/backup.rb b/app/models/backup.rb index d242fd62c..20e6e8aa8 100644 --- a/app/models/backup.rb +++ b/app/models/backup.rb @@ -17,6 +17,6 @@ class Backup < ApplicationRecord belongs_to :user, inverse_of: :backups - has_attached_file :dump + has_attached_file :dump, s3_permissions: 'private' do_not_validate_attachment_file_type :dump end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 473622edf..6a05f8163 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -17,8 +17,8 @@ class Form::AccountBatch unfollow! when 'remove_from_followers' remove_from_followers! - when 'block_domains' - block_domains! + when 'remove_domains_from_followers' + remove_domains_from_followers! when 'approve' approve! when 'reject' @@ -35,9 +35,15 @@ class Form::AccountBatch private def follow! + error = nil + accounts.each do |target_account| FollowService.new.call(current_account, target_account) + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e + error ||= e end + + raise error if error.present? end def unfollow! @@ -50,10 +56,8 @@ class Form::AccountBatch RemoveFromFollowersService.new.call(current_account, account_ids) end - def block_domains! - AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain| - [current_account.id, domain] - end + def remove_domains_from_followers! + RemoveDomainsFromFollowersService.new.call(current_account, account_domains) end def account_domains diff --git a/app/models/user.rb b/app/models/user.rb index c767f8984..efbe29f44 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -504,11 +504,14 @@ class User < ApplicationRecord def prepare_new_user! BootstrapTimelineWorker.perform_async(account_id) ActivityTracker.increment('activity:accounts:local') + ActivityTracker.record('activity:logins', id) UserMailer.welcome(self).deliver_later TriggerWebhookWorker.perform_async('account.approved', 'Account', account_id) end def prepare_returning_user! + return unless confirmed? + ActivityTracker.record('activity:logins', id) regenerate_feed! if needs_feed_update? end diff --git a/app/services/follow_migration_service.rb b/app/services/follow_migration_service.rb new file mode 100644 index 000000000..cfe9093cb --- /dev/null +++ b/app/services/follow_migration_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class FollowMigrationService < FollowService + # Follow an account with the same settings as another account, and unfollow the old account once the request is sent + # @param [Account] source_account From which to follow + # @param [Account] target_account Account to follow + # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one + # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked + def call(source_account, target_account, old_target_account, bypass_locked: false) + @old_target_account = old_target_account + + follow = source_account.active_relationships.find_by(target_account: old_target_account) + reblogs = follow&.show_reblogs? + notify = follow&.notify? + languages = follow&.languages + + super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true) + end + + private + + def request_follow! + follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit])) + + if @target_account.local? + LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request') + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + elsif @target_account.activitypub? + ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id) + end + + follow_request + end + + def direct_follow! + follow = super + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + follow + end +end diff --git a/app/services/remove_domains_from_followers_service.rb b/app/services/remove_domains_from_followers_service.rb new file mode 100644 index 000000000..d76763409 --- /dev/null +++ b/app/services/remove_domains_from_followers_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class RemoveDomainsFromFollowersService < BaseService + include Payloadable + + def call(source_account, target_domains) + source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow| + follow.destroy + + create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub? + end + end + + private + + def create_notification(follow) + ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url) + end + + def build_json(follow) + Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)) + end +end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 8354f0b9f..425472abd 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -12,7 +12,7 @@ - unless @system_checks.empty? .flash-message-stack - @system_checks.each do |message| - .flash-message.warning + .flash-message{ class: message.critical ? 'alert' : 'warning' } = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil) - if message.action = link_to t("admin.system_checks.#{message.key}.action"), message.action diff --git a/app/views/admin/reports/actions/preview.html.haml b/app/views/admin/reports/actions/preview.html.haml index 58745319c..70edb48d8 100644 --- a/app/views/admin/reports/actions/preview.html.haml +++ b/app/views/admin/reports/actions/preview.html.haml @@ -54,15 +54,15 @@ .strike-card__statuses-list__item - if (status = status_map[status_id.to_i]) .one-liner - = link_to short_account_status_url(@report.target_account, status_id), class: 'emojify' do - = one_line_preview(status) + .emojify= one_line_preview(status) - - status.ordered_media_attachments.each do |media_attachment| - %abbr{ title: media_attachment.description } - = fa_icon 'link' - = media_attachment.file_file_name + - status.ordered_media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name .strike-card__statuses-list__item__meta - %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do + %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) - unless status.application.nil? · = status.application.name diff --git a/app/views/admin/statuses/show.html.haml b/app/views/admin/statuses/show.html.haml index 62b49de8c..4631e97f1 100644 --- a/app/views/admin/statuses/show.html.haml +++ b/app/views/admin/statuses/show.html.haml @@ -34,7 +34,7 @@ %td - if @status.trend.allowed? %abbr{ title: t('admin.trends.tags.current_score', score: @status.trend.score) }= t('admin.trends.tags.trending_rank', rank: @status.trend.rank) - - elsif @status.trend.requires_review? + - elsif @status.requires_review? = t('admin.trends.pending_review') - else = t('admin.trends.not_allowed_to_trend') diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 6d18668b0..9d0efa7e1 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -3,7 +3,7 @@ = image_tag @instance_presenter.thumbnail&.file&.url(:'@1x') || asset_pack_path('media/images/preview.png'), alt: @instance_presenter.title .hero-widget__text - %p= @instance_presenter.description.html_safe.presence || t('about.about_mastodon_html') + %p= @instance_presenter.description.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - trends = Trends.tags.query.allowed.limit(3) diff --git a/app/views/disputes/strikes/show.html.haml b/app/views/disputes/strikes/show.html.haml index 7797348dd..ce52e470d 100644 --- a/app/views/disputes/strikes/show.html.haml +++ b/app/views/disputes/strikes/show.html.haml @@ -50,15 +50,15 @@ .strike-card__statuses-list__item - if (status = status_map[status_id.to_i]) .one-liner - = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do - = one_line_preview(status) + .emojify= one_line_preview(status) - - status.ordered_media_attachments.each do |media_attachment| - %abbr{ title: media_attachment.description } - = fa_icon 'link' - = media_attachment.file_file_name + - status.ordered_media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name .strike-card__statuses-list__item__meta - %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do + %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) - unless status.application.nil? · = status.application.name diff --git a/app/views/relationships/show.html.haml b/app/views/relationships/show.html.haml index 2899cd514..f08e9c1df 100644 --- a/app/views/relationships/show.html.haml +++ b/app/views/relationships/show.html.haml @@ -48,7 +48,7 @@ = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('relationships.confirm_remove_selected_followers') } unless following_relationship? - = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? + = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? .batch-table__body - if @accounts.empty? = nothing_here 'nothing-here--under-tabs' diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index c49613fdc..d7b59af27 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -64,6 +64,6 @@ %td= l backup.created_at - if backup.processed? %td= number_to_human_size backup.dump_file_size - %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url + %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup) - else %td{ colspan: 2 }= t('exports.archive_takeout.in_progress') diff --git a/app/views/shared/_og.html.haml b/app/views/shared/_og.html.haml index 2941b566e..a5d99ae33 100644 --- a/app/views/shared/_og.html.haml +++ b/app/views/shared/_og.html.haml @@ -1,5 +1,5 @@ - thumbnail = @instance_presenter.thumbnail -- description ||= strip_tags(@instance_presenter.description.presence || t('about.about_mastodon_html')) +- description ||= @instance_presenter.description.presence || strip_tags(t('about.about_mastodon_html')) %meta{ name: 'description', content: description }/ diff --git a/app/views/user_mailer/backup_ready.html.haml b/app/views/user_mailer/backup_ready.html.haml index 85140b08b..465ead2c8 100644 --- a/app/views/user_mailer/backup_ready.html.haml +++ b/app/views/user_mailer/backup_ready.html.haml @@ -55,5 +55,5 @@ %tbody %tr %td.button-primary - = link_to full_asset_url(@backup.dump.url) do + = link_to download_backup_url(@backup) do %span= t 'exports.archive_takeout.download' diff --git a/app/views/user_mailer/backup_ready.text.erb b/app/views/user_mailer/backup_ready.text.erb index eb89e7d74..8ebbaae85 100644 --- a/app/views/user_mailer/backup_ready.text.erb +++ b/app/views/user_mailer/backup_ready.text.erb @@ -4,4 +4,4 @@ <%= t 'user_mailer.backup_ready.explanation' %> -=> <%= full_asset_url(@backup.dump.url) %> +=> <%= download_backup_url(@backup) %> diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index d9153132b..7c1c14766 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -10,6 +10,16 @@ class ActivityPub::DeliveryWorker sidekiq_options queue: 'push', retry: 16, dead: false + # Unfortunately, we cannot control Sidekiq's jitter, so add our own + sidekiq_retry_in do |count| + # This is Sidekiq's default delay + delay = (count**4) + 15 + # Our custom jitter, that will be added to Sidekiq's built-in one. + # Sidekiq's built-in jitter is `rand(10) * (count + 1)` + jitter = rand(0.5 * (count**4)) + delay + jitter + end + HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze def perform(json, source_account_id, inbox_url, options = {}) diff --git a/app/workers/activitypub/migrated_follow_delivery_worker.rb b/app/workers/activitypub/migrated_follow_delivery_worker.rb new file mode 100644 index 000000000..17a9e515e --- /dev/null +++ b/app/workers/activitypub/migrated_follow_delivery_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker + def perform(json, source_account_id, inbox_url, old_target_account_id, options = {}) + super(json, source_account_id, inbox_url, options) + unfollow_old_account!(old_target_account_id) + end + + private + + def unfollow_old_account!(old_target_account_id) + old_target_account = Account.find(old_target_account_id) + UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true) + rescue StandardError + true + end +end diff --git a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb index bd92fe32c..d245f6bbd 100644 --- a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb +++ b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb @@ -7,7 +7,7 @@ class Scheduler::AccountsStatusesCleanupScheduler # This limit is mostly to be nice to the fediverse at large and not # generate too much traffic. # This also helps limiting the running time of the scheduler itself. - MAX_BUDGET = 50 + MAX_BUDGET = 150 # This is an attempt to spread the load across instances, as various # accounts are likely to have various followers. @@ -15,28 +15,22 @@ class Scheduler::AccountsStatusesCleanupScheduler # This is an attempt to limit the workload generated by status removal # jobs to something the particular instance can handle. - PER_THREAD_BUDGET = 5 + PER_THREAD_BUDGET = 6 # Those avoid loading an instance that is already under load - MAX_DEFAULT_SIZE = 2 + MAX_DEFAULT_SIZE = 200 MAX_DEFAULT_LATENCY = 5 - MAX_PUSH_SIZE = 5 + MAX_PUSH_SIZE = 500 MAX_PUSH_LATENCY = 10 + # 'pull' queue has lower priority jobs, and it's unlikely that pushing # deletes would cause much issues with this queue if it didn't cause issues # with default and push. Yet, do not enqueue deletes if the instance is # lagging behind too much. - MAX_PULL_SIZE = 500 - MAX_PULL_LATENCY = 300 + MAX_PULL_SIZE = 10_000 + MAX_PULL_LATENCY = 5.minutes.to_i - # This is less of an issue in general, but deleting old statuses is likely - # to cause delivery errors, and thus increase the number of jobs to be retried. - # This doesn't directly translate to load, but connection errors and a high - # number of dead instances may lead to this spiraling out of control if - # unchecked. - MAX_RETRY_SIZE = 50_000 - - sidekiq_options retry: 0, lock: :until_executed + sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i def perform return if under_load? @@ -72,7 +66,6 @@ class Scheduler::AccountsStatusesCleanupScheduler end def under_load? - return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY) end diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 7203b4888..a4d57839d 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -10,13 +10,7 @@ class UnfollowFollowWorker old_target_account = Account.find(old_target_account_id) new_target_account = Account.find(new_target_account_id) - follow = follower_account.active_relationships.find_by(target_account: old_target_account) - reblogs = follow&.show_reblogs? - notify = follow&.notify? - languages = follow&.languages - - FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true) - UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) + FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true end diff --git a/bin/tootctl b/bin/tootctl index a9ebb22c6..9c7ae8b87 100755 --- a/bin/tootctl +++ b/bin/tootctl @@ -5,7 +5,9 @@ require_relative '../config/boot' require_relative '../lib/cli' begin - Mastodon::CLI.start(ARGV) + Chewy.strategy(:mastodon) do + Mastodon::CLI.start(ARGV) + end rescue Interrupt exit(130) end diff --git a/config/application.rb b/config/application.rb index c51eacd68..43631c551 100644 --- a/config/application.rb +++ b/config/application.rb @@ -35,6 +35,7 @@ require_relative '../lib/terrapin/multi_pipe_extensions' require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/mastodon/rack_middleware' +require_relative '../lib/public_file_server_middleware' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' require_relative '../lib/chewy/strategy/mastodon' @@ -181,6 +182,10 @@ module Mastodon config.active_job.queue_adapter = :sidekiq config.action_mailer.deliver_later_queue_name = 'mailers' + # We use our own middleware for this + config.public_file_server.enabled = false + + config.middleware.use PublicFileServerMiddleware if Rails.env.development? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true' config.middleware.use Rack::Attack config.middleware.use Mastodon::RackMiddleware diff --git a/config/database.yml b/config/database.yml index bfb53f21b..34acf2f19 100644 --- a/config/database.yml +++ b/config/database.yml @@ -5,6 +5,7 @@ default: &default connect_timeout: 15 encoding: unicode sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> + application_name: '' development: <<: *default diff --git a/config/environments/development.rb b/config/environments/development.rb index de8762ff7..c7b4a5d03 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -16,12 +16,7 @@ Rails.application.configure do # Run rails dev:cache to toggle caching. if Rails.root.join('tmp/caching-dev.txt').exist? config.action_controller.perform_caching = true - config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS - - config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{2.days.to_i}", - } else config.action_controller.perform_caching = false diff --git a/config/environments/production.rb b/config/environments/production.rb index 99c9bb40c..00d783477 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -19,27 +19,16 @@ Rails.application.configure do # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). # config.require_master_key = true - # Disable serving static files from the `/public` folder by default since - # Apache or NGINX already handles this. - config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? - ActiveSupport::Logger.new(STDOUT).tap do |logger| logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - # Compress JavaScripts and CSS. - # config.assets.js_compressor = Uglifier.new(mangle: false) - # config.assets.css_compressor = :sass - # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = false - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb - # Specifies the header that your server uses for sending files. - # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache - config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + config.action_dispatch.x_sendfile_header = ENV['SENDFILE_HEADER'] if ENV['SENDFILE_HEADER'].present? # Allow to specify public IP of reverse proxy if it's needed config.action_dispatch.trusted_proxies = ENV['TRUSTED_PROXY_IP'].split(/(?:\s*,\s*|\s+)/).map { |item| IPAddr.new(item) } if ENV['TRUSTED_PROXY_IP'].present? @@ -67,7 +56,7 @@ Rails.application.configure do # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # English when a translation cannot be found). - config.i18n.fallbacks = [:en] + config.i18n.fallbacks = true # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify @@ -128,6 +117,7 @@ Rails.application.configure do enable_starttls_auto: enable_starttls_auto, tls: ENV['SMTP_TLS'].presence && ENV['SMTP_TLS'] == 'true', ssl: ENV['SMTP_SSL'].presence && ENV['SMTP_SSL'] == 'true', + read_timeout: 20, } config.action_mailer.delivery_method = ENV.fetch('SMTP_DELIVERY_METHOD', 'smtp').to_sym diff --git a/config/environments/test.rb b/config/environments/test.rb index ef3cb2e48..44786962a 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -12,11 +12,6 @@ Rails.application.configure do # preloads Rails for running tests, you may have to set it to true. config.eager_load = false - # Configure public file server for tests with Cache-Control for performance. - config.public_file_server.enabled = true - config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{1.hour.to_i}" - } config.assets.digest = false # Show full error reports and disable caching. diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index 752fc3c6d..2a91efefc 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -19,7 +19,6 @@ Chewy.settings = { # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :mastodon Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false diff --git a/config/locales/en.yml b/config/locales/en.yml index 39ff4236a..b1e2a8568 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -805,6 +805,12 @@ en: message_html: You haven't defined any server rules. sidekiq_process_check: message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration + upload_check_privacy_error: + action: Check here for more information + message_html: "Your web server is misconfigured. The privacy of your users is at risk." + upload_check_privacy_error_object_storage: + action: Check here for more information + message_html: "Your object storage is misconfigured. The privacy of your users is at risk." tags: review: Review status updated_msg: Hashtag settings updated successfully @@ -1399,6 +1405,7 @@ en: confirm_remove_selected_followers: Are you sure you want to remove selected followers? confirm_remove_selected_follows: Are you sure you want to remove selected follows? dormant: Dormant + follow_failure: Could not follow some of the selected accounts. follow_selected_followers: Follow selected followers followers: Followers following: Following diff --git a/config/puma.rb b/config/puma.rb index e59295445..c4e2b0b85 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -22,3 +22,5 @@ on_worker_boot do end plugin :tmp_restart + +set_remote_address(proxy_protocol: :v1) if ENV['PROXY_PROTO_V1'] == 'true' diff --git a/config/routes.rb b/config/routes.rb index 319f0c7d1..7958c6db8 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -110,6 +110,8 @@ Rails.application.routes.draw do resource :inbox, only: [:create], module: :activitypub + get '/:encoded_at(*path)', to: redirect("/@%{path}"), constraints: { encoded_at: /%40/ } + constraints(username: /[^@\/.]+/) do get '/@:username', to: 'accounts#show', as: :short_account get '/@:username/with_replies', to: 'accounts#show', as: :short_account_with_replies @@ -218,6 +220,7 @@ Rails.application.routes.draw do resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update] get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false + get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false resource :authorize_interaction, only: [:show, :create] resource :share, only: [:show, :create] @@ -470,7 +473,9 @@ Rails.application.routes.draw do resources :list, only: :show end - resources :streaming, only: [:index] + get '/streaming', to: 'streaming#index' + get '/streaming/(*any)', to: 'streaming#index' + resources :custom_emojis, only: [:index] resources :suggestions, only: [:index, :destroy] resources :scheduled_statuses, only: [:index, :show, :update, :destroy] diff --git a/docker-compose.yml b/docker-compose.yml index c534286c7..f603c2f7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: tootsuite/mastodon + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: tootsuite/mastodon + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: tootsuite/mastodon + image: ghcr.io/mastodon/mastodon restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 34afbc699..be93bba59 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -372,16 +372,16 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean - desc 'refresh [USERNAME]', 'Fetch remote user data and files' + desc 'refresh [USERNAMES]', 'Fetch remote user data and files' long_desc <<-LONG_DESC Fetch remote user data and files for one or multiple accounts. With the --all option, all remote accounts will be processed. Through the --domain option, this can be narrowed down to a - specific domain only. Otherwise, a single remote account must - be specified with USERNAME. + specific domain only. Otherwise, remote accounts must be + specified with space-separated USERNAMES. LONG_DESC - def refresh(username = nil) + def refresh(*usernames) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' if options[:domain] || options[:all] @@ -397,19 +397,25 @@ module Mastodon end say("Refreshed #{processed} accounts#{dry_run}", :green, true) - elsif username.present? - username, domain = username.split('@') - account = Account.find_remote(username, domain) + elsif !usernames.empty? + usernames.each do |user| + user, domain = user.split('@') + account = Account.find_remote(user, domain) - if account.nil? - say('No such account', :red) - exit(1) - end + if account.nil? + say('No such account', :red) + exit(1) + end - unless options[:dry_run] - account.reset_avatar! - account.reset_header! - account.save + next if options[:dry_run] + + begin + account.reset_avatar! + account.reset_header! + account.save + rescue Mastodon::UnexpectedResponseError + say("Account failed: #{user}@#{domain}", :red) + end end say("OK#{dry_run}", :green) @@ -631,7 +637,7 @@ module Mastodon exit(1) end - unless options[:force] || migration.target_acount_id == account.moved_to_account_id + unless options[:force] || migration.target_account_id == account.moved_to_account_id say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red) exit(1) end diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index c75e8401f..9832e1a27 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -3,8 +3,8 @@ class Mastodon::SidekiqMiddleware BACKTRACE_LIMIT = 3 - def call(*) - yield + def call(*, &block) + Chewy.strategy(:mastodon, &block) rescue Mastodon::HostValidationError # Do not retry rescue => e diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index b86e87748..a9569fcf2 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def flags diff --git a/lib/public_file_server_middleware.rb b/lib/public_file_server_middleware.rb new file mode 100644 index 000000000..3799230a2 --- /dev/null +++ b/lib/public_file_server_middleware.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'action_dispatch/middleware/static' + +class PublicFileServerMiddleware + SERVICE_WORKER_TTL = 7.days.to_i + CACHE_TTL = 28.days.to_i + + def initialize(app) + @app = app + @file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first) + end + + def call(env) + file = @file_handler.attempt(env) + + # If the request is not a static file, move on! + return @app.call(env) if file.nil? + + status, headers, response = file + + # Set cache headers on static files. Some paths require different cache headers + headers['Cache-Control'] = begin + request_path = env['REQUEST_PATH'] + + if request_path.start_with?('/sw.js') + "public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate" + elsif request_path.start_with?(paperclip_root_url) + "public, max-age=#{CACHE_TTL}, immutable" + else + "public, max-age=#{CACHE_TTL}, must-revalidate" + end + end + + [status, headers, response] + end + + private + + def paperclip_root_url + ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + end +end diff --git a/spec/controllers/relationships_controller_spec.rb b/spec/controllers/relationships_controller_spec.rb index 2056a2ac2..bcdcfa905 100644 --- a/spec/controllers/relationships_controller_spec.rb +++ b/spec/controllers/relationships_controller_spec.rb @@ -55,7 +55,7 @@ describe RelationshipsController do end context 'when select parameter is provided' do - subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, block_domains: '' } } + subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, remove_domains_from_followers: '' } } it 'soft-blocks followers from selected domains' do poopfeast.follow!(user.account) @@ -66,6 +66,15 @@ describe RelationshipsController do expect(poopfeast.following?(user.account)).to be false end + it 'does not unfollow users from selected domains' do + user.account.follow!(poopfeast) + + sign_in user, scope: :user + subject + + expect(user.account.following?(poopfeast)).to be true + end + include_examples 'authenticate user' include_examples 'redirects back to followers page' end diff --git a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb index fe53b4dfc..269c4d685 100644 --- a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb +++ b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb @@ -248,7 +248,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do post :create, params: { credential: new_webauthn_credential, nickname: 'USB Key' } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end @@ -268,7 +268,7 @@ describe Settings::TwoFactorAuthentication::WebauthnCredentialsController do post :create, params: { credential: new_webauthn_credential, nickname: nickname } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end diff --git a/spec/lib/plain_text_formatter_spec.rb b/spec/lib/plain_text_formatter_spec.rb index c3d0ee630..81e4ae286 100644 --- a/spec/lib/plain_text_formatter_spec.rb +++ b/spec/lib/plain_text_formatter_spec.rb @@ -4,7 +4,7 @@ RSpec.describe PlainTextFormatter do describe '#to_s' do subject { described_class.new(status.text, status.local?).to_s } - context 'given a post with local status' do + context 'when status is local' do let(:status) { Fabricate(:status, text: '

a text by a nerd who uses an HTML tag in text

', uri: nil) } it 'returns the raw text' do @@ -12,12 +12,63 @@ RSpec.describe PlainTextFormatter do end end - context 'given a post with remote status' do + context 'when status is remote' do let(:remote_account) { Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') } - let(:status) { Fabricate(:status, account: remote_account, text: '

Hello

') } - it 'returns tag-stripped text' do - is_expected.to eq 'Hello' + context 'when text contains inline HTML tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the tags' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains

tags' do + let(:status) { Fabricate(:status, account: remote_account, text: '

Lorem

ipsum

') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains a single
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem
ipsum') } + + it 'inserts a newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains consecutive
tag' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem


ipsum') } + + it 'inserts a single newline' do + expect(subject).to eq "Lorem\nipsum" + end + end + + context 'when text contains HTML entity' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem & ipsum ❤') } + + it 'unescapes the entity' do + expect(subject).to eq 'Lorem & ipsum ❤' + end + end + + context 'when text contains ipsum') } + + it 'strips the tag and its contents' do + expect(subject).to eq 'Lorem ipsum' + end + end + + context 'when text contains an HTML comment tags' do + let(:status) { Fabricate(:status, account: remote_account, text: 'Lorem ipsum') } + + it 'strips the comment' do + expect(subject).to eq 'Lorem ipsum' + end end end end diff --git a/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb index 8f20725c8..d953cc39d 100644 --- a/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb +++ b/spec/workers/scheduler/accounts_statuses_cleanup_scheduler_spec.rb @@ -23,7 +23,6 @@ describe Scheduler::AccountsStatusesCleanupScheduler do }, ] end - let(:retry_size) { 0 } before do queue_stub = double @@ -33,7 +32,6 @@ describe Scheduler::AccountsStatusesCleanupScheduler do allow(Sidekiq::ProcessSet).to receive(:new).and_return(process_set_stub) sidekiq_stats_stub = double - allow(sidekiq_stats_stub).to receive(:retry_size).and_return(retry_size) allow(Sidekiq::Stats).to receive(:new).and_return(sidekiq_stats_stub) # Create a bunch of old statuses @@ -70,14 +68,6 @@ describe Scheduler::AccountsStatusesCleanupScheduler do expect(subject.under_load?).to be true end end - - context 'when there is a huge amount of jobs to retry' do - let(:retry_size) { 1_000_000 } - - it 'returns true' do - expect(subject.under_load?).to be true - end - end end describe '#get_budget' do diff --git a/streaming/index.js b/streaming/index.js index a8708ec78..32e3babaa 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -856,15 +856,15 @@ const startWorker = async (workerId) => { res.write('# TYPE connected_channels gauge\n'); res.write('# HELP connected_channels The number of Redis channels the streaming server is subscribed to\n'); res.write(`connected_channels ${Object.keys(subs).length}.0\n`); - res.write('# TYPE pg.pool.total_connections gauge \n'); - res.write('# HELP pg.pool.total_connections The total number of clients existing within the pool\n'); - res.write(`pg.pool.total_connections ${pgPool.totalCount}.0\n`); - res.write('# TYPE pg.pool.idle_connections gauge \n'); - res.write('# HELP pg.pool.idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); - res.write(`pg.pool.idle_connections ${pgPool.idleCount}.0\n`); - res.write('# TYPE pg.pool.waiting_queries gauge \n'); - res.write('# HELP pg.pool.waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); - res.write(`pg.pool.waiting_queries ${pgPool.waitingCount}.0\n`); + res.write('# TYPE pg_pool_total_connections gauge\n'); + res.write('# HELP pg_pool_total_connections The total number of clients existing within the pool\n'); + res.write(`pg_pool_total_connections ${pgPool.totalCount}.0\n`); + res.write('# TYPE pg_pool_idle_connections gauge\n'); + res.write('# HELP pg_pool_idle_connections The number of clients which are not checked out but are currently idle in the pool\n'); + res.write(`pg_pool_idle_connections ${pgPool.idleCount}.0\n`); + res.write('# TYPE pg_pool_waiting_queries gauge\n'); + res.write('# HELP pg_pool_waiting_queries The number of queued requests waiting on a client when all clients are checked out\n'); + res.write(`pg_pool_waiting_queries ${pgPool.waitingCount}.0\n`); res.write('# EOF\n'); res.end(); }));