From 097d86454a137495b1462e42ad11098507f4a08c Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Tue, 29 Dec 2020 20:51:05 +0100 Subject: [PATCH 001/666] Add signatures to requests to mastodon to support authorized fetch mode When mastodon is in authorized fetch mode any request has to be signed or it fails with 401. This adds the needed signature to the requests made to discover the actor when receiving something from mastodon (such as a follow request) --- bookwyrm/activitypub/base_activity.py | 40 +++++++++++++++++++++++++++ bookwyrm/models/activitypub_mixin.py | 2 +- bookwyrm/signatures.py | 12 +++++--- bookwyrm/tests/test_signing.py | 2 +- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 24d383ac7..c3287d45d 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -8,6 +8,10 @@ from django.db import IntegrityError, transaction from bookwyrm.connectors import ConnectorException, get_data from bookwyrm.tasks import app +import requests +from django.utils.http import http_date +from bookwyrm import models +from bookwyrm.signatures import make_signature class ActivitySerializerError(ValueError): """routine problems serializing activitypub json""" @@ -284,6 +288,12 @@ def resolve_remote_id( # load the data and create the object try: data = get_data(remote_id) + except requests.HTTPError as e: + if e.response.status_code == 401: + ''' This most likely means it's a mastodon with secure fetch enabled. Need to be specific ''' + data = get_activitypub_data(remote_id) + else: + raise e except ConnectorException: raise ActivitySerializerError( f"Could not connect to host for remote_id: {remote_id}" @@ -304,3 +314,33 @@ def resolve_remote_id( # if we're refreshing, "result" will be set and we'll update it return item.to_model(model=model, instance=result, save=save) + +def get_activitypub_data(url): + ''' wrapper for request.get ''' + now = http_date() + + # XXX TEMP!! + sender = models.User.objects.get(id=1) + if not sender.key_pair.private_key: + # this shouldn't happen. it would be bad if it happened. + raise ValueError('No private key found for sender') + + try: + resp = requests.get( + url, + headers={ + 'Accept': 'application/json; charset=utf-8', + 'Date': now, + 'Signature': make_signature('get', sender, url, now), + }, + ) + except RequestError: + raise ConnectorException() + if not resp.ok: + resp.raise_for_status() + try: + data = resp.json() + except ValueError: + raise ConnectorException() + + return data diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 402cb040b..ee0b2c40d 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -533,7 +533,7 @@ def sign_and_send(sender, data, destination): headers={ "Date": now, "Digest": digest, - "Signature": make_signature(sender, destination, now, digest), + "Signature": make_signature("post", sender, destination, now, digest), "Content-Type": "application/activity+json; charset=utf-8", "User-Agent": USER_AGENT, }, diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 61cafe71f..27c6357f6 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -22,27 +22,31 @@ def create_key_pair(): return private_key, public_key -def make_signature(sender, destination, date, digest): +def make_signature(method, sender, destination, date, digest): """uses a private key to sign an outgoing message""" inbox_parts = urlparse(destination) signature_headers = [ - f"(request-target): post {inbox_parts.path}", + f"(request-target): {method} {inbox_parts.path}", f"host: {inbox_parts.netloc}", f"date: {date}", f"digest: {digest}", ] + headers = "(request-target) host date" + if digest is not None: + signature_headers.append("digest: %s" % digest) + headers = "(request-target) host date digest" + message_to_sign = "\n".join(signature_headers) signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key)) signed_message = signer.sign(SHA256.new(message_to_sign.encode("utf8"))) signature = { "keyId": f"{sender.remote_id}#main-key", "algorithm": "rsa-sha256", - "headers": "(request-target) host date digest", + "headers": headers, "signature": b64encode(signed_message).decode("utf8"), } return ",".join(f'{k}="{v}"' for (k, v) in signature.items()) - def make_digest(data): """creates a message digest for signing""" return "SHA-256=" + b64encode(hashlib.sha256(data.encode("utf-8")).digest()).decode( diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index d33687a59..afcfb6907 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -85,7 +85,7 @@ class Signature(TestCase): now = date or http_date() data = json.dumps(get_follow_activity(sender, self.rat)) digest = digest or make_digest(data) - signature = make_signature(signer or sender, self.rat.inbox, now, digest) + signature = make_signature("post", signer or sender, self.rat.inbox, now, digest) with patch("bookwyrm.views.inbox.activity_task.delay"): with patch("bookwyrm.models.user.set_remote_server.delay"): return self.send(signature, now, send_data or data, digest) From e2ee3d27a7e40b877d8ddba90c056aa3f7418d25 Mon Sep 17 00:00:00 2001 From: "Renato \"Lond\" Cerqueira" Date: Wed, 5 Jan 2022 15:42:54 +0100 Subject: [PATCH 002/666] WIP --- bookwyrm/activitypub/base_activity.py | 12 ++++++++++-- bookwyrm/tests/activitypub/test_base_activity.py | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index c3287d45d..f58b0bde9 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -12,6 +12,7 @@ import requests from django.utils.http import http_date from bookwyrm import models from bookwyrm.signatures import make_signature +from bookwyrm.settings import DOMAIN class ActivitySerializerError(ValueError): """routine problems serializing activitypub json""" @@ -315,12 +316,19 @@ def resolve_remote_id( # if we're refreshing, "result" will be set and we'll update it return item.to_model(model=model, instance=result, save=save) +def get_representative(): + try: + models.User.objects.get(id=-99) + except models.User.DoesNotExist: + username = "%s@%s" % (DOMAIN, DOMAIN) + email = "representative@%s" % (DOMAIN) + models.User.objects.create_user(id=-99, username=username, email=email, local=True, localname=DOMAIN) + def get_activitypub_data(url): ''' wrapper for request.get ''' now = http_date() - # XXX TEMP!! - sender = models.User.objects.get(id=1) + sender = get_representative() if not sender.key_pair.private_key: # this shouldn't happen. it would be bad if it happened. raise ValueError('No private key found for sender') diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index b951c7ab4..0eca7f7aa 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -14,6 +14,7 @@ from bookwyrm.activitypub.base_activity import ( ActivityObject, resolve_remote_id, set_related_field, + get_representative ) from bookwyrm.activitypub import ActivitySerializerError from bookwyrm import models @@ -51,6 +52,10 @@ class BaseActivity(TestCase): image.save(output, format=image.format) self.image_data = output.getvalue() + def test_get_representative_not_existing(self, _): + representative = get_representative() + self.assertIsInstance(representative, models.User) + def test_init(self, *_): """simple successfuly init""" instance = ActivityObject(id="a", type="b") From 7ae0db7f4ae3edffdd5d54e25d3fb5fe9339e590 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Thu, 5 May 2022 13:29:07 -0700 Subject: [PATCH 003/666] Ignore VariableDoesNotExist errors in debug logging They're so noisy as to make debug logging useless otherwise --- bookwyrm/settings.py | 4 ++++ bookwyrm/utils/log.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 bookwyrm/utils/log.py diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index e16c576e1..236413fe8 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -147,6 +147,9 @@ LOGGING = { "require_debug_true": { "()": "django.utils.log.RequireDebugTrue", }, + "ignore_missing_variable": { + "()": "bookwyrm.utils.log.IgnoreVariableDoesNotExist", + }, }, "handlers": { # Overrides the default handler to make it log to console @@ -154,6 +157,7 @@ LOGGING = { # console if DEBUG=False) "console": { "level": LOG_LEVEL, + "filters": ["ignore_missing_variable"], "class": "logging.StreamHandler", }, # This is copied as-is from the default logger, and is diff --git a/bookwyrm/utils/log.py b/bookwyrm/utils/log.py new file mode 100644 index 000000000..8ad86895c --- /dev/null +++ b/bookwyrm/utils/log.py @@ -0,0 +1,12 @@ +import logging + + +class IgnoreVariableDoesNotExist(logging.Filter): + def filter(self, record): + if(record.exc_info): + (errType, errValue, _) = record.exc_info + while errValue: + if type(errValue).__name__ == 'VariableDoesNotExist': + return False + errValue = errValue.__context__ + return True From 7014786fe03a73c7c8fe525aaad19384aa05c159 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 5 Jun 2022 13:41:00 -0700 Subject: [PATCH 004/666] Run formatters --- bookwyrm/utils/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/utils/log.py b/bookwyrm/utils/log.py index 8ad86895c..4ea24d81d 100644 --- a/bookwyrm/utils/log.py +++ b/bookwyrm/utils/log.py @@ -3,10 +3,10 @@ import logging class IgnoreVariableDoesNotExist(logging.Filter): def filter(self, record): - if(record.exc_info): + if record.exc_info: (errType, errValue, _) = record.exc_info while errValue: - if type(errValue).__name__ == 'VariableDoesNotExist': + if type(errValue).__name__ == "VariableDoesNotExist": return False errValue = errValue.__context__ return True From c9adb7ff129bdb75fcbe97c896347cfb889683a0 Mon Sep 17 00:00:00 2001 From: Ell Bradshaw Date: Mon, 14 Nov 2022 00:48:59 -0800 Subject: [PATCH 005/666] Linting fixes --- bookwyrm/utils/log.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bookwyrm/utils/log.py b/bookwyrm/utils/log.py index 4ea24d81d..70f32ef03 100644 --- a/bookwyrm/utils/log.py +++ b/bookwyrm/utils/log.py @@ -1,12 +1,20 @@ +""" Logging utilities """ import logging class IgnoreVariableDoesNotExist(logging.Filter): + """ + Filter to ignore VariableDoesNotExist errors + + We intentionally pass nonexistent variables to templates a lot, so + these errors are not useful to us. + """ + def filter(self, record): if record.exc_info: - (errType, errValue, _) = record.exc_info - while errValue: - if type(errValue).__name__ == "VariableDoesNotExist": + (_, err_value, _) = record.exc_info + while err_value: + if type(err_value).__name__ == "VariableDoesNotExist": return False - errValue = errValue.__context__ - return True + err_value = err_value.__context__ + return True From b812a5c73eed4e26c365fb99d3d2c252a6092c16 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 14 Nov 2022 12:07:00 -0800 Subject: [PATCH 006/666] Adds management command to revoke preview image tasks --- .../commands/revoke_preview_image_tasks.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 bookwyrm/management/commands/revoke_preview_image_tasks.py diff --git a/bookwyrm/management/commands/revoke_preview_image_tasks.py b/bookwyrm/management/commands/revoke_preview_image_tasks.py new file mode 100644 index 000000000..6d6e59e8f --- /dev/null +++ b/bookwyrm/management/commands/revoke_preview_image_tasks.py @@ -0,0 +1,31 @@ +""" Actually let's not generate those preview images """ +import json +from django.core.management.base import BaseCommand +from bookwyrm.tasks import app + + +class Command(BaseCommand): + """Find and revoke image tasks""" + + # pylint: disable=unused-argument + def handle(self, *args, **options): + """reveoke nonessential low priority tasks""" + types = [ + "bookwyrm.preview_images.generate_edition_preview_image_task", + "bookwyrm.preview_images.generate_user_preview_image_task", + ] + self.stdout.write(" | Finding tasks of types:") + self.stdout.write("\n".join(types)) + with app.pool.acquire(block=True) as conn: + tasks = conn.default_channel.client.lrange("low_priority", 0, -1) + self.stdout.write(f" | Found {len(tasks)} task(s) in low priority queue") + + revoke_ids = [] + for task in tasks: + task_json = json.loads(task) + task_type = task_json.get("headers", {}).get("task") + if task_type in types: + revoke_ids.append(task_json.get("headers", {}).get("id")) + self.stdout.write(".", ending="") + self.stdout.write(f"\n | Revoking {len(revoke_ids)} task(s)") + app.control.revoke(revoke_ids) From e655f5c2f399563afbd37906d2adf52d7484d979 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 14 Nov 2022 16:06:40 -0800 Subject: [PATCH 007/666] Fixes typo in url regex --- bookwyrm/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 7af123016..a1e0ef844 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -297,7 +297,7 @@ urlpatterns = [ name="settings-imports", ), re_path( - r"^settings/imports/(?P\d+)/complete?$", + r"^settings/imports/(?P\d+)/complete/?$", views.ImportList.as_view(), name="settings-imports-complete", ), From 5b358094ab1a09ed4b7821bc118ffb6af6a2b187 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 14 Nov 2022 18:03:36 -0800 Subject: [PATCH 008/666] Fixes report emails always claiming to be about links --- bookwyrm/emailing.py | 1 + bookwyrm/templates/email/moderation_report/html_content.html | 2 +- bookwyrm/templates/email/moderation_report/text_content.html | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bookwyrm/emailing.py b/bookwyrm/emailing.py index e767d5374..80aacf7f4 100644 --- a/bookwyrm/emailing.py +++ b/bookwyrm/emailing.py @@ -48,6 +48,7 @@ def moderation_report_email(report): if report.user: data["reportee"] = report.user.localname or report.user.username data["report_link"] = report.remote_id + data["link_domain"] = report.links.exists() for admin in models.User.objects.filter( groups__name__in=["admin", "moderator"] diff --git a/bookwyrm/templates/email/moderation_report/html_content.html b/bookwyrm/templates/email/moderation_report/html_content.html index 3828ff70c..0e604ebf8 100644 --- a/bookwyrm/templates/email/moderation_report/html_content.html +++ b/bookwyrm/templates/email/moderation_report/html_content.html @@ -3,7 +3,7 @@ {% block content %}

-{% if report_link %} +{% if link_domain %} {% blocktrans trimmed %} @{{ reporter }} has flagged a link domain for moderation. diff --git a/bookwyrm/templates/email/moderation_report/text_content.html b/bookwyrm/templates/email/moderation_report/text_content.html index 764a3c72a..351ab58ed 100644 --- a/bookwyrm/templates/email/moderation_report/text_content.html +++ b/bookwyrm/templates/email/moderation_report/text_content.html @@ -2,7 +2,7 @@ {% load i18n %} {% block content %} -{% if report_link %} +{% if link_domain %} {% blocktrans trimmed %} @{{ reporter }} has flagged a link domain for moderation. {% endblocktrans %} From dbe74f63885441498acd633b4d5b11ba48f1b2db Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 14 Nov 2022 18:18:27 -0800 Subject: [PATCH 009/666] Uses the same snippet for the footer across different templates --- bookwyrm/templates/layout.html | 48 +------------------ .../snippets/{2fa_footer.html => footer.html} | 0 .../two_factor_auth/two_factor_login.html | 2 +- .../two_factor_auth/two_factor_prompt.html | 2 +- 4 files changed, 3 insertions(+), 49 deletions(-) rename bookwyrm/templates/snippets/{2fa_footer.html => footer.html} (100%) diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index 9e954411e..e58f65edd 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -192,53 +192,7 @@

- +{% include 'snippets/footer.html' %} {% endblock %} diff --git a/bookwyrm/templates/ostatus/template.html b/bookwyrm/templates/ostatus/template.html index eb904a693..25d2430c0 100644 --- a/bookwyrm/templates/ostatus/template.html +++ b/bookwyrm/templates/ostatus/template.html @@ -11,7 +11,7 @@ {% block title %}{% endblock %} - diff --git a/bookwyrm/templates/settings/dashboard/registration_chart.html b/bookwyrm/templates/settings/dashboard/registration_chart.html index 3b258fec8..bb51ed8bc 100644 --- a/bookwyrm/templates/settings/dashboard/registration_chart.html +++ b/bookwyrm/templates/settings/dashboard/registration_chart.html @@ -1,5 +1,5 @@ {% load i18n %} -