Merge tag 'v0.6.0' into nix

.env.example

@@ -32,12 +32,17 @@ REDIS_ACTIVITY_PORT=6379
 REDIS_ACTIVITY_PASSWORD=redispassword345
 # Optional, use a different redis database (defaults to 0)
 # REDIS_ACTIVITY_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_ACTIVITY_URL=
 
 # Redis as celery broker
+REDIS_BROKER_HOST=redis_broker
 REDIS_BROKER_PORT=6379
 REDIS_BROKER_PASSWORD=redispassword123
 # Optional, use a different redis database (defaults to 0)
 # REDIS_BROKER_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_BROKER_URL=
 
 # Monitoring for celery
 FLOWER_PORT=8888
@@ -60,7 +65,7 @@ SEARCH_TIMEOUT=5
 QUERY_TIMEOUT=5
 
 # Thumbnails Generation
-ENABLE_THUMBNAIL_GENERATION=false
+ENABLE_THUMBNAIL_GENERATION=true
 
 # S3 configuration
 USE_S3=false
@@ -115,3 +120,14 @@ OTEL_SERVICE_NAME=
 # for your instance:
 # https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
 HTTP_X_FORWARDED_PROTO=false
+
+# TOTP settings
+# TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side
+# which will be accepted.
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
+TWO_FACTOR_LOGIN_MAX_SECONDS=60
+
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
+# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
+# Value should be a comma-separated list of host names.
+CSP_ADDITIONAL_HOSTS=

.github/workflows/black.yml

@@ -10,6 +10,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
-      - uses: psf/black@21.4b2
+      - uses: psf/black@22.12.0
+        with:
+          version: 22.12.0

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -23,9 +23,9 @@ jobs:
         ports:
           - 5432:5432
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install Dependencies

.github/workflows/lint-frontend.yaml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install modules
         run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint

.github/workflows/prettier.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install modules
         run: npm install prettier

.github/workflows/pylint.yml

@@ -12,9 +12,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install Dependencies

bookwyrm/activitypub/__init__.py

@@ -3,7 +3,7 @@ import inspect
 import sys
 
 from .base_activity import ActivityEncoder, Signature, naive_parse
-from .base_activity import Link, Mention
+from .base_activity import Link, Mention, Hashtag
 from .base_activity import ActivitySerializerError, resolve_remote_id
 from .image import Document, Image
 from .note import Note, GeneratedNote, Article, Comment, Quotation

bookwyrm/activitypub/base_activity.py

@@ -2,11 +2,16 @@
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+import requests
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
+from django.utils.http import http_date
 
+from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
 from bookwyrm.tasks import app, MEDIUM
 
 logger = logging.getLogger(__name__)
@@ -95,9 +100,27 @@ class ActivityObject:
 
     # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
     def to_model(
-        self, model=None, instance=None, allow_create=True, save=True, overwrite=True
+        self,
+        model=None,
+        instance=None,
+        allow_create=True,
+        save=True,
+        overwrite=True,
+        allow_external_connections=True,
     ):
-        """convert from an activity to a model instance"""
+        """convert from an activity to a model instance. Args:
+        model: the django model that this object is being converted to
+            (will guess if not known)
+        instance: an existing database entry that is going to be updated by
+            this activity
+        allow_create: whether a new object should be created if there is no
+            existing object is provided or found matching the remote_id
+        save: store in the database if true, return an unsaved model obj if false
+        overwrite: replace fields in the database with this activity if true,
+            only update blank fields if false
+        allow_external_connections: look up missing data if true,
+            throw an exception if false and an external connection is needed
+        """
         model = model or get_model_from_type(self.type)
 
         # only reject statuses if we're potentially creating them
@@ -122,7 +145,10 @@ class ActivityObject:
         for field in instance.simple_fields:
             try:
                 changed = field.set_field_from_activity(
-                    instance, self, overwrite=overwrite
+                    instance,
+                    self,
+                    overwrite=overwrite,
+                    allow_external_connections=allow_external_connections,
                 )
                 if changed:
                     update_fields.append(field.name)
@@ -133,7 +159,11 @@ class ActivityObject:
         # too early and jank up users
         for field in instance.image_fields:
             changed = field.set_field_from_activity(
-                instance, self, save=save, overwrite=overwrite
+                instance,
+                self,
+                save=save,
+                overwrite=overwrite,
+                allow_external_connections=allow_external_connections,
             )
             if changed:
                 update_fields.append(field.name)
@@ -156,8 +186,12 @@ class ActivityObject:
 
             # add many to many fields, which have to be set post-save
             for field in instance.many_to_many_fields:
-                # mention books/users, for example
+                # mention books/users/hashtags, for example
-                field.set_field_from_activity(instance, self)
+                field.set_field_from_activity(
+                    instance,
+                    self,
+                    allow_external_connections=allow_external_connections,
+                )
 
         # reversed relationships in the models
         for (
@@ -207,7 +241,7 @@ class ActivityObject:
         return data
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 @transaction.atomic
 def set_related_field(
     model_name, origin_model_name, related_field_name, related_remote_id, data
@@ -246,10 +280,10 @@ def set_related_field(
 
 def get_model_from_type(activity_type):
     """given the activity, what type of model"""
-    models = apps.get_models()
+    activity_models = apps.get_models()
     model = [
         m
-        for m in models
+        for m in activity_models
         if hasattr(m, "activity_serializer")
         and hasattr(m.activity_serializer, "type")
         and m.activity_serializer.type == activity_type
@@ -261,10 +295,22 @@ def get_model_from_type(activity_type):
     return model[0]
 
 
+# pylint: disable=too-many-arguments
 def resolve_remote_id(
-    remote_id, model=None, refresh=False, save=True, get_activity=False
+    remote_id,
+    model=None,
+    refresh=False,
+    save=True,
+    get_activity=False,
+    allow_external_connections=True,
 ):
-    """take a remote_id and return an instance, creating if necessary"""
+    """take a remote_id and return an instance, creating if necessary. Args:
+    remote_id: the unique url for looking up the object in the db or by http
+    model: a string or object representing the model that corresponds to the object
+    save: whether to return an unsaved database entry or a saved one
+    get_activity: whether to return the activitypub object or the model object
+    allow_external_connections: whether to make http connections
+    """
     if model:  # a bonus check we can do if we already know the model
         if isinstance(model, str):
             model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@@ -272,13 +318,26 @@ def resolve_remote_id(
         if result and not refresh:
             return result if not get_activity else result.to_activity_dataclass()
 
+    # The above block will return the object if it already exists in the database.
+    # If it doesn't, an external connection would be needed, so check if that's cool
+    if not allow_external_connections:
+        raise ActivitySerializerError(
+            "Unable to serialize object without making external HTTP requests"
+        )
+
     # load the data and create the object
     try:
         data = get_data(remote_id)
-    except ConnectorException:
+    except ConnectionError:
         logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
-
+    except requests.HTTPError as e:
+        if (e.response is not None) and e.response.status_code == 401:
+            # This most likely means it's a mastodon with secure fetch enabled.
+            data = get_activitypub_data(remote_id)
+        else:
+            logger.info("Could not connect to host for remote_id: %s", remote_id)
+            return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -297,6 +356,51 @@ def resolve_remote_id(
     return item.to_model(model=model, instance=result, save=save)
 
 
+def get_representative():
+    """Get or create an actor representing the instance
+    to sign requests to 'secure mastodon' servers"""
+    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
+    email = "bookwyrm@localhost"
+    try:
+        user = models.User.objects.get(username=username)
+    except models.User.DoesNotExist:
+        user = models.User.objects.create_user(
+            username=username,
+            email=email,
+            local=True,
+            localname=INSTANCE_ACTOR_USERNAME,
+        )
+    return user
+
+
+def get_activitypub_data(url):
+    """wrapper for request.get"""
+    now = http_date()
+    sender = get_representative()
+    if not sender.key_pair.private_key:
+        # this shouldn't happen. it would be bad if it happened.
+        raise ValueError("No private key found for sender")
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                "Accept": "application/json; charset=utf-8",
+                "Date": now,
+                "Signature": make_signature("get", sender, url, now),
+            },
+        )
+    except requests.RequestException:
+        raise ConnectorException()
+    if not resp.ok:
+        resp.raise_for_status()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
+    return data
+
+
 @dataclass(init=False)
 class Link(ActivityObject):
     """for tagging a book in a status"""
@@ -322,3 +426,10 @@ class Mention(Link):
     """a subtype of Link for mentioning an actor"""
 
     type: str = "Mention"
+
+
+@dataclass(init=False)
+class Hashtag(Link):
+    """a subtype of Link for mentioning a hashtag"""
+
+    type: str = "Hashtag"

bookwyrm/activitypub/book.py

@@ -92,3 +92,4 @@ class Author(BookData):
     bio: str = ""
     wikipediaLink: str = ""
     type: str = "Author"
+    website: str = ""

bookwyrm/activitypub/note.py

@@ -1,9 +1,12 @@
 """ note serializer and children thereof """
 from dataclasses import dataclass, field
 from typing import Dict, List
-from django.apps import apps
+import re
 
-from .base_activity import ActivityObject, Link
+from django.apps import apps
+from django.db import IntegrityError, transaction
+
+from .base_activity import ActivityObject, ActivitySerializerError, Link
 from .image import Document
 
 
@@ -38,6 +41,47 @@ class Note(ActivityObject):
     updated: str = None
     type: str = "Note"
 
+    # pylint: disable=too-many-arguments
+    def to_model(
+        self,
+        model=None,
+        instance=None,
+        allow_create=True,
+        save=True,
+        overwrite=True,
+        allow_external_connections=True,
+    ):
+        instance = super().to_model(
+            model, instance, allow_create, save, overwrite, allow_external_connections
+        )
+
+        if instance is None:
+            return instance
+
+        # Replace links to hashtags in content with local URLs
+        changed_content = False
+        for hashtag in instance.mention_hashtags.all():
+            updated_content = re.sub(
+                rf'(<a href=")[^"]*(" data-mention="hashtag">{hashtag.name}</a>)',
+                rf"\1{hashtag.remote_id}\2",
+                instance.content,
+                flags=re.IGNORECASE,
+            )
+            if instance.content != updated_content:
+                instance.content = updated_content
+                changed_content = True
+
+        if not save or not changed_content:
+            return instance
+
+        with transaction.atomic():
+            try:
+                instance.save(broadcast=False, update_fields=["content"])
+            except IntegrityError as e:
+                raise ActivitySerializerError(e)
+
+        return instance
+
 
 @dataclass(init=False)
 class Article(Note):

bookwyrm/activitypub/verbs.py

@@ -14,12 +14,12 @@ class Verb(ActivityObject):
     actor: str
     object: ActivityObject
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """usually we just want to update and save"""
         # self.object may return None if the object is invalid in an expected way
         # ie, Question type
         if self.object:
-            self.object.to_model()
+            self.object.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -42,7 +42,7 @@ class Delete(Verb):
     cc: List[str] = field(default_factory=lambda: [])
     type: str = "Delete"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and delete the activity object"""
         if not self.object:
             return
@@ -52,7 +52,11 @@ class Delete(Verb):
             model = apps.get_model("bookwyrm.User")
             obj = model.find_existing_by_remote_id(self.object)
         else:
-            obj = self.object.to_model(save=False, allow_create=False)
+            obj = self.object.to_model(
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
 
         if obj:
             obj.delete()
@@ -67,11 +71,13 @@ class Update(Verb):
     to: List[str]
     type: str = "Update"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """update a model instance from the dataclass"""
         if not self.object:
             return
-        self.object.to_model(allow_create=False)
+        self.object.to_model(
+            allow_create=False, allow_external_connections=allow_external_connections
+        )
 
 
 @dataclass(init=False)
@@ -80,10 +86,10 @@ class Undo(Verb):
 
     type: str = "Undo"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         if isinstance(self.object, str):
-            # it may be that sometihng should be done with these, but idk what
+            # it may be that something should be done with these, but idk what
             # this seems just to be coming from pleroma
             return
 
@@ -92,13 +98,28 @@ class Undo(Verb):
         model = None
         if self.object.type == "Follow":
             model = apps.get_model("bookwyrm.UserFollows")
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
             if not obj:
-                # this could be a folloq request not a follow proper
+                # this could be a follow request not a follow proper
                 model = apps.get_model("bookwyrm.UserFollowRequest")
-                obj = self.object.to_model(model=model, save=False, allow_create=False)
+                obj = self.object.to_model(
+                    model=model,
+                    save=False,
+                    allow_create=False,
+                    allow_external_connections=allow_external_connections,
+                )
         else:
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
         if not obj:
             # if we don't have the object, we can't undo it. happens a lot with boosts
             return
@@ -112,9 +133,9 @@ class Follow(Verb):
     object: str
     type: str = "Follow"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -124,9 +145,9 @@ class Block(Verb):
     object: str
     type: str = "Block"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -136,7 +157,7 @@ class Accept(Verb):
     object: Follow
     type: str = "Accept"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """accept a request"""
         obj = self.object.to_model(save=False, allow_create=True)
         obj.accept()
@@ -149,7 +170,7 @@ class Reject(Verb):
     object: Follow
     type: str = "Reject"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """reject a follow request"""
         obj = self.object.to_model(save=False, allow_create=False)
         obj.reject()
@@ -163,7 +184,7 @@ class Add(Verb):
     object: CollectionItem
     type: str = "Add"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """figure out the target to assign the item to a collection"""
         target = resolve_remote_id(self.target)
         item = self.object.to_model(save=False)
@@ -177,7 +198,7 @@ class Remove(Add):
 
     type: str = "Remove"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         obj = self.object.to_model(save=False, allow_create=False)
         if obj:
@@ -191,9 +212,9 @@ class Like(Verb):
     object: str
     type: str = "Like"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """like"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -207,6 +228,6 @@ class Announce(Verb):
     object: str
     type: str = "Announce"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """boost"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)

bookwyrm/activitystreams.py

@@ -13,18 +13,18 @@ from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
 class ActivityStream(RedisStore):
     """a category of activity stream (like home, local, books)"""
 
-    def stream_id(self, user):
+    def stream_id(self, user_id):
         """the redis key for this user's instance of this stream"""
-        return f"{user.id}-{self.key}"
+        return f"{user_id}-{self.key}"
 
-    def unread_id(self, user):
+    def unread_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread"
 
-    def unread_by_status_type_id(self, user):
+    def unread_by_status_type_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
     def get_rank(self, obj):  # pylint: disable=no-self-use
@@ -37,12 +37,12 @@ class ActivityStream(RedisStore):
         pipeline = self.add_object_to_related_stores(status, execute=False)
 
         if increment_unread:
-            for user in self.get_audience(status):
+            for user_id in self.get_audience(status):
                 # add to the unread status count
-                pipeline.incr(self.unread_id(user))
+                pipeline.incr(self.unread_id(user_id))
                 # add to the unread status count for status type
                 pipeline.hincrby(
-                    self.unread_by_status_type_id(user), get_status_type(status), 1
+                    self.unread_by_status_type_id(user_id), get_status_type(status), 1
                 )
 
         # and go!
@@ -52,21 +52,21 @@ class ActivityStream(RedisStore):
         """add a user's statuses to another user's feed"""
         # only add the statuses that the viewer should be able to see (ie, not dms)
         statuses = models.Status.privacy_filter(viewer).filter(user=user)
-        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer))
+        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
 
     def remove_user_statuses(self, viewer, user):
         """remove a user's status from another user's feed"""
         # remove all so that followers only statuses are removed
         statuses = user.status_set.all()
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer))
+        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
 
     def get_activity_stream(self, user):
         """load the statuses to be displayed"""
         # clear unreads for this feed
-        r.set(self.unread_id(user), 0)
+        r.set(self.unread_id(user.id), 0)
-        r.delete(self.unread_by_status_type_id(user))
+        r.delete(self.unread_by_status_type_id(user.id))
 
-        statuses = self.get_store(self.stream_id(user))
+        statuses = self.get_store(self.stream_id(user.id))
         return (
             models.Status.objects.select_subclasses()
             .filter(id__in=statuses)
@@ -83,11 +83,11 @@ class ActivityStream(RedisStore):
 
     def get_unread_count(self, user):
         """get the unread status count for this user's feed"""
-        return int(r.get(self.unread_id(user)) or 0)
+        return int(r.get(self.unread_id(user.id)) or 0)
 
     def get_unread_count_by_status_type(self, user):
         """get the unread status count for this user's feed's status types"""
-        status_types = r.hgetall(self.unread_by_status_type_id(user))
+        status_types = r.hgetall(self.unread_by_status_type_id(user.id))
         return {
             str(key.decode("utf-8")): int(value) or 0
             for key, value in status_types.items()
@@ -95,9 +95,9 @@ class ActivityStream(RedisStore):
 
     def populate_streams(self, user):
         """go from zero to a timeline"""
-        self.populate_store(self.stream_id(user))
+        self.populate_store(self.stream_id(user.id))
 
-    def get_audience(self, status):  # pylint: disable=no-self-use
+    def _get_audience(self, status):  # pylint: disable=no-self-use
         """given a status, what users should see it"""
         # direct messages don't appeard in feeds, direct comments/reviews/etc do
         if status.privacy == "direct" and status.status_type == "Note":
@@ -136,8 +136,12 @@ class ActivityStream(RedisStore):
             )
         return audience.distinct()
 
+    def get_audience(self, status):  # pylint: disable=no-self-use
+        """given a status, what users should see it"""
+        return [user.id for user in self._get_audience(status)]
+
     def get_stores_for_object(self, obj):
-        return [self.stream_id(u) for u in self.get_audience(obj)]
+        return [self.stream_id(user_id) for user_id in self.get_audience(obj)]
 
     def get_statuses_for_user(self, user):  # pylint: disable=no-self-use
         """given a user, what statuses should they see on this stream"""
@@ -157,13 +161,14 @@ class HomeStream(ActivityStream):
     key = "home"
 
     def get_audience(self, status):
-        audience = super().get_audience(status)
+        audience = super()._get_audience(status)
         if not audience:
             return []
-        return audience.filter(
+        # if the user is the post's author
-            Q(id=status.user.id)  # if the user is the post's author
+        ids_self = [user.id for user in audience.filter(Q(id=status.user.id))]
-            | Q(following=status.user)  # if the user is following the author
+        # if the user is following the author
-        ).distinct()
+        ids_following = [user.id for user in audience.filter(Q(following=status.user))]
+        return ids_self + ids_following
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -183,11 +188,11 @@ class LocalStream(ActivityStream):
 
     key = "local"
 
-    def get_audience(self, status):
+    def _get_audience(self, status):
         # this stream wants no part in non-public statuses
         if status.privacy != "public" or not status.user.local:
             return []
-        return super().get_audience(status)
+        return super()._get_audience(status)
 
     def get_statuses_for_user(self, user):
         # all public statuses by a local user
@@ -202,7 +207,7 @@ class BooksStream(ActivityStream):
 
     key = "books"
 
-    def get_audience(self, status):
+    def _get_audience(self, status):
         """anyone with the mentioned book on their shelves"""
         # only show public statuses on the books feed,
         # and only statuses that mention books
@@ -217,7 +222,7 @@ class BooksStream(ActivityStream):
             else status.mention_books.first().parent_work
         )
 
-        audience = super().get_audience(status)
+        audience = super()._get_audience(status)
         if not audience:
             return []
         return audience.filter(shelfbook__book__parent_work=work).distinct()
@@ -244,38 +249,38 @@ class BooksStream(ActivityStream):
     def add_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
+        statuses = models.Status.privacy_filter(
-            models.Status.privacy_filter(
+            user,
-                user,
+            privacy_levels=["public"],
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
         )
-        self.bulk_add_objects_to_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
 
     def remove_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
+        statuses = models.Status.privacy_filter(
-            models.Status.privacy_filter(
+            user,
-                user,
+            privacy_levels=["public"],
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
         )
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
 
 
 # determine which streams are enabled in settings.py
@@ -318,6 +323,10 @@ def add_status_on_create_command(sender, instance, created):
     if instance.published_date < timezone.now() - timedelta(
         days=1
     ) or instance.created_date < instance.published_date - timedelta(days=1):
+        # a backdated status from a local user is an import, don't add it
+        if instance.user.local:
+            return
+        # an out of date remote status is a low priority but should be added
         priority = LOW
 
     add_status_task.apply_async(
@@ -462,7 +471,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
 # ---- TASKS
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def add_book_statuses_task(user_id, book_id):
     """add statuses related to a book on shelve"""
     user = models.User.objects.get(id=user_id)
@@ -470,7 +479,7 @@ def add_book_statuses_task(user_id, book_id):
     BooksStream().add_book_statuses(user, book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def remove_book_statuses_task(user_id, book_id):
     """remove statuses about a book from a user's books feed"""
     user = models.User.objects.get(id=user_id)
@@ -478,7 +487,7 @@ def remove_book_statuses_task(user_id, book_id):
     BooksStream().remove_book_statuses(user, book)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def populate_stream_task(stream, user_id):
     """background task for populating an empty activitystream"""
     user = models.User.objects.get(id=user_id)
@@ -486,7 +495,7 @@ def populate_stream_task(stream, user_id):
     stream.populate_streams(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def remove_status_task(status_ids):
     """remove a status from any stream it might be in"""
     # this can take an id or a list of ids
@@ -499,7 +508,7 @@ def remove_status_task(status_ids):
             stream.remove_object_from_related_stores(status)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=HIGH, ignore_result=True)
 def add_status_task(status_id, increment_unread=False):
     """add a status to any stream it should be in"""
     status = models.Status.objects.select_subclasses().get(id=status_id)
@@ -511,7 +520,7 @@ def add_status_task(status_id, increment_unread=False):
         stream.add_status(status, increment_unread=increment_unread)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
     """remove all statuses by a user from a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -521,7 +530,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.remove_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def add_user_statuses_task(viewer_id, user_id, stream_list=None):
     """add all statuses by a user to a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -531,7 +540,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.add_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def handle_boost_task(boost_id):
     """remove the original post and other, earlier boosts"""
     instance = models.Status.objects.get(id=boost_id)

bookwyrm/book_search.py

@@ -20,7 +20,7 @@ def search(query, min_confidence=0, filters=None, return_first=False):
     query = query.strip()
 
     results = None
-    # first, try searching unqiue identifiers
+    # first, try searching unique identifiers
     # unique identifiers never have spaces, title/author usually do
     if not " " in query:
         results = search_identifiers(query, *filters, return_first=return_first)

bookwyrm/connectors/abstract_connector.py

@@ -1,5 +1,6 @@
 """ functionality outline for a book data connector """
 from abc import ABC, abstractmethod
+from urllib.parse import quote_plus
 import imghdr
 import logging
 import re
@@ -48,7 +49,7 @@ class AbstractMinimalConnector(ABC):
             return f"{self.isbn_search_url}{normalized_query}"
         # NOTE: previously, we tried searching isbn and if that produces no results,
         # searched as free text. This, instead, only searches isbn if it's isbn-y
-        return f"{self.search_url}{query}"
+        return f"{self.search_url}{quote_plus(query)}"
 
     def process_search_response(self, query, data, min_confidence):
         """Format the search results based on the formt of the query"""
@@ -244,7 +245,11 @@ def get_data(url, params=None, timeout=settings.QUERY_TIMEOUT):
         raise ConnectorException(err)
 
     if not resp.ok:
-        raise ConnectorException()
+        if resp.status_code == 401:
+            # this is probably an AUTHORIZED_FETCH issue
+            resp.raise_for_status()
+        else:
+            raise ConnectorException()
     try:
         data = resp.json()
     except ValueError as err:

bookwyrm/connectors/connector_manager.py

@@ -143,7 +143,7 @@ def get_or_create_connector(remote_id):
     return load_connector(connector_info)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def load_more_data(connector_id, book_id):
     """background the work of getting all 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
@@ -152,7 +152,7 @@ def load_more_data(connector_id, book_id):
     connector.expand_book_data(book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def create_edition_task(connector_id, work_id, data):
     """separate task for each of the 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)

bookwyrm/emailing.py

@@ -75,7 +75,7 @@ def format_email(email_name, data):
     return (subject, html_content, text_content)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=HIGH, ignore_result=True)
 def send_email(recipient, subject, html_content, text_content):
     """use a task to send the email"""
     email = EmailMultiAlternatives(

bookwyrm/forms/admin.py

@@ -91,6 +91,7 @@ class RegistrationForm(CustomForm):
             "invite_request_question",
             "invite_question_text",
             "require_confirm_email",
+            "default_user_auth_group",
         ]
 
         widgets = {

bookwyrm/forms/author.py

@@ -15,6 +15,7 @@ class AuthorForm(CustomForm):
             "aliases",
             "bio",
             "wikipedia_link",
+            "website",
             "born",
             "died",
             "openlibrary_key",
@@ -31,10 +32,11 @@ class AuthorForm(CustomForm):
             "wikipedia_link": forms.TextInput(
                 attrs={"aria-describedby": "desc_wikipedia_link"}
             ),
+            "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
             "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
             "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
-            "oepnlibrary_key": forms.TextInput(
+            "openlibrary_key": forms.TextInput(
-                attrs={"aria-describedby": "desc_oepnlibrary_key"}
+                attrs={"aria-describedby": "desc_openlibrary_key"}
             ),
             "inventaire_id": forms.TextInput(
                 attrs={"aria-describedby": "desc_inventaire_id"}

bookwyrm/forms/landing.py

@@ -8,6 +8,7 @@ import pyotp
 
 from bookwyrm import models
 from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import TWO_FACTOR_LOGIN_VALIDITY_WINDOW
 from .custom_form import CustomForm
 
 
@@ -108,7 +109,7 @@ class Confirm2FAForm(CustomForm):
         otp = self.data.get("otp")
         totp = pyotp.TOTP(self.instance.otp_secret)
 
-        if not totp.verify(otp):
+        if not totp.verify(otp, valid_window=TWO_FACTOR_LOGIN_VALIDITY_WINDOW):
 
             if self.instance.hotp_secret:
                 # maybe it's a backup code?

bookwyrm/forms/status.py

@@ -53,6 +53,7 @@ class QuotationForm(CustomForm):
             "sensitive",
             "privacy",
             "position",
+            "endposition",
             "position_mode",
         ]
 

bookwyrm/importers/importer.py

@@ -1,7 +1,8 @@
 """ handle reading a csv from an external service, defaults are from Goodreads """
 import csv
+from datetime import timedelta
 from django.utils import timezone
-from bookwyrm.models import ImportJob, ImportItem
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings
 
 
 class Importer:
@@ -33,6 +34,7 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
+    # pylint: disable=too-many-locals
     def create_job(self, user, csv_file, include_reviews, privacy):
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
@@ -49,7 +51,13 @@ class Importer:
             source=self.service,
         )
 
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
         for index, entry in rows:
+            if enforce_limit and index >= allowed_imports:
+                break
             self.create_item(job, index, entry)
         return job
 
@@ -99,6 +107,24 @@ class Importer:
         """use the dataclass to create the formatted row of data"""
         return {k: entry.get(v) for k, v in mappings.items()}
 
+    def get_import_limit(self, user):  # pylint: disable=no-self-use
+        """check if import limit is set and return how many imports are left"""
+        site_settings = SiteSettings.objects.get()
+        import_size_limit = site_settings.import_size_limit
+        import_limit_reset = site_settings.import_limit_reset
+        enforce_limit = import_size_limit and import_limit_reset
+        allowed_imports = 0
+
+        if enforce_limit:
+            time_range = timezone.now() - timedelta(days=import_limit_reset)
+            import_jobs = ImportJob.objects.filter(
+                user=user, created_date__gte=time_range
+            )
+            # pylint: disable=consider-using-generator
+            imported_books = sum([job.successful_item_count for job in import_jobs])
+            allowed_imports = import_size_limit - imported_books
+        return enforce_limit, allowed_imports
+
     def create_retry_job(self, user, original_job, items):
         """retry items that didn't import"""
         job = ImportJob.objects.create(
@@ -110,7 +136,13 @@ class Importer:
             mappings=original_job.mappings,
             retry=True,
         )
-        for item in items:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, item in enumerate(items):
+            if enforce_limit and index >= allowed_imports:
+                break
             # this will re-normalize the raw data
             self.create_item(job, item.index, item.data)
         return job

bookwyrm/lists_stream.py

@@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
 
 
 # ---- TASKS
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def populate_lists_task(user_id):
     """background task for populating an empty list stream"""
     user = models.User.objects.get(id=user_id)
     ListsStream().populate_lists(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def remove_list_task(list_id, re_add=False):
     """remove a list from any stream it might be in"""
     stores = models.User.objects.filter(local=True, is_active=True).values_list(
@@ -239,14 +239,14 @@ def remove_list_task(list_id, re_add=False):
         add_list_task.delay(list_id)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=HIGH, ignore_result=True)
 def add_list_task(list_id):
     """add a list to any stream it should be in"""
     book_list = models.List.objects.get(id=list_id)
     ListsStream().add_list(book_list)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     """remove all lists by a user from a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)
@@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def add_user_lists_task(viewer_id, user_id):
     """add all lists by a user to a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)

bookwyrm/management/commands/compile_themes.py

@@ -0,0 +1,48 @@
+""" Our own command to all scss themes """
+import glob
+import os
+
+import sass
+
+from django.core.management.base import BaseCommand
+
+from sass_processor.apps import APPS_INCLUDE_DIRS
+from sass_processor.processor import SassProcessor
+from sass_processor.utils import get_custom_functions
+
+from bookwyrm import settings
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "SCSS compile all BookWyrm themes"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """compile"""
+        themes_dir = os.path.join(
+            settings.BASE_DIR, "bookwyrm", "static", "css", "themes", "*.scss"
+        )
+        for theme_scss in glob.glob(themes_dir):
+            basename, _ = os.path.splitext(theme_scss)
+            theme_css = f"{basename}.css"
+            self.compile_sass(theme_scss, theme_css)
+
+    def compile_sass(self, sass_path, css_path):
+        compile_kwargs = {
+            "filename": sass_path,
+            "include_paths": SassProcessor.include_paths + APPS_INCLUDE_DIRS,
+            "custom_functions": get_custom_functions(),
+            "precision": getattr(settings, "SASS_PRECISION", 8),
+            "output_style": getattr(
+                settings,
+                "SASS_OUTPUT_STYLE",
+                "nested" if settings.DEBUG else "compressed",
+            ),
+        }
+
+        content = sass.compile(**compile_kwargs)
+        with open(css_path, "w") as f:
+            f.write(content)
+        self.stdout.write("Compiled SASS/SCSS file: '{0}'\n".format(sass_path))

bookwyrm/management/commands/initdb.py

@@ -117,10 +117,12 @@ def init_connectors():
 
 def init_settings():
     """info about the instance"""
+    group_editor = Group.objects.filter(name="editor").first()
     models.SiteSettings.objects.create(
         support_link="https://www.patreon.com/bookwyrm",
         support_title="Patreon",
         install_mode=True,
+        default_user_auth_group=group_editor,
     )
 
 

bookwyrm/management/commands/remove_remote_user_preview_images.py

@@ -0,0 +1,40 @@
+""" Remove preview images for remote users """
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from bookwyrm import models, preview_images
+
+
+# pylint: disable=line-too-long
+class Command(BaseCommand):
+    """Remove preview images for remote users"""
+
+    help = "Remove preview images for remote users"
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """generate preview images"""
+        self.stdout.write(
+            "   | Hello! I will be removing preview images from remote users."
+        )
+        self.stdout.write(
+            "🧑‍🚒 ⎨ This might take quite long if your instance has a lot of remote users."
+        )
+        self.stdout.write("   | ✧ Thank you for your patience ✧")
+
+        users = models.User.objects.filter(local=False).exclude(
+            Q(preview_image="") | Q(preview_image=None)
+        )
+
+        if len(users) > 0:
+            self.stdout.write(
+                f"   → Remote user preview images ({len(users)}): ", ending=""
+            )
+            for user in users:
+                preview_images.remove_user_preview_image_task.delay(user.id)
+                self.stdout.write(".", ending="")
+            self.stdout.write(" OK 🖼")
+        else:
+            self.stdout.write(f"   | There was no remote users with preview images.")
+
+        self.stdout.write("🧑‍🚒 ⎨ I’m all done! ✧ Enjoy ✧")

bookwyrm/migrations/0167_sitesettings_import_size_limit.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-12-05 13:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_size_limit",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_limit_reset",
+            field=models.IntegerField(default=0),
+        ),
+    ]

bookwyrm/migrations/0171_alter_user_preferred_timezone.py

@@ -0,0 +1,631 @@
+# Generated by Django 3.2.16 on 2022-12-19 15:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0171_merge_20221219_2020.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-19 20:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_sitesettings_import_size_limit"),
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0172_alter_user_preferred_language.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2022-12-21 18:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_alter_user_preferred_timezone"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_author_website.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.16 on 2023-01-15 08:38
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="website",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_default_user_auth_group_setting.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.16 on 2022-12-27 21:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def backfill_sitesettings(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    group_model = apps.get_model("auth", "Group")
+    editor_group = group_model.objects.using(db_alias).filter(name="editor").first()
+
+    sitesettings_model = apps.get_model("bookwyrm", "SiteSettings")
+    sitesettings_model.objects.update(default_user_auth_group=editor_group)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0175_merge_0173_author_website_0174_merge_20230111_1523"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                to="auth.group",
+            ),
+        ),
+        migrations.RunPython(backfill_sitesettings, migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0173_merge_20230102_1444.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-02 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_merge_20221219_2020"),
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = []

bookwyrm/migrations/0174_auto_20230130_1240.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.16 on 2023-01-30 12:40
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        ("bookwyrm", "0173_default_user_auth_group_setting"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="quotation",
+            name="endposition",
+            field=models.IntegerField(
+                blank=True,
+                null=True,
+                validators=[django.core.validators.MinValueValidator(0)],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="auth.group",
+            ),
+        ),
+    ]

bookwyrm/migrations/0174_auto_20230222_1742.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.18 on 2023-02-22 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230130_1240"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="related_link_domains",
+            field=models.ManyToManyField(to="bookwyrm.LinkDomain"),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="notification_type",
+            field=models.CharField(
+                choices=[
+                    ("FAVORITE", "Favorite"),
+                    ("REPLY", "Reply"),
+                    ("MENTION", "Mention"),
+                    ("TAG", "Tag"),
+                    ("FOLLOW", "Follow"),
+                    ("FOLLOW_REQUEST", "Follow Request"),
+                    ("BOOST", "Boost"),
+                    ("IMPORT", "Import"),
+                    ("ADD", "Add"),
+                    ("REPORT", "Report"),
+                    ("LINK_DOMAIN", "Link Domain"),
+                    ("INVITE", "Invite"),
+                    ("ACCEPT", "Accept"),
+                    ("JOIN", "Join"),
+                    ("LEAVE", "Leave"),
+                    ("REMOVE", "Remove"),
+                    ("GROUP_PRIVACY", "Group Privacy"),
+                    ("GROUP_NAME", "Group Name"),
+                    ("GROUP_DESCRIPTION", "Group Description"),
+                ],
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0174_merge_20230111_1523.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.16 on 2023-01-11 15:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0173_merge_20230102_1444"),
+    ]
+
+    operations = []

bookwyrm/migrations/0175_merge_0173_author_website_0174_merge_20230111_1523.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-19 20:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0173_author_website"),
+        ("bookwyrm", "0174_merge_20230111_1523"),
+    ]
+
+    operations = []

bookwyrm/migrations/0176_hashtag_support.py

@@ -0,0 +1,53 @@
+# Generated by Django 3.2.16 on 2022-12-17 19:28
+
+import bookwyrm.models.fields
+import django.contrib.postgres.fields.citext
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230130_1240"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Hashtag",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_date", models.DateTimeField(auto_now_add=True)),
+                ("updated_date", models.DateTimeField(auto_now=True)),
+                (
+                    "remote_id",
+                    bookwyrm.models.fields.RemoteIdField(
+                        max_length=255,
+                        null=True,
+                        validators=[bookwyrm.models.fields.validate_remote_id],
+                    ),
+                ),
+                (
+                    "name",
+                    django.contrib.postgres.fields.citext.CICharField(max_length=256),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="status",
+            name="mention_hashtags",
+            field=bookwyrm.models.fields.TagField(
+                related_name="mention_hashtag", to="bookwyrm.Hashtag"
+            ),
+        ),
+    ]

bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.18 on 2023-03-12 23:41
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0174_auto_20230222_1742"),
+        ("bookwyrm", "0176_hashtag_support"),
+    ]
+
+    operations = []

bookwyrm/models/__init__.py

@@ -34,6 +34,8 @@ from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
 
 from .notification import Notification
 
+from .hashtag import Hashtag
+
 cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 activity_models = {
     c[1].activity_serializer.__name__: c[1]

bookwyrm/models/activitypub_mixin.py

@@ -21,7 +21,7 @@ from django.utils.http import http_date
 from bookwyrm import activitypub
 from bookwyrm.settings import USER_AGENT, PAGE_LENGTH
 from bookwyrm.signatures import make_signature, make_digest
-from bookwyrm.tasks import app, MEDIUM
+from bookwyrm.tasks import app, MEDIUM, BROADCAST
 from bookwyrm.models.fields import ImageField, ManyToManyField
 
 logger = logging.getLogger(__name__)
@@ -126,7 +126,7 @@ class ActivitypubMixin:
         # there OUGHT to be only one match
         return match.first()
 
-    def broadcast(self, activity, sender, software=None, queue=MEDIUM):
+    def broadcast(self, activity, sender, software=None, queue=BROADCAST):
         """send out an activity"""
         broadcast_task.apply_async(
             args=(
@@ -198,7 +198,7 @@ class ActivitypubMixin:
 class ObjectMixin(ActivitypubMixin):
     """add this mixin for object models that are AP serializable"""
 
-    def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs):
+    def save(self, *args, created=None, software=None, priority=BROADCAST, **kwargs):
         """broadcast created/updated/deleted objects as appropriate"""
         broadcast = kwargs.get("broadcast", True)
         # this bonus kwarg would cause an error in the base save method
@@ -506,7 +506,7 @@ def unfurl_related_field(related_field, sort_field=None):
     return related_field.remote_id
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=BROADCAST, ignore_result=True)
 def broadcast_task(sender_id: int, activity: str, recipients: List[str]):
     """the celery task for broadcast"""
     user_model = apps.get_model("bookwyrm.User", require_ready=True)
@@ -543,7 +543,7 @@ async def sign_and_send(
     headers = {
         "Date": now,
         "Digest": digest,
-        "Signature": make_signature(sender, destination, now, digest),
+        "Signature": make_signature("post", sender, destination, now, digest),
         "Content-Type": "application/activity+json; charset=utf-8",
         "User-Agent": USER_AGENT,
     }

bookwyrm/models/annual_goal.py

@@ -52,7 +52,7 @@ class AnnualGoal(BookWyrmModel):
             user=self.user,
             book__in=book_ids,
         )
-        return {r.book.id: r.rating for r in reviews}
+        return {r.book_id: r.rating for r in reviews}
 
     @property
     def progress(self):

bookwyrm/models/antispam.py

@@ -65,7 +65,7 @@ class AutoMod(AdminModel):
     created_by = models.ForeignKey("User", on_delete=models.PROTECT)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def automod_task():
     """Create reports"""
     if not AutoMod.objects.exists():

bookwyrm/models/author.py

@@ -1,8 +1,6 @@
 """ database schema for info about authors """
 import re
 from django.contrib.postgres.indexes import GinIndex
-from django.core.cache import cache
-from django.core.cache.utils import make_template_fragment_key
 from django.db import models
 
 from bookwyrm import activitypub
@@ -27,6 +25,10 @@ class Author(BookDataModel):
     isfdb = fields.CharField(
         max_length=255, blank=True, null=True, deduplication_field=True
     )
+
+    website = fields.CharField(
+        max_length=255, blank=True, null=True, deduplication_field=True
+    )
     # idk probably other keys would be useful here?
     born = fields.DateTimeField(blank=True, null=True)
     died = fields.DateTimeField(blank=True, null=True)
@@ -37,16 +39,7 @@ class Author(BookDataModel):
     bio = fields.HtmlField(null=True, blank=True)
 
     def save(self, *args, **kwargs):
-        """clear related template caches"""
+        """normalize isni format"""
-        # clear template caches
-        if self.id:
-            cache_keys = [
-                make_template_fragment_key("titleby", [book])
-                for book in self.book_set.values_list("id", flat=True)
-            ]
-            cache.delete_many(cache_keys)
-
-        # normalize isni format
         if self.isni:
             self.isni = re.sub(r"\s", "", self.isni)
 

bookwyrm/models/book.py

@@ -4,7 +4,6 @@ import re
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.cache import cache
-from django.core.cache.utils import make_template_fragment_key
 from django.db import models, transaction
 from django.db.models import Prefetch
 from django.dispatch import receiver
@@ -208,10 +207,6 @@ class Book(BookDataModel):
         if not isinstance(self, Edition) and not isinstance(self, Work):
             raise ValueError("Books should be added as Editions or Works")
 
-        # clear template caches
-        cache_key = make_template_fragment_key("titleby", [self.id])
-        cache.delete(cache_key)
-
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):

bookwyrm/models/fields.py

@@ -7,6 +7,7 @@ from urllib.parse import urljoin
 import dateutil.parser
 from dateutil.parser import ParserError
 from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+from django.contrib.postgres.fields import CICharField as DjangoCICharField
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.forms import ClearableFileInput, ImageField as DjangoImageField
@@ -67,7 +68,9 @@ class ActivitypubFieldMixin:
             self.activitypub_field = activitypub_field
         super().__init__(*args, **kwargs)
 
-    def set_field_from_activity(self, instance, data, overwrite=True):
+    def set_field_from_activity(
+        self, instance, data, overwrite=True, allow_external_connections=True
+    ):
         """helper function for assinging a value to the field. Returns if changed"""
         try:
             value = getattr(data, self.get_activitypub_field())
@@ -76,7 +79,9 @@ class ActivitypubFieldMixin:
             if self.get_activitypub_field() != "attributedTo":
                 raise
             value = getattr(data, "actor")
-        formatted = self.field_from_activity(value)
+        formatted = self.field_from_activity(
+            value, allow_external_connections=allow_external_connections
+        )
         if formatted is None or formatted is MISSING or formatted == {}:
             return False
 
@@ -116,7 +121,8 @@ class ActivitypubFieldMixin:
             return {self.activitypub_wrapper: value}
         return value
 
-    def field_from_activity(self, value):
+    # pylint: disable=unused-argument
+    def field_from_activity(self, value, allow_external_connections=True):
         """formatter to convert activitypub into a model value"""
         if value and hasattr(self, "activitypub_wrapper"):
             value = value.get(self.activitypub_wrapper)
@@ -138,7 +144,7 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
         self.load_remote = load_remote
         super().__init__(*args, **kwargs)
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         if not value:
             return None
 
@@ -159,7 +165,11 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
         if not self.load_remote:
             # only look in the local database
             return related_model.find_existing_by_remote_id(value)
-        return activitypub.resolve_remote_id(value, model=related_model)
+        return activitypub.resolve_remote_id(
+            value,
+            model=related_model,
+            allow_external_connections=allow_external_connections,
+        )
 
 
 class RemoteIdField(ActivitypubFieldMixin, models.CharField):
@@ -219,7 +229,9 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
         super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public")
 
     # pylint: disable=invalid-name
-    def set_field_from_activity(self, instance, data, overwrite=True):
+    def set_field_from_activity(
+        self, instance, data, overwrite=True, allow_external_connections=True
+    ):
         if not overwrite:
             return False
 
@@ -234,7 +246,11 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
                 break
         if not user_field:
             raise ValidationError("No user field found for privacy", data)
-        user = activitypub.resolve_remote_id(getattr(data, user_field), model="User")
+        user = activitypub.resolve_remote_id(
+            getattr(data, user_field),
+            model="User",
+            allow_external_connections=allow_external_connections,
+        )
 
         if to == [self.public]:
             setattr(instance, self.name, "public")
@@ -295,13 +311,17 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
         self.link_only = link_only
         super().__init__(*args, **kwargs)
 
-    def set_field_from_activity(self, instance, data, overwrite=True):
+    def set_field_from_activity(
+        self, instance, data, overwrite=True, allow_external_connections=True
+    ):
         """helper function for assigning a value to the field"""
         if not overwrite and getattr(instance, self.name).exists():
             return False
 
         value = getattr(data, self.get_activitypub_field())
-        formatted = self.field_from_activity(value)
+        formatted = self.field_from_activity(
+            value, allow_external_connections=allow_external_connections
+        )
         if formatted is None or formatted is MISSING:
             return False
         getattr(instance, self.name).set(formatted)
@@ -313,7 +333,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             return f"{value.instance.remote_id}/{self.name}"
         return [i.remote_id for i in value.all()]
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         if value is None or value is MISSING:
             return None
         if not isinstance(value, list):
@@ -326,7 +346,11 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
             except ValidationError:
                 continue
             items.append(
-                activitypub.resolve_remote_id(remote_id, model=self.related_model)
+                activitypub.resolve_remote_id(
+                    remote_id,
+                    model=self.related_model,
+                    allow_external_connections=allow_external_connections,
+                )
             )
         return items
 
@@ -353,7 +377,7 @@ class TagField(ManyToManyField):
             )
         return tags
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         if not isinstance(value, list):
             return None
         items = []
@@ -365,9 +389,22 @@ class TagField(ManyToManyField):
             if tag_type != self.related_model.activity_serializer.type:
                 # tags can contain multiple types
                 continue
-            items.append(
+
-                activitypub.resolve_remote_id(link.href, model=self.related_model)
+            if tag_type == "Hashtag":
-            )
+                # we already have all data to create hashtags,
+                # no need to fetch from remote
+                item = self.related_model.activity_serializer(**link_json)
+                hashtag = item.to_model(model=self.related_model, save=True)
+                items.append(hashtag)
+            else:
+                # for other tag types we fetch them remotely
+                items.append(
+                    activitypub.resolve_remote_id(
+                        link.href,
+                        model=self.related_model,
+                        allow_external_connections=allow_external_connections,
+                    )
+                )
         return items
 
 
@@ -390,11 +427,15 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
         self.alt_field = alt_field
         super().__init__(*args, **kwargs)
 
-    # pylint: disable=arguments-differ,arguments-renamed
+    # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments
-    def set_field_from_activity(self, instance, data, save=True, overwrite=True):
+    def set_field_from_activity(
+        self, instance, data, save=True, overwrite=True, allow_external_connections=True
+    ):
         """helper function for assinging a value to the field"""
         value = getattr(data, self.get_activitypub_field())
-        formatted = self.field_from_activity(value)
+        formatted = self.field_from_activity(
+            value, allow_external_connections=allow_external_connections
+        )
         if formatted is None or formatted is MISSING:
             return False
 
@@ -426,7 +467,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
 
         return activitypub.Document(url=url, name=alt)
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         image_slug = value
         # when it's an inline image (User avatar/icon, Book cover), it's a json
         # blob, but when it's an attached image, it's just a url
@@ -481,7 +522,7 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
             return None
         return value.isoformat()
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         try:
             date_value = dateutil.parser.parse(value)
             try:
@@ -495,7 +536,7 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
 class HtmlField(ActivitypubFieldMixin, models.TextField):
     """a text field for storing html"""
 
-    def field_from_activity(self, value):
+    def field_from_activity(self, value, allow_external_connections=True):
         if not value or value == MISSING:
             return None
         return clean(value)
@@ -515,6 +556,10 @@ class CharField(ActivitypubFieldMixin, models.CharField):
     """activitypub-aware char field"""
 
 
+class CICharField(ActivitypubFieldMixin, DjangoCICharField):
+    """activitypub-aware cichar field"""
+
+
 class URLField(ActivitypubFieldMixin, models.URLField):
     """activitypub-aware url field"""
 

bookwyrm/models/hashtag.py

@@ -0,0 +1,23 @@
+""" model for tags """
+from bookwyrm import activitypub
+from .activitypub_mixin import ActivitypubMixin
+from .base_model import BookWyrmModel
+from .fields import CICharField
+
+
+class Hashtag(ActivitypubMixin, BookWyrmModel):
+    "a hashtag which can be used in statuses"
+
+    name = CICharField(
+        max_length=256,
+        blank=False,
+        null=False,
+        activitypub_field="name",
+        deduplication_field=True,
+    )
+
+    name_field = "name"
+    activity_serializer = activitypub.Hashtag
+
+    def __repr__(self):
+        return f"<{self.__class__} id={self.id} name={self.name}>"

bookwyrm/models/import_job.py

@@ -19,7 +19,7 @@ from bookwyrm.models import (
     Review,
     ReviewRating,
 )
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, LOW, IMPORTS
 from .fields import PrivacyLevels
 
 
@@ -74,8 +74,7 @@ class ImportJob(models.Model):
         task = start_import_task.delay(self.id)
         self.task_id = task.id
 
-        self.status = "active"
+        self.save(update_fields=["task_id"])
-        self.save(update_fields=["status", "task_id"])
 
     def complete_job(self):
         """Report that the job has completed"""
@@ -328,10 +327,12 @@ class ImportItem(models.Model):
         )
 
 
-@app.task(queue=LOW)
+@app.task(queue=IMPORTS, ignore_result=True)
 def start_import_task(job_id):
     """trigger the child tasks for each row"""
     job = ImportJob.objects.get(id=job_id)
+    job.status = "active"
+    job.save(update_fields=["status"])
     # don't start the job if it was stopped from the UI
     if job.complete:
         return
@@ -345,7 +346,7 @@ def start_import_task(job_id):
     job.save()
 
 
-@app.task(queue=LOW)
+@app.task(queue=IMPORTS, ignore_result=True)
 def import_item_task(item_id):
     """resolve a row into a book"""
     item = ImportItem.objects.get(id=item_id)

bookwyrm/models/notification.py

@@ -2,8 +2,8 @@
 from django.db import models, transaction
 from django.dispatch import receiver
 from .base_model import BookWyrmModel
-from . import Boost, Favorite, GroupMemberInvitation, ImportJob, ListItem, Report
+from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
-from . import Status, User, UserFollowRequest
+from . import ListItem, Report, Status, User, UserFollowRequest
 
 
 class Notification(BookWyrmModel):
@@ -28,6 +28,7 @@ class Notification(BookWyrmModel):
 
     # Admin
     REPORT = "REPORT"
+    LINK_DOMAIN = "LINK_DOMAIN"
 
     # Groups
     INVITE = "INVITE"
@@ -43,7 +44,7 @@ class Notification(BookWyrmModel):
     NotificationType = models.TextChoices(
         # there has got be a better way to do this
         "NotificationType",
-        f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
+        f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
     )
 
     user = models.ForeignKey("User", on_delete=models.CASCADE)
@@ -64,6 +65,7 @@ class Notification(BookWyrmModel):
         "ListItem", symmetrical=False, related_name="notifications"
     )
     related_reports = models.ManyToManyField("Report", symmetrical=False)
+    related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
 
     @classmethod
     @transaction.atomic
@@ -241,6 +243,26 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
         notification.related_reports.add(instance)
 
 
+@receiver(models.signals.post_save, sender=LinkDomain)
+@transaction.atomic
+# pylint: disable=unused-argument
+def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
+    """a new link domain needs to be verified"""
+    if not created:
+        # otherwise you'll get a notification when you approve a domain
+        return
+
+    # moderators and superusers should be notified
+    admins = User.admins()
+    for admin in admins:
+        notification, _ = Notification.objects.get_or_create(
+            user=admin,
+            notification_type=Notification.LINK_DOMAIN,
+            read=False,
+        )
+        notification.related_link_domains.add(instance)
+
+
 @receiver(models.signals.post_save, sender=GroupMemberInvitation)
 # pylint: disable=unused-argument
 def notify_user_on_group_invite(sender, instance, *args, **kwargs):

bookwyrm/models/readthrough.py

@@ -32,7 +32,7 @@ class ReadThrough(BookWyrmModel):
 
     def save(self, *args, **kwargs):
         """update user active time"""
-        cache.delete(f"latest_read_through-{self.user.id}-{self.book.id}")
+        cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}")
         self.user.update_active_date()
         # an active readthrough must have an unset finish date
         if self.finish_date or self.stopped_date:

bookwyrm/models/relationship.py

@@ -4,6 +4,7 @@ from django.db import models, transaction, IntegrityError
 from django.db.models import Q
 
 from bookwyrm import activitypub
+from bookwyrm.tasks import HIGH
 from .activitypub_mixin import ActivitypubMixin, ActivityMixin
 from .activitypub_mixin import generate_activity
 from .base_model import BookWyrmModel
@@ -139,8 +140,9 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
             )
         super().save(*args, **kwargs)
 
+        # a local user is following a remote user
         if broadcast and self.user_subject.local and not self.user_object.local:
-            self.broadcast(self.to_activity(), self.user_subject)
+            self.broadcast(self.to_activity(), self.user_subject, queue=HIGH)
 
         if self.user_object.local:
             manually_approves = self.user_object.manually_approves_followers
@@ -157,18 +159,23 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
     def accept(self, broadcast_only=False):
         """turn this request into the real deal"""
         user = self.user_object
+        # broadcast when accepting a remote request
         if not self.user_subject.local:
             activity = activitypub.Accept(
                 id=self.get_accept_reject_id(status="accepts"),
                 actor=self.user_object.remote_id,
                 object=self.to_activity(),
             ).serialize()
-            self.broadcast(activity, user)
+            self.broadcast(activity, user, queue=HIGH)
         if broadcast_only:
             return
 
         with transaction.atomic():
-            UserFollows.from_request(self)
+            try:
+                UserFollows.from_request(self)
+            except IntegrityError:
+                # this just means we already saved this relationship
+                pass
             if self.id:
                 self.delete()
 
@@ -180,7 +187,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
                 actor=self.user_object.remote_id,
                 object=self.to_activity(),
             ).serialize()
-            self.broadcast(activity, self.user_object)
+            self.broadcast(activity, self.user_object, queue=HIGH)
 
         self.delete()
 

bookwyrm/models/shelf.py

@@ -7,6 +7,7 @@ from django.utils import timezone
 
 from bookwyrm import activitypub
 from bookwyrm.settings import DOMAIN
+from bookwyrm.tasks import LOW
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
 from .base_model import BookWyrmModel
 from . import fields
@@ -39,9 +40,9 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
 
     activity_serializer = activitypub.Shelf
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, priority=LOW, **kwargs):
         """set the identifier"""
-        super().save(*args, **kwargs)
+        super().save(*args, priority=priority, **kwargs)
         if not self.identifier:
             self.identifier = self.get_identifier()
             super().save(*args, **kwargs, broadcast=False)
@@ -99,24 +100,24 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
     activity_serializer = activitypub.ShelfItem
     collection_field = "shelf"
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, priority=LOW, **kwargs):
         if not self.user:
             self.user = self.shelf.user
         if self.id and self.user.local:
             # remove all caches related to all editions of this book
             cache.delete_many(
                 [
-                    f"book-on-shelf-{book.id}-{self.shelf.id}"
+                    f"book-on-shelf-{book.id}-{self.shelf_id}"
                     for book in self.book.parent_work.editions.all()
                 ]
             )
-        super().save(*args, **kwargs)
+        super().save(*args, priority=priority, **kwargs)
 
     def delete(self, *args, **kwargs):
         if self.id and self.user.local:
             cache.delete_many(
                 [
-                    f"book-on-shelf-{book}-{self.shelf.id}"
+                    f"book-on-shelf-{book}-{self.shelf_id}"
                     for book in self.book.parent_work.editions.values_list(
                         "id", flat=True
                     )

bookwyrm/models/site.py

@@ -3,6 +3,7 @@ import datetime
 from urllib.parse import urljoin
 import uuid
 
+import django.contrib.auth.models as auth_models
 from django.core.exceptions import PermissionDenied
 from django.db import models, IntegrityError
 from django.dispatch import receiver
@@ -70,6 +71,9 @@ class SiteSettings(SiteModel):
     allow_invite_requests = models.BooleanField(default=True)
     invite_request_question = models.BooleanField(default=False)
     require_confirm_email = models.BooleanField(default=True)
+    default_user_auth_group = models.ForeignKey(
+        auth_models.Group, null=True, blank=True, on_delete=models.RESTRICT
+    )
 
     invite_question_text = models.CharField(
         max_length=255, blank=True, default="What is your favourite book?"
@@ -90,6 +94,8 @@ class SiteSettings(SiteModel):
 
     # controls
     imports_enabled = models.BooleanField(default=True)
+    import_size_limit = models.IntegerField(default=0)
+    import_limit_reset = models.IntegerField(default=0)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])
 

bookwyrm/models/status.py

@@ -34,6 +34,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     raw_content = models.TextField(blank=True, null=True)
     mention_users = fields.TagField("User", related_name="mention_user")
     mention_books = fields.TagField("Edition", related_name="mention_book")
+    mention_hashtags = fields.TagField("Hashtag", related_name="mention_hashtag")
     local = models.BooleanField(default=True)
     content_warning = fields.CharField(
         max_length=500, blank=True, null=True, activitypub_field="summary"
@@ -80,7 +81,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     def save(self, *args, **kwargs):
         """save and notify"""
         if self.reply_parent:
-            self.thread_id = self.reply_parent.thread_id or self.reply_parent.id
+            self.thread_id = self.reply_parent.thread_id or self.reply_parent_id
 
         super().save(*args, **kwargs)
 
@@ -329,6 +330,9 @@ class Quotation(BookStatus):
     position = models.IntegerField(
         validators=[MinValueValidator(0)], null=True, blank=True
     )
+    endposition = models.IntegerField(
+        validators=[MinValueValidator(0)], null=True, blank=True
+    )
     position_mode = models.CharField(
         max_length=3,
         choices=ProgressMode.choices,

bookwyrm/models/user.py

@@ -3,9 +3,9 @@ import re
 from urllib.parse import urlparse
 
 from django.apps import apps
-from django.contrib.auth.models import AbstractUser, Group
+from django.contrib.auth.models import AbstractUser
 from django.contrib.postgres.fields import ArrayField, CICharField
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
 from django.dispatch import receiver
 from django.db import models, transaction
 from django.utils import timezone
@@ -356,8 +356,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
 
             # make users editors by default
             try:
-                self.groups.add(Group.objects.get(name="editor"))
+                group = (
-            except Group.DoesNotExist:
+                    apps.get_model("bookwyrm.SiteSettings")
+                    .objects.get()
+                    .default_user_auth_group
+                )
+                if group:
+                    self.groups.add(group)
+            except ObjectDoesNotExist:
                 # this should only happen in tests
                 pass
 
@@ -373,6 +379,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
         """We don't actually delete the database entry"""
         # pylint: disable=attribute-defined-outside-init
         self.is_active = False
+        self.avatar = ""
         # skip the logic in this class's save()
         super().save(*args, **kwargs)
 
@@ -462,7 +469,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         return super().save(*args, **kwargs)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def set_remote_server(user_id):
     """figure out the user's remote server in the background"""
     user = User.objects.get(id=user_id)
@@ -506,7 +513,7 @@ def get_or_create_remote_server(domain, refresh=False):
     return server
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def get_remote_reviews(outbox):
     """ingest reviews by a new remote bookwyrm user"""
     outbox_page = outbox + "?page=true&type=Review"
@@ -525,6 +532,11 @@ def preview_image(instance, *args, **kwargs):
     """create preview images when user is updated"""
     if not ENABLE_PREVIEW_IMAGES:
         return
+
+    # don't call the task for remote users
+    if not instance.local:
+        return
+
     changed_fields = instance.field_tracker.changed()
 
     if len(changed_fields) > 0:

bookwyrm/preview_images.py

@@ -71,20 +71,29 @@ def get_wrapped_text(text, font, content_width):
     low = 0
     high = len(text)
 
+    draw = ImageDraw.Draw(Image.new("RGB", (100, 100)))
+
     try:
         # ideal length is determined via binary search
         while low < high:
             mid = math.floor(low + high)
             wrapped_text = textwrap.fill(text, width=mid)
-            width = font.getsize_multiline(wrapped_text)[0]
+
+            left, top, right, bottom = draw.multiline_textbbox(
+                (0, 0), wrapped_text, font=font
+            )
+            width = right - left
+            height = bottom - top
+
             if width < content_width:
                 low = mid
             else:
                 high = mid - 1
     except AttributeError:
         wrapped_text = text
+        height = 26
 
-    return wrapped_text
+    return wrapped_text, height
 
 
 def generate_texts_layer(texts, content_width):
@@ -100,47 +109,53 @@ def generate_texts_layer(texts, content_width):
     text_y = 0
 
     if "text_zero" in texts and texts["text_zero"]:
-        # Text one (Book title)
+        # Text zero (Site preview domain name)
-        text_zero = get_wrapped_text(texts["text_zero"], font_text_zero, content_width)
+        text_zero, text_height = get_wrapped_text(
+            texts["text_zero"], font_text_zero, content_width
+        )
 
         text_layer_draw.multiline_text(
             (0, text_y), text_zero, font=font_text_zero, fill=TEXT_COLOR
         )
 
         try:
-            text_y = text_y + font_text_zero.getsize_multiline(text_zero)[1] + 16
+            text_y = text_y + text_height + 16
         except (AttributeError, IndexError):
             text_y = text_y + 26
 
     if "text_one" in texts and texts["text_one"]:
-        # Text one (Book title)
+        # Text one (Book/Site title, User display name)
-        text_one = get_wrapped_text(texts["text_one"], font_text_one, content_width)
+        text_one, text_height = get_wrapped_text(
+            texts["text_one"], font_text_one, content_width
+        )
 
         text_layer_draw.multiline_text(
             (0, text_y), text_one, font=font_text_one, fill=TEXT_COLOR
         )
 
         try:
-            text_y = text_y + font_text_one.getsize_multiline(text_one)[1] + 16
+            text_y = text_y + text_height + 16
         except (AttributeError, IndexError):
             text_y = text_y + 26
 
     if "text_two" in texts and texts["text_two"]:
-        # Text one (Book subtitle)
+        # Text two (Book subtitle)
-        text_two = get_wrapped_text(texts["text_two"], font_text_two, content_width)
+        text_two, text_height = get_wrapped_text(
+            texts["text_two"], font_text_two, content_width
+        )
 
         text_layer_draw.multiline_text(
             (0, text_y), text_two, font=font_text_two, fill=TEXT_COLOR
         )
 
         try:
-            text_y = text_y + font_text_one.getsize_multiline(text_two)[1] + 16
+            text_y = text_y + text_height + 16
         except (AttributeError, IndexError):
             text_y = text_y + 26
 
     if "text_three" in texts and texts["text_three"]:
-        # Text three (Book authors)
+        # Text three (Book authors, Site tagline, User address)
-        text_three = get_wrapped_text(
+        text_three, _ = get_wrapped_text(
             texts["text_three"], font_text_three, content_width
         )
 
@@ -172,7 +187,7 @@ def generate_instance_layer(content_width):
     instance_text_x = 0
 
     if logo_img:
-        logo_img.thumbnail((50, 50), Image.ANTIALIAS)
+        logo_img.thumbnail((50, 50), Image.Resampling.LANCZOS)
 
         instance_layer.paste(logo_img, (0, 0))
 
@@ -183,7 +198,7 @@ def generate_instance_layer(content_width):
         (instance_text_x, 10), site.name, font=font_instance, fill=TEXT_COLOR
     )
 
-    line_width = 50 + 10 + font_instance.getsize(site.name)[0]
+    line_width = 50 + 10 + round(font_instance.getlength(site.name))
 
     line_layer = Image.new(
         "RGBA", (line_width, 2), color=(*(ImageColor.getrgb(TEXT_COLOR)), 50)
@@ -253,10 +268,12 @@ def generate_default_inner_img():
     default_cover_draw = ImageDraw.Draw(default_cover)
 
     text = "no image :("
-    text_dimensions = font_cover.getsize(text)
+    text_left, text_top, text_right, text_bottom = font_cover.getbbox(text)
+    text_width, text_height = text_right - text_left, text_bottom - text_top
+
     text_coords = (
-        math.floor((inner_img_width - text_dimensions[0]) / 2),
+        math.floor((inner_img_width - text_width) / 2),
-        math.floor((inner_img_height - text_dimensions[1]) / 2),
+        math.floor((inner_img_height - text_height) / 2),
     )
     default_cover_draw.text(text_coords, text, font=font_cover, fill="white")
 
@@ -273,7 +290,9 @@ def generate_preview_image(
     # Cover
     try:
         inner_img_layer = Image.open(picture)
-        inner_img_layer.thumbnail((inner_img_width, inner_img_height), Image.ANTIALIAS)
+        inner_img_layer.thumbnail(
+            (inner_img_width, inner_img_height), Image.Resampling.LANCZOS
+        )
         color_thief = ColorThief(picture)
         dominant_color = color_thief.get_color(quality=1)
     except:  # pylint: disable=bare-except
@@ -401,7 +420,7 @@ def save_and_cleanup(image, instance=None):
 
 
 # pylint: disable=invalid-name
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def generate_site_preview_image_task():
     """generate preview_image for the website"""
     if not settings.ENABLE_PREVIEW_IMAGES:
@@ -426,7 +445,7 @@ def generate_site_preview_image_task():
 
 
 # pylint: disable=invalid-name
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def generate_edition_preview_image_task(book_id):
     """generate preview_image for a book"""
     if not settings.ENABLE_PREVIEW_IMAGES:
@@ -451,14 +470,17 @@ def generate_edition_preview_image_task(book_id):
     save_and_cleanup(image, instance=book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def generate_user_preview_image_task(user_id):
-    """generate preview_image for a book"""
+    """generate preview_image for a user"""
     if not settings.ENABLE_PREVIEW_IMAGES:
         return
 
     user = models.User.objects.get(id=user_id)
 
+    if not user.local:
+        return
+
     texts = {
         "text_one": user.display_name,
         "text_three": f"@{user.localname}@{settings.DOMAIN}",
@@ -472,3 +494,25 @@ def generate_user_preview_image_task(user_id):
     image = generate_preview_image(texts=texts, picture=avatar)
 
     save_and_cleanup(image, instance=user)
+
+
+@app.task(queue=LOW, ignore_result=True)
+def remove_user_preview_image_task(user_id):
+    """remove preview_image for a user"""
+    if not settings.ENABLE_PREVIEW_IMAGES:
+        return
+
+    user = models.User.objects.get(id=user_id)
+
+    try:
+        file_name = user.preview_image.name
+    except ValueError:
+        file_name = None
+
+    # Delete image in model
+    user.preview_image.delete(save=False)
+    user.save(broadcast=False, update_fields=["preview_image"])
+
+    # Delete image file
+    if file_name and default_storage.exists(file_name):
+        default_storage.delete(file_name)

bookwyrm/settings.py

@@ -11,7 +11,7 @@ from django.utils.translation import gettext_lazy as _
 env = Env()
 env.read_env()
 DOMAIN = env("DOMAIN")
-VERSION = "0.5.3"
+VERSION = "0.6.0"
 
 RELEASE_API = env(
     "RELEASE_API",
@@ -21,7 +21,7 @@ RELEASE_API = env(
 PAGE_LENGTH = env("PAGE_LENGTH", 15)
 DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
 
-JS_CACHE = "ad848b97"
+JS_CACHE = "a7d4e720"
 
 # email
 EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend")
@@ -101,6 +101,7 @@ MIDDLEWARE = [
     "django.middleware.locale.LocaleMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
+    "csp.middleware.CSPMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "bookwyrm.middleware.TimezoneMiddleware",
     "bookwyrm.middleware.IPBlocklistMiddleware",
@@ -193,7 +194,8 @@ STATICFILES_FINDERS = [
 ]
 
 SASS_PROCESSOR_INCLUDE_FILE_PATTERN = r"^.+\.[s]{0,1}(?:a|c)ss$"
-SASS_PROCESSOR_ENABLED = True
+# when debug is disabled, make sure to compile themes once with `./bw-dev compile_themes`
+SASS_PROCESSOR_ENABLED = DEBUG
 
 # minify css is production but not dev
 if not DEBUG:
@@ -207,7 +209,10 @@ REDIS_ACTIVITY_PORT = env("REDIS_ACTIVITY_PORT", 6379)
 REDIS_ACTIVITY_PASSWORD = env("REDIS_ACTIVITY_PASSWORD", None)
 REDIS_ACTIVITY_SOCKET = env("REDIS_ACTIVITY_SOCKET", None)
 REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0)
-
+REDIS_ACTIVITY_URL = env(
+    "REDIS_ACTIVITY_URL",
+    f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}",
+)
 MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200))
 
 STREAMS = [
@@ -292,6 +297,7 @@ LANGUAGES = [
     ("ca-es", _("Català (Catalan)")),
     ("de-de", _("Deutsch (German)")),
     ("es-es", _("Español (Spanish)")),
+    ("eu-es", _("Euskara (Basque)")),
     ("gl-es", _("Galego (Galician)")),
     ("it-it", _("Italiano (Italian)")),
     ("fi-fi", _("Suomi (Finnish)")),
@@ -329,12 +335,15 @@ IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = "bookwyrm.thumbnail_generation.Strategy"
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
 PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", [])
 
 # Storage
 
 PROTOCOL = "http"
 if USE_HTTPS:
     PROTOCOL = "https"
+    SESSION_COOKIE_SECURE = True
+    CSRF_COOKIE_SECURE = True
 
 USE_S3 = env.bool("USE_S3", False)
 
@@ -358,18 +367,31 @@ if USE_S3:
     MEDIA_FULL_URL = MEDIA_URL
     STATIC_FULL_URL = STATIC_URL
     DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+    CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+    CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
 else:
     STATIC_URL = "/static/"
     MEDIA_URL = "/images/"
     MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
     STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+    CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
+    CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
+
+CSP_INCLUDE_NONCE_IN = ["script-src"]
 
 OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
 OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
 OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)
 
-TWO_FACTOR_LOGIN_MAX_SECONDS = 60
+TWO_FACTOR_LOGIN_MAX_SECONDS = env.int("TWO_FACTOR_LOGIN_MAX_SECONDS", 60)
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW = env.int("TWO_FACTOR_LOGIN_VALIDITY_WINDOW", 2)
 
 HTTP_X_FORWARDED_PROTO = env.bool("SECURE_PROXY_SSL_HEADER", False)
 if HTTP_X_FORWARDED_PROTO:
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# Instance Actor for signing GET requests to "secure mode"
+# Mastodon servers.
+# Do not change this setting unless you already have an existing
+# user with the same username - in which case you should change it!
+INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"

bookwyrm/signatures.py

@@ -15,29 +15,33 @@ MAX_SIGNATURE_AGE = 300
 def create_key_pair():
     """a new public/private key pair, used for creating new users"""
     random_generator = Random.new().read
-    key = RSA.generate(1024, random_generator)
+    key = RSA.generate(2048, random_generator)
     private_key = key.export_key().decode("utf8")
-    public_key = key.publickey().export_key().decode("utf8")
+    public_key = key.public_key().export_key().decode("utf8")
 
     return private_key, public_key
 
 
-def make_signature(sender, destination, date, digest):
+def make_signature(method, sender, destination, date, digest=None):
     """uses a private key to sign an outgoing message"""
     inbox_parts = urlparse(destination)
     signature_headers = [
-        f"(request-target): post {inbox_parts.path}",
+        f"(request-target): {method} {inbox_parts.path}",
         f"host: {inbox_parts.netloc}",
         f"date: {date}",
-        f"digest: {digest}",
     ]
+    headers = "(request-target) host date"
+    if digest is not None:
+        signature_headers.append(f"digest: {digest}")
+        headers = "(request-target) host date digest"
+
     message_to_sign = "\n".join(signature_headers)
     signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode("utf8")))
     signature = {
         "keyId": f"{sender.remote_id}#main-key",
         "algorithm": "rsa-sha256",
-        "headers": "(request-target) host date digest",
+        "headers": headers,
         "signature": b64encode(signed_message).decode("utf8"),
     }
     return ",".join(f'{k}="{v}"' for (k, v) in signature.items())

bookwyrm/static/css/bookwyrm/overrides/_bulma_overrides.scss

@@ -1,3 +1,53 @@
+.summary-on-open {
+    display: none;
+}
+
+@media only screen and (max-width: 768px) {
+    .navbar-menu {
+        text-align: right;
+        padding-right: 1rem;
+
+        .tags {
+            justify-content: flex-end;
+        }
+
+        #navbar-dropdown {
+            &[open] {
+                .summary-on-open {
+                    display: initial;
+                    position: fixed;
+                    top: 0;
+                    left: 0;
+                    right: 0;
+                    height: 3rem;
+                    z-index: 31;
+                    background-color: $dropdown-content-background-color;
+                    padding: 1rem 1.75rem;
+                    line-height: 1;
+                }
+            }
+
+            .dropdown-menu {
+                padding-top: 0;
+                top: 3rem;
+            }
+
+            .dropdown-content {
+                padding-top: 0;
+                box-shadow: none;
+                border-top-left-radius: 0;
+                border-top-right-radius: 0;
+            }
+
+            .navbar-item {
+                /* see ../components/_details.scss :: Navbar details */
+                padding-right: 1.75rem;
+                font-size: 1rem;
+            }
+        }
+    }
+}
+
 .image {
     overflow: hidden;
 }
@@ -59,3 +109,9 @@
     max-height: 35em;
     overflow: hidden;
 }
+
+.dropdown-menu .button {
+    @include mobile {
+        font-size: $size-6;
+    }
+}

bookwyrm/static/css/bookwyrm/utilities/_size.scss

@@ -40,6 +40,10 @@
     width: 500px !important;
 }
 
+.is-h-em {
+    height: 1em !important;
+}
+
 .is-h-xs {
     height: 80px !important;
 }

bookwyrm/static/css/vendor/shepherd.scss

@@ -6,16 +6,16 @@
 @use 'bulma/bulma.sass';
 
 .shepherd-button {
-    @extend .button.mr-2;
+    @extend .button, .mr-2;
 }
 
 .shepherd-button.shepherd-button-secondary {
-    @extend .button.is-light;
+    @extend .button, .is-light;
 }
 
 .shepherd-footer {
     @extend .message-body;
-    @extend .is-info.is-light;
+    @extend .is-info, .is-light;
     border-color: $info-light;
     border-radius: 0 0 4px 4px;
 }
@@ -29,7 +29,7 @@
 
 .shepherd-text {
     @extend .message-body;
-    @extend .is-info.is-light;
+    @extend .is-info, .is-light;
     border-radius: 0;
 }
 

bookwyrm/static/js/bookwyrm.js

@@ -48,6 +48,12 @@ let BookWyrm = new (class {
         document
             .querySelector("#barcode-scanner-modal")
             .addEventListener("open", this.openBarcodeScanner.bind(this));
+
+        document
+            .querySelectorAll('form[name="register"]')
+            .forEach((form) =>
+                form.addEventListener("submit", (e) => this.setPreferredTimezone(e, form))
+            );
     }
 
     /**
@@ -89,7 +95,6 @@ let BookWyrm = new (class {
 
     /**
      * Update a counter with recurring requests to the API
-     * The delay is slightly randomized and increased on each cycle.
      *
      * @param  {Object} counter - DOM node
      * @param  {int}    delay   - frequency for polling in ms
@@ -98,16 +103,19 @@ let BookWyrm = new (class {
     polling(counter, delay) {
         const bookwyrm = this;
 
-        delay = delay || 10000;
+        delay = delay || 5 * 60 * 1000 + (Math.random() - 0.5) * 30 * 1000;
-        delay += Math.random() * 1000;
 
         setTimeout(
             function () {
                 fetch("/api/updates/" + counter.dataset.poll)
                     .then((response) => response.json())
-                    .then((data) => bookwyrm.updateCountElement(counter, data));
+                    .then((data) => {
-
+                        bookwyrm.updateCountElement(counter, data);
-                bookwyrm.polling(counter, delay * 1.25);
+                        bookwyrm.polling(counter);
+                    })
+                    .catch(() => {
+                        bookwyrm.polling(counter, delay * 1.1);
+                    });
             },
             delay,
             counter
@@ -785,4 +793,16 @@ let BookWyrm = new (class {
 
         initBarcodes();
     }
+
+    /**
+     * Set preferred timezone in register form.
+     *
+     * @param  {Event} event - `submit` event fired by the register form.
+     * @return {undefined}
+     */
+    setPreferredTimezone(event, form) {
+        const tz = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+        form.querySelector('input[name="preferred_timezone"]').value = tz;
+    }
 })();

bookwyrm/static/js/forms.js

@@ -46,4 +46,16 @@
     document
         .querySelectorAll("[data-remove]")
         .forEach((node) => node.addEventListener("click", removeInput));
+
+    // Get the element, add a keypress listener...
+    document.getElementById("subjects").addEventListener("keypress", function (e) {
+        // e.target is the element where it listens!
+        // if e.target is input field within the "subjects" div, do stuff
+        if (e.target && e.target.nodeName == "INPUT") {
+            // Item found, prevent default
+            if (event.keyCode == 13) {
+                event.preventDefault();
+            }
+        }
+    });
 })();

bookwyrm/suggested_users.py

@@ -237,41 +237,41 @@ def domain_level_update(sender, instance, created, update_fields=None, **kwargs)
 # ------------------- TASKS
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def rerank_suggestions_task(user_id):
     """do the hard work in celery"""
     suggested_users.rerank_user_suggestions(user_id)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def rerank_user_task(user_id, update_only=False):
     """do the hard work in celery"""
     user = models.User.objects.get(id=user_id)
     suggested_users.rerank_obj(user, update_only=update_only)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def remove_user_task(user_id):
     """do the hard work in celery"""
     user = models.User.objects.get(id=user_id)
     suggested_users.remove_object_from_related_stores(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=MEDIUM, ignore_result=True)
 def remove_suggestion_task(user_id, suggested_user_id):
     """remove a specific user from a specific user's suggestions"""
     suggested_user = models.User.objects.get(id=suggested_user_id)
     suggested_users.remove_suggestion(user_id, suggested_user)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def bulk_remove_instance_task(instance_id):
     """remove a bunch of users from recs"""
     for user in models.User.objects.filter(federated_server__id=instance_id):
         suggested_users.remove_object_from_related_stores(user)
 
 
-@app.task(queue=LOW)
+@app.task(queue=LOW, ignore_result=True)
 def bulk_add_instance_task(instance_id):
     """remove a bunch of users from recs"""
     for user in models.User.objects.filter(federated_server__id=instance_id):

bookwyrm/tasks.py

@@ -14,3 +14,7 @@ app = Celery(
 LOW = "low_priority"
 MEDIUM = "medium_priority"
 HIGH = "high_priority"
+# import items get their own queue because they're such a pain in the ass
+IMPORTS = "imports"
+# I keep making more queues?? this one broadcasting out
+BROADCAST = "broadcast"

bookwyrm/templates/about/about.html

@@ -10,8 +10,9 @@
 {% endblock %}
 
 {% block about_content %}
+{% get_current_language as LANGUAGE_CODE %}
 {# seven day cache #}
-{% cache 604800 about_page_superlatives %}
+{% cache 604800 about_page_superlatives LANGUAGE_CODE %}
 
 {% get_book_superlatives as superlatives %}
 <section class=" pb-4">

bookwyrm/templates/annual_summary/layout.html

@@ -123,16 +123,18 @@
             </h2>
             <p class="subtitle is-5">{% trans "That’s great!" %}</p>
 
-            <p class="title is-4 is-serif">
+            {% if pages > 0 %} 
-                {% blocktrans with pages=pages_average|intcomma %}That makes an average of {{ pages }} pages per book.{% endblocktrans %}
+                <p class="title is-4 is-serif">
-            </p>
+                    {% blocktrans with pages=pages_average|intcomma %}That makes an average of {{ pages }} pages per book.{% endblocktrans %}
+                </p>
+            {% endif %}
 
             {% if no_page_number %}
                 <p class="subtitle is-6">
                     {% blocktrans trimmed count counter=no_page_number %}
-                    ({{ no_page_number }} book doesn’t have pages)
+                    (No page data was available for {{ no_page_number }} book)
                     {% plural %}
-                    ({{ no_page_number }} books don’t have pages)
+                    (No page data was available for {{ no_page_number }} books)
                     {% endblocktrans %}
                 </p>
             {% endif %}

bookwyrm/templates/author/author.html

@@ -28,7 +28,7 @@
     <meta itemprop="name" content="{{ author.name }}">
 
     {% firstof author.aliases author.born author.died as details %}
-    {% firstof author.wikipedia_link author.openlibrary_key author.inventaire_id author.isni author.isfdb as links %}
+    {% firstof author.wikipedia_link author.website author.openlibrary_key author.inventaire_id author.isni author.isfdb as links %}
     {% if details or links %}
     <div class="column is-3">
         {% if details %}
@@ -73,6 +73,14 @@
                 </div>
                 {% endif %}
 
+                {% if author.website %}
+                <div>
+                     <a itemprop="sameAs" href="{{ author.website }}" rel="nofollow noopener noreferrer" target="_blank">
+                         {% trans "Website" %}
+                     </a>
+                </div>
+                {% endif %}
+
                 {% if author.isni %}
                 <div class="mt-1">
                     <a itemprop="sameAs" href="{{ author.isni_link }}" rel="nofollow noopener noreferrer" target="_blank">

bookwyrm/templates/author/edit_author.html

@@ -57,6 +57,10 @@
 
             {% include 'snippets/form_errors.html' with errors_list=form.wikipedia_link.errors id="desc_wikipedia_link" %}
 
+            <p class="field"><label class="label" for="id_website">{% trans "Website:" %}</label> {{ form.website }}</p>
+
+            {% include 'snippets/form_errors.html' with errors_list=form.website.errors id="desc_website" %}
+
             <div class="field">
                 <label class="label" for="id_born">{% trans "Birth date:" %}</label>
                 <input type="date" name="born" value="{{ form.born.value|date:'Y-m-d' }}" class="input" id="id_born">
@@ -77,7 +81,7 @@
                 <label class="label" for="id_openlibrary_key">{% trans "Openlibrary key:" %}</label>
                 {{ form.openlibrary_key }}
 
-                {% include 'snippets/form_errors.html' with errors_list=form.oepnlibrary_key.errors id="desc_oepnlibrary_key" %}
+                {% include 'snippets/form_errors.html' with errors_list=form.openlibrary_key.errors id="desc_openlibrary_key" %}
             </div>
 
             <div class="field">

bookwyrm/templates/book/book.html

@@ -7,8 +7,8 @@
 
 {% block title %}{{ book|book_title }}{% endblock %}
 
-{% block opengraph_images %}
+{% block opengraph %}
-    {% include 'snippets/opengraph_images.html' with image=book.preview_image %}
+    {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book.preview_image %}
 {% endblock %}
 
 {% block content %}
@@ -46,7 +46,7 @@
                         <meta itemprop="isPartOf" content="{{ book.series | escape }}">
                         <meta itemprop="volumeNumber" content="{{ book.series_number }}">
 
-                        ({{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %})
+                        (<a href="{% url 'book-series-by' book.authors.first.id %}?series_name={{ book.series }}">{{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %}</a>)
                     {% endif %}
                 </p>
             {% endif %}
@@ -82,6 +82,8 @@
                             src="{% static "images/no_cover.jpg" %}"
                             alt=""
                             aria-hidden="true"
+                            loading="lazy"
+                            decoding="async"
                         >
                         <span class="cover-caption">
                             <span>{{ book.alt_text }}</span>
@@ -215,10 +217,10 @@
                 {% endif %}
 
 
-                {% with work=book.parent_work %}
+                {% with work=book.parent_work editions_count=book.parent_work.editions.count %}
                 <p>
                     <a href="{{ work.local_path }}/editions" id="tour-other-editions-link">
-                        {% blocktrans trimmed count counter=work.editions.count with count=work.editions.count|intcomma %}
+                        {% blocktrans trimmed count counter=editions_count with count=editions_count|intcomma %}
                             {{ count }} edition
                             {% plural %}
                             {{ count }} editions

bookwyrm/templates/book/cover_show_modal.html

@@ -5,7 +5,7 @@
     <div class="modal-background" data-modal-close></div><!-- modal background -->
     <div class="modal-card is-align-items-center" role="dialog" aria-modal="true" tabindex="-1" aria-label="{% trans 'Book cover preview' %}">
         <div class="cover-container">
-            <img class="book-cover" src="{% get_media_prefix %}{{ book.cover }}" itemprop="thumbnailUrl" alt="">
+            <img class="book-cover" src="{% get_media_prefix %}{{ book.cover }}" itemprop="thumbnailUrl" alt="" loading="lazy" decoding="async">
         </div>
     </div>
     <button type="button" data-modal-close class="modal-close is-large" aria-label="{% trans 'Close' %}"></button>

bookwyrm/templates/book/edit/edit_book.html

@@ -37,6 +37,14 @@
     {% endif %}
 </header>
 
+{% if form.errors %}
+<div class="block">
+    <p class="notification is-danger is-light">
+        {% trans "Failed to save book, see errors below for more information." %}
+    </p>
+</div>
+{% endif %}
+
 <form
     class="block"
     {% if book.id %}

bookwyrm/templates/book/edit/edit_book_form.html

@@ -81,7 +81,7 @@
                     {% include 'snippets/form_errors.html' with errors_list=form.languages.errors id="desc_languages" %}
                 </div>
 
-                <div class="field">
+                <div class="field" id="subjects">
                     <label class="label" for="id_add_subjects">
                         {% trans "Subjects:" %}
                     </label>

bookwyrm/templates/book/series.html

@@ -0,0 +1,35 @@
+{% extends 'layout.html' %}
+{% load i18n %}
+{% load book_display_tags %}
+
+{% block title %}{{ series_name }}{% endblock %}
+
+{% block content %}
+<div class="block">
+	<h1 class="title">{{ series_name }}</h1>
+	<div class="subtitle" dir="auto">
+		{% trans "Series by" %} <a
+			href="{{ author.local_path }}"
+			class="author {{ link_class }}"
+			itemprop="author"
+			itemscope
+			itemtype="https://schema.org/Thing"
+			><span
+					itemprop="name"
+					>{{ author.name }}</span></a>
+	</div>
+
+	<div class="columns is-multiline is-mobile">
+	{% for book in books %}
+	{% with book=book %}
+			<div class="column is-one-fifth-tablet is-half-mobile is-flex is-flex-direction-column">
+				<div class="is-flex-grow-1 mb-3">
+					<span class="subtitle">{% if book.series_number %}{% blocktrans with series_number=book.series_number %}Book {{ series_number }}{% endblocktrans %}{% else %}{% trans 'Unsorted Book' %}{% endif %}</span>
+					{% include 'landing/small-book.html' with book=book %}
+				</div>
+			</div>
+	{% endwith %}
+	{% endfor %}
+	</div>
+</div>
+{% endblock %}

bookwyrm/templates/discover/card-header.html

@@ -4,17 +4,17 @@
 {% with user_path=status.user.local_path username=status.user.display_name book_path=book.local_path book_title=book|book_title %}
 
 {% if status.status_type == 'GeneratedNote' %}
-    {% if status.content == 'wants to read' %}
+    {% if status.content == 'wants to read' or status.content == '<p>wants to read</p>' %}
         {% blocktrans trimmed %}
         <a href="{{ user_path}}">{{ username }}</a> wants to read <a href="{{ book_path }}">{{ book_title }}</a>
         {% endblocktrans %}
     {% endif %}
-    {% if status.content == 'finished reading' %}
+    {% if status.content == 'finished reading' or status.content == '<p>finished reading</p>' %}
         {% blocktrans trimmed %}
         <a href="{{ user_path}}">{{ username }}</a> finished reading <a href="{{ book_path }}">{{ book_title }}</a>
         {% endblocktrans %}
     {% endif %}
-    {% if status.content == 'started reading' %}
+    {% if status.content == 'started reading' or status.content == '<p>started reading</p>' %}
         {% blocktrans trimmed %}
         <a href="{{ user_path}}">{{ username }}</a> started reading <a href="{{ book_path }}">{{ book_title }}</a>
         {% endblocktrans %}
@@ -38,3 +38,4 @@
 {% endif %}
 
 {% endwith %}
+

bookwyrm/templates/discover/large-book.html

@@ -46,7 +46,7 @@
             </div>
 
             <div class="notification has-background-body p-2 mb-2 clip-text">
-                {% include "snippets/status/content_status.html" with hide_book=True trim_length=70 hide_more=True %}
+                {% include "snippets/status/content_status.html" with hide_book=True trim_length=70 hide_more=True expand=False %}
             </div>
             <a href="{{ status.remote_id }}">
                 <span>{% trans "View status" %}</span>

bookwyrm/templates/email/html_layout.html

@@ -2,7 +2,7 @@
 <div style="font-family: BlinkMacSystemFont,-apple-system,'Segoe UI',Roboto,Oxygen,Ubuntu,Cantarell,'Fira Sans','Droid Sans','Helvetica Neue',Helvetica,Arial,sans-serif; border-radius: 6px; background-color: #efefef; max-width: 632px;">
     <div style="padding: 1rem; overflow: auto;">
         <div style="float: left; margin-right: 1rem;">
-            <a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo"></a>
+            <a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
         </div>
         <div>
             <a style="color: black; text-decoration: none" href="https://{{ domain }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>

bookwyrm/templates/embed-layout.html

@@ -17,7 +17,7 @@
 
 <header class="section py-3">
     <a href="/" class="is-flex is-align-items-center">
-        <img class="image logo is-flex-shrink-0" style="height: 32px" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
+        <img class="image logo is-flex-shrink-0" style="height: 32px" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
         <span class="title is-5 ml-2">{{ site.name }}</span>
     </a>
 </header>

bookwyrm/templates/feed/layout.html

@@ -23,7 +23,7 @@
         {% block panel %}{% endblock %}
 
         {% if activities %}
-        {% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" %}
+        {% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" mode="chronological" %}
         {% endif %}
     </div>
 </div>

bookwyrm/templates/feed/status.html

@@ -2,15 +2,13 @@
 {% load feed_page_tags %}
 {% load i18n %}
 
-{% block opengraph_images %}
+{% block opengraph %}
-
+    {% firstof status.book status.mention_books.first as book %}
-{% firstof status.book status.mention_books.first as book %}
+    {% if book %}
-{% if book %}
+        {% include 'snippets/opengraph.html' with image=preview %}
-    {% include 'snippets/opengraph_images.html' with image=preview %}
+    {% else %}
-{% else %}
+        {% include 'snippets/opengraph.html' %}
-    {% include 'snippets/opengraph_images.html' %}
+    {% endif %}
-{% endif %}
-
 {% endblock %}
 
 
@@ -32,7 +30,7 @@
         {% endif %}
         {% endfor %}
         <div class="is-main block">
-            {% include 'snippets/status/status.html' with status=status main=True %}
+            {% include 'snippets/status/status.html' with status=status main=True expand=True %}
         </div>
 
         {% for child in children %}
@@ -44,4 +42,3 @@
 </div>
 
 {% endblock %}
-

bookwyrm/templates/get_started/layout.html

@@ -15,6 +15,8 @@
                 src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}"
                 aria-hidden="true"
                 alt="{{ site.name }}"
+                loading="lazy"
+                decoding="async"
             >
             <h1 class="modal-card-title" id="get_started_header">
                 {% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}

bookwyrm/templates/guided_tour/book.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     const tour = new Shepherd.Tour({
         exitOnEsc: true,
     });

bookwyrm/templates/guided_tour/group.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     const tour = new Shepherd.Tour({
         exitOnEsc: true,
     });

bookwyrm/templates/guided_tour/home.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
 const initiateTour = new Shepherd.Tour({
     exitOnEsc: true,
 });

bookwyrm/templates/guided_tour/lists.html

@@ -2,7 +2,7 @@
 {% load utilities %}
 {% load user_page_tags %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
 
     const tour = new Shepherd.Tour({
         exitOnEsc: true,

bookwyrm/templates/guided_tour/search.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
 
     let localResult = document.querySelector(".local-book-search-result");
     let remoteResult = document.querySelector(".remote-book-search-result");

bookwyrm/templates/guided_tour/user_books.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     const tour = new Shepherd.Tour({
         exitOnEsc: true,
     });

bookwyrm/templates/guided_tour/user_groups.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     const tour = new Shepherd.Tour({
         exitOnEsc: true,
     });

bookwyrm/templates/guided_tour/user_profile.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
 const tour = new Shepherd.Tour({
     exitOnEsc: true,
 });

bookwyrm/templates/hashtag.html

@@ -0,0 +1,32 @@
+{% extends "layout.html" %}
+{% load i18n %}
+
+{% block title %}{{ hashtag }}{% endblock %}
+
+{% block content %}
+<div class="container is-max-desktop">
+    <section class="block">
+        <header class="block content has-text-centered">
+            <h1 class="title">{{ hashtag }}</h1>
+            <p class="subtitle">
+                {% blocktrans trimmed with site_name=site.name %}
+                See tagged statuses in the local {{ site_name }} community
+                {% endblocktrans %}
+            </p>
+        </header>
+
+        {% for activity in activities %}
+        <div class="block">
+            {% include 'snippets/status/status.html' with status=activity %}
+        </div>
+        {% endfor %}
+        {% if not activities %}
+        <div class="block">
+            <p>{% trans "No activities for this hashtag yet!" %}</p>
+        </div>
+        {% endif %}
+
+        {% include 'snippets/pagination.html' with page=activities path=path %}
+    </section>
+</div>
+{% endblock %}

bookwyrm/templates/import/import.html

@@ -15,6 +15,12 @@
     {% endif %}
 
     {% if site.imports_enabled %}
+        {% if import_size_limit and import_limit_reset %}
+            <div class="notification">
+                <p>{% blocktrans %}Currently you are allowed to import {{ import_size_limit }} books every {{ import_limit_reset }} days.{% endblocktrans %}</p>
+                <p>{% blocktrans %}You have {{ allowed_imports }} left.{% endblocktrans %}</p>
+            </div>
+        {% endif %}
         {% if recent_avg_hours or recent_avg_minutes %}
         <div class="notification">
             <p>
@@ -90,7 +96,12 @@
                 </div>
             </div>
             </div>
-            <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+            {% if not import_limit_reset and not import_size_limit or allowed_imports > 0 %}
+                <button class="button is-primary" type="submit">{% trans "Import" %}</button>
+            {% else %}
+                <button class="button is-primary is-disabled" type="submit">{% trans "Import" %}</button>
+                <p>{% trans "You've reached the import limit." %}</p>
+            {% endif%}
         </form>
     {% else %}
     <div class="box notification has-text-centered is-warning m-6 content">

bookwyrm/templates/import/import_status.html

@@ -41,7 +41,7 @@
         </dl>
     </div>
 
-    {% if not job.complete and show_progress %}
+    {% if job.status == "active" and show_progress %}
     <div class="box is-processing">
         <div class="block">
             <span class="icon icon-spinner is-pulled-left" aria-hidden="true"></span>

bookwyrm/templates/landing/landing.html

@@ -10,7 +10,8 @@
 </div>
 
 {% get_current_language as LANGUAGE_CODE %}
-{% cache 60 * 60 LANGUAGE_CODE %}
+{# 1 hour cache #}
+{% cache 3600 landing LANGUAGE_CODE %}
 {% get_landing_books as books %}
 <section class="tile is-ancestor">
     <div class="tile is-vertical is-6">

bookwyrm/templates/layout.html

@@ -15,20 +15,9 @@
     <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">
     <link rel="apple-touch-icon" href="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
 
-    {% if preview_images_enabled is True %}
+    {% block opengraph %}
-    <meta name="twitter:card" content="summary_large_image">
+        {% include 'snippets/opengraph.html' %}
-    {% else %}
-    <meta name="twitter:card" content="summary">
-    {% endif %}
-    <meta name="twitter:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
-    <meta name="og:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
-    <meta name="twitter:description" content="{{ site.instance_tagline }}">
-    <meta name="og:description" content="{{ site.instance_tagline }}">
-
-    {% block opengraph_images %}
-        {% include 'snippets/opengraph_images.html' %}
     {% endblock %}
-    <meta name="twitter:image:alt" content="BookWyrm Logo">
 
     {% block head_links %}{% endblock %}
 </head>
@@ -36,9 +25,10 @@
 {% block body %}
 <nav class="navbar" aria-label="main navigation">
     <div class="container">
+        {% with notification_count=request.user.unread_notification_count has_unread_mentions=request.user.has_unread_mentions %}
         <div class="navbar-brand">
             <a class="navbar-item" href="/">
-                <img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
+                <img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
             </a>
             <form class="navbar-item column is-align-items-start pt-5" action="{% url 'search' %}">
                 <div class="field has-addons">
@@ -78,9 +68,8 @@
             >
                 <i class="icon-dots-three-vertical" aria-hidden="true"></i>
 
-                {% with request.user.unread_notification_count as notification_count %}
                 <strong
-                    class="{% if not notification_count %}is-hidden {% elif request.user.has_unread_mentions %}is-danger {% else %}is-primary {% endif %} tag is-small px-1"
+                    class="{% if not notification_count %}is-hidden {% elif has_unread_mentions %}is-danger {% else %}is-primary {% endif %} tag is-small px-1"
                     data-poll-wrapper
                 >
                     <span class="is-sr-only">{% trans "Notifications" %}</span>
@@ -88,7 +77,6 @@
                         {{ notification_count }}
                     </strong>
                 </strong>
-                {% endwith %}
             </button>
         </div>
 
@@ -119,14 +107,12 @@
                                 <span class="is-sr-only">{% trans "Notifications" %}</span>
                             </span>
                         </span>
-                        {% with request.user.unread_notification_count as notification_count %}
                         <span
-                            class="{% if not notification_count %}is-hidden {% elif request.user.has_unread_mentions %}is-danger {% endif %}tag is-medium transition-x"
+                            class="{% if not notification_count %}is-hidden {% elif has_unread_mentions %}is-danger {% endif %}tag is-medium transition-x"
                             data-poll-wrapper
                         >
                             <span data-poll="notifications">{{ notification_count }}</span>
                         </span>
-                        {% endwith %}
                     </a>
                 </div>
                 {% else %}
@@ -165,6 +151,7 @@
                 {% endif %}
             </div>
         </div>
+        {% endwith %}
     </div>
 </nav>
 
@@ -196,7 +183,7 @@
 {% include 'snippets/footer.html' %}
 {% endblock %}
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     var csrf_token = '{{ csrf_token }}';
 </script>
 

bookwyrm/templates/lists/layout.html

@@ -1,8 +1,13 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load list_page_tags %}
 
 {% block title %}{{ list.name }}{% endblock %}
 
+{% block opengraph %}
+    {% include 'snippets/opengraph.html' with title=list|opengraph_title description=list|opengraph_description %}
+{% endblock %}
+
 {% block content %}
 <header class="columns content is-mobile">
     <div class="column">

bookwyrm/templates/notifications/item.html

@@ -17,6 +17,8 @@
     {% include 'notifications/items/add.html' %}
 {% elif notification.notification_type == 'REPORT' %}
     {% include 'notifications/items/report.html' %}
+{% elif notification.notification_type == 'LINK_DOMAIN' %}
+    {% include 'notifications/items/link_domain.html' %}
 {% elif notification.notification_type == 'INVITE' %}
     {% include 'notifications/items/invite.html' %}
 {% elif notification.notification_type == 'ACCEPT' %}

bookwyrm/templates/notifications/items/link_domain.html

@@ -0,0 +1,20 @@
+{% extends 'notifications/items/layout.html' %}
+{% load humanize %}
+{% load i18n %}
+
+{% block primary_link %}{% spaceless %}
+{% url 'settings-link-domain' %}
+{% endspaceless %}{% endblock %}
+
+{% block icon %}
+<span class="icon icon-warning"></span>
+{% endblock %}
+
+{% block description %}
+    {% url 'settings-link-domain' as path %}
+    {% blocktrans trimmed count counter=notification.related_link_domains.count with display_count=notification.related_link_domains.count|intcomma %}
+    A new <a href="{{ path }}">link domain</a> needs review
+    {% plural %}
+    {{ display_count }} new <a href="{{ path }}">link domains</a> need moderation
+    {% endblocktrans %}
+{% endblock %}

bookwyrm/templates/ostatus/template.html

@@ -11,7 +11,7 @@
     <title>{% block title %}{% endblock %}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <link href="{% sass_src site_theme %}" rel="stylesheet" type="text/css" />
-    <script>
+    <script nonce="{{request.csp_nonce}}">
         function closeWindow() {
             window.close();
         }
@@ -21,7 +21,7 @@
 <nav class="navbar" aria-label="main navigation">
     <div class="container">
         <div class="navbar-brand">
-                <img class="image logo navbar-item" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static 'images/logo-small.png' %}{% endif %}" alt="Home page">
+                <img class="image logo navbar-item" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static 'images/logo-small.png' %}{% endif %}" alt="Home page" loading="lazy" decoding="async">
             <h2 class="navbar-item subtitle">{% block heading %}{% endblock %}</h2>
         </div>
     </div>
@@ -32,7 +32,7 @@
     </div>
 </div>
 
-<script>
+<script nonce="{{request.csp_nonce}}">
     var csrf_token = '{{ csrf_token }}';
 </script>
 <script src="{% static 'js/bookwyrm.js' %}?v={{ js_cache }}"></script>

bookwyrm/templates/search/barcode_modal.html

@@ -29,7 +29,7 @@
     <template id="barcode-scanning">
         <span class="icon icon-barcode"></span>
         <span class="is-size-5">{% trans "Scanning..." context "barcode scanner" %}</span><br/>
-        <span>{% trans "Align your book's barcode with the camera." %}</span>
+        <span>{% trans "Align your book's barcode with the camera." %}</span><span class="isbn"></span>
     </template>
     <template id="barcode-found">
         <span class="icon icon-check"></span>