diff --git a/.env.dev.example b/.env.dev.example index 1e4fb9812..9a4366e02 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -36,7 +36,7 @@ FLOWER_PORT=8888 #FLOWER_USER=mouse #FLOWER_PASSWORD=changeme -EMAIL_HOST="smtp.mailgun.org" +EMAIL_HOST=smtp.mailgun.org EMAIL_PORT=587 EMAIL_HOST_USER=mail@your.domain.here EMAIL_HOST_PASSWORD=emailpassword123 diff --git a/.env.prod.example b/.env.prod.example index 49729d533..56f52a286 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -36,7 +36,7 @@ FLOWER_PORT=8888 FLOWER_USER=mouse FLOWER_PASSWORD=changeme -EMAIL_HOST="smtp.mailgun.org" +EMAIL_HOST=smtp.mailgun.org EMAIL_PORT=587 EMAIL_HOST_USER=mail@your.domain.here EMAIL_HOST_PASSWORD=emailpassword123 diff --git a/.gitignore b/.gitignore index 43716f6a8..d7b495ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.swp **/__pycache__ .local +/nginx/nginx.conf # Poetry .cache/ diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 4896e07d9..4cba9939e 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -7,7 +7,7 @@ from django.utils import timezone from bookwyrm import models from bookwyrm.redis_store import RedisStore, r -from bookwyrm.tasks import app +from bookwyrm.tasks import app, LOW, MEDIUM, HIGH class ActivityStream(RedisStore): @@ -22,6 +22,11 @@ class ActivityStream(RedisStore): stream_id = self.stream_id(user) return f"{stream_id}-unread" + def unread_by_status_type_id(self, user): + """the redis key for this user's unread count for this stream""" + stream_id = self.stream_id(user) + return f"{stream_id}-unread-by-type" + def get_rank(self, obj): # pylint: disable=no-self-use """statuses are sorted by date published""" return obj.published_date.timestamp() @@ -35,6 +40,10 @@ class ActivityStream(RedisStore): for user in self.get_audience(status): # add to the unread status count pipeline.incr(self.unread_id(user)) + # add to the unread status count for status type + pipeline.hincrby( + self.unread_by_status_type_id(user), get_status_type(status), 1 + ) # and go! pipeline.execute() @@ -55,6 +64,7 @@ class ActivityStream(RedisStore): """load the statuses to be displayed""" # clear unreads for this feed r.set(self.unread_id(user), 0) + r.delete(self.unread_by_status_type_id(user)) statuses = self.get_store(self.stream_id(user)) return ( @@ -75,6 +85,14 @@ class ActivityStream(RedisStore): """get the unread status count for this user's feed""" return int(r.get(self.unread_id(user)) or 0) + def get_unread_count_by_status_type(self, user): + """get the unread status count for this user's feed's status types""" + status_types = r.hgetall(self.unread_by_status_type_id(user)) + return { + str(key.decode("utf-8")): int(value) or 0 + for key, value in status_types.items() + } + def populate_streams(self, user): """go from zero to a timeline""" self.populate_store(self.stream_id(user)) @@ -277,7 +295,18 @@ def add_status_on_create(sender, instance, created, *args, **kwargs): def add_status_on_create_command(sender, instance, created): """runs this code only after the database commit completes""" - add_status_task.delay(instance.id, increment_unread=created) + priority = HIGH + # check if this is an old status, de-prioritize if so + # (this will happen if federation is very slow, or, more expectedly, on csv import) + one_day = 60 * 60 * 24 + if (instance.created_date - instance.published_date).seconds > one_day: + priority = LOW + + add_status_task.apply_async( + args=(instance.id,), + kwargs={"increment_unread": created}, + queue=priority, + ) if sender == models.Boost: handle_boost_task.delay(instance.id) @@ -409,7 +438,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs): # ---- TASKS -@app.task(queue="low_priority") +@app.task(queue=LOW) def add_book_statuses_task(user_id, book_id): """add statuses related to a book on shelve""" user = models.User.objects.get(id=user_id) @@ -417,7 +446,7 @@ def add_book_statuses_task(user_id, book_id): BooksStream().add_book_statuses(user, book) -@app.task(queue="low_priority") +@app.task(queue=LOW) def remove_book_statuses_task(user_id, book_id): """remove statuses about a book from a user's books feed""" user = models.User.objects.get(id=user_id) @@ -425,7 +454,7 @@ def remove_book_statuses_task(user_id, book_id): BooksStream().remove_book_statuses(user, book) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def populate_stream_task(stream, user_id): """background task for populating an empty activitystream""" user = models.User.objects.get(id=user_id) @@ -433,7 +462,7 @@ def populate_stream_task(stream, user_id): stream.populate_streams(user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def remove_status_task(status_ids): """remove a status from any stream it might be in""" # this can take an id or a list of ids @@ -446,10 +475,10 @@ def remove_status_task(status_ids): stream.remove_object_from_related_stores(status) -@app.task(queue="high_priority") +@app.task(queue=HIGH) def add_status_task(status_id, increment_unread=False): """add a status to any stream it should be in""" - status = models.Status.objects.get(id=status_id) + status = models.Status.objects.select_subclasses().get(id=status_id) # we don't want to tick the unread count for csv import statuses, idk how better # to check than just to see if the states is more than a few days old if status.created_date < timezone.now() - timedelta(days=2): @@ -458,7 +487,7 @@ def add_status_task(status_id, increment_unread=False): stream.add_status(status, increment_unread=increment_unread) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def remove_user_statuses_task(viewer_id, user_id, stream_list=None): """remove all statuses by a user from a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -468,7 +497,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None): stream.remove_user_statuses(viewer, user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def add_user_statuses_task(viewer_id, user_id, stream_list=None): """add all statuses by a user to a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -478,7 +507,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None): stream.add_user_statuses(viewer, user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def handle_boost_task(boost_id): """remove the original post and other, earlier boosts""" instance = models.Status.objects.get(id=boost_id) @@ -496,3 +525,20 @@ def handle_boost_task(boost_id): stream.remove_object_from_related_stores(boosted, stores=audience) for status in old_versions: stream.remove_object_from_related_stores(status, stores=audience) + + +def get_status_type(status): + """return status type even for boosted statuses""" + status_type = status.status_type.lower() + + # Check if current status is a boost + if hasattr(status, "boost"): + # Act in accordance of your findings + if hasattr(status.boost.boosted_status, "review"): + status_type = "review" + if hasattr(status.boost.boosted_status, "comment"): + status_type = "comment" + if hasattr(status.boost.boosted_status, "quotation"): + status_type = "quotation" + + return status_type diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 6c89b61fb..e42a6d8c3 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -82,6 +82,8 @@ def search_identifiers(query, *filters, return_first=False): *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) ).distinct() if results.count() <= 1: + if return_first: + return results.first() return results # when there are multiple editions of the same work, pick the default. @@ -124,6 +126,7 @@ def search_title_author(query, min_confidence, *filters, return_first=False): result = default else: result = editions.first() + if return_first: return result list_results.append(result) diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index faed5429a..e9f538569 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -67,7 +67,7 @@ class Connector(AbstractConnector): extracted = list(data.get("entities").values()) try: data = extracted[0] - except KeyError: + except (KeyError, IndexError): raise ConnectorException("Invalid book data") # flatten the data so that images, uri, and claims are on the same level return { @@ -128,6 +128,7 @@ class Connector(AbstractConnector): def load_edition_data(self, work_uri): """get a list of editions for a work""" + # pylint: disable=line-too-long url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true" return get_data(url) diff --git a/bookwyrm/emailing.py b/bookwyrm/emailing.py index c6a197f29..08fd9ef85 100644 --- a/bookwyrm/emailing.py +++ b/bookwyrm/emailing.py @@ -10,14 +10,9 @@ from bookwyrm.settings import DOMAIN def email_data(): """fields every email needs""" site = models.SiteSettings.objects.get() - if site.logo_small: - logo_path = f"/images/{site.logo_small.url}" - else: - logo_path = "/static/images/logo-small.png" - return { "site_name": site.name, - "logo": logo_path, + "logo": site.logo_small_url, "domain": DOMAIN, "user": None, } @@ -46,6 +41,18 @@ def password_reset_email(reset_code): send_email.delay(reset_code.user.email, *format_email("password_reset", data)) +def moderation_report_email(report): + """a report was created""" + data = email_data() + data["reporter"] = report.reporter.localname or report.reporter.username + data["reportee"] = report.user.localname or report.user.username + data["report_link"] = report.remote_id + + for admin in models.User.objects.filter(groups__name__in=["admin", "moderator"]): + data["user"] = admin.display_name + send_email.delay(admin.email, *format_email("moderation_report", data)) + + def format_email(email_name, data): """render the email templates""" subject = get_template(f"email/{email_name}/subject.html").render(data).strip() diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 298f73da9..7ba7bd97b 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -9,6 +9,8 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from bookwyrm import models +from bookwyrm.models.fields import ClearableFileInputWithWarning +from bookwyrm.models.user import FeedFilterChoices class CustomForm(ModelForm): @@ -147,6 +149,17 @@ class EditUserForm(CustomForm): "preferred_language", ] help_texts = {f: None for f in fields} + widgets = { + "avatar": ClearableFileInputWithWarning( + attrs={"aria-describedby": "desc_avatar"} + ), + "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}), + "summary": forms.Textarea(attrs={"aria-describedby": "desc_summary"}), + "email": forms.EmailInput(attrs={"aria-describedby": "desc_email"}), + "discoverable": forms.CheckboxInput( + attrs={"aria-describedby": "desc_discoverable"} + ), + } class LimitedEditUserForm(CustomForm): @@ -160,6 +173,16 @@ class LimitedEditUserForm(CustomForm): "discoverable", ] help_texts = {f: None for f in fields} + widgets = { + "avatar": ClearableFileInputWithWarning( + attrs={"aria-describedby": "desc_avatar"} + ), + "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}), + "summary": forms.Textarea(attrs={"aria-describedby": "desc_summary"}), + "discoverable": forms.CheckboxInput( + attrs={"aria-describedby": "desc_discoverable"} + ), + } class DeleteUserForm(CustomForm): @@ -174,6 +197,18 @@ class UserGroupForm(CustomForm): fields = ["groups"] +class FeedStatusTypesForm(CustomForm): + class Meta: + model = models.User + fields = ["feed_status_types"] + help_texts = {f: None for f in fields} + widgets = { + "feed_status_types": widgets.CheckboxSelectMultiple( + choices=FeedFilterChoices, + ), + } + + class CoverForm(CustomForm): class Meta: model = models.Book @@ -196,18 +231,92 @@ class EditionForm(CustomForm): "connector", "search_vector", ] + widgets = { + "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}), + "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}), + "description": forms.Textarea( + attrs={"aria-describedby": "desc_description"} + ), + "series": forms.TextInput(attrs={"aria-describedby": "desc_series"}), + "series_number": forms.TextInput( + attrs={"aria-describedby": "desc_series_number"} + ), + "languages": forms.TextInput( + attrs={"aria-describedby": "desc_languages_help desc_languages"} + ), + "publishers": forms.TextInput( + attrs={"aria-describedby": "desc_publishers_help desc_publishers"} + ), + "first_published_date": forms.SelectDateWidget( + attrs={"aria-describedby": "desc_first_published_date"} + ), + "published_date": forms.SelectDateWidget( + attrs={"aria-describedby": "desc_published_date"} + ), + "cover": ClearableFileInputWithWarning( + attrs={"aria-describedby": "desc_cover"} + ), + "physical_format": forms.Select( + attrs={"aria-describedby": "desc_physical_format"} + ), + "physical_format_detail": forms.TextInput( + attrs={"aria-describedby": "desc_physical_format_detail"} + ), + "pages": forms.NumberInput(attrs={"aria-describedby": "desc_pages"}), + "isbn_13": forms.TextInput(attrs={"aria-describedby": "desc_isbn_13"}), + "isbn_10": forms.TextInput(attrs={"aria-describedby": "desc_isbn_10"}), + "openlibrary_key": forms.TextInput( + attrs={"aria-describedby": "desc_openlibrary_key"} + ), + "inventaire_id": forms.TextInput( + attrs={"aria-describedby": "desc_inventaire_id"} + ), + "oclc_number": forms.TextInput( + attrs={"aria-describedby": "desc_oclc_number"} + ), + "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}), + } class AuthorForm(CustomForm): class Meta: model = models.Author - exclude = [ - "remote_id", - "origin_id", - "created_date", - "updated_date", - "search_vector", + fields = [ + "last_edited_by", + "name", + "aliases", + "bio", + "wikipedia_link", + "born", + "died", + "openlibrary_key", + "inventaire_id", + "librarything_key", + "goodreads_key", + "isni", ] + widgets = { + "name": forms.TextInput(attrs={"aria-describedby": "desc_name"}), + "aliases": forms.TextInput(attrs={"aria-describedby": "desc_aliases"}), + "bio": forms.Textarea(attrs={"aria-describedby": "desc_bio"}), + "wikipedia_link": forms.TextInput( + attrs={"aria-describedby": "desc_wikipedia_link"} + ), + "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}), + "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}), + "oepnlibrary_key": forms.TextInput( + attrs={"aria-describedby": "desc_oepnlibrary_key"} + ), + "inventaire_id": forms.TextInput( + attrs={"aria-describedby": "desc_inventaire_id"} + ), + "librarything_key": forms.TextInput( + attrs={"aria-describedby": "desc_librarything_key"} + ), + "goodreads_key": forms.TextInput( + attrs={"aria-describedby": "desc_goodreads_key"} + ), + } class ImportForm(forms.Form): @@ -282,12 +391,37 @@ class SiteForm(CustomForm): class Meta: model = models.SiteSettings exclude = [] + widgets = { + "instance_short_description": forms.TextInput( + attrs={"aria-describedby": "desc_instance_short_description"} + ), + "require_confirm_email": forms.CheckboxInput( + attrs={"aria-describedby": "desc_require_confirm_email"} + ), + "invite_request_text": forms.Textarea( + attrs={"aria-describedby": "desc_invite_request_text"} + ), + } class AnnouncementForm(CustomForm): class Meta: model = models.Announcement exclude = ["remote_id"] + widgets = { + "preview": forms.TextInput(attrs={"aria-describedby": "desc_preview"}), + "content": forms.Textarea(attrs={"aria-describedby": "desc_content"}), + "event_date": forms.SelectDateWidget( + attrs={"aria-describedby": "desc_event_date"} + ), + "start_date": forms.SelectDateWidget( + attrs={"aria-describedby": "desc_start_date"} + ), + "end_date": forms.SelectDateWidget( + attrs={"aria-describedby": "desc_end_date"} + ), + "active": forms.CheckboxInput(attrs={"aria-describedby": "desc_active"}), + } class ListForm(CustomForm): @@ -312,6 +446,9 @@ class EmailBlocklistForm(CustomForm): class Meta: model = models.EmailBlocklist fields = ["domain"] + widgets = { + "avatar": forms.TextInput(attrs={"aria-describedby": "desc_domain"}), + } class IPBlocklistForm(CustomForm): diff --git a/bookwyrm/importers/goodreads_import.py b/bookwyrm/importers/goodreads_import.py index c62e65827..c0dc0ea28 100644 --- a/bookwyrm/importers/goodreads_import.py +++ b/bookwyrm/importers/goodreads_import.py @@ -7,10 +7,3 @@ class GoodreadsImporter(Importer): For a more complete example of overriding see librarything_import.py""" service = "Goodreads" - - def parse_fields(self, entry): - """handle the specific fields in goodreads csvs""" - entry.update({"import_source": self.service}) - # add missing 'Date Started' field - entry.update({"Date Started": None}) - return entry diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 6d898a2a3..94e6734e7 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -7,7 +7,7 @@ from django.utils.translation import gettext_lazy as _ from bookwyrm import models from bookwyrm.models import ImportJob, ImportItem -from bookwyrm.tasks import app +from bookwyrm.tasks import app, LOW logger = logging.getLogger(__name__) @@ -15,33 +15,90 @@ logger = logging.getLogger(__name__) class Importer: """Generic class for csv data import from an outside service""" - service = "Unknown" + service = "Import" delimiter = "," encoding = "UTF-8" - mandatory_fields = ["Title", "Author"] + + # these are from Goodreads + row_mappings_guesses = [ + ("id", ["id", "book id"]), + ("title", ["title"]), + ("authors", ["author", "authors", "primary author"]), + ("isbn_10", ["isbn10", "isbn"]), + ("isbn_13", ["isbn13", "isbn", "isbns"]), + ("shelf", ["shelf", "exclusive shelf", "read status"]), + ("review_name", ["review name"]), + ("review_body", ["my review", "review"]), + ("rating", ["my rating", "rating", "star rating"]), + ("date_added", ["date added", "entry date", "added"]), + ("date_started", ["date started", "started"]), + ("date_finished", ["date finished", "last date read", "date read", "finished"]), + ] + date_fields = ["date_added", "date_started", "date_finished"] + shelf_mapping_guesses = { + "to-read": ["to-read"], + "read": ["read"], + "reading": ["currently-reading", "reading"], + } def create_job(self, user, csv_file, include_reviews, privacy): """check over a csv and creates a database entry for the job""" + csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) + rows = enumerate(list(csv_reader)) job = ImportJob.objects.create( - user=user, include_reviews=include_reviews, privacy=privacy + user=user, + include_reviews=include_reviews, + privacy=privacy, + mappings=self.create_row_mappings(csv_reader.fieldnames), + source=self.service, ) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.delimiter)) - ): - if not all(x in entry for x in self.mandatory_fields): - raise ValueError("Author and title must be in data.") - entry = self.parse_fields(entry) - self.save_item(job, index, entry) + + for index, entry in rows: + self.create_item(job, index, entry) return job - def save_item(self, job, index, data): # pylint: disable=no-self-use - """creates and saves an import item""" - ImportItem(job=job, index=index, data=data).save() + def update_legacy_job(self, job): + """patch up a job that was in the old format""" + items = job.items + headers = list(items.first().data.keys()) + job.mappings = self.create_row_mappings(headers) + job.updated_date = timezone.now() + job.save() - def parse_fields(self, entry): - """updates csv data with additional info""" - entry.update({"import_source": self.service}) - return entry + for item in items.all(): + normalized = self.normalize_row(item.data, job.mappings) + normalized["shelf"] = self.get_shelf(normalized) + item.normalized_data = normalized + item.save() + + def create_row_mappings(self, headers): + """guess what the headers mean""" + mappings = {} + for (key, guesses) in self.row_mappings_guesses: + value = [h for h in headers if h.lower() in guesses] + value = value[0] if len(value) else None + if value: + headers.remove(value) + mappings[key] = value + return mappings + + def create_item(self, job, index, data): + """creates and saves an import item""" + normalized = self.normalize_row(data, job.mappings) + normalized["shelf"] = self.get_shelf(normalized) + ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() + + def get_shelf(self, normalized_row): + """determine which shelf to use""" + shelf_name = normalized_row["shelf"] + shelf = [ + s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs + ] + return shelf[0] if shelf else None + + def normalize_row(self, entry, mappings): # pylint: disable=no-self-use + """use the dataclass to create the formatted row of data""" + return {k: entry.get(v) for k, v in mappings.items()} def create_retry_job(self, user, original_job, items): """retry items that didn't import""" @@ -49,55 +106,65 @@ class Importer: user=user, include_reviews=original_job.include_reviews, privacy=original_job.privacy, + # TODO: allow users to adjust mappings + mappings=original_job.mappings, retry=True, ) for item in items: - self.save_item(job, item.index, item.data) + # this will re-normalize the raw data + self.create_item(job, item.index, item.data) return job - def start_import(self, job): + def start_import(self, job): # pylint: disable=no-self-use """initalizes a csv import job""" - result = import_data.delay(self.service, job.id) + result = start_import_task.delay(job.id) job.task_id = result.id job.save() @app.task(queue="low_priority") -def import_data(source, job_id): - """does the actual lookup work in a celery task""" +def start_import_task(job_id): + """trigger the child tasks for each row""" job = ImportJob.objects.get(id=job_id) + # these are sub-tasks so that one big task doesn't use up all the memory in celery + for item in job.items.values_list("id", flat=True).all(): + import_item_task.delay(item) + + +@app.task(queue="low_priority") +def import_item_task(item_id): + """resolve a row into a book""" + item = models.ImportItem.objects.get(id=item_id) try: - for item in job.items.all(): - try: - item.resolve() - except Exception as err: # pylint: disable=broad-except - logger.exception(err) - item.fail_reason = _("Error loading book") - item.save() - continue + item.resolve() + except Exception as err: # pylint: disable=broad-except + item.fail_reason = _("Error loading book") + item.save() + item.update_job() + raise err - if item.book or item.book_guess: - item.save() + if item.book: + # shelves book and handles reviews + handle_imported_book(item) + else: + item.fail_reason = _("Could not find a match for book") - if item.book: - # shelves book and handles reviews - handle_imported_book( - source, job.user, item, job.include_reviews, job.privacy - ) - else: - item.fail_reason = _("Could not find a match for book") - item.save() - finally: - job.complete = True - job.save() + item.save() + item.update_job() -def handle_imported_book(source, user, item, include_reviews, privacy): +def handle_imported_book(item): """process a csv and then post about it""" + job = item.job + user = job.user if isinstance(item.book, models.Work): item.book = item.book.default_edition if not item.book: + item.fail_reason = _("Error loading book") + item.save() return + if not isinstance(item.book, models.Edition): + item.book = item.book.edition existing_shelf = models.ShelfBook.objects.filter(book=item.book, user=user).exists() @@ -105,9 +172,9 @@ def handle_imported_book(source, user, item, include_reviews, privacy): if item.shelf and not existing_shelf: desired_shelf = models.Shelf.objects.get(identifier=item.shelf, user=user) shelved_date = item.date_added or timezone.now() - models.ShelfBook.objects.create( + models.ShelfBook( book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date - ) + ).save(priority=LOW) for read in item.reads: # check for an existing readthrough with the same dates @@ -122,35 +189,52 @@ def handle_imported_book(source, user, item, include_reviews, privacy): read.user = user read.save() - if include_reviews and (item.rating or item.review): + if job.include_reviews and (item.rating or item.review) and not item.linked_review: # we don't know the publication date of the review, # but "now" is a bad guess published_date_guess = item.date_read or item.date_added if item.review: # pylint: disable=consider-using-f-string - review_title = ( - "Review of {!r} on {!r}".format( - item.book.title, - source, - ) - if item.review - else "" + review_title = "Review of {!r} on {!r}".format( + item.book.title, + job.source, ) - models.Review.objects.create( + review = models.Review.objects.filter( user=user, book=item.book, name=review_title, - content=item.review, rating=item.rating, published_date=published_date_guess, - privacy=privacy, - ) + ).first() + if not review: + review = models.Review( + user=user, + book=item.book, + name=review_title, + content=item.review, + rating=item.rating, + published_date=published_date_guess, + privacy=job.privacy, + ) + review.save(software="bookwyrm", priority=LOW) else: # just a rating - models.ReviewRating.objects.create( + review = models.ReviewRating.objects.filter( user=user, book=item.book, - rating=item.rating, published_date=published_date_guess, - privacy=privacy, - ) + rating=item.rating, + ).first() + if not review: + review = models.ReviewRating( + user=user, + book=item.book, + rating=item.rating, + published_date=published_date_guess, + privacy=job.privacy, + ) + review.save(software="bookwyrm", priority=LOW) + + # only broadcast this review to other bookwyrm instances + item.linked_review = review + item.save() diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index b3175a82d..1b61a6f17 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -1,7 +1,5 @@ -""" handle reading a csv from librarything """ +""" handle reading a tsv from librarything """ import re -import math - from . import Importer @@ -11,32 +9,18 @@ class LibrarythingImporter(Importer): service = "LibraryThing" delimiter = "\t" encoding = "ISO-8859-1" - # mandatory_fields : fields matching the book title and author - mandatory_fields = ["Title", "Primary Author"] - def parse_fields(self, entry): - """custom parsing for librarything""" - data = {} - data["import_source"] = self.service - data["Book Id"] = entry["Book Id"] - data["Title"] = entry["Title"] - data["Author"] = entry["Primary Author"] - data["ISBN13"] = entry["ISBN"] - data["My Review"] = entry["Review"] - if entry["Rating"]: - data["My Rating"] = math.ceil(float(entry["Rating"])) - else: - data["My Rating"] = "" - data["Date Added"] = re.sub(r"\[|\]", "", entry["Entry Date"]) - data["Date Started"] = re.sub(r"\[|\]", "", entry["Date Started"]) - data["Date Read"] = re.sub(r"\[|\]", "", entry["Date Read"]) + def normalize_row(self, entry, mappings): # pylint: disable=no-self-use + """use the dataclass to create the formatted row of data""" + remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None + normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()} + isbn_13 = normalized["isbn_13"].split(", ") + normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None + return normalized - data["Exclusive Shelf"] = None - if data["Date Read"]: - data["Exclusive Shelf"] = "read" - elif data["Date Started"]: - data["Exclusive Shelf"] = "reading" - else: - data["Exclusive Shelf"] = "to-read" - - return data + def get_shelf(self, normalized_row): + if normalized_row["date_finished"]: + return "read" + if normalized_row["date_started"]: + return "reading" + return "to-read" diff --git a/bookwyrm/importers/storygraph_import.py b/bookwyrm/importers/storygraph_import.py index 25498432c..9368115d4 100644 --- a/bookwyrm/importers/storygraph_import.py +++ b/bookwyrm/importers/storygraph_import.py @@ -1,7 +1,4 @@ -""" handle reading a csv from librarything """ -import re -import math - +""" handle reading a csv from storygraph""" from . import Importer @@ -9,26 +6,3 @@ class StorygraphImporter(Importer): """csv downloads from librarything""" service = "Storygraph" - # mandatory_fields : fields matching the book title and author - mandatory_fields = ["Title"] - - def parse_fields(self, entry): - """custom parsing for storygraph""" - data = {} - data["import_source"] = self.service - data["Title"] = entry["Title"] - data["Author"] = entry["Authors"] if "Authors" in entry else entry["Author"] - data["ISBN13"] = entry["ISBN"] - data["My Review"] = entry["Review"] - if entry["Star Rating"]: - data["My Rating"] = math.ceil(float(entry["Star Rating"])) - else: - data["My Rating"] = "" - - data["Date Added"] = re.sub(r"[/]", "-", entry["Date Added"]) - data["Date Read"] = re.sub(r"[/]", "-", entry["Last Date Read"]) - - data["Exclusive Shelf"] = ( - {"read": "read", "currently-reading": "reading", "to-read": "to-read"} - ).get(entry["Read Status"], None) - return data diff --git a/bookwyrm/migrations/0113_auto_20211110_2104.py b/bookwyrm/migrations/0113_auto_20211110_2104.py new file mode 100644 index 000000000..572ba280c --- /dev/null +++ b/bookwyrm/migrations/0113_auto_20211110_2104.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.5 on 2021-11-10 21:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0112_auto_20211022_0844"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="normalized_data", + field=models.JSONField(default={}), + preserve_default=False, + ), + migrations.AddField( + model_name="importjob", + name="mappings", + field=models.JSONField(default={}), + preserve_default=False, + ), + ] diff --git a/bookwyrm/migrations/0114_importjob_source.py b/bookwyrm/migrations/0114_importjob_source.py new file mode 100644 index 000000000..3ec1432e3 --- /dev/null +++ b/bookwyrm/migrations/0114_importjob_source.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.5 on 2021-11-13 00:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0113_auto_20211110_2104"), + ] + + operations = [ + migrations.AddField( + model_name="importjob", + name="source", + field=models.CharField(default="Import", max_length=100), + preserve_default=False, + ), + ] diff --git a/bookwyrm/migrations/0115_importitem_linked_review.py b/bookwyrm/migrations/0115_importitem_linked_review.py new file mode 100644 index 000000000..8cff9b8c0 --- /dev/null +++ b/bookwyrm/migrations/0115_importitem_linked_review.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.5 on 2021-11-13 19:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0114_importjob_source"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="linked_review", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="bookwyrm.review", + ), + ), + ] diff --git a/bookwyrm/migrations/0116_auto_20211114_1734.py b/bookwyrm/migrations/0116_auto_20211114_1734.py new file mode 100644 index 000000000..1da001bdc --- /dev/null +++ b/bookwyrm/migrations/0116_auto_20211114_1734.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.5 on 2021-11-14 17:34 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0115_importitem_linked_review"), + ] + + operations = [ + migrations.RemoveField( + model_name="importjob", + name="task_id", + ), + migrations.AddField( + model_name="importjob", + name="updated_date", + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/bookwyrm/migrations/0117_alter_user_preferred_language.py b/bookwyrm/migrations/0117_alter_user_preferred_language.py new file mode 100644 index 000000000..c892b0516 --- /dev/null +++ b/bookwyrm/migrations/0117_alter_user_preferred_language.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.5 on 2021-11-15 18:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0116_auto_20211114_1734"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_language", + field=models.CharField( + blank=True, + choices=[ + ("en-us", "English"), + ("de-de", "Deutsch (German)"), + ("es-es", "Español (Spanish)"), + ("fr-fr", "Français (French)"), + ("lt-lt", "lietuvių (Lithuanian)"), + ("pt-br", "Português - Brasil (Brazilian Portuguese)"), + ("zh-hans", "简体中文 (Simplified Chinese)"), + ("zh-hant", "繁體中文 (Traditional Chinese)"), + ], + max_length=255, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0118_alter_user_preferred_language.py b/bookwyrm/migrations/0118_alter_user_preferred_language.py new file mode 100644 index 000000000..2019bb369 --- /dev/null +++ b/bookwyrm/migrations/0118_alter_user_preferred_language.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.5 on 2021-11-17 18:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0117_alter_user_preferred_language"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_language", + field=models.CharField( + blank=True, + choices=[ + ("en-us", "English"), + ("de-de", "Deutsch (German)"), + ("es-es", "Español (Spanish)"), + ("gl-es", "Galego (Galician)"), + ("fr-fr", "Français (French)"), + ("lt-lt", "Lietuvių (Lithuanian)"), + ("pt-br", "Português - Brasil (Brazilian Portuguese)"), + ("zh-hans", "简体中文 (Simplified Chinese)"), + ("zh-hant", "繁體中文 (Traditional Chinese)"), + ], + max_length=255, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0119_user_feed_status_types.py b/bookwyrm/migrations/0119_user_feed_status_types.py new file mode 100644 index 000000000..64fa91697 --- /dev/null +++ b/bookwyrm/migrations/0119_user_feed_status_types.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.5 on 2021-11-24 10:15 + +import bookwyrm.models.user +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0118_alter_user_preferred_language"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="feed_status_types", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("review", "Reviews"), + ("comment", "Comments"), + ("quotation", "Quotations"), + ("everything", "Everything else"), + ], + max_length=10, + ), + default=bookwyrm.models.user.get_feed_filter_choices, + size=8, + ), + ), + ] diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 3a88c5249..402cb040b 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -20,7 +20,7 @@ from django.utils.http import http_date from bookwyrm import activitypub from bookwyrm.settings import USER_AGENT, PAGE_LENGTH from bookwyrm.signatures import make_signature, make_digest -from bookwyrm.tasks import app +from bookwyrm.tasks import app, MEDIUM from bookwyrm.models.fields import ImageField, ManyToManyField logger = logging.getLogger(__name__) @@ -29,7 +29,6 @@ logger = logging.getLogger(__name__) PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) - # pylint: disable=invalid-name def set_activity_from_property_field(activity, obj, field): """assign a model property value to the activity json""" @@ -126,12 +125,15 @@ class ActivitypubMixin: # there OUGHT to be only one match return match.first() - def broadcast(self, activity, sender, software=None): + def broadcast(self, activity, sender, software=None, queue=MEDIUM): """send out an activity""" - broadcast_task.delay( - sender.id, - json.dumps(activity, cls=activitypub.ActivityEncoder), - self.get_recipients(software=software), + broadcast_task.apply_async( + args=( + sender.id, + json.dumps(activity, cls=activitypub.ActivityEncoder), + self.get_recipients(software=software), + ), + queue=queue, ) def get_recipients(self, software=None): @@ -195,7 +197,7 @@ class ActivitypubMixin: class ObjectMixin(ActivitypubMixin): """add this mixin for object models that are AP serializable""" - def save(self, *args, created=None, **kwargs): + def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs): """broadcast created/updated/deleted objects as appropriate""" broadcast = kwargs.get("broadcast", True) # this bonus kwarg would cause an error in the base save method @@ -219,15 +221,17 @@ class ObjectMixin(ActivitypubMixin): return try: - software = None # do we have a "pure" activitypub version of this for mastodon? - if hasattr(self, "pure_content"): + if software != "bookwyrm" and hasattr(self, "pure_content"): pure_activity = self.to_create_activity(user, pure=True) - self.broadcast(pure_activity, user, software="other") + self.broadcast( + pure_activity, user, software="other", queue=priority + ) + # set bookwyrm so that that type is also sent software = "bookwyrm" # sends to BW only if we just did a pure version for masto activity = self.to_create_activity(user) - self.broadcast(activity, user, software=software) + self.broadcast(activity, user, software=software, queue=priority) except AttributeError: # janky as heck, this catches the mutliple inheritence chain # for boosts and ignores this auxilliary broadcast @@ -241,8 +245,7 @@ class ObjectMixin(ActivitypubMixin): if isinstance(self, user_model): user = self # book data tracks last editor - elif hasattr(self, "last_edited_by"): - user = self.last_edited_by + user = user or getattr(self, "last_edited_by", None) # again, if we don't know the user or they're remote, don't bother if not user or not user.local: return @@ -252,7 +255,7 @@ class ObjectMixin(ActivitypubMixin): activity = self.to_delete_activity(user) else: activity = self.to_update_activity(user) - self.broadcast(activity, user) + self.broadcast(activity, user, queue=priority) def to_create_activity(self, user, **kwargs): """returns the object wrapped in a Create activity""" @@ -375,9 +378,9 @@ class CollectionItemMixin(ActivitypubMixin): activity_serializer = activitypub.CollectionItem - def broadcast(self, activity, sender, software="bookwyrm"): + def broadcast(self, activity, sender, software="bookwyrm", queue=MEDIUM): """only send book collection updates to other bookwyrm instances""" - super().broadcast(activity, sender, software=software) + super().broadcast(activity, sender, software=software, queue=queue) @property def privacy(self): @@ -396,7 +399,7 @@ class CollectionItemMixin(ActivitypubMixin): return [] return [collection_field.user] - def save(self, *args, broadcast=True, **kwargs): + def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): """broadcast updated""" # first off, we want to save normally no matter what super().save(*args, **kwargs) @@ -407,7 +410,7 @@ class CollectionItemMixin(ActivitypubMixin): # adding an obj to the collection activity = self.to_add_activity(self.user) - self.broadcast(activity, self.user) + self.broadcast(activity, self.user, queue=priority) def delete(self, *args, broadcast=True, **kwargs): """broadcast a remove activity""" @@ -440,12 +443,12 @@ class CollectionItemMixin(ActivitypubMixin): class ActivityMixin(ActivitypubMixin): """add this mixin for models that are AP serializable""" - def save(self, *args, broadcast=True, **kwargs): + def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): """broadcast activity""" super().save(*args, **kwargs) user = self.user if hasattr(self, "user") else self.user_subject if broadcast and user.local: - self.broadcast(self.to_activity(), user) + self.broadcast(self.to_activity(), user, queue=priority) def delete(self, *args, broadcast=True, **kwargs): """nevermind, undo that activity""" @@ -502,7 +505,7 @@ def unfurl_related_field(related_field, sort_field=None): return related_field.remote_id -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def broadcast_task(sender_id, activity, recipients): """the celery task for broadcast""" user_model = apps.get_model("bookwyrm.User", require_ready=True) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 53cf94ff4..6c29ac058 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -27,7 +27,7 @@ class Author(BookDataModel): # idk probably other keys would be useful here? born = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True) - name = fields.CharField(max_length=255, deduplication_field=True) + name = fields.CharField(max_length=255) aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 8ae75baf9..d97a1b8a6 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -66,9 +66,10 @@ class BookDataModel(ObjectMixin, BookWyrmModel): self.remote_id = None return super().save(*args, **kwargs) - def broadcast(self, activity, sender, software="bookwyrm"): + # pylint: disable=arguments-differ + def broadcast(self, activity, sender, software="bookwyrm", **kwargs): """only send book data updates to other bookwyrm instances""" - super().broadcast(activity, sender, software=software) + super().broadcast(activity, sender, software=software, **kwargs) class Book(BookDataModel): diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index ccd669cbf..7d14f88f9 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -3,6 +3,7 @@ from dataclasses import MISSING import imghdr import re from uuid import uuid4 +from urllib.parse import urljoin import dateutil.parser from dateutil.parser import ParserError @@ -13,11 +14,12 @@ from django.db import models from django.forms import ClearableFileInput, ImageField as DjangoImageField from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django.utils.encoding import filepath_to_uri from bookwyrm import activitypub from bookwyrm.connectors import get_image from bookwyrm.sanitize_html import InputHtmlParser -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import MEDIA_FULL_URL def validate_remote_id(value): @@ -294,7 +296,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): super().__init__(*args, **kwargs) def set_field_from_activity(self, instance, data, overwrite=True): - """helper function for assinging a value to the field""" + """helper function for assigning a value to the field""" if not overwrite and getattr(instance, self.name).exists(): return False @@ -381,17 +383,6 @@ class CustomImageField(DjangoImageField): widget = ClearableFileInputWithWarning -def image_serializer(value, alt): - """helper for serializing images""" - if value and hasattr(value, "url"): - url = value.url - else: - return None - if not url[:4] == "http": - url = f"https://{DOMAIN}{url}" - return activitypub.Document(url=url, name=alt) - - class ImageField(ActivitypubFieldMixin, models.ImageField): """activitypub-aware image field""" @@ -407,7 +398,11 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): if formatted is None or formatted is MISSING: return False - if not overwrite and hasattr(instance, self.name): + if ( + not overwrite + and hasattr(instance, self.name) + and getattr(instance, self.name) + ): return False getattr(instance, self.name).save(*formatted, save=save) @@ -424,7 +419,12 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): activity[key] = formatted def field_to_activity(self, value, alt=None): - return image_serializer(value, alt) + url = get_absolute_url(value) + + if not url: + return None + + return activitypub.Document(url=url, name=alt) def field_from_activity(self, value): image_slug = value @@ -461,6 +461,20 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): ) +def get_absolute_url(value): + """returns an absolute URL for the image""" + name = getattr(value, "name") + if not name: + return None + + url = filepath_to_uri(name) + if url is not None: + url = url.lstrip("/") + url = urljoin(MEDIA_FULL_URL, url) + + return url + + class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): """activitypub-aware datetime field""" diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 22253fef7..c46795856 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -6,20 +6,14 @@ from django.db import models from django.utils import timezone from bookwyrm.connectors import connector_manager -from bookwyrm.models import ReadThrough, User, Book +from bookwyrm.models import ReadThrough, User, Book, Edition from .fields import PrivacyLevels -# Mapping goodreads -> bookwyrm shelf titles. -GOODREADS_SHELVES = { - "read": "read", - "currently-reading": "reading", - "to-read": "to-read", -} - - def unquote_string(text): """resolve csv quote weirdness""" + if not text: + return None match = re.match(r'="([^"]*)"', text) if match: return match.group(1) @@ -41,14 +35,21 @@ class ImportJob(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) created_date = models.DateTimeField(default=timezone.now) - task_id = models.CharField(max_length=100, null=True) + updated_date = models.DateTimeField(default=timezone.now) include_reviews = models.BooleanField(default=True) + mappings = models.JSONField() complete = models.BooleanField(default=False) + source = models.CharField(max_length=100) privacy = models.CharField( max_length=255, default="public", choices=PrivacyLevels.choices ) retry = models.BooleanField(default=False) + @property + def pending_items(self): + """items that haven't been processed yet""" + return self.items.filter(fail_reason__isnull=True, book__isnull=True) + class ImportItem(models.Model): """a single line of a csv being imported""" @@ -56,6 +57,7 @@ class ImportItem(models.Model): job = models.ForeignKey(ImportJob, on_delete=models.CASCADE, related_name="items") index = models.IntegerField() data = models.JSONField() + normalized_data = models.JSONField() book = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) book_guess = models.ForeignKey( Book, @@ -65,9 +67,26 @@ class ImportItem(models.Model): related_name="book_guess", ) fail_reason = models.TextField(null=True) + linked_review = models.ForeignKey( + "Review", on_delete=models.SET_NULL, null=True, blank=True + ) + + def update_job(self): + """let the job know when the items get work done""" + job = self.job + job.updated_date = timezone.now() + job.save() + if not job.pending_items.exists() and not job.complete: + job.complete = True + job.save(update_fields=["complete"]) def resolve(self): """try various ways to lookup a book""" + # we might be calling this after manually adding the book, + # so no need to do searches + if self.book: + return + if self.isbn: self.book = self.get_book_from_isbn() else: @@ -85,6 +104,10 @@ class ImportItem(models.Model): self.isbn, min_confidence=0.999 ) if search_result: + # it's already in the right format + if isinstance(search_result, Edition): + return search_result + # it's just a search result, book needs to be created # raises ConnectorException return search_result.connector.get_or_create_book(search_result.key) return None @@ -96,6 +119,8 @@ class ImportItem(models.Model): search_term, min_confidence=0.1 ) if search_result: + if isinstance(search_result, Edition): + return (search_result, 1) # raises ConnectorException return ( search_result.connector.get_or_create_book(search_result.key), @@ -106,56 +131,62 @@ class ImportItem(models.Model): @property def title(self): """get the book title""" - return self.data["Title"] + return self.normalized_data.get("title") @property def author(self): - """get the book title""" - return self.data["Author"] + """get the book's authors""" + return self.normalized_data.get("authors") @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.data["ISBN13"]) + return unquote_string(self.normalized_data.get("isbn_13")) or unquote_string( + self.normalized_data.get("isbn_10") + ) @property def shelf(self): """the goodreads shelf field""" - if self.data["Exclusive Shelf"]: - return GOODREADS_SHELVES.get(self.data["Exclusive Shelf"]) - return None + return self.normalized_data.get("shelf") @property def review(self): """a user-written review, to be imported with the book data""" - return self.data["My Review"] + return self.normalized_data.get("review_body") @property def rating(self): """x/5 star rating for a book""" - if self.data.get("My Rating", None): - return int(self.data["My Rating"]) + if self.normalized_data.get("rating"): + return float(self.normalized_data.get("rating")) return None @property def date_added(self): """when the book was added to this dataset""" - if self.data["Date Added"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Added"])) + if self.normalized_data.get("date_added"): + return timezone.make_aware( + dateutil.parser.parse(self.normalized_data.get("date_added")) + ) return None @property def date_started(self): """when the book was started""" - if "Date Started" in self.data and self.data["Date Started"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Started"])) + if self.normalized_data.get("date_started"): + return timezone.make_aware( + dateutil.parser.parse(self.normalized_data.get("date_started")) + ) return None @property def date_read(self): """the date a book was completed""" - if self.data["Date Read"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Read"])) + if self.normalized_data.get("date_finished"): + return timezone.make_aware( + dateutil.parser.parse(self.normalized_data.get("date_finished")) + ) return None @property @@ -174,7 +205,9 @@ class ImportItem(models.Model): if start_date and start_date is not None and not self.date_read: return [ReadThrough(start_date=start_date)] if self.date_read: - start_date = start_date if start_date < self.date_read else None + start_date = ( + start_date if start_date and start_date < self.date_read else None + ) return [ ReadThrough( start_date=start_date, @@ -185,8 +218,10 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r}Item {!r}>".format(self.data["import_source"], self.data["Title"]) + return "<{!r} Item {!r}>".format(self.index, self.normalized_data.get("title")) def __str__(self): # pylint: disable=consider-using-f-string - return "{} by {}".format(self.data["Title"], self.data["Author"]) + return "{} by {}".format( + self.normalized_data.get("title"), self.normalized_data.get("authors") + ) diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 2f1aae4f3..417bf7591 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -157,9 +157,12 @@ def notify_user_on_unboost(sender, instance, *args, **kwargs): @receiver(models.signals.post_save, sender=ImportJob) # pylint: disable=unused-argument -def notify_user_on_import_complete(sender, instance, *args, **kwargs): +def notify_user_on_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): """we imported your books! aren't you proud of us""" - if not instance.complete: + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: return Notification.objects.create( user=instance.user, diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 8338fff88..5d91553e3 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -1,5 +1,6 @@ """ the particulars for this instance of BookWyrm """ import datetime +from urllib.parse import urljoin from django.db import models, IntegrityError from django.dispatch import receiver @@ -7,9 +8,10 @@ from django.utils import timezone from model_utils import FieldTracker from bookwyrm.preview_images import generate_site_preview_image_task -from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES +from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL from .base_model import BookWyrmModel, new_access_code from .user import User +from .fields import get_absolute_url class SiteSettings(models.Model): @@ -66,6 +68,28 @@ class SiteSettings(models.Model): default_settings.save() return default_settings + @property + def logo_url(self): + """helper to build the logo url""" + return self.get_url("logo", "images/logo.png") + + @property + def logo_small_url(self): + """helper to build the logo url""" + return self.get_url("logo_small", "images/logo-small.png") + + @property + def favicon_url(self): + """helper to build the logo url""" + return self.get_url("favicon", "images/favicon.png") + + def get_url(self, field, default_path): + """get a media url or a default static path""" + uploaded = getattr(self, field, None) + if uploaded: + return get_absolute_url(uploaded) + return urljoin(STATIC_FULL_URL, default_path) + class SiteInvite(models.Model): """gives someone access to create an account on the instance""" diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 2b395ec8b..c7c0a4253 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -19,7 +19,6 @@ from bookwyrm.settings import ENABLE_PREVIEW_IMAGES from .activitypub_mixin import ActivitypubMixin, ActivityMixin from .activitypub_mixin import OrderedCollectionPageMixin from .base_model import BookWyrmModel -from .fields import image_serializer from .readthrough import ProgressMode from . import fields @@ -190,15 +189,26 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): if hasattr(activity, "name"): activity.name = self.pure_name activity.type = self.pure_type - activity.attachment = [ - image_serializer(b.cover, b.alt_text) - for b in self.mention_books.all()[:4] - if b.cover - ] - if hasattr(self, "book") and self.book.cover: - activity.attachment.append( - image_serializer(self.book.cover, self.book.alt_text) - ) + book = getattr(self, "book", None) + books = [book] if book else [] + books += list(self.mention_books.all()) + if len(books) == 1 and getattr(books[0], "preview_image", None): + covers = [ + activitypub.Document( + url=fields.get_absolute_url(books[0].preview_image), + name=books[0].alt_text, + ) + ] + else: + covers = [ + activitypub.Document( + url=fields.get_absolute_url(b.cover), + name=b.alt_text, + ) + for b in books + if b and b.cover + ] + activity.attachment = covers return activity def to_activity(self, pure=False): # pylint: disable=arguments-differ diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index d7945843f..4d98f5c57 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -4,11 +4,12 @@ from urllib.parse import urlparse from django.apps import apps from django.contrib.auth.models import AbstractUser, Group -from django.contrib.postgres.fields import CICharField +from django.contrib.postgres.fields import ArrayField, CICharField from django.core.validators import MinValueValidator from django.dispatch import receiver from django.db import models, transaction from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from model_utils import FieldTracker import pytz @@ -27,6 +28,19 @@ from .federated_server import FederatedServer from . import fields, Review +FeedFilterChoices = [ + ("review", _("Reviews")), + ("comment", _("Comments")), + ("quotation", _("Quotations")), + ("everything", _("Everything else")), +] + + +def get_feed_filter_choices(): + """return a list of filter choice keys""" + return [f[0] for f in FeedFilterChoices] + + def site_link(): """helper for generating links to the site""" protocol = "https" if USE_HTTPS else "http" @@ -128,6 +142,13 @@ class User(OrderedCollectionPageMixin, AbstractUser): show_suggested_users = models.BooleanField(default=True) discoverable = fields.BooleanField(default=False) + # feed options + feed_status_types = ArrayField( + models.CharField(max_length=10, blank=False, choices=FeedFilterChoices), + size=8, + default=get_feed_filter_choices, + ) + preferred_timezone = models.CharField( choices=[(str(tz), str(tz)) for tz in pytz.all_timezones], default=str(pytz.utc), diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 8224a2787..a97ae2d5c 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -49,6 +49,28 @@ def get_font(font_name, size=28): return font +def get_wrapped_text(text, font, content_width): + """text wrap length depends on the max width of the content""" + + low = 0 + high = len(text) + + try: + # ideal length is determined via binary search + while low < high: + mid = math.floor(low + high) + wrapped_text = textwrap.fill(text, width=mid) + width = font.getsize_multiline(wrapped_text)[0] + if width < content_width: + low = mid + else: + high = mid - 1 + except AttributeError: + wrapped_text = text + + return wrapped_text + + def generate_texts_layer(texts, content_width): """Adds text for images""" font_text_zero = get_font("bold", size=20) @@ -63,7 +85,8 @@ def generate_texts_layer(texts, content_width): if "text_zero" in texts and texts["text_zero"]: # Text one (Book title) - text_zero = textwrap.fill(texts["text_zero"], width=72) + text_zero = get_wrapped_text(texts["text_zero"], font_text_zero, content_width) + text_layer_draw.multiline_text( (0, text_y), text_zero, font=font_text_zero, fill=TEXT_COLOR ) @@ -75,7 +98,8 @@ def generate_texts_layer(texts, content_width): if "text_one" in texts and texts["text_one"]: # Text one (Book title) - text_one = textwrap.fill(texts["text_one"], width=28) + text_one = get_wrapped_text(texts["text_one"], font_text_one, content_width) + text_layer_draw.multiline_text( (0, text_y), text_one, font=font_text_one, fill=TEXT_COLOR ) @@ -87,7 +111,8 @@ def generate_texts_layer(texts, content_width): if "text_two" in texts and texts["text_two"]: # Text one (Book subtitle) - text_two = textwrap.fill(texts["text_two"], width=36) + text_two = get_wrapped_text(texts["text_two"], font_text_two, content_width) + text_layer_draw.multiline_text( (0, text_y), text_two, font=font_text_two, fill=TEXT_COLOR ) @@ -99,7 +124,10 @@ def generate_texts_layer(texts, content_width): if "text_three" in texts and texts["text_three"]: # Text three (Book authors) - text_three = textwrap.fill(texts["text_three"], width=36) + text_three = get_wrapped_text( + texts["text_three"], font_text_three, content_width + ) + text_layer_draw.multiline_text( (0, text_y), text_three, font=font_text_three, fill=TEXT_COLOR ) @@ -317,15 +345,21 @@ def save_and_cleanup(image, instance=None): """Save and close the file""" if not isinstance(instance, (models.Book, models.User, models.SiteSettings)): return False - uuid = uuid4() - file_name = f"{instance.id}-{uuid}.jpg" image_buffer = BytesIO() try: try: - old_path = instance.preview_image.name + file_name = instance.preview_image.name except ValueError: - old_path = None + file_name = None + + if not file_name or file_name == "": + uuid = uuid4() + file_name = f"{instance.id}-{uuid}.jpg" + + # Clean up old file before saving + if file_name and default_storage.exists(file_name): + default_storage.delete(file_name) # Save image.save(image_buffer, format="jpeg", quality=75) @@ -345,10 +379,6 @@ def save_and_cleanup(image, instance=None): else: instance.save(update_fields=["preview_image"]) - # Clean up old file after saving - if old_path and default_storage.exists(old_path): - default_storage.delete(old_path) - finally: image_buffer.close() return True diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 3f30994ec..ec4fff0a8 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -14,7 +14,7 @@ VERSION = "0.1.0" PAGE_LENGTH = env("PAGE_LENGTH", 15) DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English") -JS_CACHE = "3eb4edb1" +JS_CACHE = "3891b373" # email EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend") @@ -166,7 +166,9 @@ LANGUAGES = [ ("en-us", _("English")), ("de-de", _("Deutsch (German)")), ("es-es", _("Español (Spanish)")), + ("gl-es", _("Galego (Galician)")), ("fr-fr", _("Français (French)")), + ("lt-lt", _("Lietuvių (Lithuanian)")), ("pt-br", _("Português - Brasil (Brazilian Portuguese)")), ("zh-hans", _("简体中文 (Simplified Chinese)")), ("zh-hant", _("繁體中文 (Traditional Chinese)")), diff --git a/bookwyrm/static/css/bookwyrm.css b/bookwyrm/static/css/bookwyrm.css index 0d280fd53..0e812e2a3 100644 --- a/bookwyrm/static/css/bookwyrm.css +++ b/bookwyrm/static/css/bookwyrm.css @@ -115,6 +115,34 @@ input[type=file]::file-selector-button:hover { color: #363636; } +details .dropdown-menu { + display: block !important; +} + +details.dropdown[open] summary.dropdown-trigger::before { + content: ""; + position: fixed; + top: 0; + bottom: 0; + left: 0; + right: 0; +} + +summary::marker { + content: none; +} + +.detail-pinned-button summary { + position: absolute; + right: 0; +} + +.detail-pinned-button form { + float: left; + width: -webkit-fill-available; + margin-top: 1em; +} + /** Shelving ******************************************************************************/ diff --git a/bookwyrm/static/js/bookwyrm.js b/bookwyrm/static/js/bookwyrm.js index 2d5b88adc..d656ed183 100644 --- a/bookwyrm/static/js/bookwyrm.js +++ b/bookwyrm/static/js/bookwyrm.js @@ -45,6 +45,13 @@ let BookWyrm = new class { 'change', this.disableIfTooLarge.bind(this) )); + + document.querySelectorAll('[data-duplicate]') + .forEach(node => node.addEventListener( + 'click', + this.duplicateInput.bind(this) + + )) } /** @@ -113,9 +120,44 @@ let BookWyrm = new class { * @return {undefined} */ updateCountElement(counter, data) { + let count = data.count; + const count_by_type = data.count_by_type; const currentCount = counter.innerText; - const count = data.count; const hasMentions = data.has_mentions; + const allowedStatusTypesEl = document.getElementById('unread-notifications-wrapper'); + + // If we're on the right counter element + if (counter.closest('[data-poll-wrapper]').contains(allowedStatusTypesEl)) { + const allowedStatusTypes = JSON.parse(allowedStatusTypesEl.textContent); + + // For keys in common between allowedStatusTypes and count_by_type + // This concerns 'review', 'quotation', 'comment' + count = allowedStatusTypes.reduce(function(prev, currentKey) { + const currentValue = count_by_type[currentKey] | 0; + + return prev + currentValue; + }, 0); + + // Add all the "other" in count_by_type if 'everything' is allowed + if (allowedStatusTypes.includes('everything')) { + // Clone count_by_type with 0 for reviews/quotations/comments + const count_by_everything_else = Object.assign( + {}, + count_by_type, + {review: 0, quotation: 0, comment: 0} + ); + + count = Object.keys(count_by_everything_else).reduce( + function(prev, currentKey) { + const currentValue = + count_by_everything_else[currentKey] | 0 + + return prev + currentValue; + }, + count + ); + } + } if (count != currentCount) { this.addRemoveClass(counter.closest('[data-poll-wrapper]'), 'is-hidden', count < 1); @@ -368,4 +410,24 @@ let BookWyrm = new class { ); } } + + duplicateInput (event ) { + const trigger = event.currentTarget; + const input_id = trigger.dataset['duplicate'] + const orig = document.getElementById(input_id); + const parent = orig.parentNode; + const new_count = parent.querySelectorAll("input").length + 1 + + let input = orig.cloneNode(); + + input.id += ("-" + (new_count)) + input.value = "" + + let label = parent.querySelector("label").cloneNode(); + + label.setAttribute("for", input.id) + + parent.appendChild(label) + parent.appendChild(input) + } }(); diff --git a/bookwyrm/static/js/status_cache.js b/bookwyrm/static/js/status_cache.js index 2a50bfcbe..dbc238c48 100644 --- a/bookwyrm/static/js/status_cache.js +++ b/bookwyrm/static/js/status_cache.js @@ -187,6 +187,7 @@ let StatusCache = new class { .forEach(item => BookWyrm.addRemoveClass(item, "is-hidden", false)); // Remove existing disabled states + button.querySelectorAll("[data-shelf-dropdown-identifier] button") .forEach(item => item.disabled = false); @@ -209,10 +210,10 @@ let StatusCache = new class { .forEach(item => BookWyrm.addRemoveClass(item, "is-hidden", true)); // Close menu - let menu = button.querySelector(".dropdown-trigger[aria-expanded=true]"); + let menu = button.querySelector("details[open]"); if (menu) { - menu.click(); + menu.removeAttribute("open"); } } diff --git a/bookwyrm/tasks.py b/bookwyrm/tasks.py index b860e0184..09e1d267e 100644 --- a/bookwyrm/tasks.py +++ b/bookwyrm/tasks.py @@ -9,3 +9,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "celerywyrm.settings") app = Celery( "tasks", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND ) + +# priorities +LOW = "low_priority" +MEDIUM = "medium_priority" +HIGH = "high_priority" diff --git a/bookwyrm/templates/author/author.html b/bookwyrm/templates/author/author.html index 6a67b50b3..b066c6ca4 100644 --- a/bookwyrm/templates/author/author.html +++ b/bookwyrm/templates/author/author.html @@ -2,6 +2,7 @@ {% load i18n %} {% load markdown %} {% load humanize %} +{% load utilities %} {% block title %}{{ author.name }}{% endblock %} @@ -25,7 +26,7 @@
- {% if author.aliases or author.born or author.died or author.wikipedia_link or author.openlibrary_key or author.inventaire_id %} + {% if author.aliases or author.born or author.died or author.wikipedia_link or author.openlibrary_key or author.inventaire_id or author.isni %}
@@ -63,6 +64,14 @@

{% endif %} + {% if author.isni %} +

+ + {% trans "View ISNI record" %} + +

+ {% endif %} + {% if author.openlibrary_key %}

diff --git a/bookwyrm/templates/author/edit_author.html b/bookwyrm/templates/author/edit_author.html index 54d7f4f1c..6f72b8700 100644 --- a/bookwyrm/templates/author/edit_author.html +++ b/bookwyrm/templates/author/edit_author.html @@ -34,47 +34,41 @@

{{ form.name }} - {% for error in form.name.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.name.errors id="desc_name" %}
{{ form.aliases }} {% trans "Separate multiple values with commas." %} - {% for error in form.aliases.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.aliases.errors id="desc_aliases" %}
{{ form.bio }} - {% for error in form.bio.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.bio.errors id="desc_bio" %}

{{ form.wikipedia_link }}

- {% for error in form.wikipedia_link.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.wikipedia_link.errors id="desc_wikipedia_link" %}
- {% for error in form.born.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.born.errors id="desc_born" %}
- {% for error in form.died.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.died.errors id="desc_died" %}
@@ -82,33 +76,36 @@
{{ form.openlibrary_key }} - {% for error in form.openlibrary_key.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.oepnlibrary_key.errors id="desc_oepnlibrary_key" %}
{{ form.inventaire_id }} - {% for error in form.inventaire_id.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.inventaire_id.errors id="desc_inventaire_id" %}
{{ form.librarything_key }} - {% for error in form.librarything_key.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.librarything_key.errors id="desc_librarything_key" %}
{{ form.goodreads_key }} - {% for error in form.goodreads_key.errors %} -

{{ error | escape }}

- {% endfor %} + + {% include 'snippets/form_errors.html' with errors_list=form.goodreads_key.errors id="desc_goodreads_key" %} +
+ +
+ + {{ form.isni }} + + {% include 'snippets/form_errors.html' with errors_list=form.isni.errors id="desc_isni" %}
diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index 36241ee26..7d4df76f7 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -153,12 +153,21 @@ {# user's relationship to the book #}
+ {% if user_shelfbooks.count > 0 %} +

+ {% trans "You have shelved this edition in:" %} +

+ + {% endif %} {% for shelf in other_edition_shelves %}

{% blocktrans with book_path=shelf.book.local_path shelf_path=shelf.shelf.local_path shelf_name=shelf.shelf.name %}A different edition of this book is on your {{ shelf_name }} shelf.{% endblocktrans %} diff --git a/bookwyrm/templates/book/edit/edit_book.html b/bookwyrm/templates/book/edit/edit_book.html index fc11208fd..3d41058e3 100644 --- a/bookwyrm/templates/book/edit/edit_book.html +++ b/bookwyrm/templates/book/edit/edit_book.html @@ -1,6 +1,7 @@ {% extends 'layout.html' %} {% load i18n %} {% load humanize %} +{% load utilities %} {% block title %}{% if book %}{% blocktrans with book_title=book.title %}Edit "{{ book_title }}"{% endblocktrans %}{% else %}{% trans "Add Book" %}{% endif %}{% endblock %} @@ -52,19 +53,29 @@ {% for author in author_matches %}

- {% blocktrans with name=author.name %}Is "{{ name }}" an existing author?{% endblocktrans %} + {% blocktrans with name=author.name %}Is "{{ name }}" one of these authors?{% endblocktrans %} {% with forloop.counter0 as counter %} {% for match in author.matches %} -