Attachments

api/funkwhale_api/music/factories.py

@@ -4,6 +4,7 @@ import factory
 
 from funkwhale_api.factories import registry, NoUpdateOnCreate
 
+from funkwhale_api.common import factories as common_factories
 from funkwhale_api.federation import factories as federation_factories
 from funkwhale_api.music import licenses
 from funkwhale_api.tags import factories as tags_factories
@@ -81,7 +82,7 @@ class AlbumFactory(
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
     release_date = factory.Faker("date_object")
-    cover = factory.django.ImageField()
+    attachment_cover = factory.SubFactory(common_factories.AttachmentFactory)
     artist = factory.SubFactory(ArtistFactory)
     release_group_id = factory.Faker("uuid4")
     fid = factory.Faker("federation_url")

api/funkwhale_api/music/migrations/0042_album_attachment_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.6 on 2019-11-12 09:56
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0004_auto_20191111_1338'),
+        ('music', '0041_auto_20191021_1705'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='attachment_cover',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='common.Attachment', blank=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0043_album_cover_attachment.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def create_attachments(apps, schema_editor):
+    Album = apps.get_model("music", "Album")
+    Attachment = apps.get_model("common", "Attachment")
+
+    album_attachment_mapping = {}
+    def get_mimetype(path):
+        if path.lower().endswith('.png'):
+            return "image/png"
+        return "image/jpeg"
+
+    for album in Album.objects.filter(attachment_cover=None).exclude(cover="").exclude(cover=None):
+        album_attachment_mapping[album] = Attachment(
+            file=album.cover,
+            size=album.cover.size,
+            mimetype=get_mimetype(album.cover.path),
+        )
+
+    Attachment.objects.bulk_create(album_attachment_mapping.values(), batch_size=2000)
+    # map each attachment to the corresponding album
+    # and bulk save
+    for album, attachment in album_attachment_mapping.items():
+        album.attachment_cover = attachment
+
+    Album.objects.bulk_update(album_attachment_mapping.keys(), fields=['attachment_cover'], batch_size=2000)
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("music", "0042_album_attachment_cover")]
+
+    operations = [migrations.RunPython(create_attachments, rewind)]

api/funkwhale_api/music/models.py

@@ -20,7 +20,6 @@ from django.urls import reverse
 from django.utils import timezone
 
 from versatileimagefield.fields import VersatileImageField
-from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
 from funkwhale_api import musicbrainz
 from funkwhale_api.common import fields
@@ -286,9 +285,17 @@ class Album(APIModelMixin):
     artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
     release_date = models.DateField(null=True, blank=True, db_index=True)
     release_group_id = models.UUIDField(null=True, blank=True)
+    # XXX: 1.0 clean this uneeded field in favor of attachment_cover
     cover = VersatileImageField(
         upload_to="albums/covers/%Y/%m/%d", null=True, blank=True
     )
+    attachment_cover = models.ForeignKey(
+        "common.Attachment",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="covered_album",
+    )
     TYPE_CHOICES = (("album", "Album"),)
     type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")
 
@@ -334,40 +341,46 @@ class Album(APIModelMixin):
     objects = AlbumQuerySet.as_manager()
 
     def get_image(self, data=None):
+        from funkwhale_api.common import tasks as common_tasks
+
+        attachment = None
         if data:
             extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
             extension = extensions.get(data["mimetype"], "jpg")
+            attachment = common_models.Attachment(mimetype=data["mimetype"])
             f = None
+            filename = "{}.{}".format(self.uuid, extension)
             if data.get("content"):
                 # we have to cover itself
                 f = ContentFile(data["content"])
+                attachment.file.save(filename, f, save=False)
             elif data.get("url"):
+                attachment.url = data.get("url")
                 # we can fetch from a url
                 try:
-                    response = session.get_session().get(
-                        data.get("url"),
-                        timeout=3,
-                        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+                    common_tasks.fetch_remote_attachment(
+                        attachment, filename=filename, save=False
                     )
-                    response.raise_for_status()
                 except Exception as e:
                     logger.warn(
                         "Cannot download cover at url %s: %s", data.get("url"), e
                     )
                     return
-                else:
-                    f = ContentFile(response.content)
-            if f:
-                self.cover.save("{}.{}".format(self.uuid, extension), f, save=False)
-                self.save(update_fields=["cover"])
-                return self.cover.file
-        if self.mbid:
+
+        elif self.mbid:
             image_data = musicbrainz.api.images.get_front(str(self.mbid))
             f = ContentFile(image_data)
-            self.cover.save("{0}.jpg".format(self.mbid), f, save=False)
-            self.save(update_fields=["cover"])
-        if self.cover:
-            return self.cover.file
+            attachment = common_models.Attachment(mimetype="image/jpeg")
+            attachment.file.save("{0}.jpg".format(self.mbid), f, save=False)
+        if attachment and attachment.file:
+            attachment.save()
+            self.attachment_cover = attachment
+            self.save(update_fields=["attachment_cover"])
+            return self.attachment_cover.file
+
+    @property
+    def cover(self):
+        return self.attachment_cover
 
     def __str__(self):
         return self.title
@@ -378,16 +391,6 @@ class Album(APIModelMixin):
     def get_moderation_url(self):
         return "/manage/library/albums/{}".format(self.pk)
 
-    @property
-    def cover_path(self):
-        if not self.cover:
-            return None
-        try:
-            return self.cover.path
-        except NotImplementedError:
-            # external storage
-            return self.cover.name
-
     @classmethod
     def get_or_create_from_title(cls, title, **kwargs):
         kwargs.update({"title": title})
@@ -415,7 +418,9 @@ def import_album(v):
 
 class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
     def for_nested_serialization(self):
-        return self.prefetch_related("artist", "album__artist")
+        return self.prefetch_related(
+            "artist", "album__artist", "album__attachment_cover"
+        )
 
     def annotate_playable_by_actor(self, actor):
 
@@ -729,7 +734,6 @@ class Upload(models.Model):
         return parsed.hostname
 
     def download_audio_from_remote(self, actor):
-        from funkwhale_api.common import session
         from funkwhale_api.federation import signing
 
         if actor:
@@ -743,7 +747,6 @@ class Upload(models.Model):
             stream=True,
             timeout=20,
             headers={"Content-Type": "application/octet-stream"},
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
         )
         with remote_response as r:
             remote_response.raise_for_status()
@@ -1307,13 +1310,3 @@ def update_request_status(sender, instance, created, **kwargs):
         # let's mark the request as imported since the import is over
         instance.import_request.status = "imported"
         return instance.import_request.save(update_fields=["status"])
-
-
-@receiver(models.signals.post_save, sender=Album)
-def warm_album_covers(sender, instance, **kwargs):
-    if not instance.cover or not settings.CREATE_IMAGE_THUMBNAILS:
-        return
-    album_covers_warmer = VersatileImageFieldWarmer(
-        instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"
-    )
-    num_created, failed_to_create = album_covers_warmer.warm()

api/funkwhale_api/music/serializers.py

@@ -4,7 +4,6 @@ from django.db import transaction
 from django import urls
 from django.conf import settings
 from rest_framework import serializers
-from versatileimagefield.serializers import VersatileImageFieldSerializer
 
 from funkwhale_api.activity import serializers as activity_serializers
 from funkwhale_api.common import serializers as common_serializers
@@ -17,7 +16,25 @@ from funkwhale_api.tags.models import Tag
 from . import filters, models, tasks
 
 
-cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
+class NullToEmptDict(object):
+    def get_attribute(self, o):
+        attr = super().get_attribute(o)
+        if attr is None:
+            return {}
+        return attr
+
+    def to_representation(self, v):
+        if not v:
+            return v
+        return super().to_representation(v)
+
+
+class CoverField(NullToEmptDict, common_serializers.AttachmentSerializer):
+    # XXX: BACKWARD COMPATIBILITY
+    pass
+
+
+cover_field = CoverField()
 
 
 def serialize_attributed_to(self, obj):
@@ -450,12 +467,12 @@ class OembedSerializer(serializers.Serializer):
             embed_type = "track"
             embed_id = track.pk
             data["title"] = "{} by {}".format(track.title, track.artist.name)
-            if track.album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    track.album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if track.album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = track.album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["description"] = track.full_name
             data["author_name"] = track.artist.name
             data["height"] = 150
@@ -476,12 +493,12 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "album"
             embed_id = album.pk
-            if album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = "{} by {}".format(album.title, album.artist.name)
             data["description"] = "{} by {}".format(album.title, album.artist.name)
             data["author_name"] = album.artist.name
@@ -501,19 +518,14 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "artist"
             embed_id = artist.pk
-            album = (
-                artist.albums.filter(cover__isnull=False)
-                .exclude(cover="")
-                .order_by("-id")
-                .first()
-            )
+            album = artist.albums.exclude(attachment_cover=None).order_by("-id").first()
 
-            if album and album.cover:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    album.cover.crop["400x400"].url
-                )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+            if album and album.attachment_cover:
+                data[
+                    "thumbnail_url"
+                ] = album.attachment_cover.download_url_medium_square_crop
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = artist.name
             data["description"] = artist.name
             data["author_name"] = artist.name
@@ -533,19 +545,22 @@ class OembedSerializer(serializers.Serializer):
                 )
             embed_type = "playlist"
             embed_id = obj.pk
-            playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
-            playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
-            playlist_tracks = playlist_tracks.select_related("track__album").order_by(
-                "index"
+            playlist_tracks = obj.playlist_tracks.exclude(
+                track__album__attachment_cover=None
             )
+            playlist_tracks = playlist_tracks.select_related(
+                "track__album__attachment_cover"
+            ).order_by("index")
             first_playlist_track = playlist_tracks.first()
 
             if first_playlist_track:
-                data["thumbnail_url"] = federation_utils.full_url(
-                    first_playlist_track.track.album.cover.crop["400x400"].url
+                data[
+                    "thumbnail_url"
+                ] = (
+                    first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop
                 )
-                data["thumbnail_width"] = 400
-                data["thumbnail_height"] = 400
+                data["thumbnail_width"] = 200
+                data["thumbnail_height"] = 200
             data["title"] = obj.name
             data["description"] = obj.name
             data["author_name"] = obj.name

api/funkwhale_api/music/spa_views.py

@@ -57,14 +57,12 @@ def library_track(request, pk):
             ),
         },
     ]
-    if obj.album.cover:
+    if obj.album.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, obj.album.cover.crop["400x400"].url
-                ),
+                "content": obj.album.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -126,14 +124,12 @@ def library_album(request, pk):
             }
         )
 
-    if obj.cover:
+    if obj.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, obj.cover.crop["400x400"].url
-                ),
+                "content": obj.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -166,7 +162,7 @@ def library_artist(request, pk):
     )
     # we use latest album's cover as artist image
     latest_album = (
-        obj.albums.exclude(cover="").exclude(cover=None).order_by("release_date").last()
+        obj.albums.exclude(attachment_cover=None).order_by("release_date").last()
     )
     metas = [
         {"tag": "meta", "property": "og:url", "content": artist_url},
@@ -174,14 +170,12 @@ def library_artist(request, pk):
         {"tag": "meta", "property": "og:type", "content": "profile"},
     ]
 
-    if latest_album and latest_album.cover:
+    if latest_album and latest_album.attachment_cover:
         metas.append(
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL, latest_album.cover.crop["400x400"].url
-                ),
+                "content": latest_album.attachment_cover.download_url_medium_square_crop,
             }
         )
 
@@ -217,8 +211,7 @@ def library_playlist(request, pk):
         utils.spa_reverse("library_playlist", kwargs={"pk": obj.pk}),
     )
     # we use the first playlist track's album's cover as image
-    playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
-    playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
+    playlist_tracks = obj.playlist_tracks.exclude(track__album__attachment_cover=None)
     playlist_tracks = playlist_tracks.select_related("track__album").order_by("index")
     first_playlist_track = playlist_tracks.first()
     metas = [
@@ -232,10 +225,7 @@ def library_playlist(request, pk):
             {
                 "tag": "meta",
                 "property": "og:image",
-                "content": utils.join_url(
-                    settings.FUNKWHALE_URL,
-                    first_playlist_track.track.album.cover.crop["400x400"].url,
-                ),
+                "content": first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop,
             }
         )
 

api/funkwhale_api/music/tasks.py

@@ -21,7 +21,6 @@ from . import licenses
 from . import models
 from . import metadata
 from . import signals
-from . import serializers
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +28,7 @@ logger = logging.getLogger(__name__)
 def update_album_cover(
     album, source=None, cover_data=None, musicbrainz=True, replace=False
 ):
-    if album.cover and not replace:
+    if album.attachment_cover and not replace:
         return
     if cover_data:
         return album.get_image(data=cover_data)
@@ -257,7 +256,7 @@ def process_upload(upload, update_denormalization=True):
         )
 
     # update album cover, if needed
-    if not track.album.cover:
+    if not track.album.attachment_cover:
         update_album_cover(
             track.album,
             source=final_metadata.get("upload_source"),
@@ -404,7 +403,7 @@ def sort_candidates(candidates, important_fields):
 @transaction.atomic
 def get_track_from_import_metadata(data, update_cover=False, attributed_to=None):
     track = _get_track(data, attributed_to=attributed_to)
-    if update_cover and track and not track.album.cover:
+    if update_cover and track and not track.album.attachment_cover:
         update_album_cover(
             track.album,
             source=data.get("upload_source"),
@@ -584,6 +583,8 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
     if not user:
         return
 
+    from . import serializers
+
     group = "user.{}.imports".format(user.pk)
     channels.group_send(
         group,

api/funkwhale_api/music/views.py

@@ -128,7 +128,9 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        albums = models.Album.objects.with_tracks_count()
+        albums = models.Album.objects.with_tracks_count().select_related(
+            "attachment_cover"
+        )
         albums = albums.annotate_playable_by_actor(
             utils.get_actor_from_request(self.request)
         )
@@ -149,7 +151,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
     queryset = (
         models.Album.objects.all()
         .order_by("-creation_date")
-        .prefetch_related("artist", "attributed_to")
+        .prefetch_related("artist", "attributed_to", "attachment_cover")
     )
     serializer_class = serializers.AlbumSerializer
     permission_classes = [oauth_permissions.ScopePermission]