Resolve "Per-user libraries" (use !368 instead)

api/funkwhale_api/common/authentication.py

@@ -56,3 +56,20 @@ class BearerTokenHeaderAuth(authentication.BaseJSONWebTokenAuthentication):
 
     def authenticate_header(self, request):
         return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
+
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth
+
+
+class JSONWebTokenAuthentication(authentication.JSONWebTokenAuthentication):
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth

api/funkwhale_api/common/channels.py

@@ -1,6 +1,25 @@
+import json
+import logging
+
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.core.serializers.json import DjangoJSONEncoder
 
+logger = logging.getLogger(__file__)
 channel_layer = get_channel_layer()
-group_send = async_to_sync(channel_layer.group_send)
 group_add = async_to_sync(channel_layer.group_add)
+
+
+def group_send(group, event):
+    # we serialize the payload ourselves and deserialize it to ensure it
+    # works with msgpack. This is dirty, but we'll find a better solution
+    # later
+    s = json.dumps(event, cls=DjangoJSONEncoder)
+    event = json.loads(s)
+    logger.debug(
+        "[channels] Dispatching %s to group %s: %s",
+        event["type"],
+        group,
+        {"type": event["data"]["type"]},
+    )
+    async_to_sync(channel_layer.group_send)(group, event)

api/funkwhale_api/common/consumers.py

@@ -16,3 +16,5 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
         super().accept()
         for group in self.groups:
             channels.group_add(group, self.channel_name)
+        for group in self.scope["user"].get_channels_groups():
+            channels.group_add(group, self.channel_name)

api/funkwhale_api/common/scripts/__init__.py

@@ -1,6 +1,7 @@
 from . import create_actors
 from . import create_image_variations
 from . import django_permissions_to_user_permissions
+from . import migrate_to_user_libraries
 from . import test
 
 
@@ -8,5 +9,6 @@ __all__ = [
     "create_actors",
     "create_image_variations",
     "django_permissions_to_user_permissions",
+    "migrate_to_user_libraries",
     "test",
 ]

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

@@ -0,0 +1,58 @@
+"""
+Mirate instance files to a library #463. For each user that imported music on an
+instance, we will create a "default" library with related files and an instance-level
+visibility.
+
+Files without any import job will be bounded to a "default" library on the first
+superuser account found. This should now happen though.
+"""
+
+from funkwhale_api.music import models
+from funkwhale_api.users.models import User
+
+
+def main(command, **kwargs):
+    importer_ids = set(
+        models.ImportBatch.objects.values_list("submitted_by", flat=True)
+    )
+    importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
+    command.stdout.write(
+        "* {} users imported music on this instance".format(len(importers))
+    )
+    files = models.TrackFile.objects.filter(
+        library__isnull=True, jobs__isnull=False
+    ).distinct()
+    command.stdout.write(
+        "* Reassigning {} files to importers libraries...".format(files.count())
+    )
+    for user in importers:
+        command.stdout.write(
+            "  * Setting up @{}'s 'default' library".format(user.username)
+        )
+        library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
+            0
+        ]
+        user_files = files.filter(jobs__batch__submitted_by=user)
+        total = user_files.count()
+        command.stdout.write(
+            "    * Reassigning {} files to the user library...".format(total)
+        )
+        user_files.update(library=library)
+
+    files = models.TrackFile.objects.filter(
+        library__isnull=True, jobs__isnull=True
+    ).distinct()
+    command.stdout.write(
+        "* Handling {} files with no import jobs...".format(files.count())
+    )
+
+    user = User.objects.order_by("id").filter(is_superuser=True).first()
+
+    command.stdout.write("  * Setting up @{}'s 'default' library".format(user.username))
+    library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
+    total = files.count()
+    command.stdout.write(
+        "    * Reassigning {} files to the user library...".format(total)
+    )
+    files.update(library=library)
+    command.stdout.write(" * Done!")

api/funkwhale_api/common/serializers.py

@@ -1,5 +1,70 @@
+import collections
+
 from rest_framework import serializers
 
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.encoding import smart_text
+from django.utils.translation import ugettext_lazy as _
+
+
+class RelatedField(serializers.RelatedField):
+    default_error_messages = {
+        "does_not_exist": _("Object with {related_field_name}={value} does not exist."),
+        "invalid": _("Invalid value."),
+    }
+
+    def __init__(self, related_field_name, serializer, **kwargs):
+        self.related_field_name = related_field_name
+        self.serializer = serializer
+        self.filters = kwargs.pop("filters", None)
+        kwargs["queryset"] = kwargs.pop(
+            "queryset", self.serializer.Meta.model.objects.all()
+        )
+        super().__init__(**kwargs)
+
+    def get_filters(self, data):
+        filters = {self.related_field_name: data}
+        if self.filters:
+            filters.update(self.filters(self.context))
+        return filters
+
+    def to_internal_value(self, data):
+        try:
+            queryset = self.get_queryset()
+            filters = self.get_filters(data)
+            return queryset.get(**filters)
+        except ObjectDoesNotExist:
+            self.fail(
+                "does_not_exist",
+                related_field_name=self.related_field_name,
+                value=smart_text(data),
+            )
+        except (TypeError, ValueError):
+            self.fail("invalid")
+
+    def to_representation(self, obj):
+        return self.serializer.to_representation(obj)
+
+    def get_choices(self, cutoff=None):
+        queryset = self.get_queryset()
+        if queryset is None:
+            # Ensure that field.choices returns something sensible
+            # even when accessed with a read-only field.
+            return {}
+
+        if cutoff is not None:
+            queryset = queryset[:cutoff]
+
+        return collections.OrderedDict(
+            [
+                (
+                    self.to_representation(item)[self.related_field_name],
+                    self.display_value(item),
+                )
+                for item in queryset
+            ]
+        )
+
 
 class Action(object):
     def __init__(self, name, allow_all=False, qs_filter=None):
@@ -21,6 +86,7 @@ class ActionSerializer(serializers.Serializer):
     objects = serializers.JSONField(required=True)
     filters = serializers.DictField(required=False)
     actions = None
+    pk_field = "pk"
 
     def __init__(self, *args, **kwargs):
         self.actions_by_name = {a.name: a for a in self.actions}
@@ -51,7 +117,9 @@ class ActionSerializer(serializers.Serializer):
         if value == "all":
             return self.queryset.all().order_by("id")
         if type(value) in [list, tuple]:
-            return self.queryset.filter(pk__in=value).order_by("id")
+            return self.queryset.filter(
+                **{"{}__in".format(self.pk_field): value}
+            ).order_by("id")
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "

api/funkwhale_api/favorites/views.py

@@ -3,9 +3,12 @@ from rest_framework.decorators import list_route
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
+from django.db.models import Prefetch
+
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.music.models import Track
+from funkwhale_api.music import utils as music_utils
 
 from . import filters, models, serializers
 
@@ -19,11 +22,7 @@ class TrackFavoriteViewSet(
 
     filter_class = filters.TrackFavoriteFilter
     serializer_class = serializers.UserTrackFavoriteSerializer
-    queryset = (
-        models.TrackFavorite.objects.all()
-        .select_related("track__artist", "track__album__artist", "user")
-        .prefetch_related("track__files")
-    )
+    queryset = models.TrackFavorite.objects.all().select_related("user")
     permission_classes = [
         permissions.ConditionalAuthentication,
         permissions.OwnerPermission,
@@ -49,9 +48,14 @@ class TrackFavoriteViewSet(
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        return queryset.filter(
+        queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
+        tracks = Track.objects.annotate_playable_by_actor(
+            music_utils.get_actor_from_request(self.request)
+        ).select_related("artist", "album__artist")
+        queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
+        return queryset
 
     def perform_create(self, serializer):
         track = Track.objects.get(pk=serializer.data["track"])

api/funkwhale_api/federation/activity.py

@@ -1,3 +1,9 @@
+import uuid
+
+from funkwhale_api.common import utils as funkwhale_utils
+
+PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
+
 ACTIVITY_TYPES = [
     "Accept",
     "Add",
@@ -58,4 +64,145 @@ def accept_follow(follow):
     from . import serializers
 
     serializer = serializers.AcceptFollowSerializer(follow)
-    return deliver(serializer.data, to=[follow.actor.url], on_behalf_of=follow.target)
+    return deliver(serializer.data, to=[follow.actor.fid], on_behalf_of=follow.target)
+
+
+def receive(activity, on_behalf_of):
+    from . import models
+    from . import serializers
+    from . import tasks
+
+    # we ensure the activity has the bare minimum structure before storing
+    # it in our database
+    serializer = serializers.BaseActivitySerializer(
+        data=activity, context={"actor": on_behalf_of, "local_recipients": True}
+    )
+    serializer.is_valid(raise_exception=True)
+    copy = serializer.save()
+    # we create inbox items for further delivery
+    items = [
+        models.InboxItem(activity=copy, actor=r, type="to")
+        for r in serializer.validated_data["recipients"]["to"]
+        if hasattr(r, "fid")
+    ]
+    items += [
+        models.InboxItem(activity=copy, actor=r, type="cc")
+        for r in serializer.validated_data["recipients"]["cc"]
+        if hasattr(r, "fid")
+    ]
+    models.InboxItem.objects.bulk_create(items)
+    # at this point, we have the activity in database. Even if we crash, it's
+    # okay, as we can retry later
+    tasks.dispatch_inbox.delay(activity_id=copy.pk)
+    return copy
+
+
+class Router:
+    def __init__(self):
+        self.routes = []
+
+    def connect(self, route, handler):
+        self.routes.append((route, handler))
+
+    def register(self, route):
+        def decorator(handler):
+            self.connect(route, handler)
+            return handler
+
+        return decorator
+
+
+class InboxRouter(Router):
+    def dispatch(self, payload, context):
+        """
+        Receives an Activity payload and some context and trigger our
+        business logic
+        """
+        for route, handler in self.routes:
+            if match_route(route, payload):
+                return handler(payload, context=context)
+
+
+class OutboxRouter(Router):
+    def dispatch(self, routing, context):
+        """
+        Receives a routing payload and some business objects in the context
+        and may yield data that should be persisted in the Activity model
+        for further delivery.
+        """
+        from . import models
+        from . import tasks
+
+        for route, handler in self.routes:
+            if match_route(route, routing):
+                activities_data = []
+                for e in handler(context):
+                    # a route can yield zero, one or more activity payloads
+                    if e:
+                        activities_data.append(e)
+
+                inbox_items_by_activity_uuid = {}
+                prepared_activities = []
+                for activity_data in activities_data:
+                    to = activity_data.pop("to", [])
+                    cc = activity_data.pop("cc", [])
+                    a = models.Activity(**activity_data)
+                    a.uuid = uuid.uuid4()
+                    to_items, new_to = prepare_inbox_items(to, "to")
+                    cc_items, new_cc = prepare_inbox_items(cc, "cc")
+                    if not to_items and not cc_items:
+                        continue
+                    inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
+                    if new_to:
+                        a.payload["to"] = new_to
+                    if new_cc:
+                        a.payload["cc"] = new_cc
+                    prepared_activities.append(a)
+
+                activities = models.Activity.objects.bulk_create(prepared_activities)
+                activities = [a for a in activities if a]
+
+                final_inbox_items = []
+                for a in activities:
+                    try:
+                        prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
+                    except KeyError:
+                        continue
+
+                    for ii in prepared_inbox_items:
+                        ii.activity = a
+                        final_inbox_items.append(ii)
+
+                # create all inbox items, in bulk
+                models.InboxItem.objects.bulk_create(final_inbox_items)
+
+                for a in activities:
+                    funkwhale_utils.on_commit(
+                        tasks.dispatch_outbox.delay, activity_id=a.pk
+                    )
+                return activities
+
+
+def match_route(route, payload):
+    for key, value in route.items():
+        if payload.get(key) != value:
+            return False
+
+    return True
+
+
+def prepare_inbox_items(recipient_list, type):
+    from . import models
+
+    items = []
+    new_list = []  # we return a list of actors url instead
+
+    for r in recipient_list:
+        if r != PUBLIC_ADDRESS:
+            item = models.InboxItem(actor=r, type=type)
+            items.append(item)
+            new_list.append(r.fid)
+        else:
+            new_list.append(r)
+
+    return items, new_list

api/funkwhale_api/federation/actors.py

@@ -3,15 +3,11 @@ import logging
 import xml
 
 from django.conf import settings
-from django.db import transaction
 from django.urls import reverse
 from django.utils import timezone
 from rest_framework.exceptions import PermissionDenied
 
 from funkwhale_api.common import preferences, session
-from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.music import models as music_models
-from funkwhale_api.music import tasks as music_tasks
 
 from . import activity, keys, models, serializers, signing, utils
 
@@ -39,9 +35,9 @@ def get_actor_data(actor_url):
         raise ValueError("Invalid actor payload: {}".format(response.text))
 
 
-def get_actor(actor_url):
+def get_actor(fid):
     try:
-        actor = models.Actor.objects.get(url=actor_url)
+        actor = models.Actor.objects.get(fid=fid)
     except models.Actor.DoesNotExist:
         actor = None
     fetch_delta = datetime.timedelta(
@@ -50,7 +46,7 @@ def get_actor(actor_url):
     if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
         # cache is hot, we can return as is
         return actor
-    data = get_actor_data(actor_url)
+    data = get_actor_data(fid)
     serializer = serializers.ActorSerializer(data=data)
     serializer.is_valid(raise_exception=True)
 
@@ -72,7 +68,7 @@ class SystemActor(object):
 
     def get_actor_instance(self):
         try:
-            return models.Actor.objects.get(url=self.get_actor_url())
+            return models.Actor.objects.get(fid=self.get_actor_id())
         except models.Actor.DoesNotExist:
             pass
         private, public = keys.get_key_pair()
@@ -83,7 +79,7 @@ class SystemActor(object):
         args["public_key"] = public.decode("utf-8")
         return models.Actor.objects.create(**args)
 
-    def get_actor_url(self):
+    def get_actor_id(self):
         return utils.full_url(
             reverse("federation:instance-actors-detail", kwargs={"actor": self.id})
         )
@@ -95,7 +91,7 @@ class SystemActor(object):
             "type": "Person",
             "name": name.format(host=settings.FEDERATION_HOSTNAME),
             "manually_approves_followers": True,
-            "url": self.get_actor_url(),
+            "fid": self.get_actor_id(),
             "shared_inbox_url": utils.full_url(
                 reverse("federation:instance-actors-inbox", kwargs={"actor": id})
             ),
@@ -178,91 +174,13 @@ class SystemActor(object):
         if ac["object"]["type"] != "Follow":
             return
 
-        if ac["object"]["actor"] != sender.url:
+        if ac["object"]["actor"] != sender.fid:
             # not the same actor, permission issue
             return
 
         self.handle_undo_follow(ac, sender)
 
 
-class LibraryActor(SystemActor):
-    id = "library"
-    name = "{host}'s library"
-    summary = "Bot account to federate with {host}'s library"
-    additional_attributes = {"manually_approves_followers": True}
-
-    def serialize(self):
-        data = super().serialize()
-        urls = data.setdefault("url", [])
-        urls.append(
-            {
-                "type": "Link",
-                "mediaType": "application/activity+json",
-                "name": "library",
-                "href": utils.full_url(reverse("federation:music:files-list")),
-            }
-        )
-        return data
-
-    @property
-    def manually_approves_followers(self):
-        return preferences.get("federation__music_needs_approval")
-
-    @transaction.atomic
-    def handle_create(self, ac, sender):
-        try:
-            remote_library = models.Library.objects.get(
-                actor=sender, federation_enabled=True
-            )
-        except models.Library.DoesNotExist:
-            logger.info("Skipping import, we're not following %s", sender.url)
-            return
-
-        if ac["object"]["type"] != "Collection":
-            return
-
-        if ac["object"]["totalItems"] <= 0:
-            return
-
-        try:
-            items = ac["object"]["items"]
-        except KeyError:
-            logger.warning("No items in collection!")
-            return
-
-        item_serializers = [
-            serializers.AudioSerializer(data=i, context={"library": remote_library})
-            for i in items
-        ]
-        now = timezone.now()
-        valid_serializers = []
-        for s in item_serializers:
-            if s.is_valid():
-                valid_serializers.append(s)
-            else:
-                logger.debug("Skipping invalid item %s, %s", s.initial_data, s.errors)
-
-        lts = []
-        for s in valid_serializers:
-            lts.append(s.save())
-
-        if remote_library.autoimport:
-            batch = music_models.ImportBatch.objects.create(source="federation")
-            for lt in lts:
-                if lt.creation_date < now:
-                    # track was already in the library, we do not trigger
-                    # an import
-                    continue
-                job = music_models.ImportJob.objects.create(
-                    batch=batch, library_track=lt, mbid=lt.mbid, source=lt.url
-                )
-                funkwhale_utils.on_commit(
-                    music_tasks.import_job_run.delay,
-                    import_job_id=job.pk,
-                    use_acoustid=False,
-                )
-
-
 class TestActor(SystemActor):
     id = "test"
     name = "{host}'s test account"
@@ -321,7 +239,7 @@ class TestActor(SystemActor):
                 {},
             ],
             "type": "Create",
-            "actor": test_actor.url,
+            "actor": test_actor.fid,
             "id": "{}/activity".format(reply_url),
             "published": now.isoformat(),
             "to": ac["actor"],
@@ -336,14 +254,14 @@ class TestActor(SystemActor):
                 "sensitive": False,
                 "url": reply_url,
                 "to": [ac["actor"]],
-                "attributedTo": test_actor.url,
+                "attributedTo": test_actor.fid,
                 "cc": [],
                 "attachment": [],
                 "tag": [
                     {
                         "type": "Mention",
                         "href": ac["actor"],
-                        "name": sender.mention_username,
+                        "name": sender.full_username,
                     }
                 ],
             },
@@ -359,7 +277,7 @@ class TestActor(SystemActor):
         )[0]
         activity.deliver(
             serializers.FollowSerializer(follow_back).data,
-            to=[follow_back.target.url],
+            to=[follow_back.target.fid],
             on_behalf_of=follow_back.actor,
         )
 
@@ -373,7 +291,7 @@ class TestActor(SystemActor):
             return
         undo = serializers.UndoFollowSerializer(follow).data
         follow.delete()
-        activity.deliver(undo, to=[sender.url], on_behalf_of=actor)
+        activity.deliver(undo, to=[sender.fid], on_behalf_of=actor)
 
 
-SYSTEM_ACTORS = {"library": LibraryActor(), "test": TestActor()}
+SYSTEM_ACTORS = {"test": TestActor()}

api/funkwhale_api/federation/admin.py

@@ -6,14 +6,14 @@ from . import models
 @admin.register(models.Actor)
 class ActorAdmin(admin.ModelAdmin):
     list_display = [
-        "url",
+        "fid",
         "domain",
         "preferred_username",
         "type",
         "creation_date",
         "last_fetch_date",
     ]
-    search_fields = ["url", "domain", "preferred_username"]
+    search_fields = ["fid", "domain", "preferred_username"]
     list_filter = ["type"]
 
 
@@ -21,14 +21,14 @@ class ActorAdmin(admin.ModelAdmin):
 class FollowAdmin(admin.ModelAdmin):
     list_display = ["actor", "target", "approved", "creation_date"]
     list_filter = ["approved"]
-    search_fields = ["actor__url", "target__url"]
+    search_fields = ["actor__fid", "target__fid"]
     list_select_related = True
 
 
 @admin.register(models.Library)
 class LibraryAdmin(admin.ModelAdmin):
     list_display = ["actor", "url", "creation_date", "fetched_date", "tracks_count"]
-    search_fields = ["actor__url", "url"]
+    search_fields = ["actor__fid", "url"]
     list_filter = ["federation_enabled", "download_files", "autoimport"]
     list_select_related = True
 

api/funkwhale_api/federation/api_serializers.py

@@ -0,0 +1,57 @@
+from rest_framework import serializers
+
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.music import models as music_models
+
+from . import serializers as federation_serializers
+from . import models
+
+
+class NestedLibraryFollowSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
+
+
+class LibrarySerializer(serializers.ModelSerializer):
+    actor = federation_serializers.APIActorSerializer()
+    files_count = serializers.SerializerMethodField()
+    follow = serializers.SerializerMethodField()
+
+    class Meta:
+        model = music_models.Library
+        fields = [
+            "fid",
+            "uuid",
+            "actor",
+            "name",
+            "description",
+            "creation_date",
+            "files_count",
+            "privacy_level",
+            "follow",
+        ]
+
+    def get_files_count(self, o):
+        return max(getattr(o, "_files_count", 0), o.files_count)
+
+    def get_follow(self, o):
+        try:
+            return NestedLibraryFollowSerializer(o._follows[0]).data
+        except (AttributeError, IndexError):
+            return None
+
+
+class LibraryFollowSerializer(serializers.ModelSerializer):
+    target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
+
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "uuid", "target", "approved"]
+        read_only_fields = ["uuid", "approved", "creation_date"]
+
+    def validate_target(self, v):
+        actor = self.context["actor"]
+        if v.received_follows.filter(actor=actor).exists():
+            raise serializers.ValidationError("You are already following this library")
+        return v

api/funkwhale_api/federation/api_urls.py

@@ -1,9 +1,9 @@
 from rest_framework import routers
 
-from . import views
+from . import api_views
 
 router = routers.SimpleRouter()
-router.register(r"libraries", views.LibraryViewSet, "libraries")
-router.register(r"library-tracks", views.LibraryTrackViewSet, "library-tracks")
+router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
+router.register(r"libraries", api_views.LibraryViewSet, "libraries")
 
 urlpatterns = router.urls

api/funkwhale_api/federation/api_views.py

@@ -0,0 +1,92 @@
+import requests.exceptions
+
+from django.db.models import Count
+
+from rest_framework import decorators
+from rest_framework import mixins
+from rest_framework import permissions
+from rest_framework import response
+from rest_framework import viewsets
+
+from funkwhale_api.music import models as music_models
+
+from . import api_serializers
+from . import filters
+from . import models
+from . import routes
+from . import serializers
+from . import utils
+
+
+class LibraryFollowViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.LibraryFollow.objects.all()
+        .order_by("-creation_date")
+        .select_related("target__actor", "actor")
+    )
+    serializer_class = api_serializers.LibraryFollowSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.LibraryFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor)
+
+    def perform_create(self, serializer):
+        follow = serializer.save(actor=self.request.user.actor)
+        routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["actor"] = self.request.user.actor
+        return context
+
+
+class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    lookup_field = "uuid"
+    queryset = (
+        music_models.Library.objects.all()
+        .order_by("-creation_date")
+        .select_related("actor")
+        .annotate(_files_count=Count("files"))
+    )
+    serializer_class = api_serializers.LibrarySerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.LibraryFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.viewable_by(actor=self.request.user.actor)
+
+    @decorators.list_route(methods=["post"])
+    def scan(self, request, *args, **kwargs):
+        try:
+            fid = request.data["fid"]
+        except KeyError:
+            return response.Response({"fid": ["This field is required"]})
+        try:
+            library = utils.retrieve(
+                fid,
+                queryset=self.queryset,
+                serializer_class=serializers.LibrarySerializer,
+            )
+        except requests.exceptions.RequestException as e:
+            return response.Response(
+                {"detail": "Error while scanning the library: {}".format(str(e))},
+                status=400,
+            )
+        except serializers.serializers.ValidationError as e:
+            return response.Response(
+                {"detail": "Invalid data in remote library: {}".format(str(e))},
+                status=400,
+            )
+        serializer = self.serializer_class(library)
+        return response.Response({"count": 1, "results": [serializer.data]})

api/funkwhale_api/federation/factories.py

@@ -8,6 +8,7 @@ from django.utils import timezone
 from django.utils.http import http_date
 
 from funkwhale_api.factories import registry
+from funkwhale_api.users import factories as user_factories
 
 from . import keys, models
 
@@ -61,6 +62,10 @@ class LinkFactory(factory.Factory):
         audio = factory.Trait(mediaType=factory.Iterator(["audio/mp3", "audio/ogg"]))
 
 
+def create_user(actor):
+    return user_factories.UserFactory(actor=actor)
+
+
 @registry.register
 class ActorFactory(factory.DjangoModelFactory):
     public_key = None
@@ -68,7 +73,7 @@ class ActorFactory(factory.DjangoModelFactory):
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
     domain = factory.Faker("domain_name")
-    url = factory.LazyAttribute(
+    fid = factory.LazyAttribute(
         lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
     )
     inbox_url = factory.LazyAttribute(
@@ -81,20 +86,34 @@ class ActorFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.Actor
 
-    class Params:
-        local = factory.Trait(
-            domain=factory.LazyAttribute(lambda o: settings.FEDERATION_HOSTNAME)
-        )
+    @factory.post_generation
+    def local(self, create, extracted, **kwargs):
+        if not extracted and not kwargs:
+            return
+        from funkwhale_api.users.factories import UserFactory
 
-    @classmethod
-    def _generate(cls, create, attrs):
-        has_public = attrs.get("public_key") is not None
-        has_private = attrs.get("private_key") is not None
-        if not has_public and not has_private:
+        self.domain = settings.FEDERATION_HOSTNAME
+        self.save(update_fields=["domain"])
+        if not create:
+            if extracted and hasattr(extracted, "pk"):
+                extracted.actor = self
+            else:
+                UserFactory.build(actor=self, **kwargs)
+        if extracted and hasattr(extracted, "pk"):
+            extracted.actor = self
+            extracted.save(update_fields=["user"])
+        else:
+            self.user = UserFactory(actor=self, **kwargs)
+
+    @factory.post_generation
+    def keys(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+        if not extracted:
             private, public = keys.get_key_pair()
-            attrs["private_key"] = private.decode("utf-8")
-            attrs["public_key"] = public.decode("utf-8")
-        return super()._generate(create, attrs)
+            self.private_key = private.decode("utf-8")
+            self.public_key = public.decode("utf-8")
 
 
 @registry.register
@@ -110,15 +129,70 @@ class FollowFactory(factory.DjangoModelFactory):
 
 
 @registry.register
-class LibraryFactory(factory.DjangoModelFactory):
+class MusicLibraryFactory(factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
-    url = factory.Faker("url")
-    federation_enabled = True
-    download_files = False
-    autoimport = False
+    privacy_level = "me"
+    name = factory.Faker("sentence")
+    description = factory.Faker("sentence")
+    files_count = 0
 
     class Meta:
-        model = models.Library
+        model = "music.Library"
+
+    @factory.post_generation
+    def fid(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        self.fid = extracted or self.get_federation_id()
+
+    @factory.post_generation
+    def followers_url(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        self.followers_url = extracted or self.fid + "/followers"
+
+
+@registry.register
+class LibraryScan(factory.django.DjangoModelFactory):
+    library = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+    total_files = factory.LazyAttribute(lambda o: o.library.files_count)
+
+    class Meta:
+        model = "music.LibraryScan"
+
+
+@registry.register
+class ActivityFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    url = factory.Faker("url")
+    payload = factory.LazyFunction(lambda: {"type": "Create"})
+
+    class Meta:
+        model = "federation.Activity"
+
+
+@registry.register
+class InboxItemFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    activity = factory.SubFactory(ActivityFactory)
+    type = "to"
+
+    class Meta:
+        model = "federation.InboxItem"
+
+
+@registry.register
+class LibraryFollowFactory(factory.DjangoModelFactory):
+    target = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = "federation.LibraryFollow"
 
 
 class ArtistMetadataFactory(factory.Factory):
@@ -161,25 +235,6 @@ class LibraryTrackMetadataFactory(factory.Factory):
         model = dict
 
 
-@registry.register
-class LibraryTrackFactory(factory.DjangoModelFactory):
-    library = factory.SubFactory(LibraryFactory)
-    url = factory.Faker("url")
-    title = factory.Faker("sentence")
-    artist_name = factory.Faker("sentence")
-    album_title = factory.Faker("sentence")
-    audio_url = factory.Faker("url")
-    audio_mimetype = "audio/ogg"
-    metadata = factory.SubFactory(LibraryTrackMetadataFactory)
-    published_date = factory.LazyFunction(timezone.now)
-
-    class Meta:
-        model = models.LibraryTrack
-
-    class Params:
-        with_audio_file = factory.Trait(audio_file=factory.django.FileField())
-
-
 @registry.register(name="federation.Note")
 class NoteFactory(factory.Factory):
     type = "Note"
@@ -192,22 +247,6 @@ class NoteFactory(factory.Factory):
         model = dict
 
 
-@registry.register(name="federation.Activity")
-class ActivityFactory(factory.Factory):
-    type = "Create"
-    id = factory.Faker("url")
-    published = factory.LazyFunction(lambda: timezone.now().isoformat())
-    actor = factory.Faker("url")
-    object = factory.SubFactory(
-        NoteFactory,
-        actor=factory.SelfAttribute("..actor"),
-        published=factory.SelfAttribute("..published"),
-    )
-
-    class Meta:
-        model = dict
-
-
 @registry.register(name="federation.AudioMetadata")
 class AudioMetadataFactory(factory.Factory):
     recording = factory.LazyAttribute(

api/funkwhale_api/federation/filters.py

@@ -1,68 +1,10 @@
 import django_filters
 
 from funkwhale_api.common import fields
-from funkwhale_api.common import search
 
 from . import models
 
 
-class LibraryFilter(django_filters.FilterSet):
-    approved = django_filters.BooleanFilter("following__approved")
-    q = fields.SearchFilter(search_fields=["actor__domain"])
-
-    class Meta:
-        model = models.Library
-        fields = {
-            "approved": ["exact"],
-            "federation_enabled": ["exact"],
-            "download_files": ["exact"],
-            "autoimport": ["exact"],
-            "tracks_count": ["exact"],
-        }
-
-
-class LibraryTrackFilter(django_filters.FilterSet):
-    library = django_filters.CharFilter("library__uuid")
-    status = django_filters.CharFilter(method="filter_status")
-    q = fields.SmartSearchFilter(
-        config=search.SearchConfig(
-            search_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name"},
-                "album": {"to": "album_title"},
-                "title": {"to": "title"},
-            },
-            filter_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name__iexact"},
-                "album": {"to": "album_title__iexact"},
-                "title": {"to": "title__iexact"},
-            },
-        )
-    )
-
-    def filter_status(self, queryset, field_name, value):
-        if value == "imported":
-            return queryset.filter(local_track_file__isnull=False)
-        elif value == "not_imported":
-            return queryset.filter(local_track_file__isnull=True).exclude(
-                import_jobs__status="pending"
-            )
-        elif value == "import_pending":
-            return queryset.filter(import_jobs__status="pending")
-        return queryset
-
-    class Meta:
-        model = models.LibraryTrack
-        fields = {
-            "library": ["exact"],
-            "artist_name": ["exact", "icontains"],
-            "title": ["exact", "icontains"],
-            "album_title": ["exact", "icontains"],
-            "audio_mimetype": ["exact", "icontains"],
-        }
-
-
 class FollowFilter(django_filters.FilterSet):
     pending = django_filters.CharFilter(method="filter_pending")
     ordering = django_filters.OrderingFilter(
@@ -84,3 +26,9 @@ class FollowFilter(django_filters.FilterSet):
         if value.lower() in ["true", "1", "yes"]:
             queryset = queryset.filter(approved__isnull=True)
         return queryset
+
+
+class LibraryFollowFilter(django_filters.FilterSet):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["approved"]

api/funkwhale_api/federation/library.py

@@ -71,8 +71,7 @@ def scan_from_account_name(account_name):
     return data
 
 
-def get_library_data(library_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_data(library_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     try:
         response = session.get_session().get(
@@ -98,8 +97,7 @@ def get_library_data(library_url):
     return serializer.validated_data
 
 
-def get_library_page(library, page_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_page(library, page_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     response = session.get_session().get(
         page_url,

api/funkwhale_api/federation/migrations/0007_auto_20180807_1748.py

@@ -0,0 +1,95 @@
+# Generated by Django 2.0.7 on 2018-08-07 17:48
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0006_auto_20180521_1702")]
+
+    operations = [
+        migrations.CreateModel(
+            name="Activity",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, unique=True)),
+                (
+                    "fid",
+                    models.URLField(blank=True, max_length=500, null=True, unique=True),
+                ),
+                ("url", models.URLField(blank=True, max_length=500, null=True)),
+                (
+                    "payload",
+                    django.contrib.postgres.fields.jsonb.JSONField(
+                        default={},
+                        encoder=django.core.serializers.json.DjangoJSONEncoder,
+                        max_length=50000,
+                    ),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("delivered", models.NullBooleanField(default=None)),
+                ("delivered_date", models.DateTimeField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="LibraryFollow",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "fid",
+                    models.URLField(blank=True, max_length=500, null=True, unique=True),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, unique=True)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("modification_date", models.DateTimeField(auto_now=True)),
+                ("approved", models.NullBooleanField(default=None)),
+            ],
+        ),
+        migrations.RenameField("actor", "url", "fid"),
+        migrations.AddField(
+            model_name="actor",
+            name="url",
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="follow",
+            name="fid",
+            field=models.URLField(blank=True, max_length=500, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name="libraryfollow",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="library_follows",
+                to="federation.Actor",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0008_auto_20180807_1748.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.0.7 on 2018-08-07 17:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0029_auto_20180807_1748"),
+        ("federation", "0007_auto_20180807_1748"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="libraryfollow",
+            name="target",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="received_follows",
+                to="music.Library",
+            ),
+        ),
+        migrations.AddField(
+            model_name="activity",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="libraryfollow", unique_together={("actor", "target")}
+        ),
+    ]

api/funkwhale_api/federation/migrations/0009_auto_20180822_1956.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.0.8 on 2018-08-22 19:56
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0008_auto_20180807_1748")]
+
+    operations = [
+        migrations.AddField(
+            model_name="activity",
+            name="recipient",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="inbox_activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="activity",
+            name="payload",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=50000,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="librarytrack",
+            name="metadata",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=10000,
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0010_auto_20180904_2011.py

@@ -0,0 +1,74 @@
+# Generated by Django 2.0.8 on 2018-09-04 20:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0009_auto_20180822_1956")]
+
+    operations = [
+        migrations.CreateModel(
+            name="InboxItem",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("is_delivered", models.BooleanField(default=False)),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[("to", "to"), ("cc", "cc")], max_length=10
+                    ),
+                ),
+                ("last_delivery_date", models.DateTimeField(blank=True, null=True)),
+                ("delivery_attempts", models.PositiveIntegerField(default=0)),
+            ],
+        ),
+        migrations.RemoveField(model_name="activity", name="delivered"),
+        migrations.RemoveField(model_name="activity", name="delivered_date"),
+        migrations.RemoveField(model_name="activity", name="recipient"),
+        migrations.AlterField(
+            model_name="activity",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="outbox_activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AddField(
+            model_name="inboxitem",
+            name="activity",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="inbox_items",
+                to="federation.Activity",
+            ),
+        ),
+        migrations.AddField(
+            model_name="inboxitem",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="inbox_items",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AddField(
+            model_name="activity",
+            name="recipients",
+            field=models.ManyToManyField(
+                related_name="inbox_activities",
+                through="federation.InboxItem",
+                to="federation.Actor",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -3,6 +3,7 @@ import uuid
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
@@ -11,6 +12,8 @@ from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music import utils as music_utils
 
+from . import utils as federation_utils
+
 TYPE_CHOICES = [
     ("Person", "Person"),
     ("Application", "Application"),
@@ -20,15 +23,43 @@ TYPE_CHOICES = [
 ]
 
 
+def empty_dict():
+    return {}
+
+
+class FederationMixin(models.Model):
+    # federation id/url
+    fid = models.URLField(unique=True, max_length=500, db_index=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
+
+    class Meta:
+        abstract = True
+
+
 class ActorQuerySet(models.QuerySet):
     def local(self, include=True):
         return self.exclude(user__isnull=include)
 
+    def with_current_usage(self):
+        qs = self
+        for s in ["pending", "skipped", "errored", "finished"]:
+            qs = qs.annotate(
+                **{
+                    "_usage_{}".format(s): models.Sum(
+                        "libraries__files__size",
+                        filter=models.Q(libraries__files__import_status=s),
+                    )
+                }
+            )
+
+        return qs
+
 
 class Actor(models.Model):
     ap_type = "Actor"
 
-    url = models.URLField(unique=True, max_length=500, db_index=True)
+    fid = models.URLField(unique=True, max_length=500, db_index=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
     outbox_url = models.URLField(max_length=500)
     inbox_url = models.URLField(max_length=500)
     following_url = models.URLField(max_length=500, null=True, blank=True)
@@ -63,11 +94,14 @@ class Actor(models.Model):
 
     @property
     def private_key_id(self):
-        return "{}#main-key".format(self.url)
+        return "{}#main-key".format(self.fid)
 
     @property
-    def mention_username(self):
-        return "@{}@{}".format(self.preferred_username, self.domain)
+    def full_username(self):
+        return "{}@{}".format(self.preferred_username, self.domain)
+
+    def __str__(self):
+        return "{}@{}".format(self.preferred_username, self.domain)
 
     def save(self, **kwargs):
         lowercase_fields = ["domain"]
@@ -104,26 +138,98 @@ class Actor(models.Model):
         follows = self.received_follows.filter(approved=True)
         return self.followers.filter(pk__in=follows.values_list("actor", flat=True))
 
+    def should_autoapprove_follow(self, actor):
+        return False
 
-class Follow(models.Model):
-    ap_type = "Follow"
+    def get_user(self):
+        try:
+            return self.user
+        except ObjectDoesNotExist:
+            return None
 
+    def get_current_usage(self):
+        actor = self.__class__.objects.filter(pk=self.pk).with_current_usage().get()
+        data = {}
+        for s in ["pending", "skipped", "errored", "finished"]:
+            data[s] = getattr(actor, "_usage_{}".format(s)) or 0
+
+        data["total"] = sum(data.values())
+        return data
+
+
+class InboxItemQuerySet(models.QuerySet):
+    def local(self, include=True):
+        return self.exclude(actor__user__isnull=include)
+
+
+class InboxItem(models.Model):
+    actor = models.ForeignKey(
+        Actor, related_name="inbox_items", on_delete=models.CASCADE
+    )
+    activity = models.ForeignKey(
+        "Activity", related_name="inbox_items", on_delete=models.CASCADE
+    )
+    is_delivered = models.BooleanField(default=False)
+    type = models.CharField(max_length=10, choices=[("to", "to"), ("cc", "cc")])
+    last_delivery_date = models.DateTimeField(null=True, blank=True)
+    delivery_attempts = models.PositiveIntegerField(default=0)
+
+    objects = InboxItemQuerySet.as_manager()
+
+
+class Activity(models.Model):
+    actor = models.ForeignKey(
+        Actor, related_name="outbox_activities", on_delete=models.CASCADE
+    )
+    recipients = models.ManyToManyField(
+        Actor, related_name="inbox_activities", through=InboxItem
+    )
     uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
+    payload = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder)
+    creation_date = models.DateTimeField(default=timezone.now)
+
+
+class AbstractFollow(models.Model):
+    ap_type = "Follow"
+    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(auto_now=True)
+    approved = models.NullBooleanField(default=None)
+
+    class Meta:
+        abstract = True
+
+    def get_federation_id(self):
+        return federation_utils.full_url(
+            "{}#follows/{}".format(self.actor.fid, self.uuid)
+        )
+
+
+class Follow(AbstractFollow):
     actor = models.ForeignKey(
         Actor, related_name="emitted_follows", on_delete=models.CASCADE
     )
     target = models.ForeignKey(
         Actor, related_name="received_follows", on_delete=models.CASCADE
     )
-    creation_date = models.DateTimeField(default=timezone.now)
-    modification_date = models.DateTimeField(auto_now=True)
-    approved = models.NullBooleanField(default=None)
 
     class Meta:
         unique_together = ["actor", "target"]
 
-    def get_federation_url(self):
-        return "{}#follows/{}".format(self.actor.url, self.uuid)
+
+class LibraryFollow(AbstractFollow):
+    actor = models.ForeignKey(
+        Actor, related_name="library_follows", on_delete=models.CASCADE
+    )
+    target = models.ForeignKey(
+        "music.Library", related_name="received_follows", on_delete=models.CASCADE
+    )
+
+    class Meta:
+        unique_together = ["actor", "target"]
 
 
 class Library(models.Model):
@@ -167,7 +273,9 @@ class LibraryTrack(models.Model):
     artist_name = models.CharField(max_length=500)
     album_title = models.CharField(max_length=500)
     title = models.CharField(max_length=500)
-    metadata = JSONField(default={}, max_length=10000, encoder=DjangoJSONEncoder)
+    metadata = JSONField(
+        default=empty_dict, max_length=10000, encoder=DjangoJSONEncoder
+    )
 
     @property
     def mbid(self):

api/funkwhale_api/federation/permissions.py

@@ -1,19 +0,0 @@
-
-from rest_framework.permissions import BasePermission
-
-from funkwhale_api.common import preferences
-
-from . import actors
-
-
-class LibraryFollower(BasePermission):
-    def has_permission(self, request, view):
-        if not preferences.get("federation__music_needs_approval"):
-            return True
-
-        actor = getattr(request, "actor", None)
-        if actor is None:
-            return False
-
-        library = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        return library.received_follows.filter(approved=True, actor=actor).exists()

api/funkwhale_api/federation/routes.py

@@ -0,0 +1,78 @@
+import logging
+
+from . import activity
+from . import serializers
+
+logger = logging.getLogger(__name__)
+inbox = activity.InboxRouter()
+outbox = activity.OutboxRouter()
+
+
+def with_recipients(payload, to=[], cc=[]):
+    if to:
+        payload["to"] = to
+    if cc:
+        payload["cc"] = cc
+    return payload
+
+
+@inbox.register({"type": "Follow"})
+def inbox_follow(payload, context):
+    context["recipient"] = [
+        ii.actor for ii in context["inbox_items"] if ii.type == "to"
+    ][0]
+    serializer = serializers.FollowSerializer(data=payload, context=context)
+    if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
+        logger.debug(
+            "Discarding invalid follow from {}: %s",
+            context["actor"].fid,
+            serializer.errors,
+        )
+        return
+
+    autoapprove = serializer.validated_data["object"].should_autoapprove_follow(
+        context["actor"]
+    )
+    follow = serializer.save(approved=autoapprove)
+
+    if autoapprove:
+        activity.accept_follow(follow)
+
+
+@inbox.register({"type": "Accept"})
+def inbox_accept(payload, context):
+    context["recipient"] = [
+        ii.actor for ii in context["inbox_items"] if ii.type == "to"
+    ][0]
+    serializer = serializers.AcceptFollowSerializer(data=payload, context=context)
+    if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
+        logger.debug(
+            "Discarding invalid accept from {}: %s",
+            context["actor"].fid,
+            serializer.errors,
+        )
+        return
+
+    serializer.save()
+
+
+@outbox.register({"type": "Accept"})
+def outbox_accept(context):
+    follow = context["follow"]
+    if follow._meta.label == "federation.LibraryFollow":
+        actor = follow.target.actor
+    else:
+        actor = follow.target
+    payload = serializers.AcceptFollowSerializer(follow, context={"actor": actor}).data
+    yield {"actor": actor, "payload": with_recipients(payload, to=[follow.actor])}
+
+
+@outbox.register({"type": "Follow"})
+def outbox_follow(context):
+    follow = context["follow"]
+    if follow._meta.label == "federation.LibraryFollow":
+        target = follow.target.actor
+    else:
+        target = follow.target
+    payload = serializers.FollowSerializer(follow, context={"actor": follow.actor}).data
+    yield {"actor": follow.actor, "payload": with_recipients(payload, to=[target])}

api/funkwhale_api/federation/serializers.py

@@ -4,15 +4,12 @@ import urllib.parse
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.paginator import Paginator
-from django.db import transaction
 from rest_framework import serializers
 
-from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.common import utils as funkwhale_utils
 from funkwhale_api.music import models as music_models
-from funkwhale_api.music import tasks as music_tasks
 
-from . import activity, filters, models, utils
+from . import activity, models, utils
 
 AP_CONTEXT = [
     "https://www.w3.org/ns/activitystreams",
@@ -38,7 +35,7 @@ class ActorSerializer(serializers.Serializer):
 
     def to_representation(self, instance):
         ret = {
-            "id": instance.url,
+            "id": instance.fid,
             "outbox": instance.outbox_url,
             "inbox": instance.inbox_url,
             "preferredUsername": instance.preferred_username,
@@ -58,9 +55,9 @@ class ActorSerializer(serializers.Serializer):
         ret["@context"] = AP_CONTEXT
         if instance.public_key:
             ret["publicKey"] = {
-                "owner": instance.url,
+                "owner": instance.fid,
                 "publicKeyPem": instance.public_key,
-                "id": "{}#main-key".format(instance.url),
+                "id": "{}#main-key".format(instance.fid),
             }
         ret["endpoints"] = {}
         if instance.shared_inbox_url:
@@ -78,7 +75,7 @@ class ActorSerializer(serializers.Serializer):
 
     def prepare_missing_fields(self):
         kwargs = {
-            "url": self.validated_data["id"],
+            "fid": self.validated_data["id"],
             "outbox_url": self.validated_data["outbox"],
             "inbox_url": self.validated_data["inbox"],
             "following_url": self.validated_data.get("following"),
@@ -91,7 +88,7 @@ class ActorSerializer(serializers.Serializer):
         maf = self.validated_data.get("manuallyApprovesFollowers")
         if maf is not None:
             kwargs["manually_approves_followers"] = maf
-        domain = urllib.parse.urlparse(kwargs["url"]).netloc
+        domain = urllib.parse.urlparse(kwargs["fid"]).netloc
         kwargs["domain"] = domain
         for endpoint, url in self.initial_data.get("endpoints", {}).items():
             if endpoint == "sharedInbox":
@@ -110,7 +107,7 @@ class ActorSerializer(serializers.Serializer):
     def save(self, **kwargs):
         d = self.prepare_missing_fields()
         d.update(kwargs)
-        return models.Actor.objects.update_or_create(url=d["url"], defaults=d)[0]
+        return models.Actor.objects.update_or_create(fid=d["fid"], defaults=d)[0]
 
     def validate_summary(self, value):
         if value:
@@ -122,6 +119,7 @@ class APIActorSerializer(serializers.ModelSerializer):
         model = models.Actor
         fields = [
             "id",
+            "fid",
             "url",
             "creation_date",
             "summary",
@@ -131,190 +129,73 @@ class APIActorSerializer(serializers.ModelSerializer):
             "domain",
             "type",
             "manually_approves_followers",
+            "full_username",
         ]
 
 
-class LibraryActorSerializer(ActorSerializer):
-    url = serializers.ListField(child=serializers.JSONField())
-
-    def validate(self, validated_data):
-        try:
-            urls = validated_data["url"]
-        except KeyError:
-            raise serializers.ValidationError("Missing URL field")
-
-        for u in urls:
-            try:
-                if u["name"] != "library":
-                    continue
-                validated_data["library_url"] = u["href"]
-                break
-            except KeyError:
-                continue
-
-        return validated_data
-
-
-class APIFollowSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = models.Follow
-        fields = [
-            "uuid",
-            "actor",
-            "target",
-            "approved",
-            "creation_date",
-            "modification_date",
-        ]
-
-
-class APILibrarySerializer(serializers.ModelSerializer):
-    actor = APIActorSerializer()
-    follow = APIFollowSerializer()
-
-    class Meta:
-        model = models.Library
-
-        read_only_fields = [
-            "actor",
-            "uuid",
-            "url",
-            "tracks_count",
-            "follow",
-            "fetched_date",
-            "modification_date",
-            "creation_date",
-        ]
-        fields = [
-            "autoimport",
-            "federation_enabled",
-            "download_files",
-        ] + read_only_fields
-
-
-class APILibraryScanSerializer(serializers.Serializer):
-    until = serializers.DateTimeField(required=False)
-
-
-class APILibraryFollowUpdateSerializer(serializers.Serializer):
-    follow = serializers.IntegerField()
-    approved = serializers.BooleanField()
-
-    def validate_follow(self, value):
-        from . import actors
-
-        library_actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        qs = models.Follow.objects.filter(pk=value, target=library_actor)
-        try:
-            return qs.get()
-        except models.Follow.DoesNotExist:
-            raise serializers.ValidationError("Invalid follow")
-
-    def save(self):
-        new_status = self.validated_data["approved"]
-        follow = self.validated_data["follow"]
-        if new_status == follow.approved:
-            return follow
-
-        follow.approved = new_status
-        follow.save(update_fields=["approved", "modification_date"])
-        if new_status:
-            activity.accept_follow(follow)
-        return follow
-
-
-class APILibraryCreateSerializer(serializers.ModelSerializer):
+class BaseActivitySerializer(serializers.Serializer):
+    id = serializers.URLField(max_length=500, required=False)
+    type = serializers.CharField(max_length=100)
     actor = serializers.URLField(max_length=500)
-    federation_enabled = serializers.BooleanField()
-    uuid = serializers.UUIDField(read_only=True)
 
-    class Meta:
-        model = models.Library
-        fields = ["uuid", "actor", "autoimport", "federation_enabled", "download_files"]
-
-    def validate(self, validated_data):
-        from . import actors
-        from . import library
-
-        actor_url = validated_data["actor"]
-        actor_data = actors.get_actor_data(actor_url)
-        acs = LibraryActorSerializer(data=actor_data)
-        acs.is_valid(raise_exception=True)
+    def validate_actor(self, v):
+        expected = self.context.get("actor")
+        if expected and expected.fid != v:
+            raise serializers.ValidationError("Invalid actor")
+        if expected:
+            # avoid a DB lookup
+            return expected
         try:
-            actor = models.Actor.objects.get(url=actor_url)
+            return models.Actor.objects.get(fid=v)
         except models.Actor.DoesNotExist:
-            actor = acs.save()
-        library_actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        validated_data["follow"] = models.Follow.objects.get_or_create(
-            actor=library_actor, target=actor
-        )[0]
-        if validated_data["follow"].approved is None:
-            funkwhale_utils.on_commit(
-                activity.deliver,
-                FollowSerializer(validated_data["follow"]).data,
-                on_behalf_of=validated_data["follow"].actor,
-                to=[validated_data["follow"].target.url],
-            )
-
-        library_data = library.get_library_data(acs.validated_data["library_url"])
-        if "errors" in library_data:
-            # we pass silently because it may means we require permission
-            # before scanning
-            pass
-        validated_data["library"] = library_data
-        validated_data["library"].setdefault("id", acs.validated_data["library_url"])
-        validated_data["actor"] = actor
-        return validated_data
+            raise serializers.ValidationError("Actor not found")
 
     def create(self, validated_data):
-        library = models.Library.objects.update_or_create(
-            url=validated_data["library"]["id"],
-            defaults={
-                "actor": validated_data["actor"],
-                "follow": validated_data["follow"],
-                "tracks_count": validated_data["library"].get("totalItems"),
-                "federation_enabled": validated_data["federation_enabled"],
-                "autoimport": validated_data["autoimport"],
-                "download_files": validated_data["download_files"],
-            },
-        )[0]
-        return library
+        return models.Activity.objects.create(
+            fid=validated_data.get("id"),
+            actor=validated_data["actor"],
+            payload=self.initial_data,
+        )
 
+    def validate(self, data):
+        data["recipients"] = self.validate_recipients(self.initial_data)
+        return super().validate(data)
 
-class APILibraryTrackSerializer(serializers.ModelSerializer):
-    library = APILibrarySerializer()
-    status = serializers.SerializerMethodField()
+    def validate_recipients(self, payload):
+        """
+        Ensure we have at least a to/cc field with valid actors
+        """
+        to = payload.get("to", [])
+        cc = payload.get("cc", [])
 
-    class Meta:
-        model = models.LibraryTrack
-        fields = [
-            "id",
-            "url",
-            "audio_url",
-            "audio_mimetype",
-            "creation_date",
-            "modification_date",
-            "fetched_date",
-            "published_date",
-            "metadata",
-            "artist_name",
-            "album_title",
-            "title",
-            "library",
-            "local_track_file",
-            "status",
-        ]
+        if not to and not cc:
+            raise serializers.ValidationError(
+                "We cannot handle an activity with no recipient"
+            )
 
-    def get_status(self, o):
-        try:
-            if o.local_track_file is not None:
-                return "imported"
-        except music_models.TrackFile.DoesNotExist:
-            pass
-        for job in o.import_jobs.all():
-            if job.status == "pending":
-                return "import_pending"
-        return "not_imported"
+        matching = models.Actor.objects.filter(fid__in=to + cc)
+        if self.context.get("local_recipients", False):
+            matching = matching.local()
+
+        if not len(matching):
+            raise serializers.ValidationError("No matching recipients found")
+
+        actors_by_fid = {a.fid: a for a in matching}
+
+        def match(recipients, actors):
+            for r in recipients:
+                if r == activity.PUBLIC_ADDRESS:
+                    yield r
+                else:
+                    try:
+                        yield actors[r]
+                    except KeyError:
+                        pass
+
+        return {
+            "to": list(match(to, actors_by_fid)),
+            "cc": list(match(cc, actors_by_fid)),
+        }
 
 
 class FollowSerializer(serializers.Serializer):
@@ -325,35 +206,61 @@ class FollowSerializer(serializers.Serializer):
 
     def validate_object(self, v):
         expected = self.context.get("follow_target")
-        if expected and expected.url != v:
+        if self.parent:
+            # it's probably an accept, so everything is inverted, the actor
+            # the recipient does not matter
+            recipient = None
+        else:
+            recipient = self.context.get("recipient")
+        if expected and expected.fid != v:
             raise serializers.ValidationError("Invalid target")
         try:
-            return models.Actor.objects.get(url=v)
+            obj = models.Actor.objects.get(fid=v)
+            if recipient and recipient.fid != obj.fid:
+                raise serializers.ValidationError("Invalid target")
+            return obj
         except models.Actor.DoesNotExist:
-            raise serializers.ValidationError("Target not found")
+            pass
+        try:
+            qs = music_models.Library.objects.filter(fid=v)
+            if recipient:
+                qs = qs.filter(actor=recipient)
+            return qs.get()
+        except music_models.Library.DoesNotExist:
+            pass
+
+        raise serializers.ValidationError("Target not found")
 
     def validate_actor(self, v):
         expected = self.context.get("follow_actor")
-        if expected and expected.url != v:
+        if expected and expected.fid != v:
             raise serializers.ValidationError("Invalid actor")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(fid=v)
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Actor not found")
 
     def save(self, **kwargs):
-        return models.Follow.objects.get_or_create(
+        target = self.validated_data["object"]
+
+        if target._meta.label == "music.Library":
+            follow_class = models.LibraryFollow
+        else:
+            follow_class = models.Follow
+        defaults = kwargs
+        defaults["fid"] = self.validated_data["id"]
+        return follow_class.objects.update_or_create(
             actor=self.validated_data["actor"],
             target=self.validated_data["object"],
-            **kwargs,  # noqa
+            defaults=defaults,
         )[0]
 
     def to_representation(self, instance):
         return {
             "@context": AP_CONTEXT,
-            "actor": instance.actor.url,
-            "id": instance.get_federation_url(),
-            "object": instance.target.url,
+            "actor": instance.actor.fid,
+            "id": instance.get_federation_id(),
+            "object": instance.target.fid,
             "type": "Follow",
         }
 
@@ -376,50 +283,66 @@ class APIFollowSerializer(serializers.ModelSerializer):
 
 
 class AcceptFollowSerializer(serializers.Serializer):
-    id = serializers.URLField(max_length=500)
+    id = serializers.URLField(max_length=500, required=False)
     actor = serializers.URLField(max_length=500)
     object = FollowSerializer()
     type = serializers.ChoiceField(choices=["Accept"])
 
     def validate_actor(self, v):
-        expected = self.context.get("follow_target")
-        if expected and expected.url != v:
+        expected = self.context.get("actor")
+        if expected and expected.fid != v:
             raise serializers.ValidationError("Invalid actor")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(fid=v)
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Actor not found")
 
     def validate(self, validated_data):
-        # we ensure the accept actor actually match the follow target
-        if validated_data["actor"] != validated_data["object"]["object"]:
+        # we ensure the accept actor actually match the follow target / library owner
+        target = validated_data["object"]["object"]
+
+        if target._meta.label == "music.Library":
+            expected = target.actor
+            follow_class = models.LibraryFollow
+        else:
+            expected = target
+            follow_class = models.Follow
+        if validated_data["actor"] != expected:
             raise serializers.ValidationError("Actor mismatch")
         try:
             validated_data["follow"] = (
-                models.Follow.objects.filter(
-                    target=validated_data["actor"],
-                    actor=validated_data["object"]["actor"],
+                follow_class.objects.filter(
+                    target=target, actor=validated_data["object"]["actor"]
                 )
                 .exclude(approved=True)
+                .select_related()
                 .get()
             )
-        except models.Follow.DoesNotExist:
+        except follow_class.DoesNotExist:
             raise serializers.ValidationError("No follow to accept")
         return validated_data
 
     def to_representation(self, instance):
+        if instance.target._meta.label == "music.Library":
+            actor = instance.target.actor
+        else:
+            actor = instance.target
+
         return {
             "@context": AP_CONTEXT,
-            "id": instance.get_federation_url() + "/accept",
+            "id": instance.get_federation_id() + "/accept",
             "type": "Accept",
-            "actor": instance.target.url,
+            "actor": actor.fid,
             "object": FollowSerializer(instance).data,
         }
 
     def save(self):
-        self.validated_data["follow"].approved = True
-        self.validated_data["follow"].save()
-        return self.validated_data["follow"]
+        follow = self.validated_data["follow"]
+        follow.approved = True
+        follow.save()
+        if follow.target._meta.label == "music.Library":
+            follow.target.schedule_scan()
+        return follow
 
 
 class UndoFollowSerializer(serializers.Serializer):
@@ -430,10 +353,10 @@ class UndoFollowSerializer(serializers.Serializer):
 
     def validate_actor(self, v):
         expected = self.context.get("follow_target")
-        if expected and expected.url != v:
+        if expected and expected.fid != v:
             raise serializers.ValidationError("Invalid actor")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(fid=v)
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Actor not found")
 
@@ -452,9 +375,9 @@ class UndoFollowSerializer(serializers.Serializer):
     def to_representation(self, instance):
         return {
             "@context": AP_CONTEXT,
-            "id": instance.get_federation_url() + "/undo",
+            "id": instance.get_federation_id() + "/undo",
             "type": "Undo",
-            "actor": instance.actor.url,
+            "actor": instance.actor.fid,
             "object": FollowSerializer(instance).data,
         }
 
@@ -488,9 +411,9 @@ class ActorWebfingerSerializer(serializers.Serializer):
         data = {}
         data["subject"] = "acct:{}".format(instance.webfinger_subject)
         data["links"] = [
-            {"rel": "self", "href": instance.url, "type": "application/activity+json"}
+            {"rel": "self", "href": instance.fid, "type": "application/activity+json"}
         ]
-        data["aliases"] = [instance.url]
+        data["aliases"] = [instance.fid]
         return data
 
 
@@ -519,7 +442,7 @@ class ActivitySerializer(serializers.Serializer):
 
     def validate_actor(self, value):
         request_actor = self.context.get("actor")
-        if request_actor and request_actor.url != value:
+        if request_actor and request_actor.fid != value:
             raise serializers.ValidationError(
                 "The actor making the request do not match" " the activity actor"
             )
@@ -560,6 +483,18 @@ class ObjectSerializer(serializers.Serializer):
 OBJECT_SERIALIZERS = {t: ObjectSerializer for t in activity.OBJECT_TYPES}
 
 
+def get_additional_fields(data):
+    UNSET = object()
+    additional_fields = {}
+    for field in ["name", "summary"]:
+        v = data.get(field, UNSET)
+        if v == UNSET:
+            continue
+        additional_fields[field] = v
+
+    return additional_fields
+
+
 class PaginatedCollectionSerializer(serializers.Serializer):
     type = serializers.ChoiceField(choices=["Collection"])
     totalItems = serializers.IntegerField(min_value=0)
@@ -575,18 +510,70 @@ class PaginatedCollectionSerializer(serializers.Serializer):
         last = funkwhale_utils.set_query_parameter(conf["id"], page=paginator.num_pages)
         d = {
             "id": conf["id"],
-            "actor": conf["actor"].url,
+            "actor": conf["actor"].fid,
             "totalItems": paginator.count,
-            "type": "Collection",
+            "type": conf.get("type", "Collection"),
             "current": current,
             "first": first,
             "last": last,
         }
+        d.update(get_additional_fields(conf))
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
 
 
+class LibrarySerializer(PaginatedCollectionSerializer):
+    type = serializers.ChoiceField(choices=["Library"])
+    name = serializers.CharField()
+    summary = serializers.CharField(allow_blank=True, allow_null=True, required=False)
+    audience = serializers.ChoiceField(
+        choices=["", None, "https://www.w3.org/ns/activitystreams#Public"],
+        required=False,
+        allow_null=True,
+        allow_blank=True,
+    )
+
+    def to_representation(self, library):
+        conf = {
+            "id": library.fid,
+            "name": library.name,
+            "summary": library.description,
+            "page_size": 100,
+            "actor": library.actor,
+            "items": library.files.filter(import_status="finished"),
+            "type": "Library",
+        }
+        r = super().to_representation(conf)
+        r["audience"] = (
+            "https://www.w3.org/ns/activitystreams#Public"
+            if library.privacy_level == "public"
+            else ""
+        )
+        return r
+
+    def create(self, validated_data):
+        actor = utils.retrieve(
+            validated_data["actor"],
+            queryset=models.Actor,
+            serializer_class=ActorSerializer,
+        )
+        library, created = music_models.Library.objects.update_or_create(
+            fid=validated_data["id"],
+            actor=actor,
+            defaults={
+                "files_count": validated_data["totalItems"],
+                "name": validated_data["name"],
+                "description": validated_data["summary"],
+                "privacy_level": "everyone"
+                if validated_data["audience"]
+                == "https://www.w3.org/ns/activitystreams#Public"
+                else "me",
+            },
+        )
+        return library
+
+
 class CollectionPageSerializer(serializers.Serializer):
     type = serializers.ChoiceField(choices=["CollectionPage"])
     totalItems = serializers.IntegerField(min_value=0)
@@ -623,7 +610,7 @@ class CollectionPageSerializer(serializers.Serializer):
         d = {
             "id": id,
             "partOf": conf["id"],
-            "actor": conf["actor"].url,
+            "actor": conf["actor"].fid,
             "totalItems": page.paginator.count,
             "type": "CollectionPage",
             "first": first,
@@ -645,7 +632,7 @@ class CollectionPageSerializer(serializers.Serializer):
             d["next"] = funkwhale_utils.set_query_parameter(
                 conf["id"], page=page.next_page_number()
             )
-
+        d.update(get_additional_fields(conf))
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
@@ -678,6 +665,7 @@ class AudioMetadataSerializer(serializers.Serializer):
 class AudioSerializer(serializers.Serializer):
     type = serializers.CharField()
     id = serializers.URLField(max_length=500)
+    library = serializers.URLField(max_length=500)
     url = serializers.JSONField()
     published = serializers.DateTimeField()
     updated = serializers.DateTimeField(required=False)
@@ -704,32 +692,40 @@ class AudioSerializer(serializers.Serializer):
 
         return v
 
+    def validate_library(self, v):
+        lb = self.context.get("library")
+        if lb:
+            if lb.fid != v:
+                raise serializers.ValidationError("Invalid library")
+            return lb
+        try:
+            return music_models.Library.objects.get(fid=v)
+        except music_models.Library.DoesNotExist:
+            raise serializers.ValidationError("Invalid library")
+
     def create(self, validated_data):
         defaults = {
-            "audio_mimetype": validated_data["url"]["mediaType"],
-            "audio_url": validated_data["url"]["href"],
-            "metadata": validated_data["metadata"],
-            "artist_name": validated_data["metadata"]["artist"]["name"],
-            "album_title": validated_data["metadata"]["release"]["title"],
-            "title": validated_data["metadata"]["recording"]["title"],
-            "published_date": validated_data["published"],
+            "mimetype": validated_data["url"]["mediaType"],
+            "source": validated_data["url"]["href"],
+            "creation_date": validated_data["published"],
             "modification_date": validated_data.get("updated"),
+            "metadata": self.initial_data,
         }
-        return models.LibraryTrack.objects.get_or_create(
-            library=self.context["library"], url=validated_data["id"], defaults=defaults
-        )[0]
+        tf, created = validated_data["library"].files.update_or_create(
+            fid=validated_data["id"], defaults=defaults
+        )
+        return tf
 
     def to_representation(self, instance):
         track = instance.track
         album = instance.track.album
         artist = instance.track.artist
-
         d = {
             "type": "Audio",
-            "id": instance.get_federation_url(),
+            "id": instance.get_federation_id(),
+            "library": instance.library.get_federation_id(),
             "name": instance.track.full_name,
             "published": instance.creation_date.isoformat(),
-            "updated": instance.modification_date.isoformat(),
             "metadata": {
                 "artist": {
                     "musicbrainz_id": str(artist.mbid) if artist.mbid else None,
@@ -748,12 +744,14 @@ class AudioSerializer(serializers.Serializer):
                 "length": instance.duration,
             },
             "url": {
-                "href": utils.full_url(instance.path),
+                "href": utils.full_url(instance.listen_url),
                 "type": "Link",
                 "mediaType": instance.mimetype,
             },
-            "attributedTo": [self.context["actor"].url],
         }
+        if instance.modification_date:
+            d["updated"] = instance.modification_date.isoformat()
+
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
@@ -763,7 +761,7 @@ class CollectionSerializer(serializers.Serializer):
     def to_representation(self, conf):
         d = {
             "id": conf["id"],
-            "actor": conf["actor"].url,
+            "actor": conf["actor"].fid,
             "totalItems": len(conf["items"]),
             "type": "Collection",
             "items": [
@@ -777,27 +775,3 @@ class CollectionSerializer(serializers.Serializer):
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
-
-
-class LibraryTrackActionSerializer(common_serializers.ActionSerializer):
-    actions = [common_serializers.Action("import", allow_all=True)]
-    filterset_class = filters.LibraryTrackFilter
-
-    @transaction.atomic
-    def handle_import(self, objects):
-        batch = music_models.ImportBatch.objects.create(
-            source="federation", submitted_by=self.context["submitted_by"]
-        )
-        jobs = []
-        for lt in objects:
-            job = music_models.ImportJob(
-                batch=batch, library_track=lt, mbid=lt.mbid, source=lt.url
-            )
-            jobs.append(job)
-
-        music_models.ImportJob.objects.bulk_create(jobs)
-        funkwhale_utils.on_commit(
-            music_tasks.import_batch_run.delay, import_batch_id=batch.pk
-        )
-
-        return {"batch": {"id": batch.pk}}

api/funkwhale_api/federation/tasks.py

@@ -1,92 +1,23 @@
 import datetime
-import json
 import logging
 import os
 
 from django.conf import settings
-from django.db.models import Q
+from django.db.models import Q, F
 from django.utils import timezone
 from dynamic_preferences.registries import global_preferences_registry
 from requests.exceptions import RequestException
 
 from funkwhale_api.common import session
+from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
 
-from . import actors
-from . import library as lb
 from . import models, signing
+from . import routes
 
 logger = logging.getLogger(__name__)
 
 
-@celery.app.task(
-    name="federation.send",
-    autoretry_for=[RequestException],
-    retry_backoff=30,
-    max_retries=5,
-)
-@celery.require_instance(models.Actor, "actor")
-def send(activity, actor, to):
-    logger.info("Preparing activity delivery to %s", to)
-    auth = signing.get_auth(actor.private_key, actor.private_key_id)
-    for url in to:
-        recipient_actor = actors.get_actor(url)
-        logger.debug("delivering to %s", recipient_actor.inbox_url)
-        logger.debug("activity content: %s", json.dumps(activity))
-        response = session.get_session().post(
-            auth=auth,
-            json=activity,
-            url=recipient_actor.inbox_url,
-            timeout=5,
-            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-            headers={"Content-Type": "application/activity+json"},
-        )
-        response.raise_for_status()
-        logger.debug("Remote answered with %s", response.status_code)
-
-
-@celery.app.task(
-    name="federation.scan_library",
-    autoretry_for=[RequestException],
-    retry_backoff=30,
-    max_retries=5,
-)
-@celery.require_instance(models.Library, "library")
-def scan_library(library, until=None):
-    if not library.federation_enabled:
-        return
-
-    data = lb.get_library_data(library.url)
-    scan_library_page.delay(library_id=library.id, page_url=data["first"], until=until)
-    library.fetched_date = timezone.now()
-    library.tracks_count = data["totalItems"]
-    library.save(update_fields=["fetched_date", "tracks_count"])
-
-
-@celery.app.task(
-    name="federation.scan_library_page",
-    autoretry_for=[RequestException],
-    retry_backoff=30,
-    max_retries=5,
-)
-@celery.require_instance(models.Library, "library")
-def scan_library_page(library, page_url, until=None):
-    if not library.federation_enabled:
-        return
-
-    data = lb.get_library_page(library, page_url)
-    lts = []
-    for item_serializer in data["items"]:
-        item_date = item_serializer.validated_data["published"]
-        if until and item_date < until:
-            return
-        lts.append(item_serializer.save())
-
-    next_page = data.get("next")
-    if next_page and next_page != page_url:
-        scan_library_page.delay(library_id=library.id, page_url=next_page)
-
-
 @celery.app.task(name="federation.clean_music_cache")
 def clean_music_cache():
     preferences = global_preferences_registry.manager()
@@ -96,23 +27,22 @@ def clean_music_cache():
     limit = timezone.now() - datetime.timedelta(minutes=delay)
 
     candidates = (
-        models.LibraryTrack.objects.filter(
+        music_models.TrackFile.objects.filter(
             Q(audio_file__isnull=False)
-            & (
-                Q(local_track_file__accessed_date__lt=limit)
-                | Q(local_track_file__accessed_date=None)
-            )
+            & (Q(accessed_date__lt=limit) | Q(accessed_date=None))
         )
+        .local(False)
         .exclude(audio_file="")
         .only("audio_file", "id")
+        .order_by("id")
     )
-    for lt in candidates:
-        lt.audio_file.delete()
+    for tf in candidates:
+        tf.audio_file.delete()
 
     # we also delete orphaned files, if any
     storage = models.LibraryTrack._meta.get_field("audio_file").storage
-    files = get_files(storage, "federation_cache")
-    existing = models.LibraryTrack.objects.filter(audio_file__in=files)
+    files = get_files(storage, "federation_cache/tracks")
+    existing = music_models.TrackFile.objects.filter(audio_file__in=files)
     missing = set(files) - set(existing.values_list("audio_file", flat=True))
     for m in missing:
         storage.delete(m)
@@ -130,3 +60,100 @@ def get_files(storage, *parts):
     for dir in dirs:
         files += get_files(storage, *(list(parts) + [dir]))
     return [os.path.join(parts[-1], path) for path in files]
+
+
+@celery.app.task(name="federation.dispatch_inbox")
+@celery.require_instance(models.Activity.objects.select_related(), "activity")
+def dispatch_inbox(activity):
+    """
+    Given an activity instance, triggers our internal delivery logic (follow
+    creation, etc.)
+    """
+
+    try:
+        routes.inbox.dispatch(
+            activity.payload,
+            context={
+                "actor": activity.actor,
+                "inbox_items": list(activity.inbox_items.local().select_related()),
+            },
+        )
+    except Exception:
+        activity.inbox_items.local().update(
+            delivery_attempts=F("delivery_attempts") + 1,
+            last_delivery_date=timezone.now(),
+        )
+        raise
+    else:
+        activity.inbox_items.local().update(
+            delivery_attempts=F("delivery_attempts") + 1,
+            last_delivery_date=timezone.now(),
+            is_delivered=True,
+        )
+
+
+@celery.app.task(name="federation.dispatch_outbox")
+@celery.require_instance(models.Activity.objects.select_related(), "activity")
+def dispatch_outbox(activity):
+    """
+    Deliver a local activity to its recipients
+    """
+    inbox_items = activity.inbox_items.all().select_related("actor")
+    local_recipients_items = [ii for ii in inbox_items if ii.actor.is_local]
+    if local_recipients_items:
+        dispatch_inbox.delay(activity_id=activity.pk)
+    remote_recipients_items = [ii for ii in inbox_items if not ii.actor.is_local]
+
+    shared_inbox_urls = {
+        ii.actor.shared_inbox_url
+        for ii in remote_recipients_items
+        if ii.actor.shared_inbox_url
+    }
+    inbox_urls = {
+        ii.actor.inbox_url
+        for ii in remote_recipients_items
+        if not ii.actor.shared_inbox_url
+    }
+    for url in shared_inbox_urls:
+        deliver_to_remote_inbox.delay(activity_id=activity.pk, shared_inbox_url=url)
+
+    for url in inbox_urls:
+        deliver_to_remote_inbox.delay(activity_id=activity.pk, inbox_url=url)
+
+
+@celery.app.task(
+    name="federation.deliver_to_remote_inbox",
+    autoretry_for=[RequestException],
+    retry_backoff=30,
+    max_retries=5,
+)
+@celery.require_instance(models.Activity.objects.select_related(), "activity")
+def deliver_to_remote_inbox(activity, inbox_url=None, shared_inbox_url=None):
+    url = inbox_url or shared_inbox_url
+    actor = activity.actor
+    inbox_items = activity.inbox_items.filter(is_delivered=False)
+    if inbox_url:
+        inbox_items = inbox_items.filter(actor__inbox_url=inbox_url)
+    else:
+        inbox_items = inbox_items.filter(actor__shared_inbox_url=shared_inbox_url)
+    logger.info("Preparing activity delivery to %s", url)
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    try:
+        response = session.get_session().post(
+            auth=auth,
+            json=activity.payload,
+            url=url,
+            timeout=5,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+            headers={"Content-Type": "application/activity+json"},
+        )
+        logger.debug("Remote answered with %s", response.status_code)
+        response.raise_for_status()
+    except Exception:
+        inbox_items.update(
+            last_delivery_date=timezone.now(),
+            delivery_attempts=F("delivery_attempts") + 1,
+        )
+        raise
+    else:
+        inbox_items.update(last_delivery_date=timezone.now(), is_delivered=True)

api/funkwhale_api/federation/urls.py

@@ -11,7 +11,7 @@ router.register(
 router.register(r"federation/actors", views.ActorViewSet, "actors")
 router.register(r".well-known", views.WellKnownViewSet, "well-known")
 
-music_router.register(r"files", views.MusicFilesViewSet, "files")
+music_router.register(r"libraries", views.MusicLibraryViewSet, "libraries")
 urlpatterns = router.urls + [
     url("federation/music/", include((music_router.urls, "music"), namespace="music"))
 ]

api/funkwhale_api/federation/utils.py

@@ -2,6 +2,10 @@ import unicodedata
 import re
 from django.conf import settings
 
+from funkwhale_api.common import session
+
+from . import signing
+
 
 def full_url(path):
     """
@@ -52,3 +56,35 @@ def slugify_username(username):
     )
     value = re.sub(r"[^\w\s-]", "", value).strip()
     return re.sub(r"[-\s]+", "_", value)
+
+
+def retrieve(fid, actor=None, serializer_class=None, queryset=None):
+    if queryset:
+        try:
+            # queryset can also be a Model class
+            existing = queryset.filter(fid=fid).first()
+        except AttributeError:
+            existing = queryset.objects.filter(fid=fid).first()
+        if existing:
+            return existing
+
+    auth = (
+        None if not actor else signing.get_auth(actor.private_key, actor.private_key_id)
+    )
+    response = session.get_session().get(
+        fid,
+        auth=auth,
+        timeout=5,
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+        headers={
+            "Accept": "application/activity+json",
+            "Content-Type": "application/activity+json",
+        },
+    )
+    response.raise_for_status()
+    data = response.json()
+    if not serializer_class:
+        return data
+    serializer = serializer_class(data=data)
+    serializer.is_valid(raise_exception=True)
+    return serializer.save()

api/funkwhale_api/federation/views.py

@@ -1,25 +1,20 @@
 from django import forms
 from django.core import paginator
-from django.db import transaction
 from django.http import HttpResponse, Http404
 from django.urls import reverse
-from rest_framework import mixins, response, viewsets
+from rest_framework import exceptions, mixins, response, viewsets
 from rest_framework.decorators import detail_route, list_route
 
 from funkwhale_api.common import preferences
 from funkwhale_api.music import models as music_models
-from funkwhale_api.users.permissions import HasUserPermission
 
 from . import (
+    activity,
     actors,
     authentication,
-    filters,
-    library,
     models,
-    permissions,
     renderers,
     serializers,
-    tasks,
     utils,
     webfinger,
 )
@@ -34,7 +29,6 @@ class FederationMixin(object):
 
 class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "preferred_username"
-    lookup_value_regex = ".*"
     authentication_classes = [authentication.SignatureAuthentication]
     permission_classes = []
     renderer_classes = [renderers.ActivityPubRenderer]
@@ -43,6 +37,15 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
 
     @detail_route(methods=["get", "post"])
     def inbox(self, request, *args, **kwargs):
+        actor = self.get_object()
+        if request.method.lower() == "post" and request.actor is None:
+            raise exceptions.AuthenticationFailed(
+                "You need a valid signature to send an activity"
+            )
+        if request.method.lower() == "post":
+            activity.receive(
+                activity=request.data, on_behalf_of=request.actor, recipient=actor
+            )
         return response.Response({}, status=200)
 
     @detail_route(methods=["get", "post"])
@@ -143,161 +146,64 @@ class WellKnownViewSet(viewsets.GenericViewSet):
         return serializers.ActorWebfingerSerializer(actor).data
 
 
-class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
-    authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = [permissions.LibraryFollower]
-    renderer_classes = [renderers.ActivityPubRenderer]
+def has_library_access(request, library):
+    if library.privacy_level == "everyone":
+        return True
+    if request.user.is_authenticated and request.user.is_superuser:
+        return True
 
-    def list(self, request, *args, **kwargs):
+    try:
+        actor = request.actor
+    except AttributeError:
+        return False
+
+    return library.received_follows.filter(actor=actor, approved=True).exists()
+
+
+class MusicLibraryViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    serializer_class = serializers.PaginatedCollectionSerializer
+    queryset = music_models.Library.objects.all().select_related("actor")
+    lookup_field = "uuid"
+
+    def retrieve(self, request, *args, **kwargs):
+        lb = self.get_object()
+
+        conf = {
+            "id": lb.get_federation_id(),
+            "actor": lb.actor,
+            "name": lb.name,
+            "summary": lb.description,
+            "items": lb.files.order_by("-creation_date"),
+            "item_serializer": serializers.AudioSerializer,
+        }
         page = request.GET.get("page")
-        library = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        qs = (
-            music_models.TrackFile.objects.order_by("-creation_date")
-            .select_related("track__artist", "track__album__artist")
-            .filter(library_track__isnull=True)
-        )
         if page is None:
-            conf = {
-                "id": utils.full_url(reverse("federation:music:files-list")),
-                "page_size": preferences.get("federation__collection_page_size"),
-                "items": qs,
-                "item_serializer": serializers.AudioSerializer,
-                "actor": library,
-            }
-            serializer = serializers.PaginatedCollectionSerializer(conf)
+            serializer = serializers.LibrarySerializer(lb)
             data = serializer.data
         else:
+            # if actor is requesting a specific page, we ensure library is public
+            # or readable by the actor
+            if not has_library_access(request, lb):
+                raise exceptions.AuthenticationFailed(
+                    "You do not have access to this library"
+                )
             try:
                 page_number = int(page)
             except Exception:
                 return response.Response({"page": ["Invalid page number"]}, status=400)
-            p = paginator.Paginator(
-                qs, preferences.get("federation__collection_page_size")
-            )
+            conf["page_size"] = preferences.get("federation__collection_page_size")
+            p = paginator.Paginator(conf["items"], conf["page_size"])
             try:
                 page = p.page(page_number)
-                conf = {
-                    "id": utils.full_url(reverse("federation:music:files-list")),
-                    "page": page,
-                    "item_serializer": serializers.AudioSerializer,
-                    "actor": library,
-                }
+                conf["page"] = page
                 serializer = serializers.CollectionPageSerializer(conf)
                 data = serializer.data
             except paginator.EmptyPage:
                 return response.Response(status=404)
 
         return response.Response(data)
-
-
-class LibraryViewSet(
-    mixins.RetrieveModelMixin,
-    mixins.UpdateModelMixin,
-    mixins.ListModelMixin,
-    viewsets.GenericViewSet,
-):
-    permission_classes = (HasUserPermission,)
-    required_permissions = ["federation"]
-    queryset = models.Library.objects.all().select_related("actor", "follow")
-    lookup_field = "uuid"
-    filter_class = filters.LibraryFilter
-    serializer_class = serializers.APILibrarySerializer
-    ordering_fields = (
-        "id",
-        "creation_date",
-        "fetched_date",
-        "actor__domain",
-        "tracks_count",
-    )
-
-    @list_route(methods=["get"])
-    def fetch(self, request, *args, **kwargs):
-        account = request.GET.get("account")
-        if not account:
-            return response.Response({"account": "This field is mandatory"}, status=400)
-
-        data = library.scan_from_account_name(account)
-        return response.Response(data)
-
-    @detail_route(methods=["post"])
-    def scan(self, request, *args, **kwargs):
-        library = self.get_object()
-        serializer = serializers.APILibraryScanSerializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        result = tasks.scan_library.delay(
-            library_id=library.pk, until=serializer.validated_data.get("until")
-        )
-        return response.Response({"task": result.id})
-
-    @list_route(methods=["get"])
-    def following(self, request, *args, **kwargs):
-        library_actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        queryset = (
-            models.Follow.objects.filter(actor=library_actor)
-            .select_related("actor", "target")
-            .order_by("-creation_date")
-        )
-        filterset = filters.FollowFilter(request.GET, queryset=queryset)
-        final_qs = filterset.qs
-        serializer = serializers.APIFollowSerializer(final_qs, many=True)
-        data = {"results": serializer.data, "count": len(final_qs)}
-        return response.Response(data)
-
-    @list_route(methods=["get", "patch"])
-    def followers(self, request, *args, **kwargs):
-        if request.method.lower() == "patch":
-            serializer = serializers.APILibraryFollowUpdateSerializer(data=request.data)
-            serializer.is_valid(raise_exception=True)
-            follow = serializer.save()
-            return response.Response(serializers.APIFollowSerializer(follow).data)
-
-        library_actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        queryset = (
-            models.Follow.objects.filter(target=library_actor)
-            .select_related("actor", "target")
-            .order_by("-creation_date")
-        )
-        filterset = filters.FollowFilter(request.GET, queryset=queryset)
-        final_qs = filterset.qs
-        serializer = serializers.APIFollowSerializer(final_qs, many=True)
-        data = {"results": serializer.data, "count": len(final_qs)}
-        return response.Response(data)
-
-    @transaction.atomic
-    def create(self, request, *args, **kwargs):
-        serializer = serializers.APILibraryCreateSerializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        serializer.save()
-        return response.Response(serializer.data, status=201)
-
-
-class LibraryTrackViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
-    permission_classes = (HasUserPermission,)
-    required_permissions = ["federation"]
-    queryset = (
-        models.LibraryTrack.objects.all()
-        .select_related("library__actor", "library__follow", "local_track_file")
-        .prefetch_related("import_jobs")
-    )
-    filter_class = filters.LibraryTrackFilter
-    serializer_class = serializers.APILibraryTrackSerializer
-    ordering_fields = (
-        "id",
-        "artist_name",
-        "title",
-        "album_title",
-        "creation_date",
-        "modification_date",
-        "fetched_date",
-        "published_date",
-    )
-
-    @list_route(methods=["post"])
-    def action(self, request, *args, **kwargs):
-        queryset = models.LibraryTrack.objects.filter(local_track_file__isnull=True)
-        serializer = serializers.LibraryTrackActionSerializer(
-            request.data, queryset=queryset, context={"submitted_by": request.user}
-        )
-        serializer.is_valid(raise_exception=True)
-        result = serializer.save()
-        return response.Response(result, status=200)

api/funkwhale_api/history/views.py

@@ -1,9 +1,12 @@
 from rest_framework import mixins, viewsets
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 
+from django.db.models import Prefetch
+
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
-
+from funkwhale_api.music.models import Track
+from funkwhale_api.music import utils as music_utils
 from . import models, serializers
 
 
@@ -15,11 +18,7 @@ class ListeningViewSet(
 ):
 
     serializer_class = serializers.ListeningSerializer
-    queryset = (
-        models.Listening.objects.all()
-        .select_related("track__artist", "track__album__artist", "user")
-        .prefetch_related("track__files")
-    )
+    queryset = models.Listening.objects.all().select_related("user")
     permission_classes = [
         permissions.ConditionalAuthentication,
         permissions.OwnerPermission,
@@ -39,9 +38,13 @@ class ListeningViewSet(
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        return queryset.filter(
+        queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
+        tracks = Track.objects.annotate_playable_by_actor(
+            music_utils.get_actor_from_request(self.request)
+        ).select_related("artist", "album__artist")
+        return queryset.prefetch_related(Prefetch("track", queryset=tracks))
 
     def get_serializer_context(self):
         context = super().get_serializer_context()

api/funkwhale_api/manage/filters.py

@@ -18,7 +18,7 @@ class ManageTrackFileFilterSet(filters.FilterSet):
 
     class Meta:
         model = music_models.TrackFile
-        fields = ["q", "track__album", "track__artist", "track", "library_track"]
+        fields = ["q", "track__album", "track__artist", "track"]
 
 
 class ManageUserFilterSet(filters.FilterSet):

api/funkwhale_api/manage/serializers.py

@@ -59,7 +59,6 @@ class ManageTrackFileSerializer(serializers.ModelSerializer):
             "bitrate",
             "size",
             "path",
-            "library_track",
         )
 
 

api/funkwhale_api/manage/views.py

@@ -15,7 +15,7 @@ class ManageTrackFileViewSet(
 ):
     queryset = (
         music_models.TrackFile.objects.all()
-        .select_related("track__artist", "track__album__artist", "library_track")
+        .select_related("track__artist", "track__album__artist")
         .order_by("-id")
     )
     serializer_class = serializers.ManageTrackFileSerializer

api/funkwhale_api/music/admin.py

@@ -65,6 +65,7 @@ class TrackFileAdmin(admin.ModelAdmin):
         "mimetype",
         "size",
         "bitrate",
+        "import_status",
     ]
     list_select_related = ["track"]
     search_fields = [
@@ -74,4 +75,12 @@ class TrackFileAdmin(admin.ModelAdmin):
         "track__album__title",
         "track__artist__name",
     ]
-    list_filter = ["mimetype"]
+    list_filter = ["mimetype", "import_status", "library__privacy_level"]
+
+
+@admin.register(models.Library)
+class LibraryAdmin(admin.ModelAdmin):
+    list_display = ["id", "name", "actor", "uuid", "privacy_level", "creation_date"]
+    list_select_related = True
+    search_fields = ["actor__username", "name", "description"]
+    list_filter = ["privacy_level"]

api/funkwhale_api/music/factories.py

@@ -3,8 +3,8 @@ import os
 import factory
 
 from funkwhale_api.factories import ManyToManyFromList, registry
-from funkwhale_api.federation.factories import LibraryTrackFactory
-from funkwhale_api.users.factories import UserFactory
+from funkwhale_api.federation import factories as federation_factories
+
 
 SAMPLES_PATH = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
@@ -51,6 +51,7 @@ class TrackFactory(factory.django.DjangoModelFactory):
 @registry.register
 class TrackFileFactory(factory.django.DjangoModelFactory):
     track = factory.SubFactory(TrackFactory)
+    library = factory.SubFactory(federation_factories.MusicLibraryFactory)
     audio_file = factory.django.FileField(
         from_path=os.path.join(SAMPLES_PATH, "test.ogg")
     )
@@ -58,67 +59,13 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
     bitrate = None
     size = None
     duration = None
+    mimetype = "audio/ogg"
 
     class Meta:
         model = "music.TrackFile"
 
     class Params:
         in_place = factory.Trait(audio_file=None)
-        federation = factory.Trait(
-            audio_file=None,
-            library_track=factory.SubFactory(LibraryTrackFactory),
-            mimetype=factory.LazyAttribute(lambda o: o.library_track.audio_mimetype),
-            source=factory.LazyAttribute(lambda o: o.library_track.audio_url),
-        )
-
-
-@registry.register
-class ImportBatchFactory(factory.django.DjangoModelFactory):
-    submitted_by = factory.SubFactory(UserFactory)
-
-    class Meta:
-        model = "music.ImportBatch"
-
-    class Params:
-        federation = factory.Trait(submitted_by=None, source="federation")
-        finished = factory.Trait(status="finished")
-
-
-@registry.register
-class ImportJobFactory(factory.django.DjangoModelFactory):
-    batch = factory.SubFactory(ImportBatchFactory)
-    source = factory.Faker("url")
-    mbid = factory.Faker("uuid4")
-    replace_if_duplicate = False
-
-    class Meta:
-        model = "music.ImportJob"
-
-    class Params:
-        federation = factory.Trait(
-            mbid=None,
-            library_track=factory.SubFactory(LibraryTrackFactory),
-            batch=factory.SubFactory(ImportBatchFactory, federation=True),
-        )
-        finished = factory.Trait(
-            status="finished", track_file=factory.SubFactory(TrackFileFactory)
-        )
-        in_place = factory.Trait(status="finished", audio_file=None)
-        with_audio_file = factory.Trait(
-            status="finished",
-            audio_file=factory.django.FileField(
-                from_path=os.path.join(SAMPLES_PATH, "test.ogg")
-            ),
-        )
-
-
-@registry.register(name="music.FileImportJob")
-class FileImportJobFactory(ImportJobFactory):
-    source = "file://"
-    mbid = None
-    audio_file = factory.django.FileField(
-        from_path=os.path.join(SAMPLES_PATH, "test.ogg")
-    )
 
 
 @registry.register

api/funkwhale_api/music/filters.py

@@ -1,85 +1,97 @@
-from django.db.models import Count
 from django_filters import rest_framework as filters
 
 from funkwhale_api.common import fields
+from funkwhale_api.common import search
 
 from . import models
+from . import utils
 
 
 class ArtistFilter(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["name"])
-    listenable = filters.BooleanFilter(name="_", method="filter_listenable")
+    playable = filters.BooleanFilter(name="_", method="filter_playable")
 
     class Meta:
         model = models.Artist
         fields = {
             "name": ["exact", "iexact", "startswith", "icontains"],
-            "listenable": "exact",
+            "playable": "exact",
         }
 
-    def filter_listenable(self, queryset, name, value):
-        queryset = queryset.annotate(files_count=Count("albums__tracks__files"))
-        if value:
-            return queryset.filter(files_count__gt=0)
-        else:
-            return queryset.filter(files_count=0)
+    def filter_playable(self, queryset, name, value):
+        actor = utils.get_actor_from_request(self.request)
+        return queryset.playable_by(actor, value)
 
 
 class TrackFilter(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["title", "album__title", "artist__name"])
-    listenable = filters.BooleanFilter(name="_", method="filter_listenable")
+    playable = filters.BooleanFilter(name="_", method="filter_playable")
 
     class Meta:
         model = models.Track
         fields = {
             "title": ["exact", "iexact", "startswith", "icontains"],
-            "listenable": ["exact"],
+            "playable": ["exact"],
             "artist": ["exact"],
             "album": ["exact"],
         }
 
-    def filter_listenable(self, queryset, name, value):
-        queryset = queryset.annotate(files_count=Count("files"))
-        if value:
-            return queryset.filter(files_count__gt=0)
-        else:
-            return queryset.filter(files_count=0)
+    def filter_playable(self, queryset, name, value):
+        actor = utils.get_actor_from_request(self.request)
+        return queryset.playable_by(actor, value)
 
 
-class ImportBatchFilter(filters.FilterSet):
-    q = fields.SearchFilter(search_fields=["submitted_by__username", "source"])
+class TrackFileFilter(filters.FilterSet):
+    library = filters.CharFilter("library__uuid")
+    track = filters.UUIDFilter("track__uuid")
+    track_artist = filters.UUIDFilter("track__artist__uuid")
+    album_artist = filters.UUIDFilter("track__album__artist__uuid")
+    library = filters.UUIDFilter("library__uuid")
+    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={
+                "track_artist": {"to": "track__artist__name"},
+                "album_artist": {"to": "track__album__artist__name"},
+                "album": {"to": "track__album__title"},
+                "title": {"to": "track__title"},
+            },
+            filter_fields={
+                "artist": {"to": "track__artist__name__iexact"},
+                "mimetype": {"to": "mimetype"},
+                "album": {"to": "track__album__title__iexact"},
+                "title": {"to": "track__title__iexact"},
+                "status": {"to": "import_status"},
+            },
+        )
+    )
 
     class Meta:
-        model = models.ImportBatch
-        fields = {"status": ["exact"], "source": ["exact"], "submitted_by": ["exact"]}
+        model = models.TrackFile
+        fields = [
+            "playable",
+            "import_status",
+            "mimetype",
+            "track",
+            "track_artist",
+            "album_artist",
+            "library",
+            "import_reference",
+        ]
 
-
-class ImportJobFilter(filters.FilterSet):
-    q = fields.SearchFilter(search_fields=["batch__submitted_by__username", "source"])
-
-    class Meta:
-        model = models.ImportJob
-        fields = {
-            "batch": ["exact"],
-            "batch__status": ["exact"],
-            "batch__source": ["exact"],
-            "batch__submitted_by": ["exact"],
-            "status": ["exact"],
-            "source": ["exact"],
-        }
+    def filter_playable(self, queryset, name, value):
+        actor = utils.get_actor_from_request(self.request)
+        return queryset.playable_by(actor, value)
 
 
 class AlbumFilter(filters.FilterSet):
-    listenable = filters.BooleanFilter(name="_", method="filter_listenable")
+    playable = filters.BooleanFilter(name="_", method="filter_playable")
     q = fields.SearchFilter(search_fields=["title", "artist__name" "source"])
 
     class Meta:
         model = models.Album
-        fields = ["listenable", "q", "artist"]
+        fields = ["playable", "q", "artist"]
 
-    def filter_listenable(self, queryset, name, value):
-        queryset = queryset.annotate(files_count=Count("tracks__files"))
-        if value:
-            return queryset.filter(files_count__gt=0)
-        else:
-            return queryset.filter(files_count=0)
+    def filter_playable(self, queryset, name, value):
+        actor = utils.get_actor_from_request(self.request)
+        return queryset.playable_by(actor, value)

api/funkwhale_api/music/migrations/0028_importjob_replace_if_duplicate.py

@@ -5,14 +5,12 @@ from django.db import migrations, models
 
 class Migration(migrations.Migration):
 
-    dependencies = [
-        ('music', '0027_auto_20180515_1808'),
-    ]
+    dependencies = [("music", "0027_auto_20180515_1808")]
 
     operations = [
         migrations.AddField(
-            model_name='importjob',
-            name='replace_if_duplicate',
+            model_name="importjob",
+            name="replace_if_duplicate",
             field=models.BooleanField(default=False),
-        ),
+        )
     ]

api/funkwhale_api/music/migrations/0029_auto_20180807_1748.py

@@ -0,0 +1,109 @@
+# Generated by Django 2.0.7 on 2018-08-07 17:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("federation", "0007_auto_20180807_1748"),
+        ("music", "0028_importjob_replace_if_duplicate"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Library",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("fid", models.URLField(db_index=True, max_length=500, unique=True)),
+                ("url", models.URLField(blank=True, max_length=500, null=True)),
+                (
+                    "uuid",
+                    models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+                ),
+                ("followers_url", models.URLField(max_length=500)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("name", models.CharField(max_length=100)),
+                (
+                    "description",
+                    models.TextField(blank=True, max_length=5000, null=True),
+                ),
+                (
+                    "privacy_level",
+                    models.CharField(
+                        choices=[
+                            ("me", "Only me"),
+                            ("instance", "Everyone on my instance, and my followers"),
+                            (
+                                "everyone",
+                                "Everyone, including people on other instances",
+                            ),
+                        ],
+                        default="me",
+                        max_length=25,
+                    ),
+                ),
+                (
+                    "actor",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="libraries",
+                        to="federation.Actor",
+                    ),
+                ),
+            ],
+            options={"abstract": False},
+        ),
+        migrations.AddField(
+            model_name="importjob",
+            name="audio_file_size",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="importbatch",
+            name="import_request",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="import_batches",
+                to="requests.ImportRequest",
+            ),
+        ),
+        migrations.AddField(
+            model_name="importbatch",
+            name="library",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="import_batches",
+                to="music.Library",
+            ),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="library",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="files",
+                to="music.Library",
+            ),
+        ),
+    ]

api/funkwhale_api/music/migrations/0030_auto_20180825_1411.py

@@ -0,0 +1,152 @@
+# Generated by Django 2.0.8 on 2018-08-25 14:11
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.music.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("federation", "0009_auto_20180822_1956"),
+        ("music", "0029_auto_20180807_1748"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="LibraryScan",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("total_files", models.PositiveIntegerField(default=0)),
+                ("processed_files", models.PositiveIntegerField(default=0)),
+                ("errored_files", models.PositiveIntegerField(default=0)),
+                ("status", models.CharField(default="pending", max_length=25)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("modification_date", models.DateTimeField(blank=True, null=True)),
+                (
+                    "actor",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="federation.Actor",
+                    ),
+                ),
+            ],
+        ),
+        migrations.RemoveField(model_name="trackfile", name="library_track"),
+        migrations.AddField(
+            model_name="library",
+            name="files_count",
+            field=models.PositiveIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="fid",
+            field=models.URLField(blank=True, max_length=500, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="import_date",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="import_details",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.music.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=50000,
+            ),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="import_metadata",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.music.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=50000,
+            ),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="import_reference",
+            field=models.CharField(
+                default=funkwhale_api.music.models.get_import_reference, max_length=50
+            ),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="import_status",
+            field=models.CharField(
+                choices=[
+                    ("pending", "Pending"),
+                    ("finished", "Finished"),
+                    ("errored", "Errored"),
+                    ("skipped", "Skipped"),
+                ],
+                default="pending",
+                max_length=25,
+            ),
+        ),
+        migrations.AddField(
+            model_name="trackfile",
+            name="metadata",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.music.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=50000,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="album",
+            name="release_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="trackfile",
+            name="audio_file",
+            field=models.FileField(
+                max_length=255, upload_to=funkwhale_api.music.models.get_file_path
+            ),
+        ),
+        migrations.AlterField(
+            model_name="trackfile",
+            name="source",
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AlterField(
+            model_name="trackfile",
+            name="track",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="files",
+                to="music.Track",
+            ),
+        ),
+        migrations.AddField(
+            model_name="libraryscan",
+            name="library",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="scans",
+                to="music.Library",
+            ),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -1,13 +1,14 @@
+import datetime
 import os
-import shutil
 import tempfile
 import uuid
 
 import markdown
 import pendulum
 from django.conf import settings
-from django.core.files import File
+from django.contrib.postgres.fields import JSONField
 from django.core.files.base import ContentFile
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@@ -18,12 +19,18 @@ from taggit.managers import TaggableManager
 from versatileimagefield.fields import VersatileImageField
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
-from funkwhale_api import downloader, musicbrainz
+from funkwhale_api import musicbrainz
+from funkwhale_api.common import fields
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import utils as federation_utils
-
 from . import importers, metadata, utils
 
 
+def empty_dict():
+    return {}
+
+
 class APIModelMixin(models.Model):
     mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
     uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
@@ -89,6 +96,23 @@ class ArtistQuerySet(models.QuerySet):
             models.Prefetch("albums", queryset=Album.objects.with_tracks_count())
         )
 
+    def annotate_playable_by_actor(self, actor):
+        tracks = (
+            Track.objects.playable_by(actor)
+            .filter(artist=models.OuterRef("id"))
+            .order_by("id")
+            .values("id")[:1]
+        )
+        subquery = models.Subquery(tracks)
+        return self.annotate(is_playable_by_actor=subquery)
+
+    def playable_by(self, actor, include=True):
+        tracks = Track.objects.playable_by(actor, include)
+        if include:
+            return self.filter(tracks__in=tracks)
+        else:
+            return self.exclude(tracks__in=tracks)
+
 
 class Artist(APIModelMixin):
     name = models.CharField(max_length=255)
@@ -140,6 +164,23 @@ class AlbumQuerySet(models.QuerySet):
     def with_tracks_count(self):
         return self.annotate(_tracks_count=models.Count("tracks"))
 
+    def annotate_playable_by_actor(self, actor):
+        tracks = (
+            Track.objects.playable_by(actor)
+            .filter(album=models.OuterRef("id"))
+            .order_by("id")
+            .values("id")[:1]
+        )
+        subquery = models.Subquery(tracks)
+        return self.annotate(is_playable_by_actor=subquery)
+
+    def playable_by(self, actor, include=True):
+        tracks = Track.objects.playable_by(actor, include)
+        if include:
+            return self.filter(tracks__in=tracks)
+        else:
+            return self.exclude(tracks__in=tracks)
+
 
 class Album(APIModelMixin):
     title = models.CharField(max_length=255)
@@ -287,11 +328,24 @@ class Lyrics(models.Model):
 
 class TrackQuerySet(models.QuerySet):
     def for_nested_serialization(self):
-        return (
-            self.select_related()
-            .select_related("album__artist", "artist")
-            .prefetch_related("files")
+        return self.select_related().select_related("album__artist", "artist")
+
+    def annotate_playable_by_actor(self, actor):
+        files = (
+            TrackFile.objects.playable_by(actor)
+            .filter(track=models.OuterRef("id"))
+            .order_by("id")
+            .values("id")[:1]
         )
+        subquery = models.Subquery(files)
+        return self.annotate(is_playable_by_actor=subquery)
+
+    def playable_by(self, actor, include=True):
+        files = TrackFile.objects.playable_by(actor, include)
+        if include:
+            return self.filter(files__in=files)
+        else:
+            return self.exclude(files__in=files)
 
 
 def get_artist(release_list):
@@ -423,12 +477,65 @@ class Track(APIModelMixin):
             },
         )
 
+    @property
+    def listen_url(self):
+        return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid})
+
+
+class TrackFileQuerySet(models.QuerySet):
+    def playable_by(self, actor, include=True):
+        if actor is None:
+            libraries = Library.objects.filter(privacy_level="everyone")
+
+        else:
+            me_query = models.Q(privacy_level="me", actor=actor)
+            instance_query = models.Q(
+                privacy_level="instance", actor__domain=actor.domain
+            )
+            libraries = Library.objects.filter(
+                me_query | instance_query | models.Q(privacy_level="everyone")
+            )
+        if include:
+            return self.filter(library__in=libraries)
+        return self.exclude(library__in=libraries)
+
+    def local(self, include=True):
+        return self.exclude(library__actor__user__isnull=include)
+
+
+TRACK_FILE_IMPORT_STATUS_CHOICES = (
+    ("pending", "Pending"),
+    ("finished", "Finished"),
+    ("errored", "Errored"),
+    ("skipped", "Skipped"),
+)
+
+
+def get_file_path(instance, filename):
+    if instance.library.actor.is_local:
+        return common_utils.ChunkedPath("tracks")(instance, filename)
+    else:
+        # we cache remote tracks in a different directory
+        return common_utils.ChunkedPath("federation_cache/tracks")(instance, filename)
+
+
+def get_import_reference():
+    return str(uuid.uuid4())
+
 
 class TrackFile(models.Model):
+    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
     uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
-    track = models.ForeignKey(Track, related_name="files", on_delete=models.CASCADE)
-    audio_file = models.FileField(upload_to="tracks/%Y/%m/%d", max_length=255)
-    source = models.URLField(null=True, blank=True, max_length=500)
+    track = models.ForeignKey(
+        Track, related_name="files", on_delete=models.CASCADE, null=True, blank=True
+    )
+    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
+    source = models.CharField(
+        # URL validators are not flexible enough for our file:// and upload:// schemes
+        null=True,
+        blank=True,
+        max_length=500,
+    )
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(auto_now=True)
     accessed_date = models.DateTimeField(null=True, blank=True)
@@ -437,35 +544,69 @@ class TrackFile(models.Model):
     bitrate = models.IntegerField(null=True, blank=True)
     acoustid_track_id = models.UUIDField(null=True, blank=True)
     mimetype = models.CharField(null=True, blank=True, max_length=200)
-
-    library_track = models.OneToOneField(
-        "federation.LibraryTrack",
-        related_name="local_track_file",
-        on_delete=models.CASCADE,
-        null=True,
-        blank=True,
+    library = models.ForeignKey(
+        "library", null=True, blank=True, related_name="files", on_delete=models.CASCADE
     )
 
-    def download_file(self):
-        # import the track file, since there is not any
-        # we create a tmp dir for the download
-        tmp_dir = tempfile.mkdtemp()
-        data = downloader.download(self.source, target_directory=tmp_dir)
-        self.duration = data.get("duration", None)
-        self.audio_file.save(
-            os.path.basename(data["audio_file_path"]),
-            File(open(data["audio_file_path"], "rb")),
+    # metadata from federation
+    metadata = JSONField(
+        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
+    )
+    import_date = models.DateTimeField(null=True, blank=True)
+    # optionnal metadata provided during import
+    import_metadata = JSONField(
+        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
+    )
+    # status / error details for the import
+    import_status = models.CharField(
+        default="pending", choices=TRACK_FILE_IMPORT_STATUS_CHOICES, max_length=25
+    )
+    # a short reference provided by the client to group multiple files
+    # in the same import
+    import_reference = models.CharField(max_length=50, default=get_import_reference)
+
+    # optionnal metadata about import results (error messages, etc.)
+    import_details = JSONField(
+        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
+    )
+
+    objects = TrackFileQuerySet.as_manager()
+
+    def download_audio_from_remote(self, user):
+        from funkwhale_api.common import session
+        from funkwhale_api.federation import signing
+
+        if user.is_authenticated and user.actor:
+            auth = signing.get_auth(user.actor.private_key, user.actor.private_key_id)
+        else:
+            auth = None
+
+        remote_response = session.get_session().get(
+            self.source,
+            auth=auth,
+            stream=True,
+            timeout=20,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
         )
-        shutil.rmtree(tmp_dir)
-        return self.audio_file
+        with remote_response as r:
+            remote_response.raise_for_status()
+            extension = utils.get_ext_from_type(self.mimetype)
+            title = " - ".join(
+                [self.track.title, self.track.album.title, self.track.artist.name]
+            )
+            filename = "{}.{}".format(title, extension)
+            tmp_file = tempfile.TemporaryFile()
+            for chunk in r.iter_content(chunk_size=512):
+                tmp_file.write(chunk)
+            self.audio_file.save(filename, tmp_file, save=False)
+            self.save(update_fields=["audio_file"])
+
+    def get_federation_id(self):
+        if self.fid:
+            return self.fid
 
-    def get_federation_url(self):
         return federation_utils.full_url("/federation/music/file/{}".format(self.uuid))
 
-    @property
-    def path(self):
-        return reverse("api:v1:trackfiles-serve", kwargs={"pk": self.pk})
-
     @property
     def filename(self):
         return "{}.{}".format(self.track.full_name, self.extension)
@@ -483,37 +624,30 @@ class TrackFile(models.Model):
         if self.source.startswith("file://"):
             return os.path.getsize(self.source.replace("file://", "", 1))
 
-        if self.library_track and self.library_track.audio_file:
-            return self.library_track.audio_file.size
-
     def get_audio_file(self):
         if self.audio_file:
             return self.audio_file.open()
         if self.source.startswith("file://"):
             return open(self.source.replace("file://", "", 1), "rb")
-        if self.library_track and self.library_track.audio_file:
-            return self.library_track.audio_file.open()
 
-    def set_audio_data(self):
+    def get_audio_data(self):
         audio_file = self.get_audio_file()
-        if audio_file:
-            with audio_file as f:
-                audio_data = utils.get_audio_file_data(f)
-            if not audio_data:
-                return
-            self.duration = int(audio_data["length"])
-            self.bitrate = audio_data["bitrate"]
-            self.size = self.get_file_size()
-        else:
-            lt = self.library_track
-            if lt:
-                self.duration = lt.get_metadata("length")
-                self.size = lt.get_metadata("size")
-                self.bitrate = lt.get_metadata("bitrate")
+        if not audio_file:
+            return
+        audio_data = utils.get_audio_file_data(audio_file)
+        if not audio_data:
+            return
+        return {
+            "duration": int(audio_data["length"]),
+            "bitrate": audio_data["bitrate"],
+            "size": self.get_file_size(),
+        }
 
     def save(self, **kwargs):
         if not self.mimetype and self.audio_file:
             self.mimetype = utils.guess_mimetype(self.audio_file)
+        if not self.size and self.audio_file:
+            self.size = self.audio_file.size
         return super().save(**kwargs)
 
     def get_metadata(self):
@@ -522,6 +656,10 @@ class TrackFile(models.Model):
             return
         return metadata.Metadata(audio_file)
 
+    @property
+    def listen_url(self):
+        return self.track.listen_url + "?file={}".format(self.uuid)
+
 
 IMPORT_STATUS_CHOICES = (
     ("pending", "Pending"),
@@ -559,6 +697,13 @@ class ImportBatch(models.Model):
         blank=True,
         on_delete=models.SET_NULL,
     )
+    library = models.ForeignKey(
+        "Library",
+        related_name="import_batches",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+    )
 
     class Meta:
         ordering = ["-creation_date"]
@@ -577,7 +722,7 @@ class ImportBatch(models.Model):
 
             tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)
 
-    def get_federation_url(self):
+    def get_federation_id(self):
         return federation_utils.full_url(
             "/federation/music/import/batch/{}".format(self.uuid)
         )
@@ -609,10 +754,100 @@ class ImportJob(models.Model):
         null=True,
         blank=True,
     )
+    audio_file_size = models.IntegerField(null=True, blank=True)
 
     class Meta:
         ordering = ("id",)
 
+    def save(self, **kwargs):
+        if self.audio_file and not self.audio_file_size:
+            self.audio_file_size = self.audio_file.size
+        return super().save(**kwargs)
+
+
+LIBRARY_PRIVACY_LEVEL_CHOICES = [
+    (k, l) for k, l in fields.PRIVACY_LEVEL_CHOICES if k != "followers"
+]
+
+
+class LibraryQuerySet(models.QuerySet):
+    def with_follows(self, actor):
+        return self.prefetch_related(
+            models.Prefetch(
+                "received_follows",
+                queryset=federation_models.LibraryFollow.objects.filter(actor=actor),
+                to_attr="_follows",
+            )
+        )
+
+
+class Library(federation_models.FederationMixin):
+    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
+    actor = models.ForeignKey(
+        "federation.Actor", related_name="libraries", on_delete=models.CASCADE
+    )
+    followers_url = models.URLField(max_length=500)
+    creation_date = models.DateTimeField(default=timezone.now)
+    name = models.CharField(max_length=100)
+    description = models.TextField(max_length=5000, null=True, blank=True)
+    privacy_level = models.CharField(
+        choices=LIBRARY_PRIVACY_LEVEL_CHOICES, default="me", max_length=25
+    )
+    files_count = models.PositiveIntegerField(default=0)
+    objects = LibraryQuerySet.as_manager()
+
+    def get_federation_id(self):
+        return federation_utils.full_url(
+            reverse("federation:music:libraries-detail", kwargs={"uuid": self.uuid})
+        )
+
+    def save(self, **kwargs):
+        if not self.pk and not self.fid and self.actor.is_local:
+            self.fid = self.get_federation_id()
+            self.followers_url = self.fid + "/followers"
+
+        return super().save(**kwargs)
+
+    def should_autoapprove_follow(self, actor):
+        if self.privacy_level == "everyone":
+            return True
+        if self.privacy_level == "instance" and actor.is_local:
+            return True
+        return False
+
+    def schedule_scan(self):
+        latest_scan = self.scans.order_by("-creation_date").first()
+        delay_between_scans = datetime.timedelta(seconds=3600 * 24)
+        now = timezone.now()
+        if latest_scan and latest_scan.creation_date + delay_between_scans > now:
+            return
+
+        scan = self.scans.create(total_files=self.files_count)
+        from . import tasks
+
+        common_utils.on_commit(tasks.start_library_scan.delay, library_scan_id=scan.pk)
+        return scan
+
+
+SCAN_STATUS = [
+    ("pending", "pending"),
+    ("scanning", "scanning"),
+    ("finished", "finished"),
+]
+
+
+class LibraryScan(models.Model):
+    actor = models.ForeignKey(
+        "federation.Actor", null=True, blank=True, on_delete=models.CASCADE
+    )
+    library = models.ForeignKey(Library, related_name="scans", on_delete=models.CASCADE)
+    total_files = models.PositiveIntegerField(default=0)
+    processed_files = models.PositiveIntegerField(default=0)
+    errored_files = models.PositiveIntegerField(default=0)
+    status = models.CharField(default="pending", max_length=25)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(null=True, blank=True)
+
 
 @receiver(post_save, sender=ImportJob)
 def update_batch_status(sender, instance, **kwargs):

api/funkwhale_api/music/permissions.py

@@ -1,24 +0,0 @@
-
-from rest_framework.permissions import BasePermission
-
-from funkwhale_api.common import preferences
-from funkwhale_api.federation import actors, models
-
-
-class Listen(BasePermission):
-    def has_permission(self, request, view):
-        if not preferences.get("common__api_authentication_required"):
-            return True
-
-        user = getattr(request, "user", None)
-        if user and user.is_authenticated:
-            return True
-
-        actor = getattr(request, "actor", None)
-        if actor is None:
-            return False
-
-        library = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-        return models.Follow.objects.filter(
-            target=library, actor=actor, approved=True
-        ).exists()

api/funkwhale_api/music/serializers.py

@@ -1,12 +1,13 @@
-from django.db.models import Q
+from django.db import transaction
 from rest_framework import serializers
 from taggit.models import Tag
 from versatileimagefield.serializers import VersatileImageFieldSerializer
 
 from funkwhale_api.activity import serializers as activity_serializers
-from funkwhale_api.users.serializers import UserBasicSerializer
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.common import utils as common_utils
 
-from . import models, tasks
+from . import filters, models, tasks
 
 
 cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
@@ -15,6 +16,7 @@ cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
 class ArtistAlbumSerializer(serializers.ModelSerializer):
     tracks_count = serializers.SerializerMethodField()
     cover = cover_field
+    is_playable = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Album
@@ -27,11 +29,18 @@ class ArtistAlbumSerializer(serializers.ModelSerializer):
             "cover",
             "creation_date",
             "tracks_count",
+            "is_playable",
         )
 
     def get_tracks_count(self, o):
         return o._tracks_count
 
+    def get_is_playable(self, obj):
+        try:
+            return bool(obj.is_playable_by_actor)
+        except AttributeError:
+            return None
+
 
 class ArtistWithAlbumsSerializer(serializers.ModelSerializer):
     albums = ArtistAlbumSerializer(many=True, read_only=True)
@@ -41,30 +50,6 @@ class ArtistWithAlbumsSerializer(serializers.ModelSerializer):
         fields = ("id", "mbid", "name", "creation_date", "albums")
 
 
-class TrackFileSerializer(serializers.ModelSerializer):
-    path = serializers.SerializerMethodField()
-
-    class Meta:
-        model = models.TrackFile
-        fields = (
-            "id",
-            "path",
-            "source",
-            "filename",
-            "mimetype",
-            "track",
-            "duration",
-            "mimetype",
-            "bitrate",
-            "size",
-        )
-        read_only_fields = ["duration", "mimetype", "bitrate", "size"]
-
-    def get_path(self, o):
-        url = o.path
-        return url
-
-
 class ArtistSimpleSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Artist
@@ -72,8 +57,9 @@ class ArtistSimpleSerializer(serializers.ModelSerializer):
 
 
 class AlbumTrackSerializer(serializers.ModelSerializer):
-    files = TrackFileSerializer(many=True, read_only=True)
     artist = ArtistSimpleSerializer(read_only=True)
+    is_playable = serializers.SerializerMethodField()
+    listen_url = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Track
@@ -84,15 +70,26 @@ class AlbumTrackSerializer(serializers.ModelSerializer):
             "album",
             "artist",
             "creation_date",
-            "files",
             "position",
+            "is_playable",
+            "listen_url",
         )
 
+    def get_is_playable(self, obj):
+        try:
+            return bool(obj.is_playable_by_actor)
+        except AttributeError:
+            return None
+
+    def get_listen_url(self, obj):
+        return obj.listen_url
+
 
 class AlbumSerializer(serializers.ModelSerializer):
     tracks = serializers.SerializerMethodField()
     artist = ArtistSimpleSerializer(read_only=True)
     cover = cover_field
+    is_playable = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Album
@@ -105,6 +102,7 @@ class AlbumSerializer(serializers.ModelSerializer):
             "release_date",
             "cover",
             "creation_date",
+            "is_playable",
         )
 
     def get_tracks(self, o):
@@ -114,6 +112,12 @@ class AlbumSerializer(serializers.ModelSerializer):
         )
         return AlbumTrackSerializer(ordered_tracks, many=True).data
 
+    def get_is_playable(self, obj):
+        try:
+            return any([bool(t.is_playable_by_actor) for t in obj.tracks.all()])
+        except AttributeError:
+            return None
+
 
 class TrackAlbumSerializer(serializers.ModelSerializer):
     artist = ArtistSimpleSerializer(read_only=True)
@@ -133,10 +137,11 @@ class TrackAlbumSerializer(serializers.ModelSerializer):
 
 
 class TrackSerializer(serializers.ModelSerializer):
-    files = TrackFileSerializer(many=True, read_only=True)
     artist = ArtistSimpleSerializer(read_only=True)
     album = TrackAlbumSerializer(read_only=True)
     lyrics = serializers.SerializerMethodField()
+    is_playable = serializers.SerializerMethodField()
+    listen_url = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Track
@@ -147,14 +152,146 @@ class TrackSerializer(serializers.ModelSerializer):
             "album",
             "artist",
             "creation_date",
-            "files",
             "position",
             "lyrics",
+            "is_playable",
+            "listen_url",
         )
 
     def get_lyrics(self, obj):
         return obj.get_lyrics_url()
 
+    def get_listen_url(self, obj):
+        return obj.listen_url
+
+    def get_is_playable(self, obj):
+        try:
+            return bool(obj.is_playable_by_actor)
+        except AttributeError:
+            return None
+
+
+class LibraryForOwnerSerializer(serializers.ModelSerializer):
+    files_count = serializers.SerializerMethodField()
+    size = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.Library
+        fields = [
+            "uuid",
+            "fid",
+            "name",
+            "description",
+            "privacy_level",
+            "files_count",
+            "size",
+            "creation_date",
+        ]
+        read_only_fields = ["fid", "uuid", "creation_date", "actor"]
+
+    def get_files_count(self, o):
+        return getattr(o, "_files_count", o.files_count)
+
+    def get_size(self, o):
+        return getattr(o, "_size", 0)
+
+
+class TrackFileSerializer(serializers.ModelSerializer):
+    track = TrackSerializer(required=False, allow_null=True)
+    library = common_serializers.RelatedField(
+        "uuid",
+        LibraryForOwnerSerializer(),
+        required=True,
+        filters=lambda context: {"actor": context["user"].actor},
+    )
+
+    class Meta:
+        model = models.TrackFile
+        fields = [
+            "uuid",
+            "filename",
+            "creation_date",
+            "mimetype",
+            "track",
+            "library",
+            "duration",
+            "mimetype",
+            "bitrate",
+            "size",
+            "import_date",
+            "import_status",
+        ]
+
+        read_only_fields = [
+            "uuid",
+            "creation_date",
+            "duration",
+            "mimetype",
+            "bitrate",
+            "size",
+            "track",
+            "import_date",
+            "import_status",
+        ]
+
+
+class TrackFileForOwnerSerializer(TrackFileSerializer):
+    class Meta(TrackFileSerializer.Meta):
+        fields = TrackFileSerializer.Meta.fields + [
+            "import_details",
+            "import_metadata",
+            "import_reference",
+            "metadata",
+            "source",
+            "audio_file",
+        ]
+        write_only_fields = ["audio_file"]
+        read_only_fields = TrackFileSerializer.Meta.read_only_fields + [
+            "import_details",
+            "import_metadata",
+            "metadata",
+        ]
+
+    def to_representation(self, obj):
+        r = super().to_representation(obj)
+        if "audio_file" in r:
+            del r["audio_file"]
+        return r
+
+    def validate(self, validated_data):
+        if "audio_file" in validated_data:
+            self.validate_upload_quota(validated_data["audio_file"])
+
+        return super().validate(validated_data)
+
+    def validate_upload_quota(self, f):
+        quota_status = self.context["user"].get_quota_status()
+        if (f.size / 1000 / 1000) > quota_status["remaining"]:
+            raise serializers.ValidationError("upload_quota_reached")
+
+        return f
+
+
+class TrackFileActionSerializer(common_serializers.ActionSerializer):
+    actions = [
+        common_serializers.Action("delete", allow_all=True),
+        common_serializers.Action("relaunch_import", allow_all=True),
+    ]
+    filterset_class = filters.TrackFileFilter
+    pk_field = "uuid"
+
+    @transaction.atomic
+    def handle_delete(self, objects):
+        return objects.delete()
+
+    @transaction.atomic
+    def handle_relaunch_import(self, objects):
+        qs = objects.exclude(import_status="finished")
+        pks = list(qs.values_list("id", flat=True))
+        qs.update(import_status="pending")
+        for pk in pks:
+            common_utils.on_commit(tasks.import_track_file.delay, track_file_id=pk)
+
 
 class TagSerializer(serializers.ModelSerializer):
     class Meta:
@@ -176,40 +313,6 @@ class LyricsSerializer(serializers.ModelSerializer):
         fields = ("id", "work", "content", "content_rendered")
 
 
-class ImportJobSerializer(serializers.ModelSerializer):
-    track_file = TrackFileSerializer(read_only=True)
-
-    class Meta:
-        model = models.ImportJob
-        fields = ("id", "mbid", "batch", "source", "status", "track_file", "audio_file")
-        read_only_fields = ("status", "track_file")
-
-
-class ImportBatchSerializer(serializers.ModelSerializer):
-    submitted_by = UserBasicSerializer(read_only=True)
-
-    class Meta:
-        model = models.ImportBatch
-        fields = (
-            "id",
-            "submitted_by",
-            "source",
-            "status",
-            "creation_date",
-            "import_request",
-        )
-        read_only_fields = ("creation_date", "submitted_by", "source")
-
-    def to_representation(self, instance):
-        repr = super().to_representation(instance)
-        try:
-            repr["job_count"] = instance.job_count
-        except AttributeError:
-            # Queryset was not annotated
-            pass
-        return repr
-
-
 class TrackActivitySerializer(activity_serializers.ModelSerializer):
     type = serializers.SerializerMethodField()
     name = serializers.CharField(source="title")
@@ -222,33 +325,3 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer):
 
     def get_type(self, obj):
         return "Audio"
-
-
-class ImportJobRunSerializer(serializers.Serializer):
-    jobs = serializers.PrimaryKeyRelatedField(
-        many=True,
-        queryset=models.ImportJob.objects.filter(status__in=["pending", "errored"]),
-    )
-    batches = serializers.PrimaryKeyRelatedField(
-        many=True, queryset=models.ImportBatch.objects.all()
-    )
-
-    def validate(self, validated_data):
-        jobs = validated_data["jobs"]
-        batches_ids = [b.pk for b in validated_data["batches"]]
-        query = Q(batch__pk__in=batches_ids)
-        query |= Q(pk__in=[j.id for j in jobs])
-        queryset = (
-            models.ImportJob.objects.filter(query)
-            .filter(status__in=["pending", "errored"])
-            .distinct()
-        )
-        validated_data["_jobs"] = queryset
-        return validated_data
-
-    def create(self, validated_data):
-        ids = validated_data["_jobs"].values_list("id", flat=True)
-        validated_data["_jobs"].update(status="pending")
-        for id in ids:
-            tasks.import_job_run.delay(import_job_id=id)
-        return {"jobs": list(ids)}

api/funkwhale_api/music/signals.py

@@ -0,0 +1,5 @@
+import django.dispatch
+
+track_file_import_status_updated = django.dispatch.Signal(
+    providing_args=["old_status", "new_status", "track_file"]
+)

api/funkwhale_api/music/tasks.py

@@ -1,20 +1,27 @@
 import logging
 import os
 
-from django.conf import settings
-from django.core.files.base import ContentFile
-from musicbrainzngs import ResponseError
+from django.utils import timezone
+from django.db import transaction
+from django.db.models import F
+from django.dispatch import receiver
 
+from musicbrainzngs import ResponseError
+from requests.exceptions import RequestException
+
+from funkwhale_api.common import channels
 from funkwhale_api.common import preferences
 from funkwhale_api.federation import activity, actors
-from funkwhale_api.federation import serializers as federation_serializers
+from funkwhale_api.federation import library as lb
+from funkwhale_api.federation import library as federation_serializers
 from funkwhale_api.providers.acoustid import get_acoustid_client
-from funkwhale_api.providers.audiofile import tasks as audiofile_tasks
 from funkwhale_api.taskapp import celery
 
 from . import lyrics as lyrics_utils
 from . import models
-from . import utils as music_utils
+from . import metadata
+from . import signals
+from . import serializers
 
 logger = logging.getLogger(__name__)
 
@@ -34,8 +41,7 @@ def set_acoustid_on_track_file(track_file):
         return update(result["id"])
 
 
-def import_track_from_remote(library_track):
-    metadata = library_track.metadata
+def import_track_from_remote(metadata):
     try:
         track_mbid = metadata["recording"]["musicbrainz_id"]
         assert track_mbid  # for null/empty values
@@ -52,7 +58,7 @@ def import_track_from_remote(library_track):
     else:
         album, _ = models.Album.get_or_create_from_api(mbid=album_mbid)
         return models.Track.get_or_create_from_title(
-            library_track.title, artist=album.artist, album=album
+            metadata["title"], artist=album.artist, album=album
         )[0]
 
     try:
@@ -63,130 +69,23 @@ def import_track_from_remote(library_track):
     else:
         artist, _ = models.Artist.get_or_create_from_api(mbid=artist_mbid)
         album, _ = models.Album.get_or_create_from_title(
-            library_track.album_title, artist=artist
+            metadata["album_title"], artist=artist
         )
         return models.Track.get_or_create_from_title(
-            library_track.title, artist=artist, album=album
+            metadata["title"], artist=artist, album=album
         )[0]
 
     # worst case scenario, we have absolutely no way to link to a
     # musicbrainz resource, we rely on the name/titles
-    artist, _ = models.Artist.get_or_create_from_name(library_track.artist_name)
+    artist, _ = models.Artist.get_or_create_from_name(metadata["artist_name"])
     album, _ = models.Album.get_or_create_from_title(
-        library_track.album_title, artist=artist
+        metadata["album_title"], artist=artist
     )
     return models.Track.get_or_create_from_title(
-        library_track.title, artist=artist, album=album
+        metadata["title"], artist=artist, album=album
     )[0]
 
 
-def _do_import(import_job, use_acoustid=False):
-    logger.info("[Import Job %s] starting job", import_job.pk)
-    from_file = bool(import_job.audio_file)
-    mbid = import_job.mbid
-    replace = import_job.replace_if_duplicate
-    acoustid_track_id = None
-    duration = None
-    track = None
-    # use_acoustid = use_acoustid and preferences.get('providers_acoustid__api_key')
-    # Acoustid is not reliable, we disable it for now.
-    use_acoustid = False
-    if not mbid and use_acoustid and from_file:
-        # we try to deduce mbid from acoustid
-        client = get_acoustid_client()
-        match = client.get_best_match(import_job.audio_file.path)
-        if match:
-            duration = match["recordings"][0]["duration"]
-            mbid = match["recordings"][0]["id"]
-            acoustid_track_id = match["id"]
-    if mbid:
-        logger.info(
-            "[Import Job %s] importing track from musicbrainz recording %s",
-            import_job.pk,
-            str(mbid),
-        )
-        track, _ = models.Track.get_or_create_from_api(mbid=mbid)
-    elif import_job.audio_file:
-        logger.info(
-            "[Import Job %s] importing track from uploaded track data at %s",
-            import_job.pk,
-            import_job.audio_file.path,
-        )
-        track = audiofile_tasks.import_track_data_from_path(import_job.audio_file.path)
-    elif import_job.library_track:
-        logger.info(
-            "[Import Job %s] importing track from federated library track %s",
-            import_job.pk,
-            import_job.library_track.pk,
-        )
-        track = import_track_from_remote(import_job.library_track)
-    elif import_job.source.startswith("file://"):
-        tf_path = import_job.source.replace("file://", "", 1)
-        logger.info(
-            "[Import Job %s] importing track from local track data at %s",
-            import_job.pk,
-            tf_path,
-        )
-        track = audiofile_tasks.import_track_data_from_path(tf_path)
-    else:
-        raise ValueError(
-            "Not enough data to process import, "
-            "add a mbid, an audio file or a library track"
-        )
-
-    track_file = None
-    if replace:
-        logger.info("[Import Job %s] deleting existing audio file", import_job.pk)
-        track.files.all().delete()
-    elif track.files.count() > 0:
-        logger.info(
-            "[Import Job %s] skipping, we already have a file for this track",
-            import_job.pk,
-        )
-        if import_job.audio_file:
-            import_job.audio_file.delete()
-        import_job.status = "skipped"
-        import_job.save()
-        return
-
-    track_file = track_file or models.TrackFile(track=track, source=import_job.source)
-    track_file.acoustid_track_id = acoustid_track_id
-    if from_file:
-        track_file.audio_file = ContentFile(import_job.audio_file.read())
-        track_file.audio_file.name = import_job.audio_file.name
-        track_file.duration = duration
-    elif import_job.library_track:
-        track_file.library_track = import_job.library_track
-        track_file.mimetype = import_job.library_track.audio_mimetype
-        if import_job.library_track.library.download_files:
-            raise NotImplementedError()
-        else:
-            # no downloading, we hotlink
-            pass
-    elif not import_job.audio_file and not import_job.source.startswith("file://"):
-        # not an inplace import, and we have a source, so let's download it
-        logger.info("[Import Job %s] downloading audio file from remote", import_job.pk)
-        track_file.download_file()
-    elif not import_job.audio_file and import_job.source.startswith("file://"):
-        # in place import, we set mimetype from extension
-        path, ext = os.path.splitext(import_job.source)
-        track_file.mimetype = music_utils.get_type_from_ext(ext)
-    track_file.set_audio_data()
-    track_file.save()
-    # if no cover is set on track album, we try to update it as well:
-    if not track.album.cover:
-        logger.info("[Import Job %s] retrieving album cover", import_job.pk)
-        update_album_cover(track.album, track_file)
-    import_job.status = "finished"
-    import_job.track_file = track_file
-    if import_job.audio_file:
-        # it's imported on the track, we don't need it anymore
-        import_job.audio_file.delete()
-    import_job.save()
-    logger.info("[Import Job %s] job finished", import_job.pk)
-    return track_file
-
-
 def update_album_cover(album, track_file, replace=False):
     if album.cover and not replace:
         return
@@ -240,37 +139,6 @@ def get_cover_from_fs(dir_path):
                 return {"mimetype": m, "content": c.read()}
 
 
-@celery.app.task(name="ImportJob.run", bind=True)
-@celery.require_instance(
-    models.ImportJob.objects.filter(status__in=["pending", "errored"]), "import_job"
-)
-def import_job_run(self, import_job, use_acoustid=False):
-    def mark_errored(exc):
-        logger.error("[Import Job %s] Error during import: %s", import_job.pk, str(exc))
-        import_job.status = "errored"
-        import_job.save(update_fields=["status"])
-
-    try:
-        tf = _do_import(import_job, use_acoustid=use_acoustid)
-        return tf.pk if tf else None
-    except Exception as exc:
-        if not settings.DEBUG:
-            try:
-                self.retry(exc=exc, countdown=30, max_retries=3)
-            except Exception:
-                mark_errored(exc)
-                raise
-        mark_errored(exc)
-        raise
-
-
-@celery.app.task(name="ImportBatch.run")
-@celery.require_instance(models.ImportBatch, "import_batch")
-def import_batch_run(import_batch):
-    for job_id in import_batch.jobs.order_by("id").values_list("id", flat=True):
-        import_job_run.delay(import_job_id=job_id)
-
-
 @celery.app.task(name="Lyrics.fetch_content")
 @celery.require_instance(models.Lyrics, "lyrics")
 def fetch_content(lyrics):
@@ -301,7 +169,7 @@ def import_batch_notify_followers(import_batch):
     collection = federation_serializers.CollectionSerializer(
         {
             "actor": library_actor,
-            "id": import_batch.get_federation_url(),
+            "id": import_batch.get_federation_id(),
             "items": track_files,
             "item_serializer": federation_serializers.AudioSerializer,
         }
@@ -312,9 +180,266 @@ def import_batch_notify_followers(import_batch):
                 "type": "Create",
                 "id": collection["id"],
                 "object": collection,
-                "actor": library_actor.url,
+                "actor": library_actor.fid,
                 "to": [f.url],
             }
         ).data
 
         activity.deliver(create, on_behalf_of=library_actor, to=[f.url])
+
+
+@celery.app.task(
+    name="music.start_library_scan",
+    retry_backoff=60,
+    max_retries=5,
+    autoretry_for=[RequestException],
+)
+@celery.require_instance(
+    models.LibraryScan.objects.select_related().filter(status="pending"), "library_scan"
+)
+def start_library_scan(library_scan):
+    data = lb.get_library_data(library_scan.library.fid, actor=library_scan.actor)
+    library_scan.modification_date = timezone.now()
+    library_scan.status = "scanning"
+    library_scan.total_files = data["totalItems"]
+    library_scan.save(update_fields=["status", "modification_date", "total_files"])
+    scan_library_page.delay(library_scan_id=library_scan.pk, page_url=data["first"])
+
+
+@celery.app.task(
+    name="music.scan_library_page",
+    retry_backoff=60,
+    max_retries=5,
+    autoretry_for=[RequestException],
+)
+@celery.require_instance(
+    models.LibraryScan.objects.select_related().filter(status="scanning"),
+    "library_scan",
+)
+def scan_library_page(library_scan, page_url):
+    data = lb.get_library_page(library_scan.library, page_url, library_scan.actor)
+    tfs = []
+
+    for item_serializer in data["items"]:
+        tf = item_serializer.save(library=library_scan.library)
+        if tf.import_status == "pending" and not tf.track:
+            # this track is not matched to any musicbrainz or other musical
+            # metadata
+            import_track_file.delay(track_file_id=tf.pk)
+        tfs.append(tf)
+
+    library_scan.processed_files = F("processed_files") + len(tfs)
+    library_scan.modification_date = timezone.now()
+    update_fields = ["modification_date", "processed_files"]
+
+    next_page = data.get("next")
+    fetch_next = next_page and next_page != page_url
+
+    if not fetch_next:
+        update_fields.append("status")
+        library_scan.status = "finished"
+    library_scan.save(update_fields=update_fields)
+
+    if fetch_next:
+        scan_library_page.delay(library_scan_id=library_scan.pk, page_url=next_page)
+
+
+def getter(data, *keys):
+    if not data:
+        return
+    v = data
+    for k in keys:
+        v = v.get(k)
+
+    return v
+
+
+class TrackFileImportError(ValueError):
+    def __init__(self, code):
+        self.code = code
+        super().__init__(code)
+
+
+def fail_import(track_file, error_code):
+    old_status = track_file.import_status
+    track_file.import_status = "errored"
+    track_file.import_details = {"error_code": error_code}
+    track_file.import_date = timezone.now()
+    track_file.save(update_fields=["import_details", "import_status", "import_date"])
+    signals.track_file_import_status_updated.send(
+        old_status=old_status,
+        new_status=track_file.import_status,
+        track_file=track_file,
+        sender=None,
+    )
+
+
+@celery.app.task(name="music.import_track_file")
+@celery.require_instance(
+    models.TrackFile.objects.filter(import_status="pending").select_related(
+        "library__actor__user"
+    ),
+    "track_file",
+)
+def import_track_file(track_file):
+    data = track_file.import_metadata or {}
+    old_status = track_file.import_status
+    try:
+        track = get_track_from_import_metadata(track_file.import_metadata or {})
+        if not track and track_file.audio_file:
+            # easy ways did not work. Now we have to be smart and use
+            # metadata from the file itself if any
+            track = import_track_data_from_file(track_file.audio_file.file, hints=data)
+        if not track and track_file.metadata:
+            # we can try to import using federation metadata
+            track = import_track_from_remote(track_file.metadata)
+    except TrackFileImportError as e:
+        return fail_import(track_file, e.code)
+    except Exception:
+        fail_import(track_file, "unknown_error")
+        raise
+    # under some situations, we want to skip the import (
+    # for instance if the user already owns the files)
+    owned_duplicates = get_owned_duplicates(track_file, track)
+    track_file.track = track
+
+    if owned_duplicates:
+        track_file.import_status = "skipped"
+        track_file.import_details = {
+            "code": "already_imported_in_owned_libraries",
+            "duplicates": list(owned_duplicates),
+        }
+        track_file.import_date = timezone.now()
+        track_file.save(
+            update_fields=["import_details", "import_status", "import_date", "track"]
+        )
+        signals.track_file_import_status_updated.send(
+            old_status=old_status,
+            new_status=track_file.import_status,
+            track_file=track_file,
+            sender=None,
+        )
+        return
+
+    # all is good, let's finalize the import
+    audio_data = track_file.get_audio_data()
+    if audio_data:
+        track_file.duration = audio_data["duration"]
+        track_file.size = audio_data["size"]
+        track_file.bitrate = audio_data["bitrate"]
+    track_file.import_status = "finished"
+    track_file.import_date = timezone.now()
+    track_file.save(
+        update_fields=[
+            "track",
+            "import_status",
+            "import_date",
+            "size",
+            "duration",
+            "bitrate",
+        ]
+    )
+    signals.track_file_import_status_updated.send(
+        old_status=old_status,
+        new_status=track_file.import_status,
+        track_file=track_file,
+        sender=None,
+    )
+
+    if not track.album.cover:
+        update_album_cover(track.album, track_file)
+
+
+def get_track_from_import_metadata(data):
+    track_mbid = getter(data, "track", "mbid")
+    track_uuid = getter(data, "track", "uuid")
+
+    if track_mbid:
+        # easiest case: there is a MBID provided in the import_metadata
+        return models.Track.get_or_create_from_api(mbid=track_mbid)[0]
+    if track_uuid:
+        # another easy case, we have a reference to a uuid of a track that
+        # already exists in our database
+        try:
+            return models.Track.objects.get(uuid=track_uuid)
+        except models.Track.DoesNotExist:
+            raise TrackFileImportError(code="track_uuid_not_found")
+
+
+def get_owned_duplicates(track_file, track):
+    """
+    Ensure we skip duplicate tracks to avoid wasting user/instance storage
+    """
+    owned_libraries = track_file.library.actor.libraries.all()
+    return (
+        models.TrackFile.objects.filter(
+            track__isnull=False, library__in=owned_libraries, track=track
+        )
+        .exclude(pk=track_file.pk)
+        .values_list("uuid", flat=True)
+    )
+
+
+@transaction.atomic
+def import_track_data_from_file(file, hints={}):
+    data = metadata.Metadata(file)
+    album = None
+    track_mbid = data.get("musicbrainz_recordingid", None)
+    album_mbid = data.get("musicbrainz_albumid", None)
+
+    if album_mbid and track_mbid:
+        # to gain performance and avoid additional mb lookups,
+        # we import from the release data, which is already cached
+        return models.Track.get_or_create_from_release(album_mbid, track_mbid)[0]
+    elif track_mbid:
+        return models.Track.get_or_create_from_api(track_mbid)[0]
+    elif album_mbid:
+        album = models.Album.get_or_create_from_api(album_mbid)[0]
+
+    artist = album.artist if album else None
+    artist_mbid = data.get("musicbrainz_artistid", None)
+    if not artist:
+        if artist_mbid:
+            artist = models.Artist.get_or_create_from_api(artist_mbid)[0]
+        else:
+            artist = models.Artist.objects.get_or_create(
+                name__iexact=data.get("artist"), defaults={"name": data.get("artist")}
+            )[0]
+
+    release_date = data.get("date", default=None)
+    if not album:
+        album = models.Album.objects.get_or_create(
+            title__iexact=data.get("album"),
+            artist=artist,
+            defaults={"title": data.get("album"), "release_date": release_date},
+        )[0]
+    position = data.get("track_number", default=None)
+    track = models.Track.objects.get_or_create(
+        title__iexact=data.get("title"),
+        album=album,
+        defaults={"title": data.get("title"), "position": position},
+    )[0]
+    return track
+
+
+@receiver(signals.track_file_import_status_updated)
+def broadcast_import_status_update_to_owner(
+    old_status, new_status, track_file, **kwargs
+):
+    user = track_file.library.actor.get_user()
+    if not user:
+        return
+    group = "user.{}.imports".format(user.pk)
+    channels.group_send(
+        group,
+        {
+            "type": "event.send",
+            "text": "",
+            "data": {
+                "type": "import.status_updated",
+                "track_file": serializers.TrackFileForOwnerSerializer(track_file).data,
+                "old_status": old_status,
+                "new_status": new_status,
+            },
+        },
+    )

api/funkwhale_api/music/utils.py

@@ -58,3 +58,13 @@ def get_audio_file_data(f):
     d["length"] = data.info.length
 
     return d
+
+
+def get_actor_from_request(request):
+    actor = None
+    if hasattr(request, "actor"):
+        actor = request.actor
+    elif request.user.is_authenticated:
+        actor = request.user.actor
+
+    return actor

api/funkwhale_api/music/views.py

@@ -1,32 +1,25 @@
-import json
 import logging
 import urllib
 
 from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
-from django.db.models import Count
+from django.db.models import Count, Prefetch, Sum
 from django.db.models.functions import Length
 from django.utils import timezone
-from musicbrainzngs import ResponseError
+
 from rest_framework import mixins
+from rest_framework import permissions
 from rest_framework import settings as rest_settings
 from rest_framework import views, viewsets
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
 from taggit.models import Tag
 
-from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common import permissions as common_permissions
 from funkwhale_api.federation.authentication import SignatureAuthentication
-from funkwhale_api.federation.models import LibraryTrack
-from funkwhale_api.musicbrainz import api
-from funkwhale_api.requests.models import ImportRequest
-from funkwhale_api.users.permissions import HasUserPermission
 
-from . import filters, importers, models
-from . import permissions as music_permissions
-from . import serializers, tasks, utils
+from . import filters, models, serializers, tasks, utils
 
 logger = logging.getLogger(__name__)
 
@@ -41,107 +34,65 @@ class TagViewSetMixin(object):
 
 
 class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = models.Artist.objects.with_albums()
+    queryset = models.Artist.objects.all()
     serializer_class = serializers.ArtistWithAlbumsSerializer
-    permission_classes = [ConditionalAuthentication]
+    permission_classes = [common_permissions.ConditionalAuthentication]
     filter_class = filters.ArtistFilter
     ordering_fields = ("id", "name", "creation_date")
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        albums = models.Album.objects.with_tracks_count()
+        return queryset.prefetch_related(Prefetch("albums", queryset=albums)).distinct()
+
 
 class AlbumViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = (
-        models.Album.objects.all()
-        .order_by("artist", "release_date")
-        .select_related()
-        .prefetch_related("tracks__artist", "tracks__files")
+        models.Album.objects.all().order_by("artist", "release_date").select_related()
     )
     serializer_class = serializers.AlbumSerializer
-    permission_classes = [ConditionalAuthentication]
+    permission_classes = [common_permissions.ConditionalAuthentication]
     ordering_fields = ("creation_date", "release_date", "title")
     filter_class = filters.AlbumFilter
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        tracks = models.Track.objects.annotate_playable_by_actor(
+            utils.get_actor_from_request(self.request)
+        ).select_related("artist")
+        qs = queryset.prefetch_related(Prefetch("tracks", queryset=tracks))
+        return qs.distinct()
 
-class ImportBatchViewSet(
+
+class LibraryViewSet(
     mixins.CreateModelMixin,
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.DestroyModelMixin,
     viewsets.GenericViewSet,
 ):
+    lookup_field = "uuid"
     queryset = (
-        models.ImportBatch.objects.select_related()
+        models.Library.objects.all()
         .order_by("-creation_date")
-        .annotate(job_count=Count("jobs"))
+        .annotate(_files_count=Count("files"))
+        .annotate(_size=Sum("files__size"))
     )
-    serializer_class = serializers.ImportBatchSerializer
-    permission_classes = (HasUserPermission,)
-    required_permissions = ["library", "upload"]
-    permission_operator = "or"
-    filter_class = filters.ImportBatchFilter
-
-    def perform_create(self, serializer):
-        serializer.save(submitted_by=self.request.user)
+    serializer_class = serializers.LibraryForOwnerSerializer
+    permission_classes = [
+        permissions.IsAuthenticated,
+        common_permissions.OwnerPermission,
+    ]
+    owner_field = "actor.user"
+    owner_checks = ["read", "write"]
 
     def get_queryset(self):
         qs = super().get_queryset()
-        # if user do not have library permission, we limit to their
-        # own jobs
-        if not self.request.user.has_permissions("library"):
-            qs = qs.filter(submitted_by=self.request.user)
-        return qs
-
-
-class ImportJobViewSet(
-    mixins.CreateModelMixin, mixins.ListModelMixin, viewsets.GenericViewSet
-):
-    queryset = models.ImportJob.objects.all().select_related()
-    serializer_class = serializers.ImportJobSerializer
-    permission_classes = (HasUserPermission,)
-    required_permissions = ["library", "upload"]
-    permission_operator = "or"
-    filter_class = filters.ImportJobFilter
-
-    def get_queryset(self):
-        qs = super().get_queryset()
-        # if user do not have library permission, we limit to their
-        # own jobs
-        if not self.request.user.has_permissions("library"):
-            qs = qs.filter(batch__submitted_by=self.request.user)
-        return qs
-
-    @list_route(methods=["get"])
-    def stats(self, request, *args, **kwargs):
-        if not request.user.has_permissions("library"):
-            return Response(status=403)
-        qs = models.ImportJob.objects.all()
-        filterset = filters.ImportJobFilter(request.GET, queryset=qs)
-        qs = filterset.qs
-        qs = qs.values("status").order_by("status")
-        qs = qs.annotate(status_count=Count("status"))
-
-        data = {}
-        for row in qs:
-            data[row["status"]] = row["status_count"]
-
-        for s, _ in models.IMPORT_STATUS_CHOICES:
-            data.setdefault(s, 0)
-
-        data["count"] = sum([v for v in data.values()])
-        return Response(data)
-
-    @list_route(methods=["post"])
-    def run(self, request, *args, **kwargs):
-        serializer = serializers.ImportJobRunSerializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        payload = serializer.save()
-
-        return Response(payload)
+        return qs.filter(actor=self.request.user.actor)
 
     def perform_create(self, serializer):
-        source = "file://" + serializer.validated_data["audio_file"].name
-        serializer.save(source=source)
-        funkwhale_utils.on_commit(
-            tasks.import_job_run.delay, import_job_id=serializer.instance.pk
-        )
+        serializer.save(actor=self.request.user.actor)
 
 
 class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
@@ -151,14 +102,13 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
 
     queryset = models.Track.objects.all().for_nested_serialization()
     serializer_class = serializers.TrackSerializer
-    permission_classes = [ConditionalAuthentication]
+    permission_classes = [common_permissions.ConditionalAuthentication]
     filter_class = filters.TrackFilter
     ordering_fields = (
         "creation_date",
         "title",
-        "album__title",
         "album__release_date",
-        "position",
+        "size",
         "artist__name",
     )
 
@@ -169,7 +119,10 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
         if user.is_authenticated and filter_favorites == "true":
             queryset = queryset.filter(track_favorites__user=user)
 
-        return queryset
+        queryset = queryset.annotate_playable_by_actor(
+            utils.get_actor_from_request(self.request)
+        )
+        return queryset.distinct()
 
     @detail_route(methods=["get"])
     @transaction.non_atomic_requests
@@ -228,40 +181,37 @@ def get_file_path(audio_file):
         return path.encode("utf-8")
 
 
-def handle_serve(track_file):
+def handle_serve(track_file, user):
     f = track_file
     # we update the accessed_date
     f.accessed_date = timezone.now()
     f.save(update_fields=["accessed_date"])
 
-    mt = f.mimetype
-    audio_file = f.audio_file
-    try:
-        library_track = f.library_track
-    except ObjectDoesNotExist:
-        library_track = None
-    if library_track and not audio_file:
-        if not library_track.audio_file:
-            # we need to populate from cache
-            with transaction.atomic():
-                # why the transaction/select_for_update?
-                # this is because browsers may send multiple requests
-                # in a short time range, for partial content,
-                # thus resulting in multiple downloads from the remote
-                qs = LibraryTrack.objects.select_for_update()
-                library_track = qs.get(pk=library_track.pk)
-                library_track.download_audio()
-            track_file.library_track = library_track
-            track_file.set_audio_data()
-            track_file.save(update_fields=["bitrate", "duration", "size"])
+    if f.audio_file:
+        file_path = get_file_path(f.audio_file)
 
-        audio_file = library_track.audio_file
-        file_path = get_file_path(audio_file)
-        mt = library_track.audio_mimetype
-    elif audio_file:
-        file_path = get_file_path(audio_file)
+    elif f.source and (
+        f.source.startswith("http://") or f.source.startswith("https://")
+    ):
+        # we need to populate from cache
+        with transaction.atomic():
+            # why the transaction/select_for_update?
+            # this is because browsers may send multiple requests
+            # in a short time range, for partial content,
+            # thus resulting in multiple downloads from the remote
+            qs = f.__class__.objects.select_for_update()
+            f = qs.get(pk=f.pk)
+            f.download_audio_from_remote(user=user)
+        data = f.get_audio_data()
+        if data:
+            f.duration = data["duration"]
+            f.size = data["size"]
+            f.bitrate = data["bitrate"]
+            f.save(update_fields=["bitrate", "duration", "size"])
+        file_path = get_file_path(f.audio_file)
     elif f.source and f.source.startswith("file://"):
         file_path = get_file_path(f.source.replace("file://", "", 1))
+    mt = f.mimetype
     if mt:
         response = Response(content_type=mt)
     else:
@@ -278,39 +228,93 @@ def handle_serve(track_file):
     return response
 
 
-class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = (
-        models.TrackFile.objects.all()
-        .select_related("track__artist", "track__album")
-        .order_by("-id")
-    )
-    serializer_class = serializers.TrackFileSerializer
+class ListenViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    queryset = models.Track.objects.all()
+    serializer_class = serializers.TrackSerializer
     authentication_classes = (
         rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES
         + [SignatureAuthentication]
     )
-    permission_classes = [music_permissions.Listen]
+    permission_classes = [common_permissions.ConditionalAuthentication]
+    lookup_field = "uuid"
 
-    @detail_route(methods=["get"])
-    def serve(self, request, *args, **kwargs):
-        queryset = models.TrackFile.objects.select_related(
-            "library_track", "track__album__artist", "track__artist"
-        )
-        try:
-            return handle_serve(queryset.get(pk=kwargs["pk"]))
-        except models.TrackFile.DoesNotExist:
+    def retrieve(self, request, *args, **kwargs):
+        track = self.get_object()
+        actor = utils.get_actor_from_request(request)
+        queryset = track.files.select_related("track__album__artist", "track__artist")
+        explicit_file = request.GET.get("file")
+        if explicit_file:
+            queryset = queryset.filter(uuid=explicit_file)
+        queryset = queryset.playable_by(actor)
+        tf = queryset.first()
+        if not tf:
             return Response(status=404)
 
+        return handle_serve(tf, user=request.user)
+
+
+class TrackFileViewSet(
+    mixins.ListModelMixin,
+    mixins.CreateModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.TrackFile.objects.all()
+        .order_by("-creation_date")
+        .select_related("library", "track__artist", "track__album__artist")
+    )
+    serializer_class = serializers.TrackFileForOwnerSerializer
+    permission_classes = [
+        permissions.IsAuthenticated,
+        common_permissions.OwnerPermission,
+    ]
+    owner_field = "library.actor.user"
+    owner_checks = ["read", "write"]
+    filter_class = filters.TrackFileFilter
+    ordering_fields = (
+        "creation_date",
+        "import_date",
+        "bitrate",
+        "size",
+        "artist__name",
+    )
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(library__actor=self.request.user.actor)
+
+    @list_route(methods=["post"])
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializers.TrackFileActionSerializer(
+            request.data, queryset=queryset
+        )
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return Response(result, status=200)
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["user"] = self.request.user
+        return context
+
+    def perform_create(self, serializer):
+        tf = serializer.save()
+        common_utils.on_commit(tasks.import_track_file.delay, track_file_id=tf.pk)
+
 
 class TagViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Tag.objects.all().order_by("name")
     serializer_class = serializers.TagSerializer
-    permission_classes = [ConditionalAuthentication]
+    permission_classes = [common_permissions.ConditionalAuthentication]
 
 
 class Search(views.APIView):
     max_results = 3
-    permission_classes = [ConditionalAuthentication]
+    permission_classes = [common_permissions.ConditionalAuthentication]
 
     def get(self, request, *args, **kwargs):
         query = request.GET["query"]
@@ -340,7 +344,6 @@ class Search(views.APIView):
             models.Track.objects.all()
             .filter(query_obj)
             .select_related("artist", "album__artist")
-            .prefetch_related("files")
         )[: self.max_results]
 
     def get_albums(self, query):
@@ -350,7 +353,7 @@ class Search(views.APIView):
             models.Album.objects.all()
             .filter(query_obj)
             .select_related()
-            .prefetch_related("tracks__files")
+            .prefetch_related("tracks")
         )[: self.max_results]
 
     def get_artists(self, query):
@@ -372,99 +375,3 @@ class Search(views.APIView):
         )
 
         return qs.filter(query_obj)[: self.max_results]
-
-
-class SubmitViewSet(viewsets.ViewSet):
-    queryset = models.ImportBatch.objects.none()
-    permission_classes = (HasUserPermission,)
-    required_permissions = ["library"]
-
-    @list_route(methods=["post"])
-    @transaction.non_atomic_requests
-    def single(self, request, *args, **kwargs):
-        try:
-            models.Track.objects.get(mbid=request.POST["mbid"])
-            return Response({})
-        except models.Track.DoesNotExist:
-            pass
-        batch = models.ImportBatch.objects.create(submitted_by=request.user)
-        job = models.ImportJob.objects.create(
-            mbid=request.POST["mbid"], batch=batch, source=request.POST["import_url"]
-        )
-        tasks.import_job_run.delay(import_job_id=job.pk)
-        serializer = serializers.ImportBatchSerializer(batch)
-        return Response(serializer.data, status=201)
-
-    def get_import_request(self, data):
-        try:
-            raw = data["importRequest"]
-        except KeyError:
-            return
-
-        pk = int(raw)
-        try:
-            return ImportRequest.objects.get(pk=pk)
-        except ImportRequest.DoesNotExist:
-            pass
-
-    @list_route(methods=["post"])
-    @transaction.non_atomic_requests
-    def album(self, request, *args, **kwargs):
-        data = json.loads(request.body.decode("utf-8"))
-        import_request = self.get_import_request(data)
-        import_data, batch = self._import_album(
-            data, request, batch=None, import_request=import_request
-        )
-        return Response(import_data)
-
-    @transaction.atomic
-    def _import_album(self, data, request, batch=None, import_request=None):
-        # we import the whole album here to prevent race conditions that occurs
-        # when using get_or_create_from_api in tasks
-        album_data = api.releases.get(
-            id=data["releaseId"], includes=models.Album.api_includes
-        )["release"]
-        cleaned_data = models.Album.clean_musicbrainz_data(album_data)
-        album = importers.load(
-            models.Album, cleaned_data, album_data, import_hooks=[models.import_tracks]
-        )
-        try:
-            album.get_image()
-        except ResponseError:
-            pass
-        if not batch:
-            batch = models.ImportBatch.objects.create(
-                submitted_by=request.user, import_request=import_request
-            )
-        for row in data["tracks"]:
-            try:
-                models.TrackFile.objects.get(track__mbid=row["mbid"])
-            except models.TrackFile.DoesNotExist:
-                job = models.ImportJob.objects.create(
-                    mbid=row["mbid"], batch=batch, source=row["source"]
-                )
-                funkwhale_utils.on_commit(
-                    tasks.import_job_run.delay, import_job_id=job.pk
-                )
-
-        serializer = serializers.ImportBatchSerializer(batch)
-        return serializer.data, batch
-
-    @list_route(methods=["post"])
-    @transaction.non_atomic_requests
-    def artist(self, request, *args, **kwargs):
-        data = json.loads(request.body.decode("utf-8"))
-        import_request = self.get_import_request(data)
-        artist_data = api.artists.get(id=data["artistId"])["artist"]
-        cleaned_data = models.Artist.clean_musicbrainz_data(artist_data)
-        importers.load(models.Artist, cleaned_data, artist_data, import_hooks=[])
-
-        import_data = []
-        batch = None
-        for row in data["albums"]:
-            row_data, batch = self._import_album(
-                row, request, batch=batch, import_request=import_request
-            )
-            import_data.append(row_data)
-
-        return Response(import_data[0])

api/funkwhale_api/playlists/filters.py

@@ -8,7 +8,7 @@ from . import models
 
 class PlaylistFilter(filters.FilterSet):
     q = filters.CharFilter(name="_", method="filter_q")
-    listenable = filters.BooleanFilter(name="_", method="filter_listenable")
+    playable = filters.BooleanFilter(name="_", method="filter_playable")
 
     class Meta:
         model = models.Playlist
@@ -16,10 +16,10 @@ class PlaylistFilter(filters.FilterSet):
             "user": ["exact"],
             "name": ["exact", "icontains"],
             "q": "exact",
-            "listenable": "exact",
+            "playable": "exact",
         }
 
-    def filter_listenable(self, queryset, name, value):
+    def filter_playable(self, queryset, name, value):
         queryset = queryset.annotate(plts_count=Count("playlist_tracks"))
         if value:
             return queryset.filter(plts_count__gt=0)

api/funkwhale_api/providers/audiofile/tasks.py

@@ -1,45 +0,0 @@
-from django.db import transaction
-
-from funkwhale_api.music import metadata, models
-
-
-@transaction.atomic
-def import_track_data_from_path(path):
-    data = metadata.Metadata(path)
-    album = None
-    track_mbid = data.get("musicbrainz_recordingid", None)
-    album_mbid = data.get("musicbrainz_albumid", None)
-
-    if album_mbid and track_mbid:
-        # to gain performance and avoid additional mb lookups,
-        # we import from the release data, which is already cached
-        return models.Track.get_or_create_from_release(album_mbid, track_mbid)[0]
-    elif track_mbid:
-        return models.Track.get_or_create_from_api(track_mbid)[0]
-    elif album_mbid:
-        album = models.Album.get_or_create_from_api(album_mbid)[0]
-
-    artist = album.artist if album else None
-    artist_mbid = data.get("musicbrainz_artistid", None)
-    if not artist:
-        if artist_mbid:
-            artist = models.Artist.get_or_create_from_api(artist_mbid)[0]
-        else:
-            artist = models.Artist.objects.get_or_create(
-                name__iexact=data.get("artist"), defaults={"name": data.get("artist")}
-            )[0]
-
-    release_date = data.get("date", default=None)
-    if not album:
-        album = models.Album.objects.get_or_create(
-            title__iexact=data.get("album"),
-            artist=artist,
-            defaults={"title": data.get("album"), "release_date": release_date},
-        )[0]
-    position = data.get("track_number", default=None)
-    track = models.Track.objects.get_or_create(
-        title__iexact=data.get("title"),
-        album=album,
-        defaults={"title": data.get("title"), "position": position},
-    )[0]
-    return track

api/funkwhale_api/providers/urls.py

@@ -1,16 +1,10 @@
 from django.conf.urls import include, url
 
 urlpatterns = [
-    url(
-        r"^youtube/",
-        include(
-            ("funkwhale_api.providers.youtube.urls", "youtube"), namespace="youtube"
-        ),
-    ),
     url(
         r"^musicbrainz/",
         include(
             ("funkwhale_api.musicbrainz.urls", "musicbrainz"), namespace="musicbrainz"
         ),
-    ),
+    )
 ]

api/funkwhale_api/providers/youtube/client.py

@@ -1,80 +0,0 @@
-import threading
-
-from apiclient.discovery import build
-from dynamic_preferences.registries import global_preferences_registry as registry
-
-YOUTUBE_API_SERVICE_NAME = "youtube"
-YOUTUBE_API_VERSION = "v3"
-VIDEO_BASE_URL = "https://www.youtube.com/watch?v={0}"
-
-
-def _do_search(query):
-    manager = registry.manager()
-    youtube = build(
-        YOUTUBE_API_SERVICE_NAME,
-        YOUTUBE_API_VERSION,
-        developerKey=manager["providers_youtube__api_key"],
-    )
-
-    return youtube.search().list(q=query, part="id,snippet", maxResults=25).execute()
-
-
-class Client(object):
-    def search(self, query):
-        search_response = _do_search(query)
-        videos = []
-        for search_result in search_response.get("items", []):
-            if search_result["id"]["kind"] == "youtube#video":
-                search_result["full_url"] = VIDEO_BASE_URL.format(
-                    search_result["id"]["videoId"]
-                )
-                videos.append(search_result)
-        return videos
-
-    def search_multiple(self, queries):
-        results = {}
-
-        def search(key, query):
-            results[key] = self.search(query)
-
-        threads = [
-            threading.Thread(target=search, args=(key, query))
-            for key, query in queries.items()
-        ]
-        for thread in threads:
-            thread.start()
-        for thread in threads:
-            thread.join()
-
-        return results
-
-    def to_funkwhale(self, result):
-        """
-        We convert youtube results to something more generic.
-
-        {
-            "id": "video id",
-            "type": "youtube#video",
-            "url": "https://www.youtube.com/watch?v=id",
-            "description": "description",
-            "channelId": "Channel id",
-            "title": "Title",
-            "channelTitle": "channel Title",
-            "publishedAt": "2012-08-22T18:41:03.000Z",
-            "cover": "http://coverurl"
-        }
-        """
-        return {
-            "id": result["id"]["videoId"],
-            "url": "https://www.youtube.com/watch?v={}".format(result["id"]["videoId"]),
-            "type": result["id"]["kind"],
-            "title": result["snippet"]["title"],
-            "description": result["snippet"]["description"],
-            "channelId": result["snippet"]["channelId"],
-            "channelTitle": result["snippet"]["channelTitle"],
-            "publishedAt": result["snippet"]["publishedAt"],
-            "cover": result["snippet"]["thumbnails"]["high"]["url"],
-        }
-
-
-client = Client()

api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py

@@ -1,16 +0,0 @@
-from django import forms
-from dynamic_preferences.registries import global_preferences_registry
-from dynamic_preferences.types import Section, StringPreference
-
-youtube = Section("providers_youtube")
-
-
-@global_preferences_registry.register
-class APIKey(StringPreference):
-    section = youtube
-    name = "api_key"
-    default = "CHANGEME"
-    verbose_name = "YouTube API key"
-    help_text = "The API key used to query YouTube. Get one at https://console.developers.google.com/."
-    widget = forms.PasswordInput
-    field_kwargs = {"required": False}

api/funkwhale_api/providers/youtube/urls.py

@@ -1,8 +0,0 @@
-from django.conf.urls import url
-
-from .views import APISearch, APISearchs
-
-urlpatterns = [
-    url(r"^search/$", APISearch.as_view(), name="search"),
-    url(r"^searchs/$", APISearchs.as_view(), name="searchs"),
-]

api/funkwhale_api/providers/youtube/views.py

@@ -1,27 +0,0 @@
-from rest_framework.response import Response
-from rest_framework.views import APIView
-
-from funkwhale_api.common.permissions import ConditionalAuthentication
-
-from .client import client
-
-
-class APISearch(APIView):
-    permission_classes = [ConditionalAuthentication]
-
-    def get(self, request, *args, **kwargs):
-        results = client.search(request.GET["query"])
-        return Response([client.to_funkwhale(result) for result in results])
-
-
-class APISearchs(APIView):
-    permission_classes = [ConditionalAuthentication]
-
-    def post(self, request, *args, **kwargs):
-        results = client.search_multiple(request.data)
-        return Response(
-            {
-                key: [client.to_funkwhale(result) for result in group]
-                for key, group in results.items()
-            }
-        )

api/funkwhale_api/subsonic/views.py

@@ -177,13 +177,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
     @find_object(music_models.Track.objects.all())
     def stream(self, request, *args, **kwargs):
         track = kwargs.pop("obj")
-        queryset = track.files.select_related(
-            "library_track", "track__album__artist", "track__artist"
-        )
+        queryset = track.files.select_related("track__album__artist", "track__artist")
         track_file = queryset.first()
         if not track_file:
             return response.Response(status=404)
-        return music_views.handle_serve(track_file)
+        return music_views.handle_serve(track_file=track_file, user=request.user)
 
     @list_route(methods=["get", "post"], url_name="star", url_path="star")
     @find_object(music_models.Track.objects.all())

api/funkwhale_api/taskapp/celery.py

@@ -2,20 +2,29 @@
 from __future__ import absolute_import
 
 import functools
+import traceback as tb
 import os
-
-from celery import Celery
+import logging
+import celery.app.task
 from django.apps import AppConfig
 from django.conf import settings
 
+
+logger = logging.getLogger("celery")
+
 if not settings.configured:
     # set the default Django settings module for the 'celery' program.
     os.environ.setdefault(
         "DJANGO_SETTINGS_MODULE", "config.settings.local"
     )  # pragma: no cover
 
+app = celery.Celery("funkwhale_api")
 
-app = Celery("funkwhale_api")
+
+@celery.signals.task_failure.connect
+def process_failure(sender, task_id, exception, args, kwargs, traceback, einfo, **kw):
+    print("[celery] Error during task {}: {}".format(task_id, einfo.exception))
+    tb.print_exc()
 
 
 class CeleryConfig(AppConfig):

api/funkwhale_api/users/dynamic_preferences_registry.py

@@ -28,3 +28,13 @@ class DefaultPermissions(common_preferences.StringListPreference):
     help_text = "A list of default preferences to give to all registered users."
     choices = [(k, c["label"]) for k, c in models.PERMISSIONS_CONFIGURATION.items()]
     field_kwargs = {"choices": choices, "required": False}
+
+
+@global_preferences_registry.register
+class UploadQuota(types.IntPreference):
+    show_in_api = True
+    section = users
+    name = "upload_quota"
+    default = 1000
+    verbose_name = "Upload quota"
+    help_text = "Default upload quota applied to each users, in MB. This can be overrided on a per-user basis."

api/funkwhale_api/users/migrations/0008_auto_20180617_1531.py

@@ -5,19 +5,21 @@ from django.db import migrations, models
 
 class Migration(migrations.Migration):
 
-    dependencies = [
-        ('users', '0007_auto_20180524_2009'),
-    ]
+    dependencies = [("users", "0007_auto_20180524_2009")]
 
     operations = [
         migrations.AddField(
-            model_name='user',
-            name='last_activity',
+            model_name="user",
+            name="last_activity",
             field=models.DateTimeField(blank=True, default=None, null=True),
         ),
         migrations.AlterField(
-            model_name='user',
-            name='permission_library',
-            field=models.BooleanField(default=False, help_text='Manage library, delete files, tracks, artists, albums...', verbose_name='Manage library'),
+            model_name="user",
+            name="permission_library",
+            field=models.BooleanField(
+                default=False,
+                help_text="Manage library, delete files, tracks, artists, albums...",
+                verbose_name="Manage library",
+            ),
         ),
     ]

api/funkwhale_api/users/migrations/0009_auto_20180619_2024.py

@@ -8,24 +8,46 @@ import django.utils.timezone
 
 class Migration(migrations.Migration):
 
-    dependencies = [
-        ('users', '0008_auto_20180617_1531'),
-    ]
+    dependencies = [("users", "0008_auto_20180617_1531")]
 
     operations = [
         migrations.CreateModel(
-            name='Invitation',
+            name="Invitation",
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
-                ('expiration_date', models.DateTimeField()),
-                ('code', models.CharField(max_length=50, unique=True)),
-                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='invitations', to=settings.AUTH_USER_MODEL)),
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("expiration_date", models.DateTimeField()),
+                ("code", models.CharField(max_length=50, unique=True)),
+                (
+                    "owner",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="invitations",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
             ],
         ),
         migrations.AddField(
-            model_name='user',
-            name='invitation',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='users', to='users.Invitation'),
+            model_name="user",
+            name="invitation",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="users",
+                to="users.Invitation",
+            ),
         ),
     ]

api/funkwhale_api/users/migrations/0010_user_avatar.py

@@ -7,14 +7,22 @@ import funkwhale_api.common.validators
 
 class Migration(migrations.Migration):
 
-    dependencies = [
-        ('users', '0009_auto_20180619_2024'),
-    ]
+    dependencies = [("users", "0009_auto_20180619_2024")]
 
     operations = [
         migrations.AddField(
-            model_name='user',
-            name='avatar',
-            field=models.ImageField(blank=True, max_length=150, null=True, upload_to=funkwhale_api.common.utils.ChunkedPath('users/avatars'), validators=[funkwhale_api.common.validators.ImageDimensionsValidator(max_height=400, max_width=400, min_height=50, min_width=50)]),
-        ),
+            model_name="user",
+            name="avatar",
+            field=models.ImageField(
+                blank=True,
+                max_length=150,
+                null=True,
+                upload_to=funkwhale_api.common.utils.ChunkedPath("users/avatars"),
+                validators=[
+                    funkwhale_api.common.validators.ImageDimensionsValidator(
+                        max_height=400, max_width=400, min_height=50, min_width=50
+                    )
+                ],
+            ),
+        )
     ]

api/funkwhale_api/users/migrations/0011_auto_20180721_1317.py

@@ -10,19 +10,41 @@ import versatileimagefield.fields
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('federation', '0006_auto_20180521_1702'),
-        ('users', '0010_user_avatar'),
+        ("federation", "0006_auto_20180521_1702"),
+        ("users", "0010_user_avatar"),
     ]
 
     operations = [
         migrations.AddField(
-            model_name='user',
-            name='actor',
-            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='user', to='federation.Actor'),
+            model_name="user",
+            name="actor",
+            field=models.OneToOneField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="user",
+                to="federation.Actor",
+            ),
         ),
         migrations.AlterField(
-            model_name='user',
-            name='avatar',
-            field=versatileimagefield.fields.VersatileImageField(blank=True, max_length=150, null=True, upload_to=funkwhale_api.common.utils.ChunkedPath('users/avatars', preserve_file_name=False), validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg', 'gif'], max_size=2097152)]),
+            model_name="user",
+            name="avatar",
+            field=versatileimagefield.fields.VersatileImageField(
+                blank=True,
+                max_length=150,
+                null=True,
+                upload_to=funkwhale_api.common.utils.ChunkedPath(
+                    "users/avatars", preserve_file_name=False
+                ),
+                validators=[
+                    funkwhale_api.common.validators.ImageDimensionsValidator(
+                        min_height=50, min_width=50
+                    ),
+                    funkwhale_api.common.validators.FileValidator(
+                        allowed_extensions=["png", "jpg", "jpeg", "gif"],
+                        max_size=2097152,
+                    ),
+                ],
+            ),
         ),
     ]

api/funkwhale_api/users/migrations/0012_user_upload_quota.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.0.7 on 2018-08-01 16:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("users", "0011_auto_20180721_1317")]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="upload_quota",
+            field=models.PositiveIntegerField(blank=True, null=True),
+        )
+    ]

api/funkwhale_api/users/models.py

@@ -122,6 +122,8 @@ class User(AbstractUser):
         blank=True,
     )
 
+    upload_quota = models.PositiveIntegerField(null=True, blank=True)
+
     def __str__(self):
         return self.username
 
@@ -182,6 +184,32 @@ class User(AbstractUser):
             self.last_activity = now
             self.save(update_fields=["last_activity"])
 
+    def create_actor(self):
+        self.actor = create_actor(self)
+        self.save(update_fields=["actor"])
+        return self.actor
+
+    def get_upload_quota(self):
+        return self.upload_quota or preferences.get("users__upload_quota")
+
+    def get_quota_status(self):
+        data = self.actor.get_current_usage()
+        max_ = self.get_upload_quota()
+        return {
+            "max": max_,
+            "remaining": max(max_ - (data["total"] / 1000 / 1000), 0),
+            "current": data["total"] / 1000 / 1000,
+            "skipped": data["skipped"] / 1000 / 1000,
+            "pending": data["pending"] / 1000 / 1000,
+            "finished": data["finished"] / 1000 / 1000,
+            "errored": data["errored"] / 1000 / 1000,
+        }
+
+    def get_channels_groups(self):
+        groups = ["imports"]
+
+        return ["user.{}.{}".format(self.pk, g) for g in groups]
+
 
 def generate_code(length=10):
     return "".join(
@@ -229,7 +257,7 @@ def create_actor(user):
         "type": "Person",
         "name": user.username,
         "manually_approves_followers": False,
-        "url": federation_utils.full_url(
+        "fid": federation_utils.full_url(
             reverse("federation:actors-detail", kwargs={"preferred_username": username})
         ),
         "shared_inbox_url": federation_utils.full_url(

api/funkwhale_api/users/serializers.py

@@ -109,6 +109,16 @@ class UserReadSerializer(serializers.ModelSerializer):
         return o.get_permissions()
 
 
+class MeSerializer(UserReadSerializer):
+    quota_status = serializers.SerializerMethodField()
+
+    class Meta(UserReadSerializer.Meta):
+        fields = UserReadSerializer.Meta.fields + ["quota_status"]
+
+    def get_quota_status(self, o):
+        return o.get_quota_status() if o.actor else 0
+
+
 class PasswordResetSerializer(PRS):
     def get_email_options(self):
         return {"extra_email_context": {"funkwhale_url": settings.FUNKWHALE_URL}}

api/funkwhale_api/users/views.py

@@ -31,7 +31,7 @@ class UserViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet):
     @list_route(methods=["get"])
     def me(self, request, *args, **kwargs):
         """Return information about the current user"""
-        serializer = serializers.UserReadSerializer(request.user)
+        serializer = serializers.MeSerializer(request.user)
         return Response(serializer.data)
 
     @detail_route(methods=["get", "post", "delete"], url_path="subsonic-token")