発散解像度 -divergence resolution-

Signed-off-by: Shin'ya Minazuki <shinyoukai@laidback.moe>

api/funkwhale_api/__init__.py

@@ -0,0 +1,4 @@
+from importlib.metadata import version as get_version
+
+version = get_version("funkwhale_api")
+__version__ = version

api/funkwhale_api/activity/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig, apps
+
+from . import record
+
+
+class ActivityConfig(AppConfig):
+    name = "funkwhale_api.activity"
+
+    def ready(self):
+        super().ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        record.registry.autodiscover(app_names)

api/funkwhale_api/activity/record.py

@@ -0,0 +1,37 @@
+import persisting_theory
+
+
+class ActivityRegistry(persisting_theory.Registry):
+    look_into = "activities"
+
+    def _register_for_model(self, model, attr, value):
+        key = model._meta.label
+        d = self.setdefault(key, {"consumers": []})
+        d[attr] = value
+
+    def register_serializer(self, serializer_class):
+        model = serializer_class.Meta.model
+        self._register_for_model(model, "serializer", serializer_class)
+        return serializer_class
+
+    def register_consumer(self, label):
+        def decorator(func):
+            consumers = self[label]["consumers"]
+            if func not in consumers:
+                consumers.append(func)
+            return func
+
+        return decorator
+
+
+registry = ActivityRegistry()
+
+
+def send(obj):
+    conf = registry[obj.__class__._meta.label]
+    consumers = conf["consumers"]
+    if not consumers:
+        return
+    serializer = conf["serializer"](obj)
+    for consumer in consumers:
+        consumer(data=serializer.data, obj=obj)

api/funkwhale_api/activity/serializers.py

@@ -0,0 +1,23 @@
+from rest_framework import serializers
+
+from funkwhale_api.activity import record
+
+
+class ModelSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(source="get_activity_url")
+    local_id = serializers.IntegerField(source="id")
+    # url = serializers.SerializerMethodField()
+
+    def get_url(self, obj):
+        return self.get_id(obj)
+
+
+class AutoSerializer(serializers.Serializer):
+    """
+    A serializer that will automatically use registered activity serializers
+    to serialize an henerogeneous list of objects (favorites, listenings, etc.)
+    """
+
+    def to_representation(self, instance):
+        serializer = record.registry[instance._meta.label]["serializer"](instance)
+        return serializer.data

api/funkwhale_api/activity/utils.py

@@ -0,0 +1,51 @@
+from django.db import models
+
+from funkwhale_api.common import fields
+from funkwhale_api.favorites.models import TrackFavorite
+from funkwhale_api.history.models import Listening
+
+
+def combined_recent(limit, **kwargs):
+    datetime_field = kwargs.pop("datetime_field", "creation_date")
+    source_querysets = {qs.model._meta.label: qs for qs in kwargs.pop("querysets")}
+    querysets = {
+        k: qs.annotate(
+            __type=models.Value(qs.model._meta.label, output_field=models.CharField())
+        ).values("pk", datetime_field, "__type")
+        for k, qs in source_querysets.items()
+    }
+    _qs_list = list(querysets.values())
+    union_qs = _qs_list[0].union(*_qs_list[1:])
+    records = []
+    for row in union_qs.order_by(f"-{datetime_field}")[:limit]:
+        records.append(
+            {"type": row["__type"], "when": row[datetime_field], "pk": row["pk"]}
+        )
+    # Now we bulk-load each object type in turn
+    to_load = {}
+    for record in records:
+        to_load.setdefault(record["type"], []).append(record["pk"])
+    fetched = {}
+
+    for key, pks in to_load.items():
+        for item in source_querysets[key].filter(pk__in=pks):
+            fetched[(key, item.pk)] = item
+
+    # Annotate 'records' with loaded objects
+    for record in records:
+        record["object"] = fetched[(record["type"], record["pk"])]
+    return records
+
+
+def get_activity(user, limit=20):
+    query = fields.privacy_level_query(user, lookup_field="user__privacy_level")
+    querysets = [
+        Listening.objects.filter(query).select_related(
+            "track", "user", "track__artist", "track__album__artist"
+        ),
+        TrackFavorite.objects.filter(query).select_related(
+            "track", "user", "track__artist", "track__album__artist"
+        ),
+    ]
+    records = combined_recent(limit=limit, querysets=querysets)
+    return [r["object"] for r in records]

api/funkwhale_api/activity/views.py

@@ -0,0 +1,20 @@
+from drf_spectacular.utils import extend_schema
+from rest_framework import viewsets
+from rest_framework.response import Response
+
+from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.favorites.models import TrackFavorite
+
+from . import serializers, utils
+
+
+class ActivityViewSet(viewsets.GenericViewSet):
+    serializer_class = serializers.AutoSerializer
+    permission_classes = [ConditionalAuthentication]
+    queryset = TrackFavorite.objects.none()
+
+    @extend_schema(operation_id="get_activity")
+    def list(self, request, *args, **kwargs):
+        activity = utils.get_activity(user=request.user)
+        serializer = self.serializer_class(activity, many=True)
+        return Response({"results": serializer.data}, status=200)

api/funkwhale_api/audio/admin.py

@@ -0,0 +1,15 @@
+from funkwhale_api.common import admin
+
+from . import models
+
+
+@admin.register(models.Channel)
+class ChannelAdmin(admin.ModelAdmin):
+    list_display = [
+        "uuid",
+        "artist",
+        "attributed_to",
+        "actor",
+        "library",
+        "creation_date",
+    ]

api/funkwhale_api/audio/categories.py

@@ -0,0 +1,113 @@
+# from https://help.apple.com/itc/podcasts_connect/#/itc9267a2f12
+ITUNES_CATEGORIES = {
+    "Arts": [
+        "Books",
+        "Design",
+        "Fashion & Beauty",
+        "Food",
+        "Performing Arts",
+        "Visual Arts",
+    ],
+    "Business": [
+        "Careers",
+        "Entrepreneurship",
+        "Investing",
+        "Management",
+        "Marketing",
+        "Non-Profit",
+    ],
+    "Comedy": ["Comedy Interviews", "Improv", "Stand-Up"],
+    "Education": ["Courses", "How To", "Language Learning", "Self-Improvement"],
+    "Fiction": ["Comedy Fiction", "Drama", "Science Fiction"],
+    "Government": [],
+    "History": [],
+    "Health & Fitness": [
+        "Alternative Health",
+        "Fitness",
+        "Medicine",
+        "Mental Health",
+        "Nutrition",
+        "Sexuality",
+    ],
+    "Kids & Family": [
+        "Education for Kids",
+        "Parenting",
+        "Pets & Animals",
+        "Stories for Kids",
+    ],
+    "Leisure": [
+        "Animation & Manga",
+        "Automotive",
+        "Aviation",
+        "Crafts",
+        "Games",
+        "Hobbies",
+        "Home & Garden",
+        "Video Games",
+    ],
+    "Music": ["Music Commentary", "Music History", "Music Interviews"],
+    "News": [
+        "Business News",
+        "Daily News",
+        "Entertainment News",
+        "News Commentary",
+        "Politics",
+        "Sports News",
+        "Tech News",
+    ],
+    "Religion & Spirituality": [
+        "Buddhism",
+        "Christianity",
+        "Hinduism",
+        "Islam",
+        "Judaism",
+        "Religion",
+        "Spirituality",
+    ],
+    "Science": [
+        "Astronomy",
+        "Chemistry",
+        "Earth Sciences",
+        "Life Sciences",
+        "Mathematics",
+        "Natural Sciences",
+        "Nature",
+        "Physics",
+        "Social Sciences",
+    ],
+    "Society & Culture": [
+        "Documentary",
+        "Personal Journals",
+        "Philosophy",
+        "Places & Travel",
+        "Relationships",
+    ],
+    "Sports": [
+        "Baseball",
+        "Basketball",
+        "Cricket",
+        "Fantasy Sports",
+        "Football",
+        "Golf",
+        "Hockey",
+        "Rugby",
+        "Running",
+        "Soccer",
+        "Swimming",
+        "Tennis",
+        "Volleyball",
+        "Wilderness",
+        "Wrestling",
+    ],
+    "Technology": [],
+    "True Crime": [],
+    "TV & Film": [
+        "After Shows",
+        "Film History",
+        "Film Interviews",
+        "Film Reviews",
+        "TV Reviews",
+    ],
+}
+
+ITUNES_SUBCATEGORIES = [s for p in ITUNES_CATEGORIES.values() for s in p]

api/funkwhale_api/audio/dynamic_preferences_registry.py

@@ -0,0 +1,25 @@
+from dynamic_preferences import types
+from dynamic_preferences.registries import global_preferences_registry
+
+audio = types.Section("audio")
+
+
+@global_preferences_registry.register
+class ChannelsEnabled(types.BooleanPreference):
+    section = audio
+    name = "channels_enabled"
+    default = True
+    verbose_name = "Enable channels"
+    help_text = (
+        "If disabled, the channels feature will be completely switched off, "
+        "and users won't be able to create channels or subscribe to them."
+    )
+
+
+@global_preferences_registry.register
+class MaxChannels(types.IntegerPreference):
+    show_in_api = True
+    section = audio
+    default = 20
+    name = "max_channels"
+    verbose_name = "Max channels allowed per user"

api/funkwhale_api/audio/factories.py

@@ -0,0 +1,68 @@
+import uuid
+
+import factory
+
+from funkwhale_api.factories import NoUpdateOnCreate, registry
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import factories as federation_factories
+from funkwhale_api.music import factories as music_factories
+
+from . import models
+
+
+def set_actor(o):
+    return models.generate_actor(str(o.uuid))
+
+
+def get_rss_channel_name():
+    return f"rssfeed-{uuid.uuid4()}"
+
+
+@registry.register
+class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    uuid = factory.Faker("uuid4")
+    attributed_to = factory.SubFactory(federation_factories.ActorFactory)
+    library = factory.SubFactory(
+        federation_factories.MusicLibraryFactory,
+        actor=factory.SelfAttribute("..attributed_to"),
+        privacy_level="everyone",
+    )
+    actor = factory.LazyAttribute(set_actor)
+    artist = factory.SubFactory(
+        music_factories.ArtistFactory,
+        attributed_to=factory.SelfAttribute("..attributed_to"),
+    )
+    rss_url = factory.Faker("url")
+    metadata = factory.LazyAttribute(lambda o: {})
+
+    class Meta:
+        model = "audio.Channel"
+
+    class Params:
+        external = factory.Trait(
+            attributed_to=factory.LazyFunction(actors.get_service_actor),
+            library__privacy_level="me",
+            actor=factory.SubFactory(
+                federation_factories.ActorFactory,
+                local=True,
+                preferred_username=factory.LazyFunction(get_rss_channel_name),
+            ),
+        )
+        local = factory.Trait(
+            attributed_to=factory.SubFactory(
+                federation_factories.ActorFactory, local=True
+            ),
+            library__privacy_level="everyone",
+            artist__local=True,
+        )
+
+
+@registry.register(name="audio.Subscription")
+class SubscriptionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    uuid = factory.Faker("uuid4")
+    approved = True
+    target = factory.LazyAttribute(lambda o: ChannelFactory().actor)
+    actor = factory.SubFactory(federation_factories.ActorFactory)
+
+    class Meta:
+        model = "federation.Follow"

api/funkwhale_api/audio/filters.py

@@ -0,0 +1,106 @@
+import django_filters
+from django.db.models import Q
+
+from funkwhale_api.common import fields
+from funkwhale_api.common import filters as common_filters
+from funkwhale_api.federation import actors
+from funkwhale_api.moderation import filters as moderation_filters
+
+from . import models
+
+
+def filter_tags(queryset, name, value):
+    non_empty_tags = [v.lower() for v in value if v]
+    for tag in non_empty_tags:
+        queryset = queryset.filter(artist__tagged_items__tag__name=tag).distinct()
+    return queryset
+
+
+TAG_FILTER = common_filters.MultipleQueryFilter(method=filter_tags)
+
+
+class ChannelFilter(moderation_filters.HiddenContentFilterSet):
+    q = fields.SearchFilter(
+        search_fields=["artist__name", "actor__summary", "actor__preferred_username"]
+    )
+    tag = TAG_FILTER
+    scope = common_filters.ActorScopeFilter(actor_field="attributed_to", distinct=True)
+    subscribed = django_filters.BooleanFilter(
+        field_name="_", method="filter_subscribed"
+    )
+    external = django_filters.BooleanFilter(field_name="_", method="filter_external")
+    ordering = django_filters.OrderingFilter(
+        # tuple-mapping retains order
+        fields=(
+            ("creation_date", "creation_date"),
+            ("artist__modification_date", "modification_date"),
+            ("?", "random"),
+        )
+    )
+
+    class Meta:
+        model = models.Channel
+        fields = []
+        hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["CHANNEL"]
+
+    def filter_subscribed(self, queryset, name, value):
+        if not self.request.user.is_authenticated:
+            return queryset.none()
+
+        emitted_follows = self.request.user.actor.emitted_follows.exclude(
+            target__channel__isnull=True
+        )
+
+        query = Q(actor__in=emitted_follows.values_list("target", flat=True))
+
+        if value:
+            return queryset.filter(query)
+        else:
+            return queryset.exclude(query)
+
+    def filter_external(self, queryset, name, value):
+        query = Q(
+            attributed_to=actors.get_service_actor(),
+            actor__preferred_username__startswith="rssfeed-",
+        )
+        if value:
+            queryset = queryset.filter(query)
+        else:
+            queryset = queryset.exclude(query)
+
+        return queryset
+
+
+class IncludeChannelsFilterSet(django_filters.FilterSet):
+    """
+
+    A filterset that include a "include_channels" param. Meant for compatibility
+    with clients that don't support channels yet:
+
+    - include_channels=false : exclude objects associated with a channel
+    - include_channels=true : don't exclude objects associated with a channel
+    - not specified: include_channels=false
+
+    Usage:
+
+    class MyFilterSet(IncludeChannelsFilterSet):
+        class Meta:
+            include_channels_field = "album__artist__channel"
+
+    """
+
+    include_channels = django_filters.BooleanFilter(
+        field_name="_", method="filter_include_channels"
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.data = self.data.copy()
+        self.data.setdefault("include_channels", False)
+
+    def filter_include_channels(self, queryset, name, value):
+        if value is True:
+            return queryset
+        else:
+            params = {self.__class__.Meta.include_channels_field: None}
+            return queryset.filter(**params)

api/funkwhale_api/audio/migrations/0001_initial.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.6 on 2019-10-29 12:57
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('federation', '0021_auto_20191029_1257'),
+        ('music', '0041_auto_20191021_1705'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Channel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='channel', to='federation.Actor')),
+                ('artist', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='channel', to='music.Artist')),
+                ('attributed_to', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='owned_channels', to='federation.Actor')),
+                ('library', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='channel', to='music.Library')),
+            ],
+        ),
+    ]

api/funkwhale_api/audio/migrations/0002_channel_metadata.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.9 on 2020-01-31 06:24
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations
+import funkwhale_api.audio.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audio', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='channel',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.audio.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+        ),
+    ]

api/funkwhale_api/audio/migrations/0003_channel_rss_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-02-06 15:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audio', '0002_channel_metadata'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='channel',
+            name='rss_url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/audio/migrations/0004_alter_channel_metadata.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.13 on 2022-06-27 19:15
+
+import django.core.serializers.json
+from django.db import migrations, models
+import funkwhale_api.audio.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audio', '0003_channel_rss_url'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='channel',
+            name='metadata',
+            field=models.JSONField(blank=True, default=funkwhale_api.audio.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+        ),
+    ]

api/funkwhale_api/audio/models.py

@@ -0,0 +1,122 @@
+import uuid
+
+from django.contrib.contenttypes.fields import GenericRelation
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.db.models import JSONField
+from django.db.models.signals import post_delete
+from django.dispatch import receiver
+from django.urls import reverse
+from django.utils import timezone
+
+from funkwhale_api.federation import keys
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.users import models as user_models
+
+
+def empty_dict():
+    return {}
+
+
+class ChannelQuerySet(models.QuerySet):
+    def external_rss(self, include=True):
+        from funkwhale_api.federation import actors
+
+        query = models.Q(
+            attributed_to=actors.get_service_actor(),
+            actor__preferred_username__startswith="rssfeed-",
+        )
+        if include:
+            return self.filter(query)
+        return self.exclude(query)
+
+    def subscribed(self, actor):
+        if not actor:
+            return self.none()
+
+        subscriptions = actor.emitted_follows.filter(
+            approved=True, target__channel__isnull=False
+        )
+        return self.filter(actor__in=subscriptions.values_list("target", flat=True))
+
+
+class Channel(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    artist = models.OneToOneField(
+        "music.Artist", on_delete=models.CASCADE, related_name="channel"
+    )
+    # the owner of the channel
+    attributed_to = models.ForeignKey(
+        "federation.Actor", on_delete=models.CASCADE, related_name="owned_channels"
+    )
+    # the federation actor created for the channel
+    # (the one people can follow to receive updates)
+    actor = models.OneToOneField(
+        "federation.Actor", on_delete=models.CASCADE, related_name="channel"
+    )
+
+    library = models.OneToOneField(
+        "music.Library", on_delete=models.CASCADE, related_name="channel"
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    rss_url = models.URLField(max_length=500, null=True, blank=True)
+
+    # metadata to enhance rss feed
+    metadata = JSONField(
+        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
+    )
+
+    fetches = GenericRelation(
+        "federation.Fetch",
+        content_type_field="object_content_type",
+        object_id_field="object_id",
+    )
+    objects = ChannelQuerySet.as_manager()
+
+    @property
+    def fid(self):
+        if not self.is_external_rss:
+            return self.actor.fid
+
+    @property
+    def is_local(self) -> bool:
+        return self.actor.is_local
+
+    @property
+    def is_external_rss(self):
+        return self.actor.preferred_username.startswith("rssfeed-")
+
+    def get_absolute_url(self):
+        suffix = self.uuid
+        if self.actor.is_local:
+            suffix = self.actor.preferred_username
+        else:
+            suffix = self.actor.full_username
+        return federation_utils.full_url(f"/channels/{suffix}")
+
+    def get_rss_url(self):
+        if not self.artist.is_local or self.is_external_rss:
+            return self.rss_url
+
+        return federation_utils.full_url(
+            reverse(
+                "api:v1:channels-rss",
+                kwargs={"composite": self.actor.preferred_username},
+            )
+        )
+
+
+def generate_actor(username, **kwargs):
+    actor_data = user_models.get_actor_data(username, **kwargs)
+    private, public = keys.get_key_pair()
+    actor_data["private_key"] = private.decode("utf-8")
+    actor_data["public_key"] = public.decode("utf-8")
+
+    return federation_models.Actor.objects.create(**actor_data)
+
+
+@receiver(post_delete, sender=Channel)
+def delete_channel_related_objs(instance, **kwargs):
+    instance.library.delete()
+    instance.artist.delete()

api/funkwhale_api/audio/renderers.py

@@ -0,0 +1,35 @@
+import xml.etree.ElementTree as ET
+
+from rest_framework import negotiation, renderers
+
+from funkwhale_api.subsonic.renderers import dict_to_xml_tree
+
+
+class PodcastRSSRenderer(renderers.JSONRenderer):
+    media_type = "application/rss+xml"
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if not data:
+            # when stream view is called, we don't have any data
+            return super().render(data, accepted_media_type, renderer_context)
+        final = {
+            "version": "2.0",
+            "xmlns:atom": "http://www.w3.org/2005/Atom",
+            "xmlns:itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd",
+            "xmlns:content": "http://purl.org/rss/1.0/modules/content/",
+            "xmlns:media": "http://search.yahoo.com/mrss/",
+        }
+        final.update(data)
+        tree = dict_to_xml_tree("rss", final)
+        return render_xml(tree)
+
+
+class PodcastRSSContentNegociation(negotiation.DefaultContentNegotiation):
+    def select_renderer(self, request, renderers, format_suffix=None):
+        return (PodcastRSSRenderer(), PodcastRSSRenderer.media_type)
+
+
+def render_xml(tree):
+    return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(
+        tree, encoding="utf-8"
+    )

api/funkwhale_api/audio/serializers.py

@@ -0,0 +1,998 @@
+import datetime
+import logging
+import sys
+import time
+import uuid
+
+import feedparser
+import requests
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Q
+from django.templatetags.static import static
+from django.urls import reverse
+from django.utils import timezone
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
+
+from funkwhale_api.common import locales, preferences
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.common import session
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import serializers as federation_serializers
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.moderation import mrf
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music.serializers import COVER_WRITE_FIELD, CoverField
+from funkwhale_api.tags import models as tags_models
+from funkwhale_api.tags import serializers as tags_serializers
+from funkwhale_api.users import serializers as users_serializers
+
+from . import categories, models
+
+if sys.version_info < (3, 9):
+    from backports.zoneinfo import ZoneInfo
+else:
+    from zoneinfo import ZoneInfo
+
+logger = logging.getLogger(__name__)
+
+
+class ChannelMetadataSerializer(serializers.Serializer):
+    itunes_category = serializers.ChoiceField(
+        choices=categories.ITUNES_CATEGORIES, required=True
+    )
+    itunes_subcategory = serializers.CharField(required=False, allow_null=True)
+    language = serializers.ChoiceField(required=True, choices=locales.ISO_639_CHOICES)
+    copyright = serializers.CharField(required=False, allow_null=True, max_length=255)
+    owner_name = serializers.CharField(required=False, allow_null=True, max_length=255)
+    owner_email = serializers.EmailField(required=False, allow_null=True)
+    explicit = serializers.BooleanField(required=False)
+
+    def validate(self, validated_data):
+        validated_data = super().validate(validated_data)
+        subcategory = self._validate_itunes_subcategory(
+            validated_data["itunes_category"], validated_data.get("itunes_subcategory")
+        )
+        if subcategory:
+            validated_data["itunes_subcategory"] = subcategory
+        return validated_data
+
+    def _validate_itunes_subcategory(self, parent, child):
+        if not child:
+            return
+
+        if child not in categories.ITUNES_CATEGORIES[parent]:
+            raise serializers.ValidationError(
+                f'"{child}" is not a valid subcategory for "{parent}"'
+            )
+
+        return child
+
+
+class ChannelCreateSerializer(serializers.Serializer):
+    name = serializers.CharField(max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"])
+    username = serializers.CharField(
+        max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"],
+        validators=[users_serializers.ASCIIUsernameValidator()],
+    )
+    description = common_serializers.ContentSerializer(allow_null=True)
+    tags = tags_serializers.TagsListField()
+    content_category = serializers.ChoiceField(
+        choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES
+    )
+    metadata = serializers.DictField(required=False)
+    cover = COVER_WRITE_FIELD
+
+    def validate(self, validated_data):
+        existing_channels = self.context["actor"].owned_channels.count()
+        if existing_channels >= preferences.get("audio__max_channels"):
+            raise serializers.ValidationError(
+                "You have reached the maximum amount of allowed channels"
+            )
+        validated_data = super().validate(validated_data)
+        metadata = validated_data.pop("metadata", {})
+        if validated_data["content_category"] == "podcast":
+            metadata_serializer = ChannelMetadataSerializer(data=metadata)
+            metadata_serializer.is_valid(raise_exception=True)
+            metadata = metadata_serializer.validated_data
+        validated_data["metadata"] = metadata
+        return validated_data
+
+    def validate_username(self, value):
+        if value.lower() in [n.lower() for n in settings.ACCOUNT_USERNAME_BLACKLIST]:
+            raise serializers.ValidationError("This username is already taken")
+
+        matching = federation_models.Actor.objects.local().filter(
+            preferred_username__iexact=value
+        )
+        if matching.exists():
+            raise serializers.ValidationError("This username is already taken")
+        return value
+
+    @transaction.atomic
+    def create(self, validated_data):
+        from . import views
+
+        cover = validated_data.pop("cover", None)
+        description = validated_data.get("description")
+        artist = music_models.Artist.objects.create(
+            attributed_to=validated_data["attributed_to"],
+            name=validated_data["name"],
+            content_category=validated_data["content_category"],
+            attachment_cover=cover,
+        )
+        common_utils.attach_content(artist, "description", description)
+
+        if validated_data.get("tags", []):
+            tags_models.set_tags(artist, *validated_data["tags"])
+
+        channel = models.Channel(
+            artist=artist,
+            attributed_to=validated_data["attributed_to"],
+            metadata=validated_data["metadata"],
+        )
+        channel.actor = models.generate_actor(
+            validated_data["username"],
+            name=validated_data["name"],
+        )
+
+        channel.library = music_models.Library.objects.create(
+            name=channel.actor.preferred_username,
+            privacy_level="everyone",
+            actor=validated_data["attributed_to"],
+        )
+        channel.save()
+        channel = views.ChannelViewSet.queryset.get(pk=channel.pk)
+        return channel
+
+    def to_representation(self, obj):
+        return ChannelSerializer(obj, context=self.context).data
+
+
+NOOP = object()
+
+
+class ChannelUpdateSerializer(serializers.Serializer):
+    name = serializers.CharField(max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"])
+    description = common_serializers.ContentSerializer(allow_null=True)
+    tags = tags_serializers.TagsListField()
+    content_category = serializers.ChoiceField(
+        choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES
+    )
+    metadata = serializers.DictField(required=False)
+    cover = COVER_WRITE_FIELD
+
+    def validate(self, validated_data):
+        validated_data = super().validate(validated_data)
+        require_metadata_validation = False
+        new_content_category = validated_data.get("content_category")
+        metadata = validated_data.pop("metadata", NOOP)
+        if (
+            new_content_category == "podcast"
+            and self.instance.artist.content_category != "postcast"
+        ):
+            # updating channel, setting as podcast
+            require_metadata_validation = True
+        elif self.instance.artist.content_category == "postcast" and metadata != NOOP:
+            # channel is podcast, and metadata was updated
+            require_metadata_validation = True
+        else:
+            metadata = self.instance.metadata
+
+        if require_metadata_validation:
+            metadata_serializer = ChannelMetadataSerializer(data=metadata)
+            metadata_serializer.is_valid(raise_exception=True)
+            metadata = metadata_serializer.validated_data
+
+        validated_data["metadata"] = metadata
+        return validated_data
+
+    @transaction.atomic
+    def update(self, obj, validated_data):
+        if validated_data.get("tags") is not None:
+            tags_models.set_tags(obj.artist, *validated_data["tags"])
+        actor_update_fields = []
+        artist_update_fields = []
+
+        obj.metadata = validated_data["metadata"]
+        obj.save(update_fields=["metadata"])
+
+        if "description" in validated_data:
+            common_utils.attach_content(
+                obj.artist, "description", validated_data["description"]
+            )
+
+        if "name" in validated_data:
+            actor_update_fields.append(("name", validated_data["name"]))
+            artist_update_fields.append(("name", validated_data["name"]))
+
+        if "content_category" in validated_data:
+            artist_update_fields.append(
+                ("content_category", validated_data["content_category"])
+            )
+
+        if "cover" in validated_data:
+            artist_update_fields.append(("attachment_cover", validated_data["cover"]))
+
+        if actor_update_fields:
+            for field, value in actor_update_fields:
+                setattr(obj.actor, field, value)
+            obj.actor.save(update_fields=[f for f, _ in actor_update_fields])
+
+        if artist_update_fields:
+            for field, value in artist_update_fields:
+                setattr(obj.artist, field, value)
+            obj.artist.save(update_fields=[f for f, _ in artist_update_fields])
+
+        return obj
+
+    def to_representation(self, obj):
+        return ChannelSerializer(obj, context=self.context).data
+
+
+class SimpleChannelArtistSerializer(serializers.Serializer):
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.CharField()
+    name = serializers.CharField()
+    creation_date = serializers.DateTimeField()
+    modification_date = serializers.DateTimeField()
+    is_local = serializers.BooleanField()
+    content_category = serializers.CharField()
+    description = common_serializers.ContentSerializer(allow_null=True, required=False)
+    cover = CoverField(allow_null=True, required=False)
+    channel = serializers.UUIDField(allow_null=True, required=False)
+    tracks_count = serializers.IntegerField(source="_tracks_count", required=False)
+    tags = serializers.ListField(
+        child=serializers.CharField(), source="_prefetched_tagged_items", required=False
+    )
+
+
+class ChannelSerializer(serializers.ModelSerializer):
+    artist = SimpleChannelArtistSerializer()
+    actor = serializers.SerializerMethodField()
+    downloads_count = serializers.SerializerMethodField()
+    attributed_to = federation_serializers.APIActorSerializer()
+    rss_url = serializers.CharField(source="get_rss_url")
+    url = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.Channel
+        fields = [
+            "uuid",
+            "artist",
+            "attributed_to",
+            "actor",
+            "creation_date",
+            "metadata",
+            "rss_url",
+            "url",
+            "downloads_count",
+        ]
+
+    def to_representation(self, obj):
+        data = super().to_representation(obj)
+        if self.context.get("subscriptions_count"):
+            data["subscriptions_count"] = self.get_subscriptions_count(obj)
+        return data
+
+    def get_subscriptions_count(self, obj) -> int:
+        return obj.actor.received_follows.exclude(approved=False).count()
+
+    def get_downloads_count(self, obj) -> int:
+        return getattr(obj, "_downloads_count", None) or 0
+
+    @extend_schema_field(federation_serializers.APIActorSerializer)
+    def get_actor(self, obj):
+        if obj.attributed_to == actors.get_service_actor():
+            return None
+        return federation_serializers.APIActorSerializer(obj.actor).data
+
+    @extend_schema_field(OpenApiTypes.URI)
+    def get_url(self, obj):
+        return obj.actor.url
+
+
+class InlineSubscriptionSerializer(serializers.Serializer):
+    uuid = serializers.UUIDField()
+    channel = serializers.UUIDField(source="target__channel__uuid")
+
+
+class AllSubscriptionsSerializer(serializers.Serializer):
+    results = InlineSubscriptionSerializer(source="*", many=True)
+    count = serializers.SerializerMethodField()
+
+    def get_count(self, o) -> int:
+        return len(o)
+
+
+class SubscriptionSerializer(serializers.Serializer):
+    approved = serializers.BooleanField(read_only=True)
+    fid = serializers.URLField(read_only=True)
+    uuid = serializers.UUIDField(read_only=True)
+    creation_date = serializers.DateTimeField(read_only=True)
+
+    def to_representation(self, obj):
+        data = super().to_representation(obj)
+        data["channel"] = ChannelSerializer(obj.target.channel).data
+        return data
+
+
+class RssSubscribeSerializer(serializers.Serializer):
+    url = serializers.URLField()
+
+
+class FeedFetchException(Exception):
+    pass
+
+
+class BlockedFeedException(FeedFetchException):
+    pass
+
+
+def retrieve_feed(url):
+    try:
+        logger.info("Fetching RSS feed at %s", url)
+        response = session.get_session().get(url)
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        if e.response:
+            raise FeedFetchException(
+                f"Error while fetching feed: HTTP {e.response.status_code}"
+            )
+        raise FeedFetchException("Error while fetching feed: unknown error")
+    except requests.exceptions.Timeout:
+        raise FeedFetchException("Error while fetching feed: timeout")
+    except requests.exceptions.ConnectionError:
+        raise FeedFetchException("Error while fetching feed: connection error")
+    except requests.RequestException as e:
+        raise FeedFetchException(f"Error while fetching feed: {e}")
+    except Exception as e:
+        raise FeedFetchException(f"Error while fetching feed: {e}")
+
+    return response
+
+
+@transaction.atomic
+def get_channel_from_rss_url(url, raise_exception=False):
+    # first, check if the url is blocked
+    is_valid, _ = mrf.inbox.apply({"id": url})
+    if not is_valid:
+        logger.warn("Feed fetch for url %s dropped by MRF", url)
+        raise BlockedFeedException("This feed or domain is blocked")
+
+    # retrieve the XML payload at the given URL
+    response = retrieve_feed(url)
+
+    parsed_feed = feedparser.parse(response.text)
+    serializer = RssFeedSerializer(data=parsed_feed["feed"])
+    if not serializer.is_valid(raise_exception=raise_exception):
+        raise FeedFetchException(f"Invalid xml content: {serializer.errors}")
+
+    # second mrf check with validated data
+    urls_to_check = set()
+    atom_link = serializer.validated_data.get("atom_link")
+
+    if atom_link and atom_link != url:
+        urls_to_check.add(atom_link)
+
+    if serializer.validated_data["link"] != url:
+        urls_to_check.add(serializer.validated_data["link"])
+
+    for u in urls_to_check:
+        is_valid, _ = mrf.inbox.apply({"id": u})
+        if not is_valid:
+            logger.warn("Feed fetch for url %s dropped by MRF", u)
+            raise BlockedFeedException("This feed or domain is blocked")
+
+    # now, we're clear, we can save the data
+    channel = serializer.save(rss_url=url)
+
+    entries = parsed_feed.entries or []
+    uploads = []
+    track_defaults = {}
+    existing_uploads = list(
+        channel.library.uploads.all().select_related(
+            "track__description", "track__attachment_cover"
+        )
+    )
+    if parsed_feed.feed.get("rights"):
+        track_defaults["copyright"] = parsed_feed.feed.rights
+    for entry in entries[: settings.PODCASTS_RSS_FEED_MAX_ITEMS]:
+        logger.debug("Importing feed item %s", entry.id)
+        s = RssFeedItemSerializer(data=entry)
+        if not s.is_valid(raise_exception=raise_exception):
+            logger.debug("Skipping invalid RSS feed item %s, ", entry, str(s.errors))
+            continue
+        uploads.append(
+            s.save(channel, existing_uploads=existing_uploads, **track_defaults)
+        )
+
+    common_utils.on_commit(
+        music_models.TrackActor.create_entries,
+        library=channel.library,
+        delete_existing=True,
+    )
+    if uploads:
+        latest_track_date = max([upload.track.creation_date for upload in uploads])
+        common_utils.update_modification_date(channel.artist, date=latest_track_date)
+    return channel, uploads
+
+
+# RSS related stuff
+# https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS
+# is extremely useful
+
+
+class RssFeedSerializer(serializers.Serializer):
+    title = serializers.CharField()
+    link = serializers.URLField(required=False, allow_blank=True)
+    language = serializers.CharField(required=False, allow_blank=True)
+    rights = serializers.CharField(required=False, allow_blank=True)
+    itunes_explicit = serializers.BooleanField(required=False, allow_null=True)
+    tags = serializers.ListField(required=False)
+    atom_link = serializers.DictField(required=False)
+    links = serializers.ListField(required=False)
+    summary_detail = serializers.DictField(required=False)
+    author_detail = serializers.DictField(required=False)
+    image = serializers.DictField(required=False)
+
+    def validate_atom_link(self, v):
+        if (
+            v.get("rel", "self") == "self"
+            and v.get("type", "application/rss+xml") == "application/rss+xml"
+        ):
+            return v["href"]
+
+    def validate_links(self, v):
+        for link in v:
+            if link.get("rel") == "self":
+                return link.get("href")
+
+    def validate_summary_detail(self, v):
+        content = v.get("value")
+        if not content:
+            return
+        return {
+            "content_type": v.get("type", "text/plain"),
+            "text": content,
+        }
+
+    def validate_image(self, v):
+        url = v.get("href")
+        if url:
+            return {
+                "url": url,
+                "mimetype": common_utils.get_mimetype_from_ext(url) or "image/jpeg",
+            }
+
+    def validate_tags(self, v):
+        data = {}
+        for row in v:
+            if row.get("scheme") != "http://www.itunes.com/":
+                continue
+            term = row["term"]
+            if "parent" not in data and term in categories.ITUNES_CATEGORIES:
+                data["parent"] = term
+            elif "child" not in data and term in categories.ITUNES_SUBCATEGORIES:
+                data["child"] = term
+            elif (
+                term not in categories.ITUNES_SUBCATEGORIES
+                and term not in categories.ITUNES_CATEGORIES
+            ):
+                raw_tags = term.split(" ")
+                data["tags"] = []
+                tag_serializer = tags_serializers.TagNameField()
+                for tag in raw_tags:
+                    try:
+                        data["tags"].append(tag_serializer.to_internal_value(tag))
+                    except Exception:
+                        pass
+
+        return data
+
+    def validate(self, data):
+        validated_data = super().validate(data)
+        if not validated_data.get("link"):
+            validated_data["link"] = validated_data.get("links")
+        if not validated_data.get("link"):
+            raise serializers.ValidationError("Missing link")
+        return validated_data
+
+    @transaction.atomic
+    def save(self, rss_url):
+        validated_data = self.validated_data
+        # because there may be redirections from the original feed URL
+        real_rss_url = validated_data.get("atom_link", rss_url) or rss_url
+        service_actor = actors.get_service_actor()
+        author = validated_data.get("author_detail", {})
+        categories = validated_data.get("tags", {})
+        metadata = {
+            "explicit": validated_data.get("itunes_explicit", False),
+            "copyright": validated_data.get("rights"),
+            "owner_name": author.get("name"),
+            "owner_email": author.get("email"),
+            "itunes_category": categories.get("parent"),
+            "itunes_subcategory": categories.get("child"),
+            "language": validated_data.get("language"),
+        }
+        public_url = validated_data["link"]
+        existing = (
+            models.Channel.objects.external_rss()
+            .filter(
+                Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url)
+            )
+            .first()
+        )
+        channel_defaults = {
+            "rss_url": real_rss_url,
+            "metadata": metadata,
+        }
+        if existing:
+            artist_kwargs = {"channel": existing}
+            actor_kwargs = {"channel": existing}
+            actor_defaults = {"url": public_url}
+        else:
+            artist_kwargs = {"pk": None}
+            actor_kwargs = {"pk": None}
+            preferred_username = f"rssfeed-{uuid.uuid4()}"
+            actor_defaults = {
+                "preferred_username": preferred_username,
+                "type": "Application",
+                "domain": service_actor.domain,
+                "url": public_url,
+                "fid": federation_utils.full_url(
+                    reverse(
+                        "federation:actors-detail",
+                        kwargs={"preferred_username": preferred_username},
+                    )
+                ),
+            }
+            channel_defaults["attributed_to"] = service_actor
+
+        actor_defaults["last_fetch_date"] = timezone.now()
+
+        # create/update the artist profile
+        artist, created = music_models.Artist.objects.update_or_create(
+            **artist_kwargs,
+            defaults={
+                "attributed_to": service_actor,
+                "name": validated_data["title"],
+                "content_category": "podcast",
+            },
+        )
+
+        cover = validated_data.get("image")
+
+        if cover:
+            common_utils.attach_file(artist, "attachment_cover", cover)
+        tags = categories.get("tags", [])
+
+        if tags:
+            tags_models.set_tags(artist, *tags)
+
+        summary = validated_data.get("summary_detail")
+        if summary:
+            common_utils.attach_content(artist, "description", summary)
+
+        if created:
+            channel_defaults["artist"] = artist
+
+        # create/update the actor
+        actor, created = federation_models.Actor.objects.update_or_create(
+            **actor_kwargs, defaults=actor_defaults
+        )
+        if created:
+            channel_defaults["actor"] = actor
+
+        # create the library
+        if not existing:
+            channel_defaults["library"] = music_models.Library.objects.create(
+                actor=service_actor,
+                privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY,
+                name=actor_defaults["preferred_username"],
+            )
+
+        # create/update the channel
+        channel, created = models.Channel.objects.update_or_create(
+            pk=existing.pk if existing else None,
+            defaults=channel_defaults,
+        )
+        return channel
+
+
+class ItunesDurationField(serializers.CharField):
+    def to_internal_value(self, v):
+        try:
+            return int(v)
+        except (ValueError, TypeError):
+            pass
+        parts = v.split(":")
+        int_parts = []
+        for part in parts:
+            try:
+                int_parts.append(int(part))
+            except (ValueError, TypeError):
+                raise serializers.ValidationError(f"Invalid duration {v}")
+
+        if len(int_parts) == 2:
+            hours = 0
+            minutes, seconds = int_parts
+        elif len(int_parts) == 3:
+            hours, minutes, seconds = int_parts
+        else:
+            raise serializers.ValidationError(f"Invalid duration {v}")
+
+        return (hours * 3600) + (minutes * 60) + seconds
+
+
+class DummyField(serializers.Field):
+    def to_internal_value(self, v):
+        return v
+
+
+def get_cached_upload(uploads, expected_track_uuid):
+    for upload in uploads:
+        if upload.track.uuid == expected_track_uuid:
+            return upload
+
+
+class PermissiveIntegerField(serializers.IntegerField):
+    def to_internal_value(self, v):
+        try:
+            return super().to_internal_value(v)
+        except serializers.ValidationError:
+            return self.default
+
+
+class RssFeedItemSerializer(serializers.Serializer):
+    id = serializers.CharField()
+    title = serializers.CharField()
+    rights = serializers.CharField(required=False, allow_blank=True)
+    itunes_season = serializers.IntegerField(
+        required=False, allow_null=True, default=None
+    )
+    itunes_episode = PermissiveIntegerField(
+        required=False, allow_null=True, default=None
+    )
+    itunes_duration = ItunesDurationField(
+        required=False, allow_null=True, default=None, allow_blank=True
+    )
+    links = serializers.ListField()
+    tags = serializers.ListField(required=False)
+    summary_detail = serializers.DictField(required=False)
+    content = serializers.ListField(required=False)
+    published_parsed = DummyField(required=False)
+    image = serializers.DictField(required=False)
+
+    def validate_summary_detail(self, v):
+        content = v.get("value")
+        if not content:
+            return
+        return {
+            "content_type": v.get("type", "text/plain"),
+            "text": content,
+        }
+
+    def validate_content(self, v):
+        # TODO: Are there RSS feeds that use more than one content item?
+        content = v[0].get("value")
+        if not content:
+            return
+        return {
+            "content_type": v[0].get("type", "text/plain"),
+            "text": content,
+        }
+
+    def validate_image(self, v):
+        url = v.get("href")
+        if url:
+            return {
+                "url": url,
+                "mimetype": common_utils.get_mimetype_from_ext(url) or "image/jpeg",
+            }
+
+    def validate_links(self, v):
+        data = {}
+        for row in v:
+            if not row.get("type", "").startswith("audio/"):
+                continue
+            if row.get("rel") != "enclosure":
+                continue
+            try:
+                size = int(row.get("length", 0) or 0) or None
+            except (TypeError, ValueError):
+                raise serializers.ValidationError("Invalid size")
+
+            data["audio"] = {
+                "mimetype": common_utils.get_audio_mimetype(row["type"]),
+                "size": size,
+                "source": row["href"],
+            }
+
+        if not data:
+            raise serializers.ValidationError("No valid audio enclosure found")
+
+        return data
+
+    def validate_tags(self, v):
+        data = {}
+        for row in v:
+            if row.get("scheme") != "http://www.itunes.com/":
+                continue
+            term = row["term"]
+            raw_tags = term.split(" ")
+            data["tags"] = []
+            tag_serializer = tags_serializers.TagNameField()
+            for tag in raw_tags:
+                try:
+                    data["tags"].append(tag_serializer.to_internal_value(tag))
+                except Exception:
+                    pass
+
+        return data
+
+    @transaction.atomic
+    def save(self, channel, existing_uploads=[], **track_defaults):
+        validated_data = self.validated_data
+        categories = validated_data.get("tags", {})
+        expected_uuid = uuid.uuid3(
+            uuid.NAMESPACE_URL, "rss://{}-{}".format(channel.pk, validated_data["id"])
+        )
+        existing_upload = get_cached_upload(existing_uploads, expected_uuid)
+        if existing_upload:
+            existing_track = existing_upload.track
+        else:
+            existing_track = (
+                music_models.Track.objects.filter(
+                    uuid=expected_uuid, artist__channel=channel
+                )
+                .select_related("description", "attachment_cover")
+                .first()
+            )
+            if existing_track:
+                existing_upload = existing_track.uploads.filter(
+                    library=channel.library
+                ).first()
+
+        track_defaults = track_defaults
+        track_defaults.update(
+            {
+                "disc_number": validated_data.get("itunes_season", 1) or 1,
+                "position": validated_data.get("itunes_episode", 1) or 1,
+                "title": validated_data["title"],
+                "artist": channel.artist,
+            }
+        )
+        if "rights" in validated_data:
+            track_defaults["copyright"] = validated_data["rights"]
+
+        if "published_parsed" in validated_data:
+            track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
+                time.mktime(validated_data["published_parsed"])
+            ).replace(tzinfo=ZoneInfo("UTC"))
+
+        upload_defaults = {
+            "source": validated_data["links"]["audio"]["source"],
+            "size": validated_data["links"]["audio"]["size"],
+            "mimetype": validated_data["links"]["audio"]["mimetype"],
+            "duration": validated_data.get("itunes_duration") or None,
+            "import_status": "finished",
+            "library": channel.library,
+        }
+        if existing_track:
+            track_kwargs = {"pk": existing_track.pk}
+            upload_kwargs = {"track": existing_track}
+        else:
+            track_kwargs = {"pk": None}
+            track_defaults["uuid"] = expected_uuid
+            upload_kwargs = {"pk": None}
+
+        if existing_upload and existing_upload.source != upload_defaults["source"]:
+            # delete existing upload, the url to the audio file has changed
+            existing_upload.delete()
+
+        # create/update the track
+        track, created = music_models.Track.objects.update_or_create(
+            **track_kwargs,
+            defaults=track_defaults,
+        )
+        # optimisation for reducing SQL queries, because we cannot use select_related with
+        # update or create, so we restore the cache by hand
+        if existing_track:
+            for field in ["attachment_cover", "description"]:
+                cached_id_value = getattr(existing_track, f"{field}_id")
+                new_id_value = getattr(track, f"{field}_id")
+                if new_id_value and cached_id_value == new_id_value:
+                    setattr(track, field, getattr(existing_track, field))
+
+        cover = validated_data.get("image")
+
+        if cover:
+            common_utils.attach_file(track, "attachment_cover", cover)
+        tags = categories.get("tags", [])
+
+        if tags:
+            tags_models.set_tags(track, *tags)
+
+        # "content" refers to the <content:encoded> node in the RSS feed,
+        # whereas "summary_detail" refers to the <description> node.
+        # <description> is intended to be used as a short summary and is often
+        # encoded merely as plain text, whereas <content:encoded> contains
+        # the full episode description and is generally encoded as HTML.
+        #
+        # For details, see https://www.rssboard.org/rss-profile#element-channel-item-description
+        summary = validated_data.get("content")
+        if not summary:
+            summary = validated_data.get("summary_detail")
+        if summary:
+            common_utils.attach_content(track, "description", summary)
+
+        if created:
+            upload_defaults["track"] = track
+
+        # create/update the upload
+        upload, created = music_models.Upload.objects.update_or_create(
+            **upload_kwargs, defaults=upload_defaults
+        )
+
+        return upload
+
+
+def rfc822_date(dt):
+    return dt.strftime("%a, %d %b %Y %H:%M:%S %z")
+
+
+def rss_duration(seconds):
+    if not seconds:
+        return "00:00:00"
+    full_hours = seconds // 3600
+    full_minutes = (seconds - (full_hours * 3600)) // 60
+    remaining_seconds = seconds - (full_hours * 3600) - (full_minutes * 60)
+    return "{}:{}:{}".format(
+        str(full_hours).zfill(2),
+        str(full_minutes).zfill(2),
+        str(remaining_seconds).zfill(2),
+    )
+
+
+def rss_serialize_item(upload):
+    data = {
+        "title": [{"value": upload.track.title}],
+        "itunes:title": [{"value": upload.track.title}],
+        "guid": [{"cdata_value": str(upload.uuid), "isPermaLink": "false"}],
+        "pubDate": [{"value": rfc822_date(upload.creation_date)}],
+        "itunes:duration": [{"value": rss_duration(upload.duration)}],
+        "itunes:explicit": [{"value": "no"}],
+        "itunes:episodeType": [{"value": "full"}],
+        "itunes:season": [{"value": upload.track.disc_number or 1}],
+        "itunes:episode": [{"value": upload.track.position or 1}],
+        "link": [{"value": federation_utils.full_url(upload.track.get_absolute_url())}],
+        "enclosure": [
+            {
+                # we enforce MP3, since it's the only format supported everywhere
+                "url": federation_utils.full_url(
+                    reverse(
+                        "api:v1:stream-detail", kwargs={"uuid": str(upload.track.uuid)}
+                    )
+                    + ".mp3"
+                ),
+                "length": upload.size or 0,
+                "type": "audio/mpeg",
+            }
+        ],
+    }
+    if upload.track.description:
+        data["itunes:subtitle"] = [{"value": upload.track.description.truncate(254)}]
+        data["itunes:summary"] = [{"cdata_value": upload.track.description.rendered}]
+        data["description"] = [{"value": upload.track.description.as_plain_text}]
+
+    if upload.track.attachment_cover:
+        data["itunes:image"] = [
+            {"href": upload.track.attachment_cover.download_url_original}
+        ]
+
+    tagged_items = getattr(upload.track, "_prefetched_tagged_items", [])
+    if tagged_items:
+        data["itunes:keywords"] = [
+            {"value": ",".join([ti.tag.name for ti in tagged_items])}
+        ]
+
+    return data
+
+
+def rss_serialize_channel(channel):
+    metadata = channel.metadata or {}
+    explicit = metadata.get("explicit", False)
+    copyright = metadata.get("copyright", "All rights reserved")
+    owner_name = metadata.get("owner_name", channel.attributed_to.display_name)
+    owner_email = metadata.get("owner_email")
+    itunes_category = metadata.get("itunes_category")
+    itunes_subcategory = metadata.get("itunes_subcategory")
+    language = metadata.get("language")
+
+    data = {
+        "title": [{"value": channel.artist.name}],
+        "copyright": [{"value": copyright}],
+        "itunes:explicit": [{"value": "no" if not explicit else "yes"}],
+        "itunes:author": [{"value": owner_name}],
+        "itunes:owner": [{"itunes:name": [{"value": owner_name}]}],
+        "itunes:type": [{"value": "episodic"}],
+        "link": [{"value": channel.get_absolute_url()}],
+        "atom:link": [
+            {
+                "href": channel.get_rss_url(),
+                "rel": "self",
+                "type": "application/rss+xml",
+            },
+            {
+                "href": channel.actor.fid,
+                "rel": "alternate",
+                "type": "application/activity+json",
+            },
+        ],
+    }
+    if language:
+        data["language"] = [{"value": language}]
+
+    if owner_email:
+        data["itunes:owner"][0]["itunes:email"] = [{"value": owner_email}]
+
+    if itunes_category:
+        node = {"text": itunes_category}
+        if itunes_subcategory:
+            node["itunes:category"] = [{"text": itunes_subcategory}]
+        data["itunes:category"] = [node]
+
+    if channel.artist.description:
+        data["itunes:subtitle"] = [{"value": channel.artist.description.truncate(254)}]
+        data["itunes:summary"] = [{"cdata_value": channel.artist.description.rendered}]
+        data["description"] = [{"value": channel.artist.description.as_plain_text}]
+
+    if channel.artist.attachment_cover:
+        data["itunes:image"] = [
+            {"href": channel.artist.attachment_cover.download_url_original}
+        ]
+    else:
+        placeholder_url = federation_utils.full_url(
+            static("images/podcasts-cover-placeholder.png")
+        )
+        data["itunes:image"] = [{"href": placeholder_url}]
+
+    tagged_items = getattr(channel.artist, "_prefetched_tagged_items", [])
+
+    if tagged_items:
+        data["itunes:keywords"] = [
+            {"value": " ".join([ti.tag.name for ti in tagged_items])}
+        ]
+
+    return data
+
+
+def rss_serialize_channel_full(channel, uploads):
+    channel_data = rss_serialize_channel(channel)
+    channel_data["item"] = [rss_serialize_item(upload) for upload in uploads]
+    return {"channel": channel_data}
+
+
+# OPML stuff
+def get_opml_outline(channel):
+    return {
+        "title": channel.artist.name,
+        "text": channel.artist.name,
+        "type": "rss",
+        "xmlUrl": channel.get_rss_url(),
+        "htmlUrl": channel.actor.url,
+    }
+
+
+def get_opml(channels, date, title):
+    return {
+        "version": "2.0",
+        "head": [{"date": [{"value": rfc822_date(date)}], "title": [{"value": title}]}],
+        "body": [{"outline": [get_opml_outline(channel) for channel in channels]}],
+    }

api/funkwhale_api/audio/spa_views.py

@@ -0,0 +1,104 @@
+import urllib.parse
+
+from django.conf import settings
+from django.db.models import Q
+from django.urls import reverse
+from rest_framework import serializers
+
+from funkwhale_api.common import middleware, preferences, utils
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.music import spa_views
+
+from . import models
+
+
+def channel_detail(query, redirect_to_ap):
+    queryset = models.Channel.objects.filter(query).select_related(
+        "artist__attachment_cover", "actor", "library"
+    )
+    try:
+        obj = queryset.get()
+    except models.Channel.DoesNotExist:
+        return []
+
+    if redirect_to_ap:
+        raise middleware.ApiRedirect(obj.actor.fid)
+
+    obj_url = utils.join_url(
+        settings.FUNQUAIL_URL,
+        utils.spa_reverse(
+            "channel_detail", kwargs={"username": obj.actor.full_username}
+        ),
+    )
+    metas = [
+        {"tag": "meta", "property": "og:url", "content": obj_url},
+        {"tag": "meta", "property": "og:title", "content": obj.artist.name},
+        {"tag": "meta", "property": "og:type", "content": "profile"},
+    ]
+
+    if obj.artist.attachment_cover:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:image",
+                "content": obj.artist.attachment_cover.download_url_medium_square_crop,
+            }
+        )
+
+    if preferences.get("federation__enabled"):
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/activity+json",
+                "href": obj.actor.fid,
+            }
+        )
+
+    metas.append(
+        {
+            "tag": "link",
+            "rel": "alternate",
+            "type": "application/rss+xml",
+            "href": obj.get_rss_url(),
+            "title": f"{obj.artist.name} - RSS Podcast Feed",
+        },
+    )
+
+    if obj.library.uploads.all().playable_by(None).exists():
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/json+oembed",
+                "href": (
+                    utils.join_url(settings.FUNQUAIL_URL, reverse("api:v1:oembed"))
+                    + f"?format=json&url={urllib.parse.quote_plus(obj_url)}"
+                ),
+            }
+        )
+        # twitter player is also supported in various software
+        metas += spa_views.get_twitter_card_metas(type="channel", id=obj.uuid)
+    return metas
+
+
+def channel_detail_uuid(request, uuid, redirect_to_ap):
+    validator = serializers.UUIDField().to_internal_value
+    try:
+        uuid = validator(uuid)
+    except serializers.ValidationError:
+        return []
+    return channel_detail(Q(uuid=uuid), redirect_to_ap)
+
+
+def channel_detail_username(request, username, redirect_to_ap):
+    validator = federation_utils.get_actor_data_from_username
+    try:
+        username_data = validator(username)
+    except serializers.ValidationError:
+        return []
+    query = Q(
+        actor__domain=username_data["domain"],
+        actor__preferred_username__iexact=username_data["username"],
+    )
+    return channel_detail(query, redirect_to_ap)

api/funkwhale_api/audio/tasks.py

@@ -0,0 +1,50 @@
+import datetime
+import logging
+
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from funkwhale_api.taskapp import celery
+
+from . import models, serializers
+
+logger = logging.getLogger(__name__)
+
+
+@celery.app.task(name="audio.fetch_rss_feeds")
+def fetch_rss_feeds():
+    limit = timezone.now() - datetime.timedelta(
+        seconds=settings.PODCASTS_RSS_FEED_REFRESH_DELAY
+    )
+    candidates = (
+        models.Channel.objects.external_rss()
+        .filter(actor__last_fetch_date__lte=limit)
+        .values_list("rss_url", flat=True)
+    )
+
+    total = len(candidates)
+    logger.info("Refreshing %s rss feeds…", total)
+    for url in candidates:
+        fetch_rss_feed.delay(rss_url=url)
+
+
+@celery.app.task(name="audio.fetch_rss_feed")
+@transaction.atomic
+def fetch_rss_feed(rss_url):
+    channel = (
+        models.Channel.objects.external_rss()
+        .filter(rss_url=rss_url)
+        .order_by("id")
+        .first()
+    )
+    if not channel:
+        logger.warn("Cannot refresh non external feed")
+        return
+
+    try:
+        serializers.get_channel_from_rss_url(rss_url)
+    except serializers.BlockedFeedException:
+        # channel was blocked since last fetch, let's delete it
+        logger.info("Deleting blocked channel linked to %s", rss_url)
+        channel.delete()

api/funkwhale_api/audio/views.py

@@ -0,0 +1,344 @@
+from django import http
+from django.db import transaction
+from django.db.models import Count, Prefetch, Q, Sum
+from django.utils import timezone
+from drf_spectacular.utils import extend_schema, extend_schema_view
+from rest_framework import decorators, exceptions, mixins
+from rest_framework import permissions as rest_permissions
+from rest_framework import response, viewsets
+
+from funkwhale_api.common import locales, permissions, preferences
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common.mixins import MultipleLookupDetailMixin
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import routes
+from funkwhale_api.federation import tasks as federation_tasks
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import views as music_views
+from funkwhale_api.users.oauth import permissions as oauth_permissions
+
+from . import categories, filters, models, renderers, serializers
+
+ARTIST_PREFETCH_QS = (
+    music_models.Artist.objects.select_related(
+        "description",
+        "attachment_cover",
+    )
+    .prefetch_related(music_views.TAG_PREFETCH)
+    .annotate(_tracks_count=Count("tracks"))
+)
+
+
+class ChannelsMixin:
+    def dispatch(self, request, *args, **kwargs):
+        if not preferences.get("audio__channels_enabled"):
+            return http.HttpResponse(status=405)
+        return super().dispatch(request, *args, **kwargs)
+
+
+@extend_schema_view(
+    metedata_choices=extend_schema(operation_id="get_channel_metadata_choices"),
+    subscribe=extend_schema(operation_id="subscribe_channel"),
+    unsubscribe=extend_schema(operation_id="unsubscribe_channel"),
+    rss_subscribe=extend_schema(operation_id="subscribe_channel_rss"),
+)
+class ChannelViewSet(
+    ChannelsMixin,
+    MultipleLookupDetailMixin,
+    mixins.CreateModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.ListModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    url_lookups = [
+        {
+            "lookup_field": "uuid",
+            "validator": serializers.serializers.UUIDField().to_internal_value,
+        },
+        {
+            "lookup_field": "username",
+            "validator": federation_utils.get_actor_data_from_username,
+            "get_query": lambda v: Q(
+                actor__domain=v["domain"],
+                actor__preferred_username__iexact=v["username"],
+            ),
+        },
+    ]
+    filterset_class = filters.ChannelFilter
+    serializer_class = serializers.ChannelSerializer
+    queryset = (
+        models.Channel.objects.all()
+        .prefetch_related(
+            "library",
+            "attributed_to",
+            "actor",
+            Prefetch("artist", queryset=ARTIST_PREFETCH_QS),
+        )
+        .order_by("-creation_date")
+    )
+    permission_classes = [
+        oauth_permissions.ScopePermission,
+        permissions.OwnerPermission,
+    ]
+    required_scope = "libraries"
+    anonymous_policy = "setting"
+    owner_checks = ["write"]
+    owner_field = "attributed_to.user"
+    owner_exception = exceptions.PermissionDenied
+
+    def get_serializer_class(self):
+        if self.request.method.lower() in ["head", "get", "options"]:
+            return serializers.ChannelSerializer
+        elif self.action in ["update", "partial_update"]:
+            return serializers.ChannelUpdateSerializer
+        elif self.action == "create":
+            return serializers.ChannelCreateSerializer
+        return serializers.ChannelSerializer
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.action == "retrieve":
+            queryset = queryset.annotate(
+                _downloads_count=Sum("artist__tracks__downloads_count")
+            )
+        return queryset
+
+    def perform_create(self, serializer):
+        return serializer.save(attributed_to=self.request.user.actor)
+
+    def list(self, request, *args, **kwargs):
+        if self.request.GET.get("output") == "opml":
+            queryset = self.filter_queryset(self.get_queryset())[:500]
+            opml = serializers.get_opml(
+                channels=queryset,
+                date=timezone.now(),
+                title="FunQuail channels OPML export",
+            )
+            xml_body = renderers.render_xml(renderers.dict_to_xml_tree("opml", opml))
+            return http.HttpResponse(xml_body, content_type="application/xml")
+
+        else:
+            return super().list(request, *args, **kwargs)
+
+    def get_object(self):
+        obj = super().get_object()
+        if (
+            self.action == "retrieve"
+            and self.request.GET.get("refresh", "").lower() == "true"
+        ):
+            obj = music_views.refetch_obj(obj, self.get_queryset())
+        return obj
+
+    @decorators.action(
+        detail=True,
+        methods=["post"],
+        permission_classes=[rest_permissions.IsAuthenticated],
+        serializer_class=serializers.SubscriptionSerializer,
+    )
+    def subscribe(self, request, *args, **kwargs):
+        object = self.get_object()
+        subscription = federation_models.Follow(actor=request.user.actor)
+        subscription.fid = subscription.get_federation_id()
+        subscription, created = SubscriptionsViewSet.queryset.get_or_create(
+            target=object.actor,
+            actor=request.user.actor,
+            defaults={
+                "approved": True,
+                "fid": subscription.fid,
+                "uuid": subscription.uuid,
+            },
+        )
+        # prefetch stuff
+        subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
+        if not object.actor.is_local:
+            routes.outbox.dispatch({"type": "Follow"}, context={"follow": subscription})
+
+        data = serializers.SubscriptionSerializer(subscription).data
+        return response.Response(data, status=201)
+
+    @extend_schema(responses={204: None})
+    @decorators.action(
+        detail=True,
+        methods=["post", "delete"],
+        permission_classes=[rest_permissions.IsAuthenticated],
+    )
+    def unsubscribe(self, request, *args, **kwargs):
+        object = self.get_object()
+        follow_qs = request.user.actor.emitted_follows.filter(target=object.actor)
+        follow = follow_qs.first()
+        if follow:
+            if not object.actor.is_local:
+                routes.outbox.dispatch(
+                    {"type": "Undo", "object": {"type": "Follow"}},
+                    context={"follow": follow},
+                )
+            follow_qs.delete()
+        return response.Response(status=204)
+
+    @decorators.action(
+        detail=True,
+        methods=["get"],
+        content_negotiation_class=renderers.PodcastRSSContentNegociation,
+    )
+    def rss(self, request, *args, **kwargs):
+        object = self.get_object()
+        if not object.attributed_to.is_local:
+            return response.Response({"detail": "Not found"}, status=404)
+
+        if object.attributed_to == actors.get_service_actor():
+            # external feed, we redirect to the canonical one
+            return http.HttpResponseRedirect(object.rss_url)
+
+        uploads = (
+            object.library.uploads.playable_by(None)
+            .prefetch_related(
+                Prefetch(
+                    "track",
+                    queryset=music_models.Track.objects.select_related(
+                        "attachment_cover", "description"
+                    ).prefetch_related(
+                        music_views.TAG_PREFETCH,
+                    ),
+                ),
+            )
+            .select_related("track__attachment_cover", "track__description")
+            .order_by("-creation_date")
+        )[:50]
+        data = serializers.rss_serialize_channel_full(channel=object, uploads=uploads)
+        return response.Response(data, status=200)
+
+    @decorators.action(
+        methods=["get"],
+        detail=False,
+        url_path="metadata-choices",
+        url_name="metadata_choices",
+        permission_classes=[],
+    )
+    def metedata_choices(self, request, *args, **kwargs):
+        data = {
+            "language": [
+                {"value": code, "label": name} for code, name in locales.ISO_639_CHOICES
+            ],
+            "itunes_category": [
+                {"value": code, "label": code, "children": children}
+                for code, children in categories.ITUNES_CATEGORIES.items()
+            ],
+        }
+        return response.Response(data)
+
+    @decorators.action(
+        methods=["post"],
+        detail=False,
+        url_path="rss-subscribe",
+        url_name="rss_subscribe",
+    )
+    @transaction.atomic
+    def rss_subscribe(self, request, *args, **kwargs):
+        serializer = serializers.RssSubscribeSerializer(data=request.data)
+        if not serializer.is_valid():
+            return response.Response(serializer.errors, status=400)
+        channel = (
+            models.Channel.objects.filter(
+                rss_url=serializer.validated_data["url"],
+            )
+            .order_by("id")
+            .first()
+        )
+        if not channel:
+            # try to retrieve the channel via its URL and create it
+            try:
+                channel, uploads = serializers.get_channel_from_rss_url(
+                    serializer.validated_data["url"]
+                )
+            except serializers.FeedFetchException as e:
+                return response.Response(
+                    {"detail": str(e)},
+                    status=400,
+                )
+
+        subscription = federation_models.Follow(actor=request.user.actor)
+        subscription.fid = subscription.get_federation_id()
+        subscription, created = SubscriptionsViewSet.queryset.get_or_create(
+            target=channel.actor,
+            actor=request.user.actor,
+            defaults={
+                "approved": True,
+                "fid": subscription.fid,
+                "uuid": subscription.uuid,
+            },
+        )
+        # prefetch stuff
+        subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
+
+        return response.Response(
+            serializers.SubscriptionSerializer(subscription).data, status=201
+        )
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["subscriptions_count"] = self.action in [
+            "retrieve",
+            "create",
+            "update",
+            "partial_update",
+        ]
+        if self.request.user.is_authenticated:
+            context["actor"] = self.request.user.actor
+        return context
+
+    @transaction.atomic
+    def perform_destroy(self, instance):
+        instance.__class__.objects.filter(pk=instance.pk).delete()
+        common_utils.on_commit(
+            federation_tasks.remove_actor.delay, actor_id=instance.actor.pk
+        )
+
+
+class SubscriptionsViewSet(
+    ChannelsMixin,
+    mixins.RetrieveModelMixin,
+    mixins.ListModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    serializer_class = serializers.SubscriptionSerializer
+    queryset = (
+        federation_models.Follow.objects.exclude(target__channel__isnull=True)
+        .prefetch_related(
+            "target__channel__library",
+            "target__channel__attributed_to",
+            "actor",
+            Prefetch("target__channel__artist", queryset=ARTIST_PREFETCH_QS),
+        )
+        .order_by("-creation_date")
+    )
+    permission_classes = [
+        oauth_permissions.ScopePermission,
+        rest_permissions.IsAuthenticated,
+    ]
+    required_scope = "libraries"
+    anonymous_policy = False
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor)
+
+    @extend_schema(
+        responses=serializers.AllSubscriptionsSerializer(),
+        operation_id="get_all_subscriptions",
+    )
+    @decorators.action(methods=["get"], detail=False)
+    def all(self, request, *args, **kwargs):
+        """
+        Return all the subscriptions of the current user, with only limited data
+        to have a performant endpoint and avoid lots of queries just to display
+        subscription status in the UI
+        """
+        subscriptions = self.get_queryset().values("uuid", "target__channel__uuid")
+
+        payload = serializers.AllSubscriptionsSerializer(subscriptions).data
+        return response.Response(payload, status=200)

api/funkwhale_api/cli/base.py

@@ -0,0 +1,66 @@
+import functools
+
+import click
+
+
+@click.group()
+def cli():
+    pass
+
+
+def confirm_action(f, id_var, message_template="Do you want to proceed?"):
+    @functools.wraps(f)
+    def action(*args, **kwargs):
+        if id_var:
+            id_value = kwargs[id_var]
+            message = message_template.format(len(id_value))
+        else:
+            message = message_template
+        if not kwargs.pop("no_input", False) and not click.confirm(message, abort=True):
+            return
+
+        return f(*args, **kwargs)
+
+    return action
+
+
+def delete_command(
+    group,
+    id_var="id",
+    name="rm",
+    message_template="Do you want to delete {} objects? This action is irreversible.",
+):
+    """
+    Wrap a command to ensure it asks for confirmation before deletion, unless the --no-input
+    flag is provided
+    """
+
+    def decorator(f):
+        decorated = click.option("--no-input", is_flag=True)(f)
+        decorated = confirm_action(
+            decorated, id_var=id_var, message_template=message_template
+        )
+        return group.command(name)(decorated)
+
+    return decorator
+
+
+def update_command(
+    group,
+    id_var="id",
+    name="set",
+    message_template="Do you want to update {} objects? This action may have irreversible consequnces.",
+):
+    """
+    Wrap a command to ensure it asks for confirmation before deletion, unless the --no-input
+    flag is provided
+    """
+
+    def decorator(f):
+        decorated = click.option("--no-input", is_flag=True)(f)
+        decorated = confirm_action(
+            decorated, id_var=id_var, message_template=message_template
+        )
+        return group.command(name)(decorated)
+
+    return decorator

api/funkwhale_api/cli/library.py

@@ -0,0 +1,51 @@
+import click
+
+from funkwhale_api.music import tasks
+
+from . import base
+
+
+def handler_add_tags_from_tracks(
+    artists=False,
+    albums=False,
+):
+    result = None
+    if artists:
+        result = tasks.artists_set_tags_from_tracks()
+    elif albums:
+        result = tasks.albums_set_tags_from_tracks()
+    else:
+        raise click.BadOptionUsage("You must specify artists or albums")
+
+    if result is None:
+        click.echo("  No relevant tags found")
+    else:
+        click.echo(f"  Relevant tags added to {len(result)} objects")
+
+
+@base.cli.group()
+def albums():
+    """Manage albums"""
+    pass
+
+
+@base.cli.group()
+def artists():
+    """Manage artists"""
+    pass
+
+
+@albums.command(name="add-tags-from-tracks")
+def albums_add_tags_from_tracks():
+    """
+    Associate tags to album with no genre tags, assuming identical tags are found on the album tracks
+    """
+    handler_add_tags_from_tracks(albums=True)
+
+
+@artists.command(name="add-tags-from-tracks")
+def artists_add_tags_from_tracks():
+    """
+    Associate tags to artists with no genre tags, assuming identical tags are found on the artist tracks
+    """
+    handler_add_tags_from_tracks(artists=True)

api/funkwhale_api/cli/main.py

@@ -0,0 +1,22 @@
+import sys
+
+import click
+from rest_framework.exceptions import ValidationError
+
+from . import library  # noqa
+from . import media  # noqa
+from . import plugins  # noqa
+from . import users  # noqa
+from . import base
+
+
+def invoke():
+    try:
+        return base.cli()
+    except ValidationError as e:
+        click.secho("Invalid data:", fg="red")
+        for field, errors in e.detail.items():
+            click.secho(f"  {field}:", fg="red")
+            for error in errors:
+                click.secho(f"    - {error}", fg="red")
+        sys.exit(1)

api/funkwhale_api/cli/media.py

@@ -0,0 +1,61 @@
+import click
+from django.conf import settings
+from django.core.cache import cache
+from django.core.files.storage import default_storage
+from versatileimagefield import settings as vif_settings
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common.models import Attachment
+
+from . import base
+
+
+@base.cli.group()
+def media():
+    """Manage media files (avatars, covers, attachments…)"""
+    pass
+
+
+@media.command("generate-thumbnails")
+@click.option("-d", "--delete", is_flag=True)
+def generate_thumbnails(delete):
+    """
+    Generate thumbnails for all images (avatars, covers, etc.).
+
+    This can take a long time and generate a lot of I/O depending of the size
+    of your library.
+    """
+    click.echo("Deleting existing thumbnails…")
+    if delete:
+        try:
+            # FileSystemStorage doesn't support deleting a non-empty directory
+            # so we reimplemented a method to do so
+            default_storage.force_delete("__sized__")
+        except AttributeError:
+            # backends doesn't support directory deletion
+            pass
+    MODELS = [
+        (Attachment, "file", "attachment_square"),
+    ]
+    for model, attribute, key_set in MODELS:
+        click.echo(f"Generating thumbnails for {model._meta.label}.{attribute}…")
+        qs = model.objects.exclude(**{f"{attribute}__isnull": True})
+        qs = qs.exclude(**{attribute: ""})
+        cache_key = "*{}{}*".format(
+            settings.MEDIA_URL, vif_settings.VERSATILEIMAGEFIELD_SIZED_DIRNAME
+        )
+        entries = cache.keys(cache_key)
+        if entries:
+            click.echo(f"  Clearing {len(entries)} cache entries: {cache_key}…")
+            for keys in common_utils.batch(iter(entries)):
+                cache.delete_many(keys)
+        warmer = VersatileImageFieldWarmer(
+            instance_or_queryset=qs,
+            rendition_key_set=key_set,
+            image_attr=attribute,
+            verbose=True,
+        )
+        click.echo("  Creating images")
+        num_created, failed_to_create = warmer.warm()
+        click.echo(f"  {num_created} created, {len(failed_to_create)} in error")

api/funkwhale_api/cli/plugins.py

@@ -0,0 +1,34 @@
+import os
+import subprocess
+import sys
+
+import click
+from django.conf import settings
+
+from . import base
+
+
+@base.cli.group()
+def plugins():
+    """Manage plugins"""
+    pass
+
+
+@plugins.command("install")
+@click.argument("plugin", nargs=-1)
+def install(plugin):
+    """
+    Install a plugin from a given URL (zip, pip or git are supported)
+    """
+    if not plugin:
+        return click.echo("No plugin provided")
+
+    click.echo("Installing plugins…")
+    pip_install(list(plugin), settings.FUNQUAIL_PLUGINS_PATH)
+
+
+def pip_install(deps, target):
+    if not deps:
+        return
+    pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    subprocess.check_call([pip_path, "install", "-t", target] + deps)

api/funkwhale_api/cli/users.py

@@ -0,0 +1,235 @@
+import click
+from django.db import transaction
+
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.users import models, serializers, tasks
+
+from . import base, utils
+
+
+class FakeRequest:
+    def __init__(self, session={}):
+        self.session = session
+
+
+@transaction.atomic
+def handler_create_user(
+    username,
+    password,
+    email,
+    is_superuser=False,
+    is_staff=False,
+    permissions=[],
+    upload_quota=None,
+):
+    serializer = serializers.RS(
+        data={
+            "username": username,
+            "email": email,
+            "password1": password,
+            "password2": password,
+        }
+    )
+    utils.logger.debug("Validating user data…")
+    serializer.is_valid(raise_exception=True)
+
+    # Override e-mail validation, we assume accounts created from CLI have a valid e-mail
+    request = FakeRequest(session={"account_verified_email": email})
+    utils.logger.debug("Creating user…")
+    user = serializer.save(request=request)
+    utils.logger.debug("Setting permissions and other attributes…")
+    user.is_staff = is_staff or is_superuser  # Always set staff if superuser is set
+    user.upload_quota = upload_quota
+    user.is_superuser = is_superuser
+    for permission in permissions:
+        if permission in models.PERMISSIONS:
+            utils.logger.debug("Setting %s permission to True", permission)
+            setattr(user, f"permission_{permission}", True)
+        else:
+            utils.logger.warn("Unknown permission %s", permission)
+    utils.logger.debug("Creating actor…")
+    user.actor = models.create_actor(user)
+    user.save()
+    return user
+
+
+@transaction.atomic
+def handler_delete_user(usernames, soft=True):
+    for username in usernames:
+        click.echo(f"Deleting {username}…")
+        actor = None
+        user = None
+        try:
+            user = models.User.objects.get(username=username)
+        except models.User.DoesNotExist:
+            try:
+                actor = federation_models.Actor.objects.local().get(
+                    preferred_username=username
+                )
+            except federation_models.Actor.DoesNotExist:
+                click.echo("  Not found, skipping")
+                continue
+
+        actor = actor or user.actor
+        if user:
+            tasks.delete_account(user_id=user.pk)
+        if not soft:
+            click.echo("  Hard delete, removing actor")
+            actor.delete()
+        click.echo("  Done")
+
+
+@transaction.atomic
+def handler_update_user(usernames, kwargs):
+    users = models.User.objects.filter(username__in=usernames)
+    total = users.count()
+    if not total:
+        click.echo("No matching users")
+        return
+
+    final_kwargs = {}
+    supported_fields = [
+        "is_active",
+        "permission_moderation",
+        "permission_library",
+        "permission_settings",
+        "is_staff",
+        "is_superuser",
+        "upload_quota",
+        "password",
+    ]
+    for field in supported_fields:
+        try:
+            value = kwargs[field]
+        except KeyError:
+            continue
+        final_kwargs[field] = value
+
+    click.echo(
+        "Updating {} on {} matching users…".format(
+            ", ".join(final_kwargs.keys()), total
+        )
+    )
+    if "password" in final_kwargs:
+        new_password = final_kwargs.pop("password")
+        for user in users:
+            user.set_password(new_password)
+        models.User.objects.bulk_update(users, ["password"])
+    if final_kwargs:
+        users.update(**final_kwargs)
+    click.echo("Done!")
+
+
+@base.cli.group()
+def users():
+    """Manage users"""
+    pass
+
+
+@users.command()
+@click.option("--username", "-u", prompt=True, required=True)
+@click.option(
+    "-p",
+    "--password",
+    prompt="Password (leave empty to have a random one generated)",
+    hide_input=True,
+    envvar="FUNQUAIL_CLI_USER_PASSWORD",
+    default="",
+    help="If empty, a random password will be generated and displayed in console output",
+)
+@click.option(
+    "-e",
+    "--email",
+    prompt=True,
+    help="Email address to associate with the account",
+    required=True,
+)
+@click.option(
+    "-q",
+    "--upload-quota",
+    help="Upload quota (leave empty to use default pod quota)",
+    required=False,
+    default=None,
+    type=click.INT,
+)
+@click.option(
+    "--superuser/--no-superuser",
+    default=False,
+)
+@click.option(
+    "--staff/--no-staff",
+    default=False,
+)
+@click.option(
+    "--permission",
+    multiple=True,
+)
+def create(username, password, email, superuser, staff, permission, upload_quota):
+    """Create a new user"""
+    generated_password = None
+    if password == "":
+        generated_password = models.User.objects.make_random_password()
+    user = handler_create_user(
+        username=username,
+        password=password or generated_password,
+        email=email,
+        is_superuser=superuser,
+        is_staff=staff,
+        permissions=permission,
+        upload_quota=upload_quota,
+    )
+    click.echo(f"User {user.username} created!")
+    if generated_password:
+        click.echo(f"  Generated password: {generated_password}")
+
+
+@base.delete_command(group=users, id_var="username")
+@click.argument("username", nargs=-1)
+@click.option(
+    "--hard/--no-hard",
+    default=False,
+    help="Purge all user-related info (allow recreating a user with the same username)",
+)
+def delete(username, hard):
+    """Delete given users"""
+    handler_delete_user(usernames=username, soft=not hard)
+
+
+@base.update_command(group=users, id_var="username")
+@click.argument("username", nargs=-1)
+@click.option(
+    "--active/--inactive",
+    help="Mark as active or inactive (inactive users cannot login or use the service)",
+    default=None,
+)
+@click.option("--superuser/--no-superuser", default=None)
+@click.option("--staff/--no-staff", default=None)
+@click.option("--permission-library/--no-permission-library", default=None)
+@click.option("--permission-moderation/--no-permission-moderation", default=None)
+@click.option("--permission-settings/--no-permission-settings", default=None)
+@click.option("--password", default=None, envvar="FUNQUAIL_CLI_USER_UPDATE_PASSWORD")
+@click.option(
+    "-q",
+    "--upload-quota",
+    type=click.INT,
+)
+def update(username, **kwargs):
+    """Update attributes for given users"""
+    field_mapping = {
+        "active": "is_active",
+        "superuser": "is_superuser",
+        "staff": "is_staff",
+    }
+    final_kwargs = {}
+    for cli_field, value in kwargs.items():
+        if value is None:
+            continue
+        model_field = (
+            field_mapping[cli_field] if cli_field in field_mapping else cli_field
+        )
+        final_kwargs[model_field] = value
+
+    if not final_kwargs:
+        raise click.BadArgumentUsage("You need to update at least one attribute")
+
+    handler_update_user(usernames=username, kwargs=final_kwargs)

api/funkwhale_api/cli/utils.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("funkwhale_api.cli")

api/funkwhale_api/common/admin.py

@@ -0,0 +1,65 @@
+from django.contrib.admin import site  # noqa: F401
+from django.contrib.admin import ModelAdmin
+from django.contrib.admin import register as initial_register
+from django.db.models.fields.related import RelatedField
+
+from . import models, tasks
+
+
+def register(model):
+    """
+    To make the admin more performant, we ensure all the the relations
+    are listed under raw_id_fields
+    """
+
+    def decorator(modeladmin):
+        raw_id_fields = []
+        for field in model._meta.fields:
+            if isinstance(field, RelatedField):
+                raw_id_fields.append(field.name)
+        setattr(modeladmin, "raw_id_fields", raw_id_fields)
+        return initial_register(model)(modeladmin)
+
+    return decorator
+
+
+def apply(modeladmin, request, queryset):
+    queryset.update(is_approved=True)
+    for id in queryset.values_list("id", flat=True):
+        tasks.apply_mutation.delay(mutation_id=id)
+
+
+apply.short_description = "Approve and apply"
+
+
+@register(models.Mutation)
+class MutationAdmin(ModelAdmin):
+    list_display = [
+        "uuid",
+        "type",
+        "created_by",
+        "creation_date",
+        "applied_date",
+        "is_approved",
+        "is_applied",
+    ]
+    search_fields = ["created_by__preferred_username"]
+    list_filter = ["type", "is_approved", "is_applied"]
+    actions = [apply]
+
+
+@register(models.Attachment)
+class AttachmentAdmin(ModelAdmin):
+    list_display = [
+        "uuid",
+        "actor",
+        "url",
+        "file",
+        "size",
+        "mimetype",
+        "creation_date",
+        "last_fetch_date",
+    ]
+    list_select_related = True
+    search_fields = ["actor__domain__name"]
+    list_filter = ["mimetype"]

api/funkwhale_api/common/apps.py

@@ -0,0 +1,20 @@
+from django.apps import AppConfig, apps
+from django.conf import settings
+
+from config import plugins
+
+from . import mutations, utils
+
+
+class CommonConfig(AppConfig):
+    name = "funkwhale_api.common"
+
+    def ready(self):
+        super().ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        mutations.registry.autodiscover(app_names)
+        utils.monkey_patch_request_build_absolute_uri()
+        plugins.startup.autodiscover([p + ".funkwhale_ready" for p in settings.PLUGINS])
+        for p in plugins._plugins.values():
+            p["settings"] = plugins.load_settings(p["name"], p["settings"])

api/funkwhale_api/common/authentication.py

@@ -0,0 +1,64 @@
+from allauth.account.utils import send_email_confirmation
+from django.core.cache import cache
+from django.utils.translation import ugettext as _
+from oauth2_provider.contrib.rest_framework.authentication import (
+    OAuth2Authentication as BaseOAuth2Authentication,
+)
+from rest_framework import exceptions
+
+from funkwhale_api.users import models as users_models
+
+
+class UnverifiedEmail(Exception):
+    def __init__(self, user):
+        self.user = user
+
+
+def resend_confirmation_email(request, user):
+    THROTTLE_DELAY = 500
+    cache_key = f"auth:resent-email-confirmation:{user.pk}"
+    if cache.get(cache_key):
+        return False
+
+    done = send_email_confirmation(request, user)
+    cache.set(cache_key, True, THROTTLE_DELAY)
+    return done
+
+
+class OAuth2Authentication(BaseOAuth2Authentication):
+    def authenticate(self, request):
+        try:
+            return super().authenticate(request)
+        except UnverifiedEmail as e:
+            request.oauth2_error = {"error": "unverified_email"}
+            resend_confirmation_email(request, e.user)
+
+
+class ApplicationTokenAuthentication:
+    def authenticate(self, request):
+        try:
+            header = request.headers["Authorization"]
+        except KeyError:
+            return
+
+        if "Bearer" not in header:
+            return
+
+        token = header.split()[-1].strip()
+
+        try:
+            application = users_models.Application.objects.exclude(user=None).get(
+                token=token
+            )
+        except users_models.Application.DoesNotExist:
+            return
+        user = users_models.User.objects.all().for_auth().get(id=application.user_id)
+        if not user.is_active:
+            msg = _("User account is disabled.")
+            raise exceptions.AuthenticationFailed(msg)
+
+        if user.should_verify_email():
+            raise UnverifiedEmail(user)
+
+        request.scopes = application.scope.split()
+        return user, None

api/funkwhale_api/common/cache.py

@@ -0,0 +1,29 @@
+import logging
+
+from django_redis.client import default
+
+logger = logging.getLogger(__name__)
+
+
+class RedisClient(default.DefaultClient):
+    def get(self, key, default=None, version=None, client=None):
+        try:
+            return super().get(key, default=default, version=version, client=client)
+        except ValueError as e:
+            if "unsupported pickle protocol" in str(e):
+                # pickle deserialization error
+                logger.warn("Error while deserializing pickle value from cache")
+                return default
+            else:
+                raise
+
+    def get_many(self, *args, **kwargs):
+        try:
+            return super().get_many(*args, **kwargs)
+        except ValueError as e:
+            if "unsupported pickle protocol" in str(e):
+                # pickle deserialization error
+                logger.warn("Error while deserializing pickle value from cache")
+                return {}
+            else:
+                raise

api/funkwhale_api/common/channels.py

@@ -0,0 +1,26 @@
+import json
+import logging
+
+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
+from django.core.serializers.json import DjangoJSONEncoder
+
+logger = logging.getLogger(__name__)
+channel_layer = get_channel_layer()
+group_add = async_to_sync(channel_layer.group_add)
+group_discard = async_to_sync(channel_layer.group_discard)
+
+
+def group_send(group, event):
+    # we serialize the payload ourselves and deserialize it to ensure it
+    # works with msgpack. This is dirty, but we'll find a better solution
+    # later
+    s = json.dumps(event, cls=DjangoJSONEncoder)
+    event = json.loads(s)
+    logger.debug(
+        "[channels] Dispatching %s to group %s: %s",
+        event["type"],
+        group,
+        {"type": event["data"]["type"]},
+    )
+    async_to_sync(channel_layer.group_send)(group, event)

api/funkwhale_api/common/consumers.py

@@ -0,0 +1,24 @@
+from channels.generic.websocket import JsonWebsocketConsumer
+
+from funkwhale_api.common import channels
+
+
+class JsonAuthConsumer(JsonWebsocketConsumer):
+    def connect(self):
+        try:
+            assert self.scope["user"].pk is not None
+        except (AssertionError, AttributeError, KeyError):
+            return self.close()
+
+        return self.accept()
+
+    def accept(self):
+        super().accept()
+        groups = self.scope["user"].get_channels_groups() + self.groups
+        for group in groups:
+            channels.group_add(group, self.channel_name)
+
+    def disconnect(self, close_code):
+        groups = self.scope["user"].get_channels_groups() + self.groups
+        for group in groups:
+            channels.group_discard(group, self.channel_name)

api/funkwhale_api/common/decorators.py

@@ -0,0 +1,95 @@
+from django.db import transaction
+from drf_spectacular.utils import OpenApiParameter, extend_schema
+from rest_framework import decorators, exceptions, response, status
+
+from . import filters, models
+from . import mutations as common_mutations
+from . import serializers, signals, tasks, utils
+
+
+def action_route(serializer_class):
+    @decorators.action(methods=["post"], detail=False)
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializer_class(request.data, queryset=queryset)
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)
+
+    return action
+
+
+def mutations_route(types):
+    """
+    Given a queryset and a list of mutation types, return a view
+    that can be included in any viewset, and serve:
+
+    GET /{id}/mutations/ - list of mutations for the given object
+    POST /{id}/mutations/ - create a mutation for the given object
+    """
+
+    @transaction.atomic
+    def mutations(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if request.method == "GET":
+            queryset = models.Mutation.objects.get_for_target(obj).filter(
+                type__in=types
+            )
+            queryset = queryset.order_by("-creation_date")
+            filterset = filters.MutationFilter(request.GET, queryset=queryset)
+            page = self.paginate_queryset(filterset.qs)
+            if page is not None:
+                serializer = serializers.APIMutationSerializer(page, many=True)
+                return self.get_paginated_response(serializer.data)
+
+            serializer = serializers.APIMutationSerializer(queryset, many=True)
+            return response.Response(serializer.data)
+        if request.method == "POST":
+            if not request.user.is_authenticated:
+                raise exceptions.NotAuthenticated()
+            serializer = serializers.APIMutationSerializer(
+                data=request.data, context={"registry": common_mutations.registry}
+            )
+            serializer.is_valid(raise_exception=True)
+            if not common_mutations.registry.has_perm(
+                actor=request.user.actor,
+                type=serializer.validated_data["type"],
+                obj=obj,
+                perm="approve"
+                if serializer.validated_data.get("is_approved", False)
+                else "suggest",
+            ):
+                raise exceptions.PermissionDenied()
+
+            final_payload = common_mutations.registry.get_validated_payload(
+                type=serializer.validated_data["type"],
+                payload=serializer.validated_data["payload"],
+                obj=obj,
+            )
+            mutation = serializer.save(
+                created_by=request.user.actor,
+                target=obj,
+                payload=final_payload,
+                is_approved=serializer.validated_data.get("is_approved", None),
+            )
+            if mutation.is_approved:
+                utils.on_commit(tasks.apply_mutation.delay, mutation_id=mutation.pk)
+
+            utils.on_commit(
+                signals.mutation_created.send, sender=None, mutation=mutation
+            )
+            return response.Response(serializer.data, status=status.HTTP_201_CREATED)
+
+    return extend_schema(
+        methods=["post"], responses=serializers.APIMutationSerializer()
+    )(
+        extend_schema(
+            methods=["get"],
+            responses=serializers.APIMutationSerializer(many=True),
+            parameters=[OpenApiParameter("id", location="query", exclude=True)],
+        )(
+            decorators.action(
+                methods=["get", "post"], detail=True, required_scope="edits"
+            )(mutations)
+        )
+    )

api/funkwhale_api/common/dynamic_preferences_registry.py

@@ -0,0 +1,17 @@
+from dynamic_preferences import types
+from dynamic_preferences.registries import global_preferences_registry
+
+common = types.Section("common")
+
+
+@global_preferences_registry.register
+class APIAutenticationRequired(types.BooleanPreference):
+    section = common
+    name = "api_authentication_required"
+    verbose_name = "API Requires authentication"
+    default = True
+    help_text = (
+        "If disabled, anonymous users will be able to query the API "
+        "and access music data (as well as other data exposed in the API "
+        "without specific permissions)."
+    )

api/funkwhale_api/common/factories.py

@@ -0,0 +1,45 @@
+import factory
+
+from funkwhale_api.factories import NoUpdateOnCreate, registry
+from funkwhale_api.federation import factories as federation_factories
+
+
+@registry.register
+class MutationFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    fid = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    created_by = factory.SubFactory(federation_factories.ActorFactory)
+    summary = factory.Faker("paragraph")
+    type = "update"
+
+    class Meta:
+        model = "common.Mutation"
+
+
+@registry.register
+class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    url = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    actor = factory.SubFactory(federation_factories.ActorFactory)
+    file = factory.django.ImageField()
+
+    class Meta:
+        model = "common.Attachment"
+
+
+@registry.register
+class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    text = factory.Faker("paragraph")
+    content_type = "text/plain"
+
+    class Meta:
+        model = "common.Content"
+
+
+@registry.register
+class PluginConfiguration(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    code = "test"
+    conf = {"foo": "bar"}
+
+    class Meta:
+        model = "common.PluginConfiguration"

api/funkwhale_api/common/fields.py

@@ -0,0 +1,177 @@
+import django_filters
+from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from rest_framework import serializers
+
+from . import search
+
+PRIVACY_LEVEL_CHOICES = [
+    ("me", "Only me"),
+    ("followers", "Me and my followers"),
+    ("instance", "Everyone on my instance, and my followers"),
+    ("everyone", "Everyone, including people on other instances"),
+]
+
+
+def get_privacy_field():
+    return models.CharField(
+        max_length=30, choices=PRIVACY_LEVEL_CHOICES, default="instance"
+    )
+
+
+def privacy_level_query(user, lookup_field="privacy_level", user_field="user"):
+    if user.is_anonymous:
+        return models.Q(**{lookup_field: "everyone"})
+
+    return models.Q(**{f"{lookup_field}__in": ["instance", "everyone"]}) | models.Q(
+        **{lookup_field: "me", user_field: user}
+    )
+
+
+class SearchFilter(django_filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        self.search_fields = kwargs.pop("search_fields")
+        self.fts_search_fields = kwargs.pop("fts_search_fields", [])
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        if self.fts_search_fields:
+            query = search.get_fts_query(
+                value, self.fts_search_fields, model=self.parent.Meta.model
+            )
+        else:
+            query = search.get_query(value, self.search_fields)
+        return qs.filter(query)
+
+
+class SmartSearchFilter(django_filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        self.config = kwargs.pop("config")
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        try:
+            cleaned = self.config.clean(value)
+        except forms.ValidationError:
+            return qs.none()
+        return search.apply(qs, cleaned)
+
+
+def get_generic_filter_query(value, relation_name, choices):
+    parts = value.split(":", 1)
+    type = parts[0]
+    try:
+        conf = choices[type]
+    except KeyError:
+        raise forms.ValidationError("Invalid type")
+    related_queryset = conf["queryset"]
+    related_model = related_queryset.model
+    filter_query = models.Q(
+        **{
+            "{}_content_type__app_label".format(
+                relation_name
+            ): related_model._meta.app_label,
+            "{}_content_type__model".format(
+                relation_name
+            ): related_model._meta.model_name,
+        }
+    )
+    if len(parts) > 1:
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        try:
+            id_value = parts[1]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise forms.ValidationError("Invalid id")
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        obj_query = query_getter(id_attr, id_value)
+        try:
+            obj = related_queryset.get(obj_query)
+        except related_queryset.model.DoesNotExist:
+            raise forms.ValidationError("Invalid object")
+        filter_query &= models.Q(**{f"{relation_name}_id": obj.id})
+
+    return filter_query
+
+
+class GenericRelationFilter(django_filters.CharFilter):
+    def __init__(self, relation_name, choices, *args, **kwargs):
+        self.relation_name = relation_name
+        self.choices = choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        try:
+            filter_query = get_generic_filter_query(
+                value, relation_name=self.relation_name, choices=self.choices
+            )
+        except forms.ValidationError:
+            return qs.none()
+        return qs.filter(filter_query)
+
+
+class GenericRelation(serializers.JSONField):
+    def __init__(self, choices, *args, **kwargs):
+        self.choices = choices
+        self.encoder = kwargs.setdefault("encoder", DjangoJSONEncoder)
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        if not value:
+            return
+        type = None
+        id = None
+        id_attr = None
+        for key, choice in self.choices.items():
+            if isinstance(value, choice["queryset"].model):
+                type = key
+                id_attr = choice.get("id_attr", "id")
+                id = getattr(value, id_attr)
+                break
+
+        if type:
+            return {"type": type, id_attr: id}
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+
+        if not v or not isinstance(v, dict):
+            raise serializers.ValidationError("Invalid data")
+
+        try:
+            type = v["type"]
+            field = serializers.ChoiceField(choices=list(self.choices.keys()))
+            type = field.to_internal_value(type)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid type")
+
+        conf = self.choices[type]
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        queryset = conf["queryset"]
+        try:
+            id_value = v[id_attr]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError(f"Invalid {id_attr}")
+
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        query = query_getter(id_attr, id_value)
+        try:
+            obj = queryset.get(query)
+        except queryset.model.DoesNotExist:
+            raise serializers.ValidationError("Object not found")
+
+        return obj

api/funkwhale_api/common/filters.py

@@ -0,0 +1,258 @@
+from django import forms
+from django.db.models import Q
+from django.db.models.functions import Lower
+from django_filters import rest_framework as filters
+from django_filters import widgets
+from drf_spectacular.utils import extend_schema_field
+
+from . import fields, models, search, utils
+
+
+class NoneObject:
+    def __eq__(self, other):
+        return other.__class__ == NoneObject
+
+
+NONE = NoneObject()
+BOOLEAN_CHOICES = [
+    (True, True),
+    ("true", True),
+    ("True", True),
+    ("1", True),
+    ("yes", True),
+    (False, False),
+    ("false", False),
+    ("False", False),
+    ("0", False),
+    ("no", False),
+]
+NULL_BOOLEAN_CHOICES = BOOLEAN_CHOICES + [
+    ("None", NONE),
+    ("none", NONE),
+    ("Null", NONE),
+    ("null", NONE),
+]
+
+
+class CoerceChoiceField(forms.ChoiceField):
+    """
+    Same as forms.ChoiceField but will return the second value
+    in the choices tuple instead of the user provided one
+    """
+
+    def clean(self, value):
+        if value is None:
+            return value
+        v = super().clean(value)
+        try:
+            return [b for a, b in self.choices if v == a][0]
+        except IndexError:
+            raise forms.ValidationError(f"Invalid value {value}")
+
+
+@extend_schema_field(bool)
+class NullBooleanFilter(filters.ChoiceFilter):
+    field_class = CoerceChoiceField
+
+    def __init__(self, *args, **kwargs):
+        self.choices = NULL_BOOLEAN_CHOICES
+        kwargs["choices"] = self.choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if value in ["", None]:
+            return qs
+        if value == NONE:
+            value = None
+        qs = self.get_method(qs)(**{f"{self.field_name}__{self.lookup_expr}": value})
+        return qs.distinct() if self.distinct else qs
+
+
+def clean_null_boolean_filter(v):
+    v = CoerceChoiceField(choices=NULL_BOOLEAN_CHOICES).clean(v)
+    if v == NONE:
+        v = None
+
+    return v
+
+
+def clean_boolean_filter(v):
+    return CoerceChoiceField(choices=BOOLEAN_CHOICES).clean(v)
+
+
+def get_null_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_null_boolean_filter(v)})}
+
+
+def get_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_boolean_filter(v)})}
+
+
+def get_generic_relation_filter(relation_name, choices):
+    return {
+        "handler": lambda v: fields.get_generic_filter_query(
+            v, relation_name=relation_name, choices=choices
+        )
+    }
+
+
+class DummyTypedMultipleChoiceField(forms.TypedMultipleChoiceField):
+    def valid_value(self, value):
+        return True
+
+
+class QueryArrayWidget(widgets.QueryArrayWidget):
+    """
+    Until https://github.com/carltongibson/django-filter/issues/1047 is fixed
+    """
+
+    def value_from_datadict(self, data, files, name):
+        data = data.copy()
+        return super().value_from_datadict(data, files, name)
+
+
+class MultipleQueryFilter(filters.TypedMultipleChoiceFilter):
+    field_class = DummyTypedMultipleChoiceField
+
+    def __init__(self, *args, **kwargs):
+        kwargs["widget"] = QueryArrayWidget()
+        super().__init__(*args, **kwargs)
+
+
+def filter_target(value):
+    config = {
+        "artist": ["artist", "target_id", int],
+        "album": ["album", "target_id", int],
+        "track": ["track", "target_id", int],
+    }
+    parts = value.lower().split(" ")
+    if parts[0].strip() not in config:
+        raise forms.ValidationError("Improper target")
+
+    conf = config[parts[0].strip()]
+
+    query = Q(target_content_type__model=conf[0])
+    if len(parts) > 1:
+        _, lookup_field, validator = conf
+        try:
+            lookup_value = validator(parts[1].strip())
+        except TypeError:
+            raise forms.ValidationError("Imparsable target id")
+        return query & Q(**{lookup_field: lookup_value})
+
+    return query
+
+
+class MutationFilter(filters.FilterSet):
+    is_approved = NullBooleanFilter("is_approved")
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={
+                "summary": {"to": "summary"},
+                "fid": {"to": "fid"},
+                "type": {"to": "type"},
+            },
+            filter_fields={
+                "domain": {"to": "created_by__domain__name__iexact"},
+                "is_approved": get_null_boolean_filter("is_approved"),
+                "target": {"handler": filter_target},
+                "is_applied": get_boolean_filter("is_applied"),
+            },
+        )
+    )
+
+    class Meta:
+        model = models.Mutation
+        fields = ["is_approved", "is_applied", "type"]
+
+
+class EmptyQuerySet(ValueError):
+    pass
+
+
+class ActorScopeFilter(filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        self.actor_field = kwargs.pop("actor_field")
+        self.library_field = kwargs.pop("library_field", None)
+        super().__init__(*args, **kwargs)
+
+    def filter(self, queryset, value):
+        if not value:
+            return queryset
+
+        request = getattr(self.parent, "request", None)
+        if not request:
+            return queryset.none()
+
+        user = getattr(request, "user", None)
+        actor = getattr(user, "actor", None)
+        scopes = [v.strip().lower() for v in value.split(",")]
+        query = None
+        for scope in scopes:
+            try:
+                right_query = self.get_query(scope, user, actor)
+            except ValueError:
+                return queryset.none()
+            query = utils.join_queries_or(query, right_query)
+
+        return queryset.filter(query).distinct()
+
+    def get_query(self, scope, user, actor):
+        from funkwhale_api.federation import models as federation_models
+
+        if scope == "me":
+            return self.filter_me(actor)
+        elif scope == "all":
+            return Q(pk__gte=0)
+
+        elif scope == "subscribed":
+            if not actor or self.library_field is None:
+                raise EmptyQuerySet()
+            followed_libraries = federation_models.LibraryFollow.objects.filter(
+                approved=True, actor=user.actor
+            ).values_list("target_id", flat=True)
+            if not self.library_field:
+                predicate = "pk__in"
+            else:
+                predicate = f"{self.library_field}__in"
+            return Q(**{predicate: followed_libraries})
+
+        elif scope.startswith("actor:"):
+            full_username = scope.split("actor:", 1)[1]
+            username, domain = full_username.split("@")
+            try:
+                actor = federation_models.Actor.objects.get(
+                    preferred_username__iexact=username,
+                    domain_id=domain,
+                )
+            except federation_models.Actor.DoesNotExist:
+                raise EmptyQuerySet()
+
+            return Q(**{self.actor_field: actor})
+        elif scope.startswith("domain:"):
+            domain = scope.split("domain:", 1)[1]
+            return Q(**{f"{self.actor_field}__domain_id": domain})
+        else:
+            raise EmptyQuerySet()
+
+    def filter_me(self, actor):
+        if not actor:
+            raise EmptyQuerySet()
+
+        return Q(**{self.actor_field: actor})
+
+
+class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        order_by = []
+
+        if value is None:
+            return qs
+
+        for param in value:
+            if param == "name":
+                order_by.append(Lower("name"))
+            else:
+                order_by.append(self.get_ordering_value(param))
+
+        return qs.order_by(*order_by)

api/funkwhale_api/common/locales.py

@@ -0,0 +1,191 @@
+# from https://gist.github.com/carlopires/1262033/c52ef0f7ce4f58108619508308372edd8d0bd518
+
+ISO_639_CHOICES = [
+    ("ab", "Abkhaz"),
+    ("aa", "Afar"),
+    ("af", "Afrikaans"),
+    ("ak", "Akan"),
+    ("sq", "Albanian"),
+    ("am", "Amharic"),
+    ("ar", "Arabic"),
+    ("an", "Aragonese"),
+    ("hy", "Armenian"),
+    ("as", "Assamese"),
+    ("av", "Avaric"),
+    ("ae", "Avestan"),
+    ("ay", "Aymara"),
+    ("az", "Azerbaijani"),
+    ("bm", "Bambara"),
+    ("ba", "Bashkir"),
+    ("eu", "Basque"),
+    ("be", "Belarusian"),
+    ("bn", "Bengali"),
+    ("bh", "Bihari"),
+    ("bi", "Bislama"),
+    ("bs", "Bosnian"),
+    ("br", "Breton"),
+    ("bg", "Bulgarian"),
+    ("my", "Burmese"),
+    ("ca", "Catalan; Valencian"),
+    ("ch", "Chamorro"),
+    ("ce", "Chechen"),
+    ("ny", "Chichewa; Chewa; Nyanja"),
+    ("zh", "Chinese"),
+    ("cv", "Chuvash"),
+    ("kw", "Cornish"),
+    ("co", "Corsican"),
+    ("cr", "Cree"),
+    ("hr", "Croatian"),
+    ("cs", "Czech"),
+    ("da", "Danish"),
+    ("dv", "Divehi; Maldivian;"),
+    ("nl", "Dutch"),
+    ("dz", "Dzongkha"),
+    ("en", "English"),
+    ("eo", "Esperanto"),
+    ("et", "Estonian"),
+    ("ee", "Ewe"),
+    ("fo", "Faroese"),
+    ("fj", "Fijian"),
+    ("fi", "Finnish"),
+    ("fr", "French"),
+    ("ff", "Fula"),
+    ("gl", "Galician"),
+    ("ka", "Georgian"),
+    ("de", "German"),
+    ("el", "Greek, Modern"),
+    ("gn", "Guaraní"),
+    ("gu", "Gujarati"),
+    ("ht", "Haitian"),
+    ("ha", "Hausa"),
+    ("he", "Hebrew (modern)"),
+    ("hz", "Herero"),
+    ("hi", "Hindi"),
+    ("ho", "Hiri Motu"),
+    ("hu", "Hungarian"),
+    ("ia", "Interlingua"),
+    ("id", "Indonesian"),
+    ("ie", "Interlingue"),
+    ("ga", "Irish"),
+    ("ig", "Igbo"),
+    ("ik", "Inupiaq"),
+    ("io", "Ido"),
+    ("is", "Icelandic"),
+    ("it", "Italian"),
+    ("iu", "Inuktitut"),
+    ("ja", "Japanese"),
+    ("jv", "Javanese"),
+    ("kl", "Kalaallisut"),
+    ("kn", "Kannada"),
+    ("kr", "Kanuri"),
+    ("ks", "Kashmiri"),
+    ("kk", "Kazakh"),
+    ("km", "Khmer"),
+    ("ki", "Kikuyu, Gikuyu"),
+    ("rw", "Kinyarwanda"),
+    ("ky", "Kirghiz, Kyrgyz"),
+    ("kv", "Komi"),
+    ("kg", "Kongo"),
+    ("ko", "Korean"),
+    ("ku", "Kurdish"),
+    ("kj", "Kwanyama, Kuanyama"),
+    ("la", "Latin"),
+    ("lb", "Luxembourgish"),
+    ("lg", "Luganda"),
+    ("li", "Limburgish"),
+    ("ln", "Lingala"),
+    ("lo", "Lao"),
+    ("lt", "Lithuanian"),
+    ("lu", "Luba-Katanga"),
+    ("lv", "Latvian"),
+    ("gv", "Manx"),
+    ("mk", "Macedonian"),
+    ("mg", "Malagasy"),
+    ("ms", "Malay"),
+    ("ml", "Malayalam"),
+    ("mt", "Maltese"),
+    ("mi", "Māori"),
+    ("mr", "Marathi (Marāṭhī)"),
+    ("mh", "Marshallese"),
+    ("mn", "Mongolian"),
+    ("na", "Nauru"),
+    ("nv", "Navajo, Navaho"),
+    ("nb", "Norwegian Bokmål"),
+    ("nd", "North Ndebele"),
+    ("ne", "Nepali"),
+    ("ng", "Ndonga"),
+    ("nn", "Norwegian Nynorsk"),
+    ("no", "Norwegian"),
+    ("ii", "Nuosu"),
+    ("nr", "South Ndebele"),
+    ("oc", "Occitan"),
+    ("oj", "Ojibwe, Ojibwa"),
+    ("cu", "Old Church Slavonic"),
+    ("om", "Oromo"),
+    ("or", "Oriya"),
+    ("os", "Ossetian, Ossetic"),
+    ("pa", "Panjabi, Punjabi"),
+    ("pi", "Pāli"),
+    ("fa", "Persian"),
+    ("pl", "Polish"),
+    ("ps", "Pashto, Pushto"),
+    ("pt", "Portuguese"),
+    ("qu", "Quechua"),
+    ("rm", "Romansh"),
+    ("rn", "Kirundi"),
+    ("ro", "Romanian, Moldavan"),
+    ("ru", "Russian"),
+    ("sa", "Sanskrit (Saṁskṛta)"),
+    ("sc", "Sardinian"),
+    ("sd", "Sindhi"),
+    ("se", "Northern Sami"),
+    ("sm", "Samoan"),
+    ("sg", "Sango"),
+    ("sr", "Serbian"),
+    ("gd", "Scottish Gaelic"),
+    ("sn", "Shona"),
+    ("si", "Sinhala, Sinhalese"),
+    ("sk", "Slovak"),
+    ("sl", "Slovene"),
+    ("so", "Somali"),
+    ("st", "Southern Sotho"),
+    ("es", "Spanish; Castilian"),
+    ("su", "Sundanese"),
+    ("sw", "Swahili"),
+    ("ss", "Swati"),
+    ("sv", "Swedish"),
+    ("ta", "Tamil"),
+    ("te", "Telugu"),
+    ("tg", "Tajik"),
+    ("th", "Thai"),
+    ("ti", "Tigrinya"),
+    ("bo", "Tibetan"),
+    ("tk", "Turkmen"),
+    ("tl", "Tagalog"),
+    ("tn", "Tswana"),
+    ("to", "Tonga"),
+    ("tr", "Turkish"),
+    ("ts", "Tsonga"),
+    ("tt", "Tatar"),
+    ("tw", "Twi"),
+    ("ty", "Tahitian"),
+    ("ug", "Uighur, Uyghur"),
+    ("uk", "Ukrainian"),
+    ("ur", "Urdu"),
+    ("uz", "Uzbek"),
+    ("ve", "Venda"),
+    ("vi", "Vietnamese"),
+    ("vo", "Volapük"),
+    ("wa", "Walloon"),
+    ("cy", "Welsh"),
+    ("wo", "Wolof"),
+    ("fy", "Western Frisian"),
+    ("xh", "Xhosa"),
+    ("yi", "Yiddish"),
+    ("yo", "Yoruba"),
+    ("za", "Zhuang, Chuang"),
+    ("zu", "Zulu"),
+]
+
+
+ISO_639_BY_CODE = {code: name for code, name in ISO_639_CHOICES}

api/funkwhale_api/common/management/commands/createsuperuser.py

@@ -0,0 +1,22 @@
+import os
+
+from django.contrib.auth.management.commands.createsuperuser import (
+    Command as BaseCommand,
+)
+from django.core.management.base import CommandError
+
+
+class Command(BaseCommand):
+    def handle(self, *apps_label, **options):
+        """
+        Creating Django Superusers would bypass some of our username checks, which can lead to unexpected behaviour.
+        We therefore prohibit the execution of the command.
+        """
+        if not os.environ.get("FORCE") == "1":
+            raise CommandError(
+                "Running createsuperuser on your FunQuail instance bypasses some of our checks "
+                "which can lead to unexpected behavior of your instance. We therefore suggest to "
+                "run `funkwhale-manage fw users create --superuser` instead."
+            )
+
+        return super().handle(*apps_label, **options)

api/funkwhale_api/common/management/commands/gitpod.py

@@ -0,0 +1,78 @@
+import os
+
+import debugpy
+import uvicorn
+from django.core.management import call_command
+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.common import preferences
+from funkwhale_api.music.models import Library
+from funkwhale_api.users.models import User
+
+
+class Command(BaseCommand):
+    help = "Manage gitpod environment"
+
+    def add_arguments(self, parser):
+        parser.add_argument("command", nargs="?", type=str)
+
+    def handle(self, *args, **options):
+        command = options["command"]
+
+        if not command:
+            return self.show_help()
+
+        if command == "init":
+            return self.init()
+
+        if command == "dev":
+            return self.dev()
+
+    def show_help(self):
+        self.stdout.write("")
+        self.stdout.write("Available commands:")
+        self.stdout.write("init - Initialize gitpod workspace")
+        self.stdout.write("dev - Run FunQuail in development mode with debug server")
+        self.stdout.write("")
+
+    def init(self):
+        user = User.objects.get(username="gitpod")
+
+        # Allow anonymous access
+        preferences.set("common__api_authentication_required", False)
+
+        # Download music catalog
+        os.system(
+            "git clone https://dev.funkwhale.audio/funkwhale/catalog.git /tmp/catalog"
+        )
+        os.system("mv -f /tmp/catalog/music /workspace/funkwhale/data")
+        os.system("rm -rf /tmp/catalog/music")
+
+        # Import music catalog into library
+        call_command(
+            "create_library",
+            "gitpod",
+            name="funkwhale/catalog",
+            privacy_level="everyone",
+        )
+        call_command(
+            "import_files",
+            Library.objects.get(actor=user.actor).uuid,
+            "/workspace/funkwhale/data/music/",
+            recursive=True,
+            in_place=True,
+            no_input=False,
+        )
+
+    def dev(self):
+        debugpy.listen(5678)
+        uvicorn.run(
+            "config.asgi:application",
+            host="0.0.0.0",
+            port=5000,
+            reload=True,
+            reload_dirs=[
+                "/workspace/funkwhale/api/config/",
+                "/workspace/funkwhale/api/funkwhale_api/",
+            ],
+        )

api/funkwhale_api/common/management/commands/inplace_to_s3.py

@@ -0,0 +1,95 @@
+import pathlib
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.music import models
+
+
+class Command(BaseCommand):
+    help = """
+    Update the reference for Uploads that have been imported with --in-place and are now moved to s3.
+
+    Please note: This does not move any file! Make sure you already moved the files to your s3 bucket.
+
+    Specify --source to filter the reference to update to files from a specific in-place directory. If no
+    --source is given, all in-place imported track references will be updated.
+
+    Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is
+    given, the file is expected to be stored in the same path as before.
+
+    Examples:
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target unset
+
+    All files imported from /music will be updated and expected to be in the same folder structure in the bucket
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target: /in_place
+
+    The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg
+    """
+
+    def create_parser(self, *args, **kwargs):
+        parser = super().create_parser(*args, **kwargs)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            default=True,
+            help="Disable dry run mode and apply updates for real on the database",
+        )
+        parser.add_argument(
+            "--source",
+            type=pathlib.Path,
+            required=True,
+            help="Specify the path of the directory where the files originally were stored to update their reference.",
+        )
+        parser.add_argument(
+            "--target",
+            type=pathlib.Path,
+            help="Specify a subdirectory in the S3 bucket where you moved the files to.",
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        if options["dry_run"]:
+            self.stdout.write("Dry-run on, will not touch the database")
+        else:
+            self.stdout.write("Dry-run off, *changing the database*")
+        self.stdout.write("")
+
+        prefix = f"file://{options['source']}"
+
+        to_change = models.Upload.objects.filter(source__startswith=prefix)
+
+        self.stdout.write(f"Found {to_change.count()} uploads to update.")
+
+        target = options.get("target")
+        if target is None:
+            target = options["source"]
+
+        for upl in to_change:
+            upl.audio_file = str(upl.source).replace(str(prefix), str(target))
+            upl.source = None
+            self.stdout.write(f"Upload expected in {upl.audio_file}")
+            if not options["dry_run"]:
+                upl.save()
+
+        self.stdout.write("")
+        if options["dry_run"]:
+            self.stdout.write(
+                "Nothing was updated, rerun this command with --no-dry-run to apply the changes"
+            )
+        else:
+            self.stdout.write("Updating completed!")
+
+        self.stdout.write("")

api/funkwhale_api/common/management/commands/load_test_data.py

@@ -0,0 +1,322 @@
+import math
+import random
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.federation import keys
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.music import models as music_models
+from funkwhale_api.tags import models as tags_models
+from funkwhale_api.users import models as users_models
+
+BATCH_SIZE = 500
+
+
+def create_local_accounts(factories, count, dependencies):
+    password = factories["users.User"].build().password
+    users = factories["users.User"].build_batch(size=count)
+    for user in users:
+        # we set the hashed password by hand, because computing one for each user
+        # is CPU intensive
+        user.password = password
+    users = users_models.User.objects.bulk_create(users, batch_size=BATCH_SIZE)
+    actors = []
+    domain = federation_models.Domain.objects.get_or_create(
+        name=settings.FEDERATION_HOSTNAME
+    )[0]
+    users = [u for u in users if u.pk]
+    private, public = keys.get_key_pair()
+    for user in users:
+        if not user.pk:
+            continue
+        actor = federation_models.Actor(
+            private_key=private.decode("utf-8"),
+            public_key=public.decode("utf-8"),
+            **users_models.get_actor_data(user.username, domain=domain)
+        )
+        actors.append(actor)
+    actors = federation_models.Actor.objects.bulk_create(actors, batch_size=BATCH_SIZE)
+    for user, actor in zip(users, actors):
+        user.actor = actor
+    users_models.User.objects.bulk_update(users, ["actor"])
+    return actors
+
+
+def create_taggable_items(dependency):
+    def inner(factories, count, dependencies):
+        objs = []
+        tagged_objects = dependencies.get(
+            dependency, list(CONFIG_BY_ID[dependency]["model"].objects.all().only("pk"))
+        )
+        tags = dependencies.get("tags", list(tags_models.Tag.objects.all().only("pk")))
+        for i in range(count):
+            tag = random.choice(tags)
+            tagged_object = random.choice(tagged_objects)
+            objs.append(
+                factories["tags.TaggedItem"].build(
+                    content_object=tagged_object, tag=tag
+                )
+            )
+
+        return tags_models.TaggedItem.objects.bulk_create(
+            objs, batch_size=BATCH_SIZE, ignore_conflicts=True
+        )
+
+    return inner
+
+
+CONFIG = [
+    {
+        "id": "tracks",
+        "model": music_models.Track,
+        "factory": "music.Track",
+        "factory_kwargs": {"artist": None, "album": None},
+        "depends_on": [
+            {"field": "album", "id": "albums", "default_factor": 0.1},
+            {"field": "artist", "id": "artists", "default_factor": 0.05},
+        ],
+    },
+    {
+        "id": "albums",
+        "model": music_models.Album,
+        "factory": "music.Album",
+        "factory_kwargs": {"artist": None},
+        "depends_on": [{"field": "artist", "id": "artists", "default_factor": 0.3}],
+    },
+    {"id": "artists", "model": music_models.Artist, "factory": "music.Artist"},
+    {
+        "id": "local_accounts",
+        "model": federation_models.Actor,
+        "handler": create_local_accounts,
+    },
+    {
+        "id": "local_libraries",
+        "model": music_models.Library,
+        "factory": "music.Library",
+        "factory_kwargs": {"actor": None},
+        "depends_on": [{"field": "actor", "id": "local_accounts", "default_factor": 1}],
+    },
+    {
+        "id": "local_uploads",
+        "model": music_models.Upload,
+        "factory": "music.Upload",
+        "factory_kwargs": {"import_status": "finished", "library": None, "track": None},
+        "depends_on": [
+            {
+                "field": "library",
+                "id": "local_libraries",
+                "default_factor": 0.05,
+                "queryset": music_models.Library.objects.all().select_related(
+                    "actor__user"
+                ),
+            },
+            {"field": "track", "id": "tracks", "default_factor": 1},
+        ],
+    },
+    {"id": "tags", "model": tags_models.Tag, "factory": "tags.Tag"},
+    {
+        "id": "track_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="track"
+        ),
+        "handler": create_taggable_items("tracks"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "tracks",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "album_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="album"
+        ),
+        "handler": create_taggable_items("albums"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "albums",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "artist_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="artist"
+        ),
+        "handler": create_taggable_items("artists"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "artists",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+]
+
+CONFIG_BY_ID = {c["id"]: c for c in CONFIG}
+
+
+class Rollback(Exception):
+    pass
+
+
+def create_objects(row, factories, count, **factory_kwargs):
+    return factories[row["factory"]].build_batch(size=count, **factory_kwargs)
+
+
+class Command(BaseCommand):
+    help = """
+    Inject demo data into your database. Useful for load testing, or setting up a demo instance.
+
+    Use with caution and only if you know what you are doing.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            help="Commit the changes to the database",
+        )
+        parser.add_argument(
+            "--create-dependencies", action="store_true", dest="create_dependencies"
+        )
+        for row in CONFIG:
+            parser.add_argument(
+                "--{}".format(row["id"].replace("_", "-")),
+                dest=row["id"],
+                type=int,
+                help="Number of {} objects to create".format(row["id"]),
+            )
+            dependencies = row.get("depends_on", [])
+            for dependency in dependencies:
+                parser.add_argument(
+                    "--{}-{}-factor".format(row["id"], dependency["field"]),
+                    dest="{}_{}_factor".format(row["id"], dependency["field"]),
+                    type=float,
+                    help="Number of {} objects to create per {} object".format(
+                        dependency["id"], row["id"]
+                    ),
+                )
+
+    def handle(self, *args, **options):
+        from django.apps import apps
+
+        from funkwhale_api import factories
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        factories.registry.autodiscover(app_names)
+        try:
+            return self.inner_handle(*args, **options)
+        except Rollback:
+            pass
+
+    @transaction.atomic
+    def inner_handle(self, *args, **options):
+        results = {}
+        for row in CONFIG:
+            self.create_batch(row, results, options, count=options.get(row["id"]))
+
+        self.stdout.write("\nFinal state of database:\n\n")
+        for row in CONFIG:
+            qs = row.get("queryset", row["model"].objects.all())
+            total = qs.count()
+            self.stdout.write("- {} {} objects".format(total, row["id"]))
+
+        self.stdout.write("")
+        if options["dry_run"]:
+            self.stdout.write(
+                "Run this command with --no-dry-run to commit the changes to the database"
+            )
+            raise Rollback()
+
+        self.stdout.write(self.style.SUCCESS("Done!"))
+
+    def create_batch(self, row, results, options, count):
+        from funkwhale_api import factories
+
+        if row["id"] in results:
+            # already generated
+            return results[row["id"]]
+        if not count:
+            return []
+        dependencies = row.get("depends_on", [])
+        create_dependencies = options.get("create_dependencies")
+        for dependency in dependencies:
+            dep_count = options.get(dependency["id"])
+            if not create_dependencies and dep_count is None:
+                continue
+            if dep_count is None:
+                factor = options[
+                    "{}_{}_factor".format(row["id"], dependency["field"])
+                ] or dependency.get("default_factor")
+                dep_count = math.ceil(factor * count)
+
+            results[dependency["id"]] = self.create_batch(
+                CONFIG_BY_ID[dependency["id"]], results, options, count=dep_count
+            )
+        self.stdout.write("Creating {} {}…".format(count, row["id"]))
+        handler = row.get("handler")
+        if handler:
+            objects = handler(factories.registry, count, dependencies=results)
+        else:
+            objects = create_objects(
+                row, factories.registry, count, **row.get("factory_kwargs", {})
+            )
+        for dependency in dependencies:
+            if not dependency.get("set", True):
+                continue
+            if create_dependencies:
+                candidates = results[dependency["id"]]
+            else:
+                # we use existing objects in the database
+                queryset = dependency.get(
+                    "queryset", CONFIG_BY_ID[dependency["id"]]["model"].objects.all()
+                )
+                candidates = list(queryset.values_list("pk", flat=True))
+                picked_pks = [random.choice(candidates) for _ in objects]
+                picked_objects = {o.pk: o for o in queryset.filter(pk__in=picked_pks)}
+            for i, obj in enumerate(objects):
+                if create_dependencies:
+                    value = random.choice(candidates)
+                else:
+                    value = picked_objects[picked_pks[i]]
+                setattr(obj, dependency["field"], value)
+        if not handler:
+            objects = row["model"].objects.bulk_create(objects, batch_size=BATCH_SIZE)
+        results[row["id"]] = objects
+        return objects

api/funkwhale_api/common/management/commands/makemigrations.py

@@ -0,0 +1,22 @@
+import os
+
+from django.core.management.base import CommandError
+from django.core.management.commands.makemigrations import Command as BaseCommand
+
+
+class Command(BaseCommand):
+    def handle(self, *apps_label, **options):
+        """
+        Running makemigrations in production can have desastrous consequences.
+
+        We ensure the command is disabled, unless a specific env var is provided.
+        """
+        force = os.environ.get("FORCE") == "1"
+        if not force:
+            raise CommandError(
+                "Running makemigrations on your FunQuail instance can have desastrous"
+                " consequences. This command is disabled, and should only be run in "
+                "development environments."
+            )
+
+        return super().handle(*apps_label, **options)

api/funkwhale_api/common/management/commands/migrate.py

@@ -0,0 +1,29 @@
+from django.core.management.commands.migrate import Command as BaseCommand
+
+
+def patch_write(buffer):
+    """
+    Django is trying to help us when running migrate, by checking we don't have
+    model changes not included in migrations. Unfortunately, running makemigrations
+    on production instances create unwanted migrations and corrupt the database.
+
+    So we disabled the makemigrations command, and we're patching the
+    write method to ensure misleading messages are never shown to the user,
+    because https://github.com/django/django/blob/2.1.5/django/core/management/commands/migrate.py#L186
+    does not leave an easy way to disable them.
+    """
+    unpatched = buffer.write
+
+    def p(message, *args, **kwargs):
+        if "'manage.py makemigrations'" in message or "not yet reflected" in message:
+            return
+        return unpatched(message, *args, **kwargs)
+
+    setattr(buffer, "write", p)
+
+
+class Command(BaseCommand):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        patch_write(self.stdout)

api/funkwhale_api/common/management/commands/script.py

@@ -0,0 +1,68 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from funkwhale_api.common import scripts
+
+
+class Command(BaseCommand):
+    help = "Run a specific script from funkwhale_api/common/scripts/"
+
+    def add_arguments(self, parser):
+        parser.add_argument("script_name", nargs="?", type=str)
+        parser.add_argument(
+            "--noinput",
+            "--no-input",
+            action="store_false",
+            dest="interactive",
+            help="Do NOT prompt the user for input of any kind.",
+        )
+
+    def handle(self, *args, **options):
+        name = options["script_name"]
+        if not name:
+            return self.show_help()
+
+        available_scripts = self.get_scripts()
+        try:
+            script = available_scripts[name]
+        except KeyError:
+            raise CommandError(
+                "{} is not a valid script. Run funkwhale-manage script for a "
+                "list of available scripts".format(name)
+            )
+
+        self.stdout.write("")
+        if options["interactive"]:
+            message = (
+                "Are you sure you want to execute the script {}?\n\n"
+                "Type 'yes' to continue, or 'no' to cancel: "
+            ).format(name)
+            if input("".join(message)) != "yes":
+                raise CommandError("Script cancelled.")
+        script["entrypoint"](self, **options)
+
+    def show_help(self):
+        self.stdout.write("")
+        self.stdout.write("Available scripts:")
+        self.stdout.write("Launch with: funkwhale-manage script <script_name>")
+        available_scripts = self.get_scripts()
+        for name, script in sorted(available_scripts.items()):
+            self.stdout.write("")
+            self.stdout.write(self.style.SUCCESS(name))
+            self.stdout.write("")
+            for line in script["help"].splitlines():
+                self.stdout.write(f"     {line}")
+        self.stdout.write("")
+
+    def get_scripts(self):
+        available_scripts = [
+            k for k in sorted(scripts.__dict__.keys()) if not k.startswith("__")
+        ]
+        data = {}
+        for name in available_scripts:
+            module = getattr(scripts, name)
+            data[name] = {
+                "name": name,
+                "help": module.__doc__.strip(),
+                "entrypoint": module.main,
+            }
+        return data

api/funkwhale_api/common/management/commands/testdata.py

@@ -0,0 +1,43 @@
+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.federation import factories
+from funkwhale_api.federation.models import Actor
+
+
+class Command(BaseCommand):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.help = "Helper to generate randomized testdata"
+        self.type_choices = {"notifications": self.handle_notifications}
+        self.missing_args_message = f"Please specify one of the following sub-commands: { *self.type_choices.keys(), }"
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest="subcommand")
+
+        notification_parser = subparsers.add_parser("notifications")
+        notification_parser.add_argument(
+            "username", type=str, help="Username to send the notifications to"
+        )
+        notification_parser.add_argument(
+            "--count", type=int, help="Number of elements to create", default=1
+        )
+
+    def handle(self, *args, **options):
+        self.type_choices[options["subcommand"]](options)
+
+    def handle_notifications(self, options):
+        self.stdout.write(
+            f"Create {options['count']} notification(s) for {options['username']}"
+        )
+        try:
+            actor = Actor.objects.get(preferred_username=options["username"])
+        except Actor.DoesNotExist:
+            self.stdout.write(
+                "The user you want to create notifications for does not exist"
+            )
+            return
+
+        follow_activity = factories.ActivityFactory(type="Follow")
+        for _ in range(options["count"]):
+            factories.InboxItemFactory(actor=actor, activity=follow_activity)

api/funkwhale_api/common/middleware.py

@@ -0,0 +1,420 @@
+import html
+import io
+import logging
+import os
+import re
+import time
+import tracemalloc
+import urllib.parse
+import xml.sax.saxutils
+
+from django import http, urls
+from django.conf import settings
+from django.contrib import auth
+from django.core.cache import caches
+from django.middleware import csrf
+from rest_framework import views
+
+from funkwhale_api.federation import utils as federation_utils
+
+from . import preferences, session, throttling, utils
+
+EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
+
+logger = logging.getLogger(__name__)
+
+
+def should_fallback_to_spa(path):
+    if path == "/":
+        return True
+    return not any([path.startswith(m) for m in EXCLUDED_PATHS])
+
+
+def serve_spa(request):
+    html = get_spa_html(settings.FUNQUAIL_SPA_HTML_ROOT)
+    head, tail = html.split("</head>", 1)
+    if settings.FUNQUAIL_SPA_REWRITE_MANIFEST:
+        new_url = (
+            settings.FUNQUAIL_SPA_REWRITE_MANIFEST_URL
+            or federation_utils.full_url(urls.reverse("api:v1:instance:spa-manifest"))
+        )
+        title = preferences.get("instance__name")
+        if title:
+            head = replace_title(head, title)
+        head = replace_manifest_url(head, new_url)
+
+    if not preferences.get("common__api_authentication_required"):
+        try:
+            request_tags = get_request_head_tags(request) or []
+        except urls.exceptions.Resolver404:
+            # we don't have any custom tags for this route
+            request_tags = []
+    else:
+        # API is not open, we don't expose any custom data
+        request_tags = []
+    default_tags = get_default_head_tags(request.path)
+    unique_attributes = ["name", "property"]
+
+    final_tags = request_tags
+    skip = []
+
+    for t in final_tags:
+        for attr in unique_attributes:
+            if attr in t:
+                skip.append(t[attr])
+    for t in default_tags:
+        existing = False
+        for attr in unique_attributes:
+            if t.get(attr) in skip:
+                existing = True
+                break
+        if not existing:
+            final_tags.append(t)
+
+    # let's inject our meta tags in the HTML
+    head += "\n" + "\n".join(render_tags(final_tags)) + "\n</head>"
+    css = get_custom_css() or ""
+    if css:
+        # We add the style add the end of the body to ensure it has the highest
+        # priority (since it will come after other stylesheets)
+        body, tail = tail.split("</body>", 1)
+        css = f"<style>{css}</style>"
+        tail = body + "\n" + css + "\n</body>" + tail
+
+    # set a csrf token so that visitor can login / query API if needed
+    token = csrf.get_token(request)
+    response = http.HttpResponse(head + tail)
+    response.set_cookie("csrftoken", token, max_age=None)
+    return response
+
+
+MANIFEST_LINK_REGEX = re.compile(r"<link [^>]*rel=(?:'|\")?manifest(?:'|\")?[^>]*>")
+TITLE_REGEX = re.compile(r"<title>.*</title>")
+
+
+def replace_manifest_url(head, new_url):
+    replacement = f'<link rel=manifest href="{new_url}">'
+    head = MANIFEST_LINK_REGEX.sub(replacement, head)
+    return head
+
+
+def replace_title(head, new_title):
+    replacement = f"<title>{html.escape(new_title)}</title>"
+    head = TITLE_REGEX.sub(replacement, head)
+    return head
+
+
+def get_spa_html(spa_url):
+    return get_spa_file(spa_url, "index.html")
+
+
+def get_spa_file(spa_url, name):
+    if spa_url.startswith("/"):
+        # spa_url is an absolute path to index.html, on the local disk.
+        # However, we may want to access manifest.json or other files as well, so we
+        # strip the filename
+        path = os.path.join(os.path.dirname(spa_url), name)
+        # we try to open a local file
+        with open(path, "rb") as f:
+            return f.read().decode("utf-8")
+    cache_key = f"spa-file:{spa_url}:{name}"
+    cached = caches["local"].get(cache_key)
+    if cached:
+        return cached
+
+    response = session.get_session().get(
+        utils.join_url(spa_url, name),
+    )
+    response.raise_for_status()
+    response.encoding = "utf-8"
+    content = response.text
+    caches["local"].set(cache_key, content, settings.FUNQUAIL_SPA_HTML_CACHE_DURATION)
+    return content
+
+
+def get_default_head_tags(path):
+    instance_name = preferences.get("instance__name")
+    short_description = preferences.get("instance__short_description")
+    app_name = settings.APP_NAME
+
+    parts = [instance_name, app_name]
+
+    return [
+        {"tag": "meta", "property": "og:type", "content": "website"},
+        {
+            "tag": "meta",
+            "property": "og:site_name",
+            "content": " - ".join([p for p in parts if p]),
+        },
+        {"tag": "meta", "property": "og:description", "content": short_description},
+        {
+            "tag": "meta",
+            "property": "og:image",
+            "content": utils.join_url(
+                settings.FUNQUAIL_URL, "/android-chrome-512x512.png"
+            ),
+        },
+        {
+            "tag": "meta",
+            "property": "og:url",
+            "content": utils.join_url(settings.FUNQUAIL_URL, path),
+        },
+    ]
+
+
+def render_tags(tags):
+    """
+    Given a dict like {'tag': 'meta', 'hello': 'world'}
+    return a html ready tag like
+    <meta hello="world" />
+    """
+    for tag in tags:
+        yield "<{tag} {attrs} />".format(
+            tag=tag.pop("tag"),
+            attrs=" ".join(
+                [f'{a}="{html.escape(str(v))}"' for a, v in sorted(tag.items()) if v]
+            ),
+        )
+
+
+def get_request_head_tags(request):
+    accept_header = request.headers.get("Accept") or None
+    redirect_to_ap = (
+        False
+        if not accept_header
+        else not federation_utils.should_redirect_ap_to_html(accept_header)
+    )
+    match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
+    return match.func(
+        request, *match.args, redirect_to_ap=redirect_to_ap, **match.kwargs
+    )
+
+
+def get_custom_css():
+    css = preferences.get("ui__custom_css").strip()
+    if not css:
+        return
+
+    return xml.sax.saxutils.escape(css)
+
+
+class ApiRedirect(Exception):
+    def __init__(self, url):
+        self.url = url
+
+
+def get_api_response(request, url):
+    """
+    Quite ugly but we have no choice. When Accept header is set to application/activity+json
+    some clients expect to get a JSON payload (instead of the HTML we return). Since
+    redirecting to the URL does not work (because it makes the signature verification fail),
+    we grab the internal view corresponding to the URL, call it and return this as the
+    response
+    """
+    path = urllib.parse.urlparse(url).path
+
+    try:
+        match = urls.resolve(path)
+    except urls.exceptions.Resolver404:
+        return http.HttpResponseNotFound()
+    response = match.func(request, *match.args, **match.kwargs)
+    if hasattr(response, "render"):
+        response.render()
+    return response
+
+
+class SPAFallbackMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+
+        if response.status_code == 404 and should_fallback_to_spa(request.path):
+            try:
+                return serve_spa(request)
+            except ApiRedirect as e:
+                return get_api_response(request, e.url)
+
+        return response
+
+
+class DevHttpsMiddleware:
+    """
+    In development, it's sometimes difficult to have django use HTTPS
+    when we have django behind nginx behind traefix.
+
+    We thus use a simple setting (in dev ONLY) to control that.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if settings.FORCE_HTTPS_URLS:
+            setattr(request.__class__, "scheme", "https")
+            setattr(
+                request,
+                "get_host",
+                lambda: request.__class__.get_host(request).replace(":80", ":443"),
+            )
+        return self.get_response(request)
+
+
+def monkey_patch_rest_initialize_request():
+    """
+    Rest framework use it's own APIRequest, meaning we can't easily
+    access our throttling info in the middleware. So me monkey patch the
+    `initialize_request` method from rest_framework to keep a link between both requests
+    """
+    original = views.APIView.initialize_request
+
+    def replacement(self, request, *args, **kwargs):
+        r = original(self, request, *args, **kwargs)
+        setattr(request, "_api_request", r)
+        return r
+
+    setattr(views.APIView, "initialize_request", replacement)
+
+
+monkey_patch_rest_initialize_request()
+
+
+def monkey_patch_auth_get_user():
+    """
+    We need an actor on our users for many endpoints, so we monkey patch
+    auth.get_user to create it if it's missing
+    """
+    original = auth.get_user
+
+    def replacement(request):
+        r = original(request)
+        if not r.is_anonymous and not r.actor:
+            r.create_actor()
+        return r
+
+    setattr(auth, "get_user", replacement)
+
+
+monkey_patch_auth_get_user()
+
+
+class ThrottleStatusMiddleware:
+    """
+    Include useful information regarding throttling in API responses to
+    ensure clients can adapt.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        try:
+            response = self.get_response(request)
+        except throttling.TooManyRequests:
+            # manual throttling in non rest_framework view, we have to return
+            # the proper response ourselves
+            response = http.HttpResponse(status=429)
+        request_to_check = request
+        try:
+            request_to_check = request._api_request
+        except AttributeError:
+            pass
+        throttle_status = getattr(request_to_check, "_throttle_status", None)
+        if throttle_status:
+            response["X-RateLimit-Limit"] = str(throttle_status["num_requests"])
+            response["X-RateLimit-Scope"] = str(throttle_status["scope"])
+            response["X-RateLimit-Remaining"] = throttle_status["num_requests"] - len(
+                throttle_status["history"]
+            )
+            response["X-RateLimit-Duration"] = str(throttle_status["duration"])
+            if throttle_status["history"]:
+                now = int(time.time())
+                # At this point, the client can send additional requests
+                oldtest_request = throttle_status["history"][-1]
+                remaining = throttle_status["duration"] - (now - int(oldtest_request))
+                response["Retry-After"] = str(remaining)
+                # At this point, all Rate Limit is reset to 0
+                latest_request = throttle_status["history"][0]
+                remaining = throttle_status["duration"] - (now - int(latest_request))
+                response["X-RateLimit-Reset"] = str(now + remaining)
+                response["X-RateLimit-ResetSeconds"] = str(remaining)
+
+        return response
+
+
+class VerboseBadRequestsMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if response.status_code == 400:
+            logger.warning("Bad request: %s", response.content)
+        return response
+
+
+class ProfilerMiddleware:
+    """
+    from https://github.com/omarish/django-cprofile-middleware/blob/master/django_cprofile_middleware/middleware.py
+    Simple profile middleware to profile django views. To run it, add ?prof to
+    the URL like this:
+        http://localhost:8000/view/?prof
+    Optionally pass the following to modify the output:
+    ?sort => Sort the output by a given metric. Default is time.
+        See
+        http://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
+        for all sort options.
+    ?count => The number of rows to display. Default is 100.
+    ?download => Download profile file suitable for visualization. For example
+        in snakeviz or RunSnakeRun
+    This is adapted from an example found here:
+    http://www.slideshare.net/zeeg/django-con-high-performance-django-presentation.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if "prof" not in request.GET:
+            return self.get_response(request)
+        import profile
+        import pstats
+
+        profiler = profile.Profile()
+        response = profiler.runcall(self.get_response, request)
+        profiler.create_stats()
+        if "prof-download" in request.GET:
+            import marshal
+
+            output = marshal.dumps(profiler.stats)
+            response = http.HttpResponse(
+                output, content_type="application/octet-stream"
+            )
+            response["Content-Disposition"] = "attachment; filename=view.prof"
+            response["Content-Length"] = len(output)
+        stream = io.StringIO()
+        stats = pstats.Stats(profiler, stream=stream)
+
+        stats.sort_stats(request.GET.get("prof-sort", "cumtime"))
+        stats.print_stats(int(request.GET.get("count", 100)))
+
+        response = http.HttpResponse("<pre>%s</pre>" % stream.getvalue())
+
+        return response
+
+
+class PymallocMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if tracemalloc.is_tracing():
+            snapshot = tracemalloc.take_snapshot()
+            stats = snapshot.statistics("lineno")
+
+            print("Memory trace")
+            for stat in stats[:25]:
+                print(stat)
+
+        return self.get_response(request)

api/funkwhale_api/common/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.0.2 on 2018-02-27 18:43
+from django.db import migrations
+from django.contrib.postgres.operations import UnaccentExtension
+
+
+class CustomUnaccentExtension(UnaccentExtension):
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'unaccent'"
+        with schema_editor.connection.cursor() as cursor:
+            cursor.execute(check_sql)
+            result = cursor.fetchall()
+
+        if result:
+            return
+        return super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = []
+
+    operations = [CustomUnaccentExtension()]

api/funkwhale_api/common/migrations/0002_mutation.py

@@ -0,0 +1,91 @@
+# Generated by Django 2.1.5 on 2019-01-31 15:44
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("federation", "0017_auto_20190130_0926"),
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Mutation",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("fid", models.URLField(db_index=True, max_length=500, unique=True)),
+                (
+                    "uuid",
+                    models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+                ),
+                ("type", models.CharField(db_index=True, max_length=100)),
+                ("is_approved", models.BooleanField(default=None, null=True)),
+                ("is_applied", models.BooleanField(default=None, null=True)),
+                (
+                    "creation_date",
+                    models.DateTimeField(
+                        db_index=True, default=django.utils.timezone.now
+                    ),
+                ),
+                (
+                    "applied_date",
+                    models.DateTimeField(blank=True, db_index=True, null=True),
+                ),
+                ("summary", models.TextField(max_length=2000, blank=True, null=True)),
+                ("payload", django.contrib.postgres.fields.jsonb.JSONField()),
+                (
+                    "previous_state",
+                    django.contrib.postgres.fields.jsonb.JSONField(
+                        null=True, default=None
+                    ),
+                ),
+                ("target_id", models.IntegerField(null=True)),
+                (
+                    "approved_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="approved_mutations",
+                        to="federation.Actor",
+                    ),
+                ),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="created_mutations",
+                        to="federation.Actor",
+                    ),
+                ),
+                (
+                    "target_content_type",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="targeting_mutations",
+                        to="contenttypes.ContentType",
+                    ),
+                ),
+            ],
+        )
+    ]

api/funkwhale_api/common/migrations/0003_cit_extension.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.0.2 on 2018-02-27 18:43
+from django.db import migrations
+from django.contrib.postgres.operations import CITextExtension
+
+
+class CustomCITExtension(CITextExtension):
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'citext'"
+        with schema_editor.connection.cursor() as cursor:
+            cursor.execute(check_sql)
+            result = cursor.fetchall()
+
+        if result:
+            return
+        return super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("common", "0002_mutation")]
+
+    operations = [CustomCITExtension()]

api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.2.6 on 2019-11-11 13:38
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.common.models
+import funkwhale_api.common.validators
+import uuid
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0003_cit_extension'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Attachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(max_length=500, unique=True, null=True)),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('last_fetch_date', models.DateTimeField(blank=True, null=True)),
+                ('size', models.IntegerField(blank=True, null=True)),
+                ('mimetype', models.CharField(blank=True, max_length=200, null=True)),
+                ('file', versatileimagefield.fields.VersatileImageField(max_length=255, upload_to=funkwhale_api.common.models.get_file_path, validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg'], max_size=5242880)])),
+                ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='federation.Actor', null=True)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/migrations/0005_auto_20191125_1421.py

@@ -0,0 +1,37 @@
+# Generated by Django 2.2.7 on 2019-11-25 14:21
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0004_auto_20191111_1338'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mutation',
+            name='payload',
+            field=django.contrib.postgres.fields.jsonb.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='previous_state',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
+        ),
+        migrations.CreateModel(
+            name='MutationAttachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('attachment', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='mutation_attachment', to='common.Attachment')),
+                ('mutation', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='mutation_attachment', to='common.Mutation')),
+            ],
+            options={
+                'unique_together': {('attachment', 'mutation')},
+            },
+        ),
+    ]

api/funkwhale_api/common/migrations/0006_content.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.7 on 2020-01-13 10:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0005_auto_20191125_1421'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Content',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.CharField(blank=True, max_length=5000, null=True)),
+                ('content_type', models.CharField(max_length=100)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/migrations/0007_auto_20200116_1610.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.9 on 2020-01-16 16:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0006_content'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.0.8 on 2020-07-01 13:17
+
+from django.conf import settings
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('common', '0007_auto_20200116_1610'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.CreateModel(
+            name='PluginConfiguration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=100)),
+                ('conf', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+                ('enabled', models.BooleanField(default=False)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='plugins', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'code')},
+            },
+        ),
+    ]

api/funkwhale_api/common/migrations/0009_auto_20220627_1915.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.2.13 on 2022-06-27 19:15
+
+import django.core.serializers.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0008_auto_20200701_1317'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_applied',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_approved',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='payload',
+            field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='previous_state',
+            field=models.JSONField(default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
+        ),
+        migrations.AlterField(
+            model_name='pluginconfiguration',
+            name='conf',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/common/mixins.py

@@ -0,0 +1,33 @@
+from django.db.models import Q
+from django.shortcuts import get_object_or_404
+from rest_framework import serializers
+
+
+class MultipleLookupDetailMixin:
+    lookup_value_regex = "[^/]+"
+    lookup_field = "composite"
+
+    def get_object(self):
+        queryset = self.filter_queryset(self.get_queryset())
+
+        relevant_lookup = None
+        value = None
+        for lookup in self.url_lookups:
+            field_validator = lookup["validator"]
+            try:
+                value = field_validator(self.kwargs["composite"])
+            except serializers.ValidationError:
+                continue
+            else:
+                relevant_lookup = lookup
+                break
+        get_query = relevant_lookup.get(
+            "get_query", lambda value: Q(**{relevant_lookup["lookup_field"]: value})
+        )
+        query = get_query(value)
+        obj = get_object_or_404(queryset, query)
+
+        # May raise a permission denied
+        self.check_object_permissions(self.request, obj)
+
+        return obj

api/funkwhale_api/common/models.py

@@ -0,0 +1,390 @@
+import mimetypes
+import uuid
+
+import magic
+from django.conf import settings
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import connections, models, transaction
+from django.db.models import JSONField, Lookup
+from django.db.models.fields import Field
+from django.db.models.sql.compiler import SQLCompiler
+from django.dispatch import receiver
+from django.urls import reverse
+from django.utils import timezone
+from versatileimagefield.fields import VersatileImageField
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+
+from funkwhale_api.federation import utils as federation_utils
+
+from . import utils, validators
+
+CONTENT_TEXT_MAX_LENGTH = 5000
+CONTENT_TEXT_SUPPORTED_TYPES = [
+    "text/html",
+    "text/markdown",
+    "text/plain",
+]
+
+
+@Field.register_lookup
+class NotEqual(Lookup):
+    lookup_name = "ne"
+
+    def as_sql(self, compiler, connection):
+        lhs, lhs_params = self.process_lhs(compiler, connection)
+        rhs, rhs_params = self.process_rhs(compiler, connection)
+        params = lhs_params + rhs_params
+        return f"{lhs} <> {rhs}", params
+
+
+class NullsLastSQLCompiler(SQLCompiler):
+    def get_order_by(self):
+        result = super().get_order_by()
+        if result and self.connection.vendor == "postgresql":
+            return [
+                (
+                    expr,
+                    (
+                        sql + " NULLS LAST" if not sql.endswith(" NULLS LAST") else sql,
+                        params,
+                        is_ref,
+                    ),
+                )
+                for (expr, (sql, params, is_ref)) in result
+            ]
+        return result
+
+
+class NullsLastQuery(models.sql.query.Query):
+    """Use a custom compiler to inject 'NULLS LAST' (for PostgreSQL)."""
+
+    def get_compiler(self, using=None, connection=None):
+        if using is None and connection is None:
+            raise ValueError("Need either using or connection")
+        if using:
+            connection = connections[using]
+        return NullsLastSQLCompiler(self, connection, using)
+
+
+class NullsLastQuerySet(models.QuerySet):
+    def __init__(self, model=None, query=None, using=None, hints=None):
+        super().__init__(model, query, using, hints)
+        self.query = query or NullsLastQuery(self.model)
+
+
+class LocalFromFidQuerySet:
+    def local(self, include=True):
+        host = settings.FEDERATION_HOSTNAME
+        query = models.Q(fid__startswith=f"http://{host}/") | models.Q(
+            fid__startswith=f"https://{host}/"
+        )
+        if include:
+            return self.filter(query)
+        else:
+            return self.filter(~query)
+
+
+class GenericTargetQuerySet(models.QuerySet):
+    def get_for_target(self, target):
+        content_type = ContentType.objects.get_for_model(target)
+        return self.filter(target_content_type=content_type, target_id=target.pk)
+
+
+class Mutation(models.Model):
+    fid = models.URLField(unique=True, max_length=500, db_index=True)
+    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
+    created_by = models.ForeignKey(
+        "federation.Actor",
+        related_name="created_mutations",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
+    approved_by = models.ForeignKey(
+        "federation.Actor",
+        related_name="approved_mutations",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
+
+    type = models.CharField(max_length=100, db_index=True)
+    # None = no choice, True = approved, False = refused
+    is_approved = models.BooleanField(default=None, null=True)
+
+    # None = not applied, True = applied, False = failed
+    is_applied = models.BooleanField(default=None, null=True)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
+    applied_date = models.DateTimeField(null=True, blank=True, db_index=True)
+    summary = models.TextField(max_length=2000, null=True, blank=True)
+
+    payload = JSONField(encoder=DjangoJSONEncoder)
+    previous_state = JSONField(null=True, default=None, encoder=DjangoJSONEncoder)
+
+    target_id = models.IntegerField(null=True)
+    target_content_type = models.ForeignKey(
+        ContentType,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="targeting_mutations",
+    )
+    target = GenericForeignKey("target_content_type", "target_id")
+
+    objects = GenericTargetQuerySet.as_manager()
+
+    def get_federation_id(self):
+        if self.fid:
+            return self.fid
+
+        return federation_utils.full_url(
+            reverse("federation:edits-detail", kwargs={"uuid": self.uuid})
+        )
+
+    def save(self, **kwargs):
+        if not self.pk and not self.fid:
+            self.fid = self.get_federation_id()
+
+        return super().save(**kwargs)
+
+    @transaction.atomic
+    def apply(self):
+        from . import mutations
+
+        if self.is_applied:
+            raise ValueError("Mutation was already applied")
+
+        previous_state = mutations.registry.apply(
+            type=self.type, obj=self.target, payload=self.payload
+        )
+        self.previous_state = previous_state
+        self.is_applied = True
+        self.applied_date = timezone.now()
+        self.save(update_fields=["is_applied", "applied_date", "previous_state"])
+        return previous_state
+
+
+def get_file_path(instance, filename):
+    return utils.ChunkedPath("attachments")(instance, filename)
+
+
+class AttachmentQuerySet(models.QuerySet):
+    def attached(self, include=True):
+        related_fields = [
+            "covered_album",
+            "mutation_attachment",
+            "covered_track",
+            "covered_artist",
+            "iconed_actor",
+        ]
+        query = None
+        for field in related_fields:
+            field_query = ~models.Q(**{field: None})
+            query = query | field_query if query else field_query
+
+        if not include:
+            query = ~query
+
+        return self.filter(query)
+
+    def local(self, include=True):
+        if include:
+            return self.filter(actor__domain_id=settings.FEDERATION_HOSTNAME)
+        else:
+            return self.exclude(actor__domain_id=settings.FEDERATION_HOSTNAME)
+
+
+class Attachment(models.Model):
+    # Remote URL where the attachment can be fetched
+    url = models.URLField(max_length=500, null=True, blank=True)
+    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
+    # Actor associated with the attachment
+    actor = models.ForeignKey(
+        "federation.Actor",
+        related_name="attachments",
+        on_delete=models.CASCADE,
+        null=True,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    last_fetch_date = models.DateTimeField(null=True, blank=True)
+    # File size
+    size = models.IntegerField(null=True, blank=True)
+    mimetype = models.CharField(null=True, blank=True, max_length=200)
+
+    file = VersatileImageField(
+        upload_to=get_file_path,
+        max_length=255,
+        validators=[
+            validators.ImageDimensionsValidator(min_width=50, min_height=50),
+            validators.FileValidator(
+                allowed_extensions=["png", "jpg", "jpeg"],
+                max_size=1024 * 1024 * 5,
+            ),
+        ],
+    )
+
+    objects = AttachmentQuerySet.as_manager()
+
+    def save(self, **kwargs):
+        if self.file and not self.size:
+            self.size = self.file.size
+
+        if self.file and not self.mimetype:
+            self.mimetype = self.guess_mimetype()
+
+        return super().save()
+
+    @property
+    def is_local(self):
+        return federation_utils.is_local(self.fid)
+
+    def guess_mimetype(self):
+        f = self.file
+        b = min(1000000, f.size)
+        t = magic.from_buffer(f.read(b), mime=True)
+        if not t.startswith("image/"):
+            # failure, we try guessing by extension
+            mt, _ = mimetypes.guess_type(f.name)
+            if mt:
+                t = mt
+        return t
+
+    @property
+    def download_url_original(self):
+        if self.file:
+            return utils.media_url(self.file.url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=original")
+
+    @property
+    def download_url_medium_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["200x200"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=medium_square_crop")
+
+    @property
+    def download_url_large_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["600x600"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=large_square_crop")
+
+
+class MutationAttachment(models.Model):
+    """
+    When using attachments in mutations, we need to keep a reference to
+    the attachment to ensure it is not pruned by common/tasks.py.
+
+    This is what this model does.
+    """
+
+    attachment = models.OneToOneField(
+        Attachment, related_name="mutation_attachment", on_delete=models.CASCADE
+    )
+    mutation = models.OneToOneField(
+        Mutation, related_name="mutation_attachment", on_delete=models.CASCADE
+    )
+
+    class Meta:
+        unique_together = ("attachment", "mutation")
+
+
+class Content(models.Model):
+    """
+    A text content that can be associated to other models, like a description, a summary, etc.
+    """
+
+    text = models.CharField(max_length=CONTENT_TEXT_MAX_LENGTH, blank=True, null=True)
+    content_type = models.CharField(max_length=100)
+
+    @property
+    def rendered(self):
+        from . import utils
+
+        return utils.render_html(self.text, self.content_type)
+
+    @property
+    def as_plain_text(self):
+        from . import utils
+
+        return utils.render_plain_text(self.rendered)
+
+    def truncate(self, length):
+        text = self.as_plain_text
+        truncated = text[:length]
+        if len(truncated) < len(text):
+            truncated += "…"
+
+        return truncated
+
+
+@receiver(models.signals.post_save, sender=Attachment)
+def warm_attachment_thumbnails(sender, instance, **kwargs):
+    if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS:
+        return
+    warmer = VersatileImageFieldWarmer(
+        instance_or_queryset=instance,
+        rendition_key_set="attachment_square",
+        image_attr="file",
+    )
+    num_created, failed_to_create = warmer.warm()
+
+
+@receiver(models.signals.post_save, sender=Mutation)
+def trigger_mutation_post_init(sender, instance, created, **kwargs):
+    if not created:
+        return
+
+    from . import mutations
+
+    try:
+        conf = mutations.registry.get_conf(instance.type, instance.target)
+    except mutations.ConfNotFound:
+        return
+    serializer = conf["serializer_class"]()
+    try:
+        handler = serializer.mutation_post_init
+    except AttributeError:
+        return
+    handler(instance)
+
+
+CONTENT_FKS = {
+    "music.Track": ["description"],
+    "music.Album": ["description"],
+    "music.Artist": ["description"],
+}
+
+
+@receiver(models.signals.post_delete, sender=None)
+def remove_attached_content(sender, instance, **kwargs):
+    fk_fields = CONTENT_FKS.get(instance._meta.label, [])
+    for field in fk_fields:
+        if getattr(instance, f"{field}_id"):
+            try:
+                getattr(instance, field).delete()
+            except Content.DoesNotExist:
+                pass
+
+
+class PluginConfiguration(models.Model):
+    """
+    Store plugin configuration in DB
+    """
+
+    code = models.CharField(max_length=100)
+    user = models.ForeignKey(
+        "users.User",
+        related_name="plugins",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    conf = JSONField(null=True, blank=True)
+    enabled = models.BooleanField(default=False)
+    creation_date = models.DateTimeField(default=timezone.now)
+
+    class Meta:
+        unique_together = ("user", "code")

api/funkwhale_api/common/mutations.py

@@ -0,0 +1,171 @@
+import persisting_theory
+from django.db import models, transaction
+from rest_framework import serializers
+
+
+class ConfNotFound(KeyError):
+    pass
+
+
+class Registry(persisting_theory.Registry):
+    look_into = "mutations"
+
+    def connect(self, type, klass, perm_checkers=None):
+        def decorator(serializer_class):
+            t = self.setdefault(type, {})
+            t[klass] = {
+                "serializer_class": serializer_class,
+                "perm_checkers": perm_checkers or {},
+            }
+            return serializer_class
+
+        return decorator
+
+    @transaction.atomic
+    def apply(self, type, obj, payload):
+        conf = self.get_conf(type, obj)
+        serializer = conf["serializer_class"](obj, data=payload)
+        serializer.is_valid(raise_exception=True)
+        previous_state = serializer.get_previous_state(obj, serializer.validated_data)
+        serializer.apply(obj, serializer.validated_data)
+        return previous_state
+
+    def is_valid(self, type, obj, payload):
+        conf = self.get_conf(type, obj)
+        serializer = conf["serializer_class"](obj, data=payload)
+        return serializer.is_valid(raise_exception=True)
+
+    def get_validated_payload(self, type, obj, payload):
+        conf = self.get_conf(type, obj)
+        serializer = conf["serializer_class"](obj, data=payload)
+        serializer.is_valid(raise_exception=True)
+        return serializer.payload_serialize(serializer.validated_data)
+
+    def has_perm(self, perm, type, obj, actor):
+        if perm not in ["approve", "suggest"]:
+            raise ValueError(f"Invalid permission {perm}")
+        conf = self.get_conf(type, obj)
+        checker = conf["perm_checkers"].get(perm)
+        if not checker:
+            return False
+        return checker(obj=obj, actor=actor)
+
+    def get_conf(self, type, obj):
+        try:
+            type_conf = self[type]
+        except KeyError:
+            raise ConfNotFound(f"{type} is not a registered mutation")
+
+        try:
+            conf = type_conf[obj.__class__]
+        except KeyError:
+            try:
+                conf = type_conf[None]
+            except KeyError:
+                raise ConfNotFound(
+                    f"No mutation configuration found for {obj.__class__}"
+                )
+        return conf
+
+
+class MutationSerializer(serializers.Serializer):
+    def apply(self, obj, validated_data):
+        raise NotImplementedError()
+
+    def post_apply(self, obj, validated_data):
+        pass
+
+    def get_previous_state(self, obj, validated_data):
+        return
+
+    def payload_serialize(self, data):
+        return data
+
+
+class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
+    def __init__(self, *args, **kwargs):
+        # we force partial mode, because update mutations are partial
+        kwargs.setdefault("partial", True)
+        super().__init__(*args, **kwargs)
+
+    @transaction.atomic
+    def apply(self, obj, validated_data):
+        r = self.update(obj, validated_data)
+        self.post_apply(r, validated_data)
+        return r
+
+    def validate(self, validated_data):
+        if not validated_data:
+            raise serializers.ValidationError("You must update at least one field")
+
+        return super().validate(validated_data)
+
+    def db_serialize(self, validated_data):
+        serialized_relations = self.get_serialized_relations()
+        data = {}
+        # ensure model fields are serialized properly
+        for key, value in list(validated_data.items()):
+            if not isinstance(value, models.Model):
+                data[key] = value
+                continue
+            field = serialized_relations[key]
+            data[key] = getattr(value, field)
+        return data
+
+    def payload_serialize(self, data):
+        data = super().payload_serialize(data)
+        # we use our serialized_relations configuration
+        # to ensure we store ids instead of model instances in our json
+        # payload
+        for field, attr in self.get_serialized_relations().items():
+            try:
+                obj = data[field]
+            except KeyError:
+                continue
+            if obj is None:
+                data[field] = None
+            else:
+                data[field] = getattr(obj, attr)
+        return data
+
+    def create(self, validated_data):
+        validated_data = self.db_serialize(validated_data)
+        return super().create(validated_data)
+
+    def get_previous_state(self, obj, validated_data):
+        return get_update_previous_state(
+            obj,
+            *list(validated_data.keys()),
+            serialized_relations=self.get_serialized_relations(),
+            handlers=self.get_previous_state_handlers(),
+        )
+
+    def get_serialized_relations(self):
+        return {}
+
+    def get_previous_state_handlers(self):
+        return {}
+
+
+def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
+    if not fields:
+        raise ValueError("You need to provide at least one field")
+
+    state = {}
+    for field in fields:
+        if field in handlers:
+            state[field] = handlers[field](obj)
+            continue
+        value = getattr(obj, field)
+        if isinstance(value, models.Model):
+            # we store the related object id and repr for better UX
+            id_field = serialized_relations[field]
+            related_value = getattr(value, id_field)
+            state[field] = {"value": related_value, "repr": str(value)}
+        else:
+            state[field] = {"value": value}
+
+    return state
+
+
+registry = Registry()

api/funkwhale_api/common/pagination.py

@@ -0,0 +1,29 @@
+from rest_framework.pagination import PageNumberPagination, _positive_int
+
+
+class FunQuailPagination(PageNumberPagination):
+    page_size_query_param = "page_size"
+    default_max_page_size = 50
+    default_page_size = None
+    view = None
+
+    def paginate_queryset(self, queryset, request, view=None):
+        self.view = view
+        return super().paginate_queryset(queryset, request, view)
+
+    def get_page_size(self, request):
+        max_page_size = (
+            getattr(self.view, "max_page_size", 0) or self.default_max_page_size
+        )
+        page_size = getattr(self.view, "default_page_size", 0) or max_page_size
+        if self.page_size_query_param:
+            try:
+                return _positive_int(
+                    request.query_params[self.page_size_query_param],
+                    strict=True,
+                    cutoff=max_page_size,
+                )
+            except (KeyError, ValueError):
+                pass
+
+        return page_size

api/funkwhale_api/common/permissions.py

@@ -0,0 +1,58 @@
+import operator
+
+from django.core.exceptions import ObjectDoesNotExist
+from django.http import Http404
+from rest_framework.permissions import BasePermission
+
+from funkwhale_api.common import preferences
+
+
+class ConditionalAuthentication(BasePermission):
+    def has_permission(self, request, view):
+        if preferences.get("common__api_authentication_required"):
+            return (request.user and request.user.is_authenticated) or (
+                hasattr(request, "actor") and request.actor
+            )
+        return True
+
+
+class OwnerPermission(BasePermission):
+    """
+    Ensure the request user is the owner of the object.
+
+    Usage:
+
+    class MyView(APIView):
+        model = MyModel
+        permission_classes = [OwnerPermission]
+        owner_field = 'owner'
+        owner_checks = ['read', 'write']
+    """
+
+    perms_map = {
+        "GET": "read",
+        "OPTIONS": "read",
+        "HEAD": "read",
+        "POST": "write",
+        "PUT": "write",
+        "PATCH": "write",
+        "DELETE": "write",
+    }
+
+    def has_object_permission(self, request, view, obj):
+        method_check = self.perms_map[request.method]
+        owner_checks = getattr(view, "owner_checks", ["read", "write"])
+        if method_check not in owner_checks:
+            # check not enabled
+            return True
+
+        owner_field = getattr(view, "owner_field", "user")
+        owner_exception = getattr(view, "owner_exception", Http404)
+        try:
+            owner = operator.attrgetter(owner_field)(obj)
+        except ObjectDoesNotExist:
+            raise owner_exception
+
+        if not owner or not request.user.is_authenticated or owner != request.user:
+            raise owner_exception
+        return True

api/funkwhale_api/common/preferences.py

@@ -0,0 +1,107 @@
+import json
+
+from django import forms
+from django.conf import settings
+from django.contrib.postgres.forms import JSONField
+from dynamic_preferences import serializers, types
+from dynamic_preferences.registries import global_preferences_registry
+
+
+class DefaultFromSettingMixin:
+    def get_default(self):
+        return getattr(settings, self.setting)
+
+
+def get(pref):
+    manager = global_preferences_registry.manager()
+    return manager[pref]
+
+
+def all():
+    manager = global_preferences_registry.manager()
+    return manager.all()
+
+
+def set(pref, value):
+    manager = global_preferences_registry.manager()
+    manager[pref] = value
+
+
+class StringListSerializer(serializers.BaseSerializer):
+    separator = ","
+    sort = True
+
+    @classmethod
+    def to_db(cls, value, **kwargs):
+        if not value:
+            return
+
+        if type(value) not in [list, tuple]:
+            raise cls.exception(
+                f"Cannot serialize, value {value} is not a list or a tuple"
+            )
+
+        if cls.sort:
+            value = sorted(value)
+        return cls.separator.join(value)
+
+    @classmethod
+    def to_python(cls, value, **kwargs):
+        if not value:
+            return []
+        return value.split(",")
+
+
+class StringListPreference(types.BasePreferenceType):
+    serializer = StringListSerializer
+    field_class = forms.MultipleChoiceField
+
+    def get_api_additional_data(self):
+        d = super().get_api_additional_data()
+        d["choices"] = self.get("choices")
+        return d
+
+
+class JSONSerializer(serializers.BaseSerializer):
+    required = True
+
+    @classmethod
+    def to_db(cls, value, **kwargs):
+        if not cls.required and value is None:
+            return json.dumps(value)
+        data_serializer = cls.data_serializer_class(data=value)
+        if not data_serializer.is_valid():
+            raise cls.exception(
+                f"{value} is not a valid value: {data_serializer.errors}"
+            )
+        value = data_serializer.validated_data
+        try:
+            return json.dumps(value, sort_keys=True)
+        except TypeError:
+            raise cls.exception(
+                f"Cannot serialize, value {value} is not JSON serializable"
+            )
+
+    @classmethod
+    def to_python(cls, value, **kwargs):
+        return json.loads(value)
+
+
+class SerializedPreference(types.BasePreferenceType):
+    """
+    A preference that store arbitrary JSON and validate it using a rest_framework
+    serializer
+    """
+
+    serializer = JSONSerializer
+    data_serializer_class = None
+    field_class = JSONField
+    widget = forms.Textarea
+
+    @property
+    def serializer(self):
+        class _internal(JSONSerializer):
+            data_serializer_class = self.data_serializer_class
+            required = self.get("required")
+
+        return _internal

api/funkwhale_api/common/renderers.py

@@ -0,0 +1,5 @@
+from rest_framework.renderers import JSONRenderer
+
+
+class ActivityStreamRenderer(JSONRenderer):
+    media_type = "application/activity+json"

api/funkwhale_api/common/routers.py

@@ -0,0 +1,7 @@
+from rest_framework.routers import DefaultRouter
+
+
+class OptionalSlashRouter(DefaultRouter):
+    def __init__(self):
+        super().__init__()
+        self.trailing_slash = "/?"

api/funkwhale_api/common/scripts/__init__.py

@@ -0,0 +1,15 @@
+from . import (
+    create_actors,
+    delete_pre_017_federated_uploads,
+    django_permissions_to_user_permissions,
+    migrate_to_user_libraries,
+    test,
+)
+
+__all__ = [
+    "create_actors",
+    "django_permissions_to_user_permissions",
+    "migrate_to_user_libraries",
+    "delete_pre_017_federated_uploads",
+    "test",
+]

api/funkwhale_api/common/scripts/create_actors.py

@@ -0,0 +1,21 @@
+"""
+Compute different sizes of image used for Album covers and User avatars
+"""
+from django.db.utils import IntegrityError
+
+from funkwhale_api.users.models import User, create_actor
+
+
+def main(command, **kwargs):
+    qs = User.objects.filter(actor__isnull=True).order_by("username")
+    total = len(qs)
+    command.stdout.write(f"{total} users found without actors")
+    for i, user in enumerate(qs):
+        command.stdout.write(f"{i + 1}/{total} creating actor for {user.username}")
+        try:
+            user.actor = create_actor(user)
+        except IntegrityError as e:
+            # somehow, an actor with the the url exists in the database
+            command.stderr.write(f"Error while creating actor: {str(e)}")
+            continue
+        user.save(update_fields=["actor"])

api/funkwhale_api/common/scripts/create_image_variations.py

@@ -0,0 +1,28 @@
+"""
+Compute different sizes of image used for Album covers and User avatars
+"""
+
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+
+from funkwhale_api.common.models import Attachment
+
+MODELS = [
+    (Attachment, "file", "attachment_square"),
+]
+
+
+def main(command, **kwargs):
+    for model, attribute, key_set in MODELS:
+        qs = model.objects.exclude(**{f"{attribute}__isnull": True})
+        qs = qs.exclude(**{attribute: ""})
+        warmer = VersatileImageFieldWarmer(
+            instance_or_queryset=qs,
+            rendition_key_set=key_set,
+            image_attr=attribute,
+            verbose=True,
+        )
+        command.stdout.write(f"Creating images for {model.__name__} / {attribute}")
+        num_created, failed_to_create = warmer.warm()
+        command.stdout.write(
+            f"  {num_created} created, {len(failed_to_create)} in error"
+        )

api/funkwhale_api/common/scripts/delete_pre_017_federated_uploads.py

@@ -0,0 +1,14 @@
+"""
+Compute different sizes of image used for Album covers and User avatars
+"""
+
+from funkwhale_api.music.models import Upload
+
+
+def main(command, **kwargs):
+    queryset = Upload.objects.filter(
+        source__startswith="http", source__contains="/federation/music/file/"
+    ).exclude(source__contains="youtube")
+    total = queryset.count()
+    command.stdout.write(f"{total} uploads found")
+    queryset.delete()

api/funkwhale_api/common/scripts/django_permissions_to_user_permissions.py

@@ -0,0 +1,28 @@
+"""
+Convert django permissions to user permissions in the database,
+following the work done in #152.
+"""
+from django.contrib.auth.models import Permission
+from django.db.models import Q
+
+from funkwhale_api.users import models
+
+mapping = {
+    "dynamic_preferences.change_globalpreferencemodel": "settings",
+    "music.add_importbatch": "library",
+}
+
+
+def main(command, **kwargs):
+    for codename, user_permission in sorted(mapping.items()):
+        app_label, c = codename.split(".")
+        p = Permission.objects.get(content_type__app_label=app_label, codename=c)
+        users = models.User.objects.filter(
+            Q(groups__permissions=p) | Q(user_permissions=p)
+        ).distinct()
+        total = users.count()
+
+        command.stdout.write(
+            f"Updating {total} users with {user_permission} permission..."
+        )
+        users.update(**{f"permission_{user_permission}": True})

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

@@ -0,0 +1,155 @@
+"""
+Mirate instance files to a library #463. For each user that imported music on an
+instance, we will create a "default" library with related files and an instance-level
+visibility (unless instance has common__api_authentication_required set to False,
+in which case the libraries will be public).
+
+Files without any import job will be bounded to a "default" library on the first
+superuser account found. This should now happen though.
+
+This command will also generate federation ids for existing resources.
+
+"""
+
+from django.conf import settings
+from django.db.models import CharField, F, Value, functions
+
+from funkwhale_api.common import preferences
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.music import models
+from funkwhale_api.users.models import User
+
+
+def create_libraries(open_api, stdout):
+    local_actors = federation_models.Actor.objects.exclude(user=None).only("pk", "user")
+    privacy_level = "everyone" if open_api else "instance"
+    stdout.write(
+        "* Creating {} libraries with {} visibility".format(
+            len(local_actors), privacy_level
+        )
+    )
+    libraries_by_user = {}
+
+    for a in local_actors:
+        library, created = models.Library.objects.get_or_create(
+            name="default", actor=a, defaults={"privacy_level": privacy_level}
+        )
+        libraries_by_user[library.actor.user.pk] = library.pk
+        if created:
+            stdout.write(f"  * Created library {library.pk} for user {a.user.pk}")
+        else:
+            stdout.write(
+                "  * Found existing library {} for user {}".format(
+                    library.pk, a.user.pk
+                )
+            )
+
+    return libraries_by_user
+
+
+def update_uploads(libraries_by_user, stdout):
+    stdout.write("* Updating uploads with proper libraries...")
+    for user_id, library_id in libraries_by_user.items():
+        jobs = models.ImportJob.objects.filter(
+            upload__library=None, batch__submitted_by=user_id
+        )
+        candidates = models.Upload.objects.filter(
+            pk__in=jobs.values_list("upload", flat=True)
+        )
+        total = candidates.update(library=library_id, import_status="finished")
+        if total:
+            stdout.write(f"  * Assigned {total} uploads to user {user_id}'s library")
+        else:
+            stdout.write(f"  * No uploads to assign to user {user_id}'s library")
+
+
+def update_orphan_uploads(open_api, stdout):
+    privacy_level = "everyone" if open_api else "instance"
+    first_superuser = (
+        User.objects.filter(is_superuser=True)
+        .exclude(actor=None)
+        .order_by("pk")
+        .first()
+    )
+    if not first_superuser:
+        stdout.write("* No superuser found, skipping update orphan uploads")
+        return
+    library, _ = models.Library.objects.get_or_create(
+        name="default",
+        actor=first_superuser.actor,
+        defaults={"privacy_level": privacy_level},
+    )
+    candidates = (
+        models.Upload.objects.filter(library=None, jobs__isnull=True)
+        .exclude(audio_file=None)
+        .exclude(audio_file="")
+    )
+
+    total = candidates.update(library=library, import_status="finished")
+    if total:
+        stdout.write(
+            "* Assigned {} orphaned uploads to superuser {}".format(
+                total, first_superuser.pk
+            )
+        )
+    else:
+        stdout.write("* No orphaned uploads found")
+
+
+def set_fid(queryset, path, stdout):
+    model = queryset.model._meta.label
+    qs = queryset.filter(fid=None)
+    base_url = f"{settings.FUNQUAIL_URL}{path}"
+    stdout.write(f"* Assigning federation ids to {model} entries (path: {base_url})")
+    new_fid = functions.Concat(Value(base_url), F("uuid"), output_field=CharField())
+    total = qs.update(fid=new_fid)
+
+    stdout.write(f"  * {total} entries updated")
+
+
+def update_shared_inbox_url(stdout):
+    stdout.write("* Update shared inbox url for local actors...")
+    candidates = federation_models.Actor.objects.local()
+    url = federation_models.get_shared_inbox_url()
+    candidates.update(shared_inbox_url=url)
+
+
+def generate_actor_urls(part, stdout):
+    field = f"{part}_url"
+    stdout.write(f"* Update {field} for local actors...")
+
+    queryset = federation_models.Actor.objects.local().filter(**{field: None})
+    base_url = f"{settings.FUNQUAIL_URL}/federation/actors/"
+
+    new_field = functions.Concat(
+        Value(base_url),
+        F("preferred_username"),
+        Value(f"/{part}"),
+        output_field=CharField(),
+    )
+
+    queryset.update(**{field: new_field})
+
+
+def main(command, **kwargs):
+    open_api = not preferences.get("common__api_authentication_required")
+    libraries_by_user = create_libraries(open_api, command.stdout)
+    update_uploads(libraries_by_user, command.stdout)
+    update_orphan_uploads(open_api, command.stdout)
+
+    set_fid_params = [
+        (
+            models.Upload.objects.exclude(library__actor__user=None),
+            "/federation/music/uploads/",
+        ),
+        (models.Artist.objects.all(), "/federation/music/artists/"),
+        (models.Album.objects.all(), "/federation/music/albums/"),
+        (models.Track.objects.all(), "/federation/music/tracks/"),
+    ]
+    for qs, path in set_fid_params:
+        set_fid(qs, path, command.stdout)
+
+    update_shared_inbox_url(command.stdout)
+
+    for part in ["followers", "following"]:
+        generate_actor_urls(part, command.stdout)

api/funkwhale_api/common/scripts/test.py

@@ -0,0 +1,8 @@
+"""
+This is a test script that does nothing.
+You can launch it just to check how it works.
+"""
+
+
+def main(command, **kwargs):
+    command.stdout.write("Test script run successfully")

api/funkwhale_api/common/search.py

@@ -0,0 +1,246 @@
+import re
+
+from django.contrib.postgres.search import SearchQuery
+from django.db.models import Q
+
+from . import utils
+
+QUERY_REGEX = re.compile(r'(((?P<key>\w+):)?(?P<value>"[^"]+"|[\S]+))')
+
+
+def parse_query(query):
+    """
+    Given a search query such as "hello is:issue status:opened",
+    returns a list of dictionaries describing each query token
+    """
+    matches = [m.groupdict() for m in QUERY_REGEX.finditer(query.lower())]
+    for m in matches:
+        if m["value"].startswith('"') and m["value"].endswith('"'):
+            m["value"] = m["value"][1:-1]
+    return matches
+
+
+def normalize_query(
+    query_string,
+    findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
+    normspace=re.compile(r"\s{2,}").sub,
+):
+    """Splits the query string in individual keywords, getting rid of unnecessary spaces
+    and grouping quoted words together.
+    Example:
+
+    >>> normalize_query('  some random  words "with   quotes  " and   spaces')
+    ['some', 'random', 'words', 'with quotes', 'and', 'spaces']
+
+    """
+    return [normspace(" ", (t[0] or t[1]).strip()) for t in findterms(query_string)]
+
+
+def get_query(query_string, search_fields):
+    """Returns a query, that is a combination of Q objects. That combination
+    aims to search keywords within a model by testing the given search fields.
+
+    """
+    query = None  # Query to search for every search term
+    terms = normalize_query(query_string)
+    for term in terms:
+        or_query = None  # Query to search for a given term in each field
+        for field_name in search_fields:
+            q = Q(**{"%s__icontains" % field_name: term})
+            if or_query is None:
+                or_query = q
+            else:
+                or_query = or_query | q
+        if query is None:
+            query = or_query
+        else:
+            query = query & or_query
+    return query
+
+
+def remove_chars(string, chars):
+    for char in chars:
+        string = string.replace(char, "")
+    return string
+
+
+def get_fts_query(query_string, fts_fields=["body_text"], model=None):
+    search_type = "raw"
+    if query_string.startswith('"') and query_string.endswith('"'):
+        # we pass the query directly to the FTS engine
+        query_string = query_string[1:-1]
+    else:
+        query_string = remove_chars(query_string, ['"', "&", "(", ")", "!", "'"])
+        parts = query_string.replace(":", "").split(" ")
+        parts = [f"{p}:*" for p in parts if p]
+        if not parts:
+            return Q(pk=None)
+
+        query_string = "&".join(parts)
+
+    if not fts_fields or not query_string.strip():
+        return Q(pk=None)
+    query = None
+    for field in fts_fields:
+        if "__" in field and model:
+            # When we have a nested lookup, we switch to a subquery for enhanced performance
+            fk_field_name, lookup = (
+                field.split("__")[0],
+                "__".join(field.split("__")[1:]),
+            )
+            fk_field = model._meta.get_field(fk_field_name)
+            related_model = fk_field.related_model
+            subquery = related_model.objects.filter(
+                **{
+                    lookup: SearchQuery(
+                        query_string, search_type=search_type, config="english_nostop"
+                    )
+                }
+            ).values_list("pk", flat=True)
+            new_query = Q(**{f"{fk_field_name}__in": list(subquery)})
+        else:
+            new_query = Q(
+                **{
+                    field: SearchQuery(
+                        query_string, search_type=search_type, config="english_nostop"
+                    )
+                }
+            )
+        query = utils.join_queries_or(query, new_query)
+
+    return query
+
+
+def filter_tokens(tokens, valid):
+    return [t for t in tokens if t["key"] in valid]
+
+
+def apply(qs, config_data):
+    for k in ["filter_query", "search_query"]:
+        q = config_data.get(k)
+        if q:
+            qs = qs.filter(q)
+    distinct = config_data.get("distinct", False)
+    if distinct:
+        qs = qs.distinct()
+    return qs
+
+
+class SearchConfig:
+    def __init__(self, search_fields={}, filter_fields={}, types=[]):
+        self.filter_fields = filter_fields
+        self.search_fields = search_fields
+        self.types = types
+
+    def clean(self, query):
+        tokens = parse_query(query)
+        cleaned_data = {}
+        cleaned_data["types"] = self.clean_types(filter_tokens(tokens, ["is"]))
+        cleaned_data["search_query"] = self.clean_search_query(
+            filter_tokens(tokens, [None, "in"] + list(self.search_fields.keys()))
+        )
+        unhandled_tokens = [
+            t
+            for t in tokens
+            if t["key"] not in [None, "is", "in"] + list(self.search_fields.keys())
+        ]
+        cleaned_data["filter_query"], matching_filters = self.clean_filter_query(
+            unhandled_tokens
+        )
+        if matching_filters:
+            cleaned_data["distinct"] = any(
+                [
+                    self.filter_fields[k].get("distinct", False)
+                    for k in matching_filters
+                    if k in self.filter_fields
+                ]
+            )
+        else:
+            cleaned_data["distinct"] = False
+        return cleaned_data
+
+    def clean_search_query(self, tokens):
+        if not self.search_fields or not tokens:
+            return
+
+        fields_subset = {
+            f for t in filter_tokens(tokens, ["in"]) for f in t["value"].split(",")
+        } or set(self.search_fields.keys())
+        fields_subset = set(self.search_fields.keys()) & fields_subset
+        to_fields = [self.search_fields[k]["to"] for k in fields_subset]
+
+        specific_field_query = None
+        for token in tokens:
+            if token["key"] not in self.search_fields:
+                continue
+            to = self.search_fields[token["key"]]["to"]
+            try:
+                field = token["field"]
+                value = field.clean(token["value"])
+            except KeyError:
+                # no cleaning to apply
+                value = token["value"]
+            q = Q(**{f"{to}__icontains": value})
+            if not specific_field_query:
+                specific_field_query = q
+            else:
+                specific_field_query &= q
+        query_string = " ".join([t["value"] for t in filter_tokens(tokens, [None])])
+        unhandled_tokens_query = get_query(query_string, sorted(to_fields))
+
+        if specific_field_query and unhandled_tokens_query:
+            return unhandled_tokens_query & specific_field_query
+        elif specific_field_query:
+            return specific_field_query
+        elif unhandled_tokens_query:
+            return unhandled_tokens_query
+        return None
+
+    def clean_filter_query(self, tokens):
+        if not self.filter_fields or not tokens:
+            return None, []
+
+        matching = [t for t in tokens if t["key"] in self.filter_fields]
+        queries = [self.get_filter_query(token) for token in matching]
+        query = None
+        for q in queries:
+            if not query:
+                query = q
+            else:
+                query = query & q
+        return query, [m["key"] for m in matching]
+
+    def get_filter_query(self, token):
+        raw_value = token["value"]
+        try:
+            field = self.filter_fields[token["key"]]["field"]
+            value = field.clean(raw_value)
+        except KeyError:
+            # no cleaning to apply
+            value = raw_value
+        try:
+            query_field = self.filter_fields[token["key"]]["to"]
+            return Q(**{query_field: value})
+        except KeyError:
+            pass
+
+        # we don't have a basic filter -> field mapping, this likely means we
+        # have a dynamic handler in the config
+        handler = self.filter_fields[token["key"]]["handler"]
+        value = handler(value)
+        return value
+
+    def clean_types(self, tokens):
+        if not self.types:
+            return []
+
+        if not tokens:
+            # no filtering on type, we return all types
+            return [t for key, t in self.types]
+        types = []
+        for token in tokens:
+            for key, t in self.types:
+                if key.lower() == token["value"]:
+                    types.append(t)
+
+        return types

api/funkwhale_api/common/serializers.py

@@ -0,0 +1,367 @@
+import collections
+import io
+import os
+
+import PIL
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.utils.encoding import smart_text
+from django.utils.translation import ugettext_lazy as _
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
+
+from . import models, utils
+
+
+class RelatedField(serializers.RelatedField):
+    default_error_messages = {
+        "does_not_exist": _("Object with {related_field_name}={value} does not exist."),
+        "invalid": _("Invalid value."),
+    }
+
+    def __init__(self, related_field_name, serializer, **kwargs):
+        self.related_field_name = related_field_name
+        self.serializer = serializer
+        self.filters = kwargs.pop("filters", None)
+        self.queryset_filter = kwargs.pop("queryset_filter", None)
+        try:
+            kwargs["queryset"] = kwargs.pop("queryset")
+        except KeyError:
+            kwargs["queryset"] = self.serializer.Meta.model.objects.all()
+        super().__init__(**kwargs)
+
+    def get_filters(self, data):
+        filters = {self.related_field_name: data}
+        if self.filters:
+            filters.update(self.filters(self.context))
+        return filters
+
+    def filter_queryset(self, queryset):
+        if self.queryset_filter:
+            queryset = self.queryset_filter(queryset, self.context)
+        return queryset
+
+    def to_internal_value(self, data):
+        try:
+            queryset = self.get_queryset()
+            filters = self.get_filters(data)
+            queryset = self.filter_queryset(queryset)
+            return queryset.get(**filters)
+        except ObjectDoesNotExist:
+            self.fail(
+                "does_not_exist",
+                related_field_name=self.related_field_name,
+                value=smart_text(data),
+            )
+        except (TypeError, ValueError):
+            self.fail("invalid")
+
+    def to_representation(self, obj):
+        return self.serializer.to_representation(obj)
+
+    def get_choices(self, cutoff=None):
+        queryset = self.get_queryset()
+        if queryset is None:
+            # Ensure that field.choices returns something sensible
+            # even when accessed with a read-only field.
+            return {}
+
+        if cutoff is not None:
+            queryset = queryset[:cutoff]
+
+        return collections.OrderedDict(
+            [
+                (
+                    self.to_representation(item)[self.related_field_name],
+                    self.display_value(item),
+                )
+                for item in queryset
+                if self.serializer
+            ]
+        )
+
+
+class Action:
+    def __init__(self, name, allow_all=False, qs_filter=None):
+        self.name = name
+        self.allow_all = allow_all
+        self.qs_filter = qs_filter
+
+    def __repr__(self):
+        return f"<Action {self.name}>"
+
+
+class ActionSerializer(serializers.Serializer):
+    """
+    A special serializer that can operate on a list of objects
+    and apply actions on it.
+    """
+
+    action = serializers.CharField(required=True)
+    objects = serializers.JSONField(required=True)
+    filters = serializers.DictField(required=False)
+    actions = None
+    pk_field = "pk"
+
+    def __init__(self, *args, **kwargs):
+        self.actions_by_name = {a.name: a for a in self.actions}
+        self.queryset = kwargs.pop("queryset")
+        if self.actions is None:
+            raise ValueError(
+                "You must declare a list of actions on " "the serializer class"
+            )
+
+        for action in self.actions_by_name.keys():
+            handler_name = f"handle_{action}"
+            assert hasattr(self, handler_name), "{} miss a {} method".format(
+                self.__class__.__name__, handler_name
+            )
+        super().__init__(self, *args, **kwargs)
+
+    def validate_action(self, value):
+        try:
+            return self.actions_by_name[value]
+        except KeyError:
+            raise serializers.ValidationError(
+                "{} is not a valid action. Pick one of {}.".format(
+                    value, ", ".join(self.actions_by_name.keys())
+                )
+            )
+
+    def validate_objects(self, value):
+        if value == "all":
+            return self.queryset.all().order_by("id")
+        if type(value) in [list, tuple]:
+            return self.queryset.filter(**{f"{self.pk_field}__in": value}).order_by(
+                self.pk_field
+            )
+
+        raise serializers.ValidationError(
+            "{} is not a valid value for objects. You must provide either a "
+            'list of identifiers or the string "all".'.format(value)
+        )
+
+    def validate(self, data):
+        allow_all = data["action"].allow_all
+        if not allow_all and self.initial_data["objects"] == "all":
+            raise serializers.ValidationError(
+                "You cannot apply this action on all objects"
+            )
+        final_filters = data.get("filters", {}) or {}
+        if self.filterset_class and final_filters:
+            qs_filterset = self.filterset_class(final_filters, queryset=data["objects"])
+            try:
+                assert qs_filterset.form.is_valid()
+            except (AssertionError, TypeError):
+                raise serializers.ValidationError("Invalid filters")
+            data["objects"] = qs_filterset.qs
+
+        if data["action"].qs_filter:
+            data["objects"] = data["action"].qs_filter(data["objects"])
+
+        data["count"] = data["objects"].count()
+        if data["count"] < 1:
+            raise serializers.ValidationError("No object matching your request")
+        return data
+
+    def save(self):
+        handler_name = "handle_{}".format(self.validated_data["action"].name)
+        handler = getattr(self, handler_name)
+        result = handler(self.validated_data["objects"])
+        payload = {
+            "updated": self.validated_data["count"],
+            "action": self.validated_data["action"].name,
+            "result": result,
+        }
+        return payload
+
+
+def track_fields_for_update(*fields):
+    """
+    Apply this decorator to serializer to call function when specific values
+    are updated on an object:
+
+    .. code-block:: python
+
+        @track_fields_for_update('privacy_level')
+        class LibrarySerializer(serializers.ModelSerializer):
+            def on_updated_privacy_level(self, obj, old_value, new_value):
+                print('Do someting')
+    """
+
+    def decorator(serializer_class):
+        original_update = serializer_class.update
+
+        def new_update(self, obj, validated_data):
+            tracked_fields_before = {f: getattr(obj, f) for f in fields}
+            obj = original_update(self, obj, validated_data)
+            tracked_fields_after = {f: getattr(obj, f) for f in fields}
+
+            if tracked_fields_before != tracked_fields_after:
+                self.on_updated_fields(obj, tracked_fields_before, tracked_fields_after)
+            return obj
+
+        serializer_class.update = new_update
+        return serializer_class
+
+    return decorator
+
+
+class StripExifImageField(serializers.ImageField):
+    def to_internal_value(self, data):
+        file_obj = super().to_internal_value(data)
+
+        image = PIL.Image.open(file_obj)
+        data = list(image.getdata())
+        image_without_exif = PIL.Image.new(image.mode, image.size)
+        image_without_exif.putdata(data)
+
+        with io.BytesIO() as output:
+            image_without_exif.save(
+                output,
+                format=PIL.Image.EXTENSION[os.path.splitext(file_obj.name)[-1].lower()],
+                quality=100,
+            )
+            content = output.getvalue()
+
+        return SimpleUploadedFile(
+            file_obj.name, content, content_type=file_obj.content_type
+        )
+
+
+from funkwhale_api.federation import serializers as federation_serializers  # noqa
+
+TARGET_ID_TYPE_MAPPING = {
+    "music.Track": ("id", "track"),
+    "music.Artist": ("id", "artist"),
+    "music.Album": ("id", "album"),
+}
+
+
+class APIMutationSerializer(serializers.ModelSerializer):
+    created_by = federation_serializers.APIActorSerializer(read_only=True)
+    target = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.Mutation
+        fields = [
+            "fid",
+            "uuid",
+            "type",
+            "creation_date",
+            "applied_date",
+            "is_approved",
+            "is_applied",
+            "created_by",
+            "approved_by",
+            "summary",
+            "payload",
+            "previous_state",
+            "target",
+        ]
+        read_only_fields = [
+            "uuid",
+            "creation_date",
+            "fid",
+            "is_applied",
+            "created_by",
+            "approved_by",
+            "previous_state",
+        ]
+
+    @extend_schema_field(OpenApiTypes.OBJECT)
+    def get_target(self, obj):
+        target = obj.target
+        if not target:
+            return
+
+        id_field, type = TARGET_ID_TYPE_MAPPING[target._meta.label]
+        return {"type": type, "id": getattr(target, id_field), "repr": str(target)}
+
+    def validate_type(self, value):
+        if value not in self.context["registry"]:
+            raise serializers.ValidationError(f"Invalid mutation type {value}")
+        return value
+
+
+class AttachmentSerializer(serializers.Serializer):
+    uuid = serializers.UUIDField(read_only=True)
+    size = serializers.IntegerField(read_only=True)
+    mimetype = serializers.CharField(read_only=True)
+    creation_date = serializers.DateTimeField(read_only=True)
+    file = StripExifImageField(write_only=True)
+    urls = serializers.SerializerMethodField()
+
+    @extend_schema_field(OpenApiTypes.OBJECT)
+    def get_urls(self, o):
+        urls = {}
+        urls["source"] = o.url
+        urls["original"] = o.download_url_original
+        urls["medium_square_crop"] = o.download_url_medium_square_crop
+        urls["large_square_crop"] = o.download_url_large_square_crop
+        return urls
+
+    def create(self, validated_data):
+        return models.Attachment.objects.create(
+            file=validated_data["file"], actor=validated_data["actor"]
+        )
+
+
+class ContentSerializer(serializers.Serializer):
+    text = serializers.CharField(
+        max_length=models.CONTENT_TEXT_MAX_LENGTH, allow_null=True
+    )
+    content_type = serializers.ChoiceField(
+        choices=models.CONTENT_TEXT_SUPPORTED_TYPES,
+    )
+    html = serializers.SerializerMethodField()
+
+    def get_html(self, o) -> str:
+        return utils.render_html(o.text, o.content_type)
+
+
+class NullToEmptDict:
+    def get_attribute(self, o):
+        attr = super().get_attribute(o)
+        if attr is None:
+            return {}
+        return attr
+
+    def to_representation(self, v):
+        if not v:
+            return v
+        return super().to_representation(v)
+
+
+class ScopesSerializer(serializers.Serializer):
+    id = serializers.CharField()
+    rate = serializers.CharField()
+    description = serializers.CharField()
+    limit = serializers.IntegerField()
+    duration = serializers.IntegerField()
+    remaining = serializers.IntegerField()
+    available = serializers.IntegerField()
+    available_seconds = serializers.IntegerField()
+    reset = serializers.IntegerField()
+    reset_seconds = serializers.IntegerField()
+
+
+class IdentSerializer(serializers.Serializer):
+    type = serializers.CharField()
+    id = serializers.CharField()
+
+
+class RateLimitSerializer(serializers.Serializer):
+    enabled = serializers.BooleanField()
+    ident = IdentSerializer()
+    scopes = serializers.ListField(child=ScopesSerializer())
+
+
+class ErrorDetailSerializer(serializers.Serializer):
+    detail = serializers.CharField(source="*")
+
+
+class TextPreviewSerializer(serializers.Serializer):
+    rendered = serializers.CharField(read_only=True, source="*")
+    text = serializers.CharField(write_only=True)

api/funkwhale_api/common/session.py

@@ -0,0 +1,23 @@
+import requests
+from django.conf import settings
+
+import funkwhale_api
+
+
+class FunQuailSession(requests.Session):
+    def request(self, *args, **kwargs):
+        kwargs.setdefault("verify", settings.EXTERNAL_REQUESTS_VERIFY_SSL)
+        kwargs.setdefault("timeout", settings.EXTERNAL_REQUESTS_TIMEOUT)
+        return super().request(*args, **kwargs)
+
+
+def get_user_agent():
+    return "python-requests (funkwhale/{}; +{})".format(
+        funkwhale_api.__version__, settings.FUNQUAIL_URL
+    )
+
+
+def get_session():
+    s = FunQuailSession()
+    s.headers["User-Agent"] = get_user_agent()
+    return s

api/funkwhale_api/common/signals.py

@@ -0,0 +1,6 @@
+import django.dispatch
+
+mutation_created = django.dispatch.Signal(providing_args=["mutation"])
+mutation_updated = django.dispatch.Signal(
+    providing_args=["mutation", "old_is_approved", "new_is_approved"]
+)

api/funkwhale_api/common/storage.py

@@ -0,0 +1,33 @@
+import os
+import shutil
+
+import slugify
+from django.core.files.storage import FileSystemStorage
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+def asciionly(name):
+    """
+    Convert unicode characters in name to ASCII characters.
+    """
+    return slugify.slugify(name, ok=slugify.SLUG_OK + ".", only_ascii=True)
+
+
+class ASCIIFileSystemStorage(FileSystemStorage):
+    def get_valid_name(self, name):
+        return super().get_valid_name(asciionly(name))
+
+    def force_delete(self, name):
+        path = self.path(name)
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                return super().delete(name)
+        except FileNotFoundError:
+            pass
+
+
+class ASCIIS3Boto3Storage(S3Boto3Storage):
+    def get_valid_name(self, name):
+        return super().get_valid_name(asciionly(name))

api/funkwhale_api/common/tasks.py

@@ -0,0 +1,98 @@
+import datetime
+import logging
+import tempfile
+
+from django.conf import settings
+from django.core.files import File
+from django.db import transaction
+from django.dispatch import receiver
+from django.utils import timezone
+
+from funkwhale_api.common import channels
+from funkwhale_api.taskapp import celery
+
+from . import models, serializers, session, signals
+
+logger = logging.getLogger(__name__)
+
+
+@celery.app.task(name="common.apply_mutation")
+@transaction.atomic
+@celery.require_instance(
+    models.Mutation.objects.exclude(is_applied=True).select_for_update(), "mutation"
+)
+def apply_mutation(mutation):
+    mutation.apply()
+
+
+@receiver(signals.mutation_created)
+def broadcast_mutation_created(mutation, **kwargs):
+    group = "instance_activity"
+    channels.group_send(
+        group,
+        {
+            "type": "event.send",
+            "text": "",
+            "data": {
+                "type": "mutation.created",
+                "mutation": serializers.APIMutationSerializer(mutation).data,
+                "pending_review_count": models.Mutation.objects.filter(
+                    is_approved=None
+                ).count(),
+            },
+        },
+    )
+
+
+@receiver(signals.mutation_updated)
+def broadcast_mutation_update(mutation, old_is_approved, new_is_approved, **kwargs):
+    group = "instance_activity"
+    channels.group_send(
+        group,
+        {
+            "type": "event.send",
+            "text": "",
+            "data": {
+                "type": "mutation.updated",
+                "mutation": serializers.APIMutationSerializer(mutation).data,
+                "pending_review_count": models.Mutation.objects.filter(
+                    is_approved=None
+                ).count(),
+                "old_is_approved": old_is_approved,
+                "new_is_approved": new_is_approved,
+            },
+        },
+    )
+
+
+def fetch_remote_attachment(attachment, filename=None, save=True):
+    if attachment.file:
+        # already there, no need to fetch
+        return
+
+    s = session.get_session()
+    attachment.last_fetch_date = timezone.now()
+    with tempfile.TemporaryFile() as tf:
+        with s.get(attachment.url, timeout=5, stream=True) as r:
+            for chunk in r.iter_content(chunk_size=1024 * 100):
+                tf.write(chunk)
+            tf.seek(0)
+            if not filename:
+                filename = attachment.url.split("/")[-1]
+                filename = filename[-50:]
+            attachment.file.save(filename, File(tf), save=save)
+
+
+@celery.app.task(name="common.prune_unattached_attachments")
+def prune_unattached_attachments():
+    limit = timezone.now() - datetime.timedelta(
+        seconds=settings.ATTACHMENTS_UNATTACHED_PRUNE_DELAY
+    )
+    candidates = models.Attachment.objects.attached(False).filter(
+        creation_date__lte=limit
+    )
+
+    total = candidates.count()
+    logger.info("Deleting %s unattached attachments…", total)
+    result = candidates.delete()
+    logger.info("Deletion done: %s", result)

api/funkwhale_api/common/throttling.py

@@ -0,0 +1,150 @@
+import collections
+
+from django.conf import settings
+from django.core.cache import cache
+from rest_framework import throttling as rest_throttling
+
+
+def get_ident(user, request):
+    if user and user.is_authenticated:
+        return {"type": "authenticated", "id": f"{user.pk}"}
+    ident = rest_throttling.BaseThrottle().get_ident(request)
+
+    return {"type": "anonymous", "id": ident}
+
+
+def get_cache_key(scope, ident):
+    parts = ["throttling", scope, ident["type"], str(ident["id"])]
+    return ":".join(parts)
+
+
+def get_scope_for_action_and_ident_type(action, ident_type, view_conf={}):
+    config = collections.ChainMap(view_conf, settings.THROTTLING_SCOPES)
+
+    try:
+        action_config = config[action]
+    except KeyError:
+        action_config = config.get("*", {})
+
+    try:
+        return action_config[ident_type]
+    except KeyError:
+        return
+
+
+def get_status(ident, now):
+    data = []
+    throttle = FunQuailThrottle()
+    for key in sorted(settings.THROTTLING_RATES.keys()):
+        conf = settings.THROTTLING_RATES[key]
+        row_data = {"id": key, "rate": conf["rate"], "description": conf["description"]}
+        if conf["rate"]:
+            num_requests, duration = throttle.parse_rate(conf["rate"])
+            history = cache.get(get_cache_key(key, ident)) or []
+
+            relevant_history = [h for h in history if h > now - duration]
+            row_data["limit"] = num_requests
+            row_data["duration"] = duration
+            row_data["remaining"] = num_requests - len(relevant_history)
+            if relevant_history and len(relevant_history) >= num_requests:
+                # At this point, the endpoint becomes available again
+                now_request = relevant_history[-1]
+                remaining = duration - (now - int(now_request))
+                row_data["available"] = int(now + remaining) or None
+                row_data["available_seconds"] = int(remaining) or None
+            else:
+                row_data["available"] = None
+                row_data["available_seconds"] = None
+
+            if relevant_history:
+                # At this point, all Rate Limit is reset to 0
+                latest_request = relevant_history[0]
+                remaining = duration - (now - int(latest_request))
+                row_data["reset"] = int(now + remaining)
+                row_data["reset_seconds"] = int(remaining)
+            else:
+                row_data["reset"] = None
+                row_data["reset_seconds"] = None
+        else:
+            row_data["limit"] = None
+            row_data["duration"] = None
+            row_data["remaining"] = None
+            row_data["available"] = None
+            row_data["available_seconds"] = None
+            row_data["reset"] = None
+            row_data["reset_seconds"] = None
+
+        data.append(row_data)
+
+    return data
+
+
+class FunQuailThrottle(rest_throttling.SimpleRateThrottle):
+    def __init__(self):
+        pass
+
+    def get_cache_key(self, request, view):
+        return get_cache_key(self.scope, self.ident)
+
+    def allow_request(self, request, view):
+        self.request = request
+        self.ident = get_ident(getattr(request, "user", None), request)
+        action = getattr(view, "action", "*")
+        view_scopes = getattr(view, "throttling_scopes", {})
+        if view_scopes is None:
+            return True
+        self.scope = get_scope_for_action_and_ident_type(
+            action=action, ident_type=self.ident["type"], view_conf=view_scopes
+        )
+        if not self.scope or self.scope not in settings.THROTTLING_RATES:
+            return True
+        self.rate = settings.THROTTLING_RATES[self.scope].get("rate")
+        self.num_requests, self.duration = self.parse_rate(self.rate)
+        self.request = request
+
+        return super().allow_request(request, view)
+
+    def attach_info(self):
+        info = {
+            "num_requests": self.num_requests,
+            "duration": self.duration,
+            "scope": self.scope,
+            "history": self.history or [],
+            "wait": self.wait(),
+        }
+        setattr(self.request, "_throttle_status", info)
+
+    def throttle_success(self):
+        self.attach_info()
+        return super().throttle_success()
+
+    def throttle_failure(self):
+        self.attach_info()
+        return super().throttle_failure()
+
+
+class TooManyRequests(Exception):
+    pass
+
+
+DummyView = collections.namedtuple("DummyView", "action throttling_scopes")
+
+
+def check_request(request, scope):
+    """
+    A simple wrapper around FunQuailThrottle for views that aren't API views
+    or cannot use rest_framework automatic throttling.
+
+    Raise TooManyRequests if limit is reached.
+    """
+    if not settings.THROTTLING_ENABLED:
+        return True
+
+    view = DummyView(
+        action=scope,
+        throttling_scopes={scope: {"anonymous": scope, "authenticated": scope}},
+    )
+    throttle = FunQuailThrottle()
+    if not throttle.allow_request(request, view):
+        raise TooManyRequests()
+    return True

api/funkwhale_api/common/utils.py

@@ -0,0 +1,489 @@
+import datetime
+import hashlib
+import logging
+import os
+import shutil
+import uuid
+import xml.etree.ElementTree as ET
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
+
+import bleach.sanitizer
+import markdown
+from django import urls
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.db import models, transaction
+from django.http import request
+from django.utils import timezone
+from django.utils.deconstruct import deconstructible
+
+logger = logging.getLogger(__name__)
+
+
+def batch(iterable, n=1):
+    has_entries = True
+    while has_entries:
+        current = []
+        for i in range(0, n):
+            try:
+                current.append(next(iterable))
+            except StopIteration:
+                has_entries = False
+        yield current
+
+
+def rename_file(instance, field_name, new_name, allow_missing_file=False):
+    field = getattr(instance, field_name)
+    current_name, extension = os.path.splitext(field.name)
+
+    new_name_with_extension = f"{new_name}{extension}"
+    try:
+        shutil.move(field.path, new_name_with_extension)
+    except FileNotFoundError:
+        if not allow_missing_file:
+            raise
+        print("Skipped missing file", field.path)
+    initial_path = os.path.dirname(field.name)
+    field.name = os.path.join(initial_path, new_name_with_extension)
+    instance.save()
+    return new_name_with_extension
+
+
+def on_commit(f, *args, **kwargs):
+    return transaction.on_commit(lambda: f(*args, **kwargs))
+
+
+def set_query_parameter(url, **kwargs):
+    """Given a URL, set or replace a query parameter and return the
+    modified URL.
+
+    >>> set_query_parameter('http://example.com?foo=bar&biz=baz', 'foo', 'stuff')
+    'http://example.com?foo=stuff&biz=baz'
+    """
+    scheme, netloc, path, query_string, fragment = urlsplit(url)
+    query_params = parse_qs(query_string)
+
+    for param_name, param_value in kwargs.items():
+        query_params[param_name] = [param_value]
+    new_query_string = urlencode(query_params, doseq=True)
+
+    return urlunsplit((scheme, netloc, path, new_query_string, fragment))
+
+
+@deconstructible
+class ChunkedPath:
+    def sanitize_filename(self, filename):
+        return filename.replace("/", "-")
+
+    def __init__(self, root, preserve_file_name=True):
+        self.root = root
+        self.preserve_file_name = preserve_file_name
+
+    def __call__(self, instance, filename):
+        self.sanitize_filename(filename)
+        uid = str(uuid.uuid4())
+        chunk_size = 2
+        chunks = [uid[i : i + chunk_size] for i in range(0, len(uid), chunk_size)]
+        if self.preserve_file_name:
+            parts = chunks[:3] + [filename]
+        else:
+            ext = os.path.splitext(filename)[1][1:].lower()
+            new_filename = "".join(chunks[3:]) + f".{ext}"
+            parts = chunks[:3] + [new_filename]
+        return os.path.join(self.root, *parts)
+
+
+def chunk_queryset(source_qs, chunk_size):
+    """
+    From https://github.com/peopledoc/django-chunkator/blob/master/chunkator/__init__.py
+    """
+    pk = None
+    # In django 1.9, _fields is always present and `None` if 'values()' is used
+    # In Django 1.8 and below, _fields will only be present if using `values()`
+    has_fields = hasattr(source_qs, "_fields") and source_qs._fields
+    if has_fields:
+        if "pk" not in source_qs._fields:
+            raise ValueError("The values() call must include the `pk` field")
+
+    field = source_qs.model._meta.pk
+    # set the correct field name:
+    # for ForeignKeys, we want to use `model_id` field, and not `model`,
+    # to bypass default ordering on related model
+    order_by_field = field.attname
+
+    source_qs = source_qs.order_by(order_by_field)
+    queryset = source_qs
+    while True:
+        if pk:
+            queryset = source_qs.filter(pk__gt=pk)
+        page = queryset[:chunk_size]
+        page = list(page)
+        nb_items = len(page)
+
+        if nb_items == 0:
+            return
+
+        last_item = page[-1]
+        # source_qs._fields exists *and* is not none when using "values()"
+        if has_fields:
+            pk = last_item["pk"]
+        else:
+            pk = last_item.pk
+
+        yield page
+
+        if nb_items < chunk_size:
+            return
+
+
+def join_url(start, end):
+    if end.startswith("http://") or end.startswith("https://"):
+        # already a full URL, joining makes no sense
+        return end
+    if start.endswith("/") and end.startswith("/"):
+        return start + end[1:]
+
+    if not start.endswith("/") and not end.startswith("/"):
+        return start + "/" + end
+
+    return start + end
+
+
+def media_url(path):
+    if settings.MEDIA_URL.startswith("http://") or settings.MEDIA_URL.startswith(
+        "https://"
+    ):
+        return join_url(settings.MEDIA_URL, path)
+
+    from funkwhale_api.federation import utils as federation_utils
+
+    return federation_utils.full_url(path)
+
+
+def spa_reverse(name, args=[], kwargs={}):
+    return urls.reverse(name, urlconf=settings.SPA_URLCONF, args=args, kwargs=kwargs)
+
+
+def spa_resolve(path):
+    return urls.resolve(path, urlconf=settings.SPA_URLCONF)
+
+
+def parse_meta(html):
+    # dirty but this is only for testing so we don't really care,
+    # we convert the html string to xml so it can be parsed as xml
+    html = '<?xml version="1.0"?>' + html
+    tree = ET.fromstring(html)
+
+    meta = [elem for elem in tree.iter() if elem.tag in ["meta", "link"]]
+
+    return [dict([("tag", elem.tag)] + list(elem.items())) for elem in meta]
+
+
+def order_for_search(qs, field):
+    """
+    When searching, it's often more useful to have short results first,
+    this function will order the given qs based on the length of the given field
+    """
+    return qs.annotate(__size=models.functions.Length(field)).order_by("__size", "pk")
+
+
+def recursive_getattr(obj, key, permissive=False):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} or and object and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        try:
+            if hasattr(v, "get"):
+                v = v.get(k)
+            else:
+                v = getattr(v, k)
+        except (TypeError, AttributeError):
+            if not permissive:
+                raise
+            return
+        if v is None:
+            return
+
+    return v
+
+
+def replace_prefix(queryset, field, old, new):
+    """
+    Given a queryset of objects and a field name, will find objects
+    for which the field have the given value, and replace the old prefix by
+    the new one.
+
+    This is especially useful to find/update bad federation ids, to replace:
+
+    http://wrongprotocolanddomain/path
+
+    by
+
+    https://goodprotocalanddomain/path
+
+    on a whole table with a single query.
+    """
+    qs = queryset.filter(**{f"{field}__startswith": old})
+    # we extract the part after the old prefix, and Concat it with our new prefix
+    update = models.functions.Concat(
+        models.Value(new),
+        models.functions.Substr(field, len(old) + 1, output_field=models.CharField()),
+    )
+    return qs.update(**{field: update})
+
+
+def concat_dicts(*dicts):
+    n = {}
+    for d in dicts:
+        n.update(d)
+
+    return n
+
+
+def get_updated_fields(conf, data, obj):
+    """
+    Given a list of fields, a dict and an object, will return the dict keys/values
+    that differ from the corresponding fields on the object.
+    """
+    final_conf = []
+    for c in conf:
+        if isinstance(c, str):
+            final_conf.append((c, c))
+        else:
+            final_conf.append(c)
+
+    final_data = {}
+
+    for data_field, obj_field in final_conf:
+        try:
+            data_value = data[data_field]
+        except KeyError:
+            continue
+        if obj.pk:
+            obj_value = getattr(obj, obj_field)
+            if obj_value != data_value:
+                final_data[obj_field] = data_value
+        else:
+            final_data[obj_field] = data_value
+
+    return final_data
+
+
+def join_queries_or(left, right):
+    if left:
+        return left | right
+    else:
+        return right
+
+
+MARKDOWN_RENDERER = markdown.Markdown(extensions=settings.MARKDOWN_EXTENSIONS)
+
+
+def render_markdown(text):
+    return MARKDOWN_RENDERER.convert(text)
+
+
+SAFE_TAGS = [
+    "p",
+    "a",
+    "abbr",
+    "acronym",
+    "b",
+    "blockquote",
+    "br",
+    "code",
+    "em",
+    "i",
+    "li",
+    "ol",
+    "strong",
+    "ul",
+]
+HTMl_CLEANER = bleach.sanitizer.Cleaner(strip=True, tags=SAFE_TAGS)
+
+HTML_PERMISSIVE_CLEANER = bleach.sanitizer.Cleaner(
+    strip=True,
+    tags=SAFE_TAGS + ["h1", "h2", "h3", "h4", "h5", "h6", "div", "section", "article"],
+    attributes=["class", "rel", "alt", "title", "href"],
+)
+
+# support for additional tlds
+# cf https://github.com/mozilla/bleach/issues/367#issuecomment-384631867
+ALL_TLDS = set(settings.LINKIFIER_SUPPORTED_TLDS + bleach.linkifier.TLDS)
+URL_RE = bleach.linkifier.build_url_re(tlds=sorted(ALL_TLDS, reverse=True))
+HTML_LINKER = bleach.linkifier.Linker(url_re=URL_RE)
+
+
+def clean_html(html, permissive=False):
+    return (
+        HTML_PERMISSIVE_CLEANER.clean(html) if permissive else HTMl_CLEANER.clean(html)
+    )
+
+
+def render_html(text, content_type, permissive=False):
+    if not text:
+        return ""
+    rendered = render_markdown(text)
+    if content_type == "text/html":
+        rendered = text
+    elif content_type == "text/markdown":
+        rendered = render_markdown(text)
+    else:
+        rendered = render_markdown(text)
+    rendered = HTML_LINKER.linkify(rendered)
+    return clean_html(rendered, permissive=permissive).strip().replace("\n", "")
+
+
+def render_plain_text(html):
+    if not html:
+        return ""
+    return bleach.clean(html, tags=[], strip=True)
+
+
+def same_content(old, text=None, content_type=None):
+    return old.text == text and old.content_type == content_type
+
+
+@transaction.atomic
+def attach_content(obj, field, content_data):
+    from . import models
+
+    content_data = content_data or {}
+    existing = getattr(obj, f"{field}_id")
+
+    if existing:
+        if same_content(getattr(obj, field), **content_data):
+            # optimization to avoid a delete/save if possible
+            return getattr(obj, field)
+        getattr(obj, field).delete()
+        setattr(obj, field, None)
+
+    if not content_data:
+        return
+
+    content_obj = models.Content.objects.create(
+        text=content_data["text"][: models.CONTENT_TEXT_MAX_LENGTH],
+        content_type=content_data["content_type"],
+    )
+    setattr(obj, field, content_obj)
+    obj.save(update_fields=[field])
+    return content_obj
+
+
+@transaction.atomic
+def attach_file(obj, field, file_data, fetch=False):
+    from . import models, tasks
+
+    existing = getattr(obj, f"{field}_id")
+    if existing:
+        getattr(obj, field).delete()
+
+    if not file_data:
+        return
+
+    if isinstance(file_data, models.Attachment):
+        attachment = file_data
+    else:
+        extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
+        extension = extensions.get(file_data["mimetype"], "jpg")
+        attachment = models.Attachment(mimetype=file_data["mimetype"])
+        name_fields = ["uuid", "full_username", "pk"]
+        name = [
+            getattr(obj, field) for field in name_fields if getattr(obj, field, None)
+        ][0]
+        filename = f"{field}-{name}.{extension}"
+        if "url" in file_data:
+            attachment.url = file_data["url"]
+        else:
+            f = ContentFile(file_data["content"])
+            attachment.file.save(filename, f, save=False)
+
+        if not attachment.file and fetch:
+            try:
+                tasks.fetch_remote_attachment(attachment, filename=filename, save=False)
+            except Exception as e:
+                logger.warn(
+                    "Cannot download attachment at url %s: %s", attachment.url, e
+                )
+                attachment = None
+
+        if attachment:
+            attachment.save()
+
+    setattr(obj, field, attachment)
+    obj.save(update_fields=[field])
+    return attachment
+
+
+def get_mimetype_from_ext(path):
+    parts = path.lower().split(".")
+    ext = parts[-1]
+    match = {
+        "jpeg": "image/jpeg",
+        "jpg": "image/jpeg",
+        "png": "image/png",
+        "gif": "image/gif",
+    }
+    return match.get(ext)
+
+
+def get_audio_mimetype(mt):
+    aliases = {"audio/x-mp3": "audio/mpeg", "audio/mpeg3": "audio/mpeg"}
+    return aliases.get(mt, mt)
+
+
+def update_modification_date(obj, field="modification_date", date=None):
+    IGNORE_DELAY = 60
+    current_value = getattr(obj, field)
+    date = date or timezone.now()
+    ignore = current_value is not None and current_value < date - datetime.timedelta(
+        seconds=IGNORE_DELAY
+    )
+    if ignore:
+        setattr(obj, field, date)
+        obj.__class__.objects.filter(pk=obj.pk).update(**{field: date})
+
+    return date
+
+
+def monkey_patch_request_build_absolute_uri():
+    """
+    Since we have FUNQUAIL_HOSTNAME and PROTOCOL hardcoded in settings, we can
+    override django's multisite logic which can break when reverse proxy aren't configured
+    properly.
+    """
+    builtin_scheme = request.HttpRequest.scheme
+
+    def scheme(self):
+        if settings.IGNORE_FORWARDED_HOST_AND_PROTO:
+            return settings.FUNQUAIL_PROTOCOL
+        return builtin_scheme.fget(self)
+
+    builtin_get_host = request.HttpRequest.get_host
+
+    def get_host(self):
+        if settings.IGNORE_FORWARDED_HOST_AND_PROTO:
+            return settings.FUNQUAIL_HOSTNAME
+        return builtin_get_host(self)
+
+    request.HttpRequest.scheme = property(scheme)
+    request.HttpRequest.get_host = get_host
+
+
+def get_file_hash(file, algo=None, chunk_size=None, full_read=False):
+    algo = algo or settings.HASHING_ALGORITHM
+    chunk_size = chunk_size or settings.HASHING_CHUNK_SIZE
+    hasher = hashlib.new(algo)
+    file.seek(0)
+    if full_read:
+        for byte_block in iter(lambda: file.read(chunk_size), b""):
+            hasher.update(byte_block)
+    else:
+        # sometimes, it's useful to only hash the beginning of the file, e.g
+        # to avoid a lot of I/O when crawling large libraries
+        hasher.update(file.read(chunk_size))
+    return f"{algo}:{hasher.hexdigest()}"

api/funkwhale_api/common/validators.py

@@ -0,0 +1,167 @@
+import mimetypes
+from os.path import splitext
+
+from django.core import validators
+from django.core.exceptions import ValidationError
+from django.core.files.images import get_image_dimensions
+from django.template.defaultfilters import filesizeformat
+from django.utils.deconstruct import deconstructible
+from django.utils.translation import ugettext_lazy as _
+
+
+@deconstructible
+class ImageDimensionsValidator:
+    """
+    ImageField dimensions validator.
+
+    from https://gist.github.com/emilio-rst/4f81ea2718736a6aaf9bdb64d5f2ea6c
+    """
+
+    def __init__(
+        self,
+        width=None,
+        height=None,
+        min_width=None,
+        max_width=None,
+        min_height=None,
+        max_height=None,
+    ):
+        """
+        Constructor
+
+        Args:
+            width (int): exact width
+            height (int): exact height
+            min_width (int): minimum width
+            min_height (int): minimum height
+            max_width (int): maximum width
+            max_height (int): maximum height
+        """
+
+        self.width = width
+        self.height = height
+        self.min_width = min_width
+        self.max_width = max_width
+        self.min_height = min_height
+        self.max_height = max_height
+
+    def __call__(self, image):
+        w, h = get_image_dimensions(image)
+
+        if self.width is not None and w != self.width:
+            raise ValidationError(_("Width must be %dpx.") % (self.width,))
+
+        if self.height is not None and h != self.height:
+            raise ValidationError(_("Height must be %dpx.") % (self.height,))
+
+        if self.min_width is not None and w < self.min_width:
+            raise ValidationError(_("Minimum width must be %dpx.") % (self.min_width,))
+
+        if self.min_height is not None and h < self.min_height:
+            raise ValidationError(
+                _("Minimum height must be %dpx.") % (self.min_height,)
+            )
+
+        if self.max_width is not None and w > self.max_width:
+            raise ValidationError(_("Maximum width must be %dpx.") % (self.max_width,))
+
+        if self.max_height is not None and h > self.max_height:
+            raise ValidationError(
+                _("Maximum height must be %dpx.") % (self.max_height,)
+            )
+
+
+@deconstructible
+class FileValidator:
+    """
+    Taken from https://gist.github.com/jrosebr1/2140738
+    Validator for files, checking the size, extension and mimetype.
+    Initialization parameters:
+        allowed_extensions: iterable with allowed file extensions
+            ie. ('txt', 'doc')
+        allowd_mimetypes: iterable with allowed mimetypes
+            ie. ('image/png', )
+        min_size: minimum number of bytes allowed
+            ie. 100
+        max_size: maximum number of bytes allowed
+            ie. 24*1024*1024 for 24 MB
+    Usage example::
+        MyModel(models.Model):
+            myfile = FileField(validators=FileValidator(max_size=24*1024*1024), ...)
+    """
+
+    extension_message = _(
+        "Extension '%(extension)s' not allowed. Allowed extensions are: '%(allowed_extensions)s.'"
+    )
+    mime_message = _(
+        "MIME type '%(mimetype)s' is not valid. Allowed types are: %(allowed_mimetypes)s."
+    )
+    min_size_message = _(
+        "The current file %(size)s, which is too small. The minimum file size is %(allowed_size)s."
+    )
+    max_size_message = _(
+        "The current file %(size)s, which is too large. The maximum file size is %(allowed_size)s."
+    )
+
+    def __init__(self, *args, **kwargs):
+        self.allowed_extensions = kwargs.pop("allowed_extensions", None)
+        self.allowed_mimetypes = kwargs.pop("allowed_mimetypes", None)
+        self.min_size = kwargs.pop("min_size", 0)
+        self.max_size = kwargs.pop("max_size", None)
+
+    def __call__(self, value):
+        """
+        Check the extension, content type and file size.
+        """
+
+        # Check the extension
+        ext = splitext(value.name)[1][1:].lower()
+        if self.allowed_extensions and ext not in self.allowed_extensions:
+            message = self.extension_message % {
+                "extension": ext,
+                "allowed_extensions": ", ".join(self.allowed_extensions),
+            }
+
+            raise ValidationError(message)
+
+        # Check the content type
+        mimetype = mimetypes.guess_type(value.name)[0]
+        if self.allowed_mimetypes and mimetype not in self.allowed_mimetypes:
+            message = self.mime_message % {
+                "mimetype": mimetype,
+                "allowed_mimetypes": ", ".join(self.allowed_mimetypes),
+            }
+
+            raise ValidationError(message)
+
+        # Check the file size
+        filesize = len(value)
+        if self.max_size and filesize > self.max_size:
+            message = self.max_size_message % {
+                "size": filesizeformat(filesize),
+                "allowed_size": filesizeformat(self.max_size),
+            }
+
+            raise ValidationError(message)
+
+        elif filesize < self.min_size:
+            message = self.min_size_message % {
+                "size": filesizeformat(filesize),
+                "allowed_size": filesizeformat(self.min_size),
+            }
+
+            raise ValidationError(message)
+
+
+class DomainValidator(validators.URLValidator):
+    message = "Enter a valid domain name."
+
+    def __call__(self, value):
+        """
+        This is a bit hackish but since we don't have any built-in domain validator,
+        we use the url one, and prepend http:// in front of it.
+
+        If it fails, we know the domain is not valid.
+        """
+        super().__call__(f"http://{value}")
+        return value

api/funkwhale_api/common/views.py

@@ -0,0 +1,321 @@
+import logging
+import time
+
+from django.conf import settings
+from django.db import transaction
+from drf_spectacular.utils import extend_schema
+from rest_framework import (
+    exceptions,
+    generics,
+    mixins,
+    permissions,
+    response,
+    views,
+    viewsets,
+)
+from rest_framework.decorators import action
+
+from config import plugins
+from funkwhale_api.common.serializers import (
+    ErrorDetailSerializer,
+    TextPreviewSerializer,
+)
+from funkwhale_api.users.oauth import permissions as oauth_permissions
+
+from . import filters, models, mutations, serializers, signals, tasks, throttling, utils
+
+logger = logging.getLogger(__name__)
+
+
+class SkipFilterForGetObject:
+    def get_object(self, *args, **kwargs):
+        setattr(self.request, "_skip_filters", True)
+        return super().get_object(*args, **kwargs)
+
+    def filter_queryset(self, queryset):
+        if getattr(self.request, "_skip_filters", False):
+            return queryset
+        return super().filter_queryset(queryset)
+
+
+class MutationViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.Mutation.objects.all()
+        .exclude(target_id=None)
+        .order_by("-creation_date")
+        .select_related("created_by", "approved_by")
+        .prefetch_related("target")
+    )
+    serializer_class = serializers.APIMutationSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    ordering_fields = ("creation_date",)
+    filterset_class = filters.MutationFilter
+
+    def perform_destroy(self, instance):
+        if instance.is_applied:
+            raise exceptions.PermissionDenied("You cannot delete an applied mutation")
+
+        actor = self.request.user.actor
+        is_owner = actor == instance.created_by
+
+        if not any(
+            [
+                is_owner,
+                mutations.registry.has_perm(
+                    perm="approve", type=instance.type, obj=instance.target, actor=actor
+                ),
+            ]
+        ):
+            raise exceptions.PermissionDenied()
+
+        return super().perform_destroy(instance)
+
+    @extend_schema(operation_id="approve_mutation")
+    @action(detail=True, methods=["post"])
+    @transaction.atomic
+    def approve(self, request, *args, **kwargs):
+        instance = self.get_object()
+        if instance.is_applied:
+            return response.Response(
+                {"error": "This mutation was already applied"}, status=403
+            )
+        actor = self.request.user.actor
+        can_approve = mutations.registry.has_perm(
+            perm="approve", type=instance.type, obj=instance.target, actor=actor
+        )
+
+        if not can_approve:
+            raise exceptions.PermissionDenied()
+        previous_is_approved = instance.is_approved
+        instance.approved_by = actor
+        instance.is_approved = True
+        instance.save(update_fields=["approved_by", "is_approved"])
+        utils.on_commit(tasks.apply_mutation.delay, mutation_id=instance.id)
+        utils.on_commit(
+            signals.mutation_updated.send,
+            sender=None,
+            mutation=instance,
+            old_is_approved=previous_is_approved,
+            new_is_approved=instance.is_approved,
+        )
+        return response.Response({}, status=200)
+
+    @extend_schema(operation_id="reject_mutation")
+    @action(detail=True, methods=["post"])
+    @transaction.atomic
+    def reject(self, request, *args, **kwargs):
+        instance = self.get_object()
+        if instance.is_applied:
+            return response.Response(
+                {"error": "This mutation was already applied"}, status=403
+            )
+        actor = self.request.user.actor
+        can_approve = mutations.registry.has_perm(
+            perm="approve", type=instance.type, obj=instance.target, actor=actor
+        )
+
+        if not can_approve:
+            raise exceptions.PermissionDenied()
+        previous_is_approved = instance.is_approved
+        instance.approved_by = actor
+        instance.is_approved = False
+        instance.save(update_fields=["approved_by", "is_approved"])
+        utils.on_commit(
+            signals.mutation_updated.send,
+            sender=None,
+            mutation=instance,
+            old_is_approved=previous_is_approved,
+            new_is_approved=instance.is_approved,
+        )
+        return response.Response({}, status=200)
+
+
+class RateLimitView(views.APIView):
+    permission_classes = []
+    throttle_classes = []
+    serializer_class = serializers.RateLimitSerializer
+
+    def get(self, request, *args, **kwargs):
+        ident = throttling.get_ident(getattr(request, "user", None), request)
+        data = {
+            "enabled": settings.THROTTLING_ENABLED,
+            "ident": ident,
+            "scopes": throttling.get_status(ident, time.time()),
+        }
+        return response.Response(serializers.RateLimitSerializer(data).data, status=200)
+
+
+class AttachmentViewSet(
+    mixins.RetrieveModelMixin,
+    mixins.CreateModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = models.Attachment.objects.all()
+    serializer_class = serializers.AttachmentSerializer
+    permission_classes = [oauth_permissions.ScopePermission]
+    required_scope = "libraries"
+    anonymous_policy = "setting"
+
+    @action(
+        detail=True, methods=["get"], permission_classes=[], authentication_classes=[]
+    )
+    @transaction.atomic
+    def proxy(self, request, *args, **kwargs):
+        instance = self.get_object()
+        if not settings.EXTERNAL_MEDIA_PROXY_ENABLED:
+            r = response.Response(status=302)
+            r["Location"] = instance.url
+            return r
+
+        size = request.GET.get("next", "original").lower()
+        if size not in ["original", "medium_square_crop", "large_square_crop"]:
+            size = "original"
+
+        try:
+            tasks.fetch_remote_attachment(instance)
+        except Exception:
+            logger.exception("Error while fetching attachment %s", instance.url)
+            return response.Response(status=500)
+        data = self.serializer_class(instance).data
+        redirect = response.Response(status=302)
+        redirect["Location"] = data["urls"][size]
+        return redirect
+
+    def perform_create(self, serializer):
+        return serializer.save(actor=self.request.user.actor)
+
+    def perform_destroy(self, instance):
+        if instance.actor is None or instance.actor != self.request.user.actor:
+            raise exceptions.PermissionDenied()
+        instance.delete()
+
+
+class TextPreviewView(generics.GenericAPIView):
+    permission_classes = []
+    serializer_class = TextPreviewSerializer
+
+    @extend_schema(
+        operation_id="preview_text",
+        responses={200: TextPreviewSerializer, 400: ErrorDetailSerializer},
+    )
+    def post(self, request, *args, **kwargs):
+        payload = request.data
+        if "text" not in payload:
+            return response.Response(
+                ErrorDetailSerializer("Invalid input").data, status=400
+            )
+
+        permissive = payload.get("permissive", False)
+        data = TextPreviewSerializer(
+            utils.render_html(payload["text"], "text/markdown", permissive=permissive)
+        ).data
+        return response.Response(data, status=200)
+
+
+class PluginViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
+    required_scope = "plugins"
+    serializer_class = serializers.serializers.Serializer
+    queryset = models.PluginConfiguration.objects.none()
+
+    def list(self, request, *args, **kwargs):
+        user = request.user
+        user_plugins = [p for p in plugins._plugins.values() if p["user"] is True]
+
+        return response.Response(
+            [
+                plugins.serialize_plugin(p, confs=plugins.get_confs(user=user))
+                for p in user_plugins
+            ]
+        )
+
+    def retrieve(self, request, *args, **kwargs):
+        user = request.user
+        user_plugin = [
+            p
+            for p in plugins._plugins.values()
+            if p["user"] is True and p["name"] == kwargs["pk"]
+        ]
+        if not user_plugin:
+            return response.Response(status=404)
+
+        return response.Response(
+            plugins.serialize_plugin(user_plugin[0], confs=plugins.get_confs(user=user))
+        )
+
+    def post(self, request, *args, **kwargs):
+        return self.create(request, *args, **kwargs)
+
+    def create(self, request, *args, **kwargs):
+        user = request.user
+        confs = plugins.get_confs(user=user)
+
+        user_plugin = [
+            p
+            for p in plugins._plugins.values()
+            if p["user"] is True and p["name"] == kwargs["pk"]
+        ]
+        if kwargs["pk"] not in confs:
+            return response.Response(status=404)
+        plugins.set_conf(kwargs["pk"], request.data, user)
+        return response.Response(
+            plugins.serialize_plugin(user_plugin[0], confs=plugins.get_confs(user=user))
+        )
+
+    def delete(self, request, *args, **kwargs):
+        user = request.user
+        confs = plugins.get_confs(user=user)
+        if kwargs["pk"] not in confs:
+            return response.Response(status=404)
+
+        user.plugins.filter(code=kwargs["pk"]).delete()
+        return response.Response(status=204)
+
+    @extend_schema(operation_id="enable_plugin")
+    @action(detail=True, methods=["post"])
+    def enable(self, request, *args, **kwargs):
+        user = request.user
+        if kwargs["pk"] not in plugins._plugins:
+            return response.Response(status=404)
+        plugins.enable_conf(kwargs["pk"], True, user)
+        return response.Response({}, status=200)
+
+    @extend_schema(operation_id="disable_plugin")
+    @action(detail=True, methods=["post"])
+    def disable(self, request, *args, **kwargs):
+        user = request.user
+        if kwargs["pk"] not in plugins._plugins:
+            return response.Response(status=404)
+        plugins.enable_conf(kwargs["pk"], False, user)
+        return response.Response({}, status=200)
+
+    @action(detail=True, methods=["post"])
+    def scan(self, request, *args, **kwargs):
+        user = request.user
+        if kwargs["pk"] not in plugins._plugins:
+            return response.Response(status=404)
+        conf = plugins.get_conf(kwargs["pk"], user=user)
+
+        if not conf["enabled"]:
+            return response.Response(status=405)
+
+        library = request.user.actor.libraries.get(uuid=conf["conf"]["library"])
+        hook = [
+            hook
+            for p, hook in plugins._hooks.get(plugins.SCAN, [])
+            if p == kwargs["pk"]
+        ]
+
+        if not hook:
+            return response.Response(status=405)
+
+        hook[0](library=library, conf=conf["conf"])
+
+        return response.Response({}, status=200)

api/funkwhale_api/contrib/listenbrainz/client.py

@@ -0,0 +1,168 @@
+# Copyright (c) 2018 Philipp Wolfer <ph.wolfer@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import json
+import logging
+import ssl
+import time
+from http.client import HTTPSConnection
+
+HOST_NAME = "api.listenbrainz.org"
+PATH_SUBMIT = "/1/submit-listens"
+SSL_CONTEXT = ssl.create_default_context()
+
+
+class Track:
+    """
+    Represents a single track to submit.
+
+    See https://listenbrainz.readthedocs.io/en/latest/dev/json.html
+    """
+
+    def __init__(self, artist_name, track_name, release_name=None, additional_info={}):
+        """
+        Create a new Track instance
+        @param artist_name as str
+        @param track_name as str
+        @param release_name as str
+        @param additional_info as dict
+        """
+        self.artist_name = artist_name
+        self.track_name = track_name
+        self.release_name = release_name
+        self.additional_info = additional_info
+
+    @staticmethod
+    def from_dict(data):
+        return Track(
+            data["artist_name"],
+            data["track_name"],
+            data.get("release_name", None),
+            data.get("additional_info", {}),
+        )
+
+    def to_dict(self):
+        return {
+            "artist_name": self.artist_name,
+            "track_name": self.track_name,
+            "release_name": self.release_name,
+            "additional_info": self.additional_info,
+        }
+
+    def __repr__(self):
+        return f"Track({self.artist_name}, {self.track_name})"
+
+
+class ListenBrainzClient:
+    """
+    Submit listens to ListenBrainz.org.
+
+    See https://listenbrainz.readthedocs.io/en/latest/dev/api.html
+    """
+
+    def __init__(self, user_token, logger=logging.getLogger(__name__)):
+        self.__next_request_time = 0
+        self.user_token = user_token
+        self.logger = logger
+
+    def listen(self, listened_at, track):
+        """
+        Submit a listen for a track
+        @param listened_at as int
+        @param entry as Track
+        """
+        payload = _get_payload(track, listened_at)
+        return self._submit("single", [payload])
+
+    def playing_now(self, track):
+        """
+        Submit a playing now notification for a track
+        @param track as Track
+        """
+        payload = _get_payload(track)
+        return self._submit("playing_now", [payload])
+
+    def import_tracks(self, tracks):
+        """
+        Import a list of tracks as (listened_at, Track) pairs
+        @param track as [(int, Track)]
+        """
+        payload = _get_payload_many(tracks)
+        return self._submit("import", payload)
+
+    def _submit(self, listen_type, payload, retry=0):
+        self._wait_for_ratelimit()
+        self.logger.debug("ListenBrainz %s: %r", listen_type, payload)
+        data = {"listen_type": listen_type, "payload": payload}
+        headers = {
+            "Authorization": "Token %s" % self.user_token,
+            "Content-Type": "application/json",
+        }
+        body = json.dumps(data)
+        conn = HTTPSConnection(HOST_NAME, context=SSL_CONTEXT)
+        conn.request("POST", PATH_SUBMIT, body, headers)
+        response = conn.getresponse()
+        response_text = response.read()
+        try:
+            response_data = json.loads(response_text)
+        except json.decoder.JSONDecodeError:
+            response_data = response_text
+
+        self._handle_ratelimit(response)
+        log_msg = f"Response {response.status}: {response_data!r}"
+        if response.status == 429 and retry < 5:  # Too Many Requests
+            self.logger.warning(log_msg)
+            return self._submit(listen_type, payload, retry + 1)
+        elif response.status == 200:
+            self.logger.debug(log_msg)
+        else:
+            self.logger.error(log_msg)
+        return response
+
+    def _wait_for_ratelimit(self):
+        now = time.time()
+        if self.__next_request_time > now:
+            delay = self.__next_request_time - now
+            self.logger.debug("Rate limit applies, delay %d", delay)
+            time.sleep(delay)
+
+    def _handle_ratelimit(self, response):
+        remaining = int(response.getheader("X-RateLimit-Remaining", 0))
+        reset_in = int(response.getheader("X-RateLimit-Reset-In", 0))
+        self.logger.debug("X-RateLimit-Remaining: %i", remaining)
+        self.logger.debug("X-RateLimit-Reset-In: %i", reset_in)
+        if remaining == 0:
+            self.__next_request_time = time.time() + reset_in
+
+
+def _get_payload_many(tracks):
+    payload = []
+    for listened_at, track in tracks:
+        data = _get_payload(track, listened_at)
+        payload.append(data)
+    return payload
+
+
+def _get_payload(track, listened_at=None):
+    data = {"track_metadata": track.to_dict()}
+    if listened_at is not None:
+        data["listened_at"] = listened_at
+    return data

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -0,0 +1,50 @@
+import funkwhale_api
+from config import plugins
+
+from .client import ListenBrainzClient, Track
+from .funkwhale_startup import PLUGIN
+
+
+@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
+def submit_listen(listening, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token:
+        return
+
+    logger = PLUGIN["logger"]
+    logger.info("Submitting listen to ListenBrainz")
+    client = ListenBrainzClient(user_token=user_token, logger=logger)
+    track = get_track(listening.track)
+    client.listen(int(listening.creation_date.timestamp()), track)
+
+
+def get_track(track):
+    artist = track.artist.name
+    title = track.title
+    album = None
+    additional_info = {
+        "media_player": "FunQuail",
+        "media_player_version": funkwhale_api.__version__,
+        "submission_client": "FunQuail ListenBrainz plugin",
+        "submission_client_version": PLUGIN["version"],
+        "tracknumber": track.position,
+        "discnumber": track.disc_number,
+    }
+
+    if track.mbid:
+        additional_info["recording_mbid"] = str(track.mbid)
+
+    if track.album:
+        if track.album.title:
+            album = track.album.title
+        if track.album.mbid:
+            additional_info["release_mbid"] = str(track.album.mbid)
+
+    if track.artist.mbid:
+        additional_info["artist_mbids"] = [str(track.artist.mbid)]
+
+    upload = track.uploads.filter(duration__gte=0).first()
+    if upload:
+        additional_info["duration"] = upload.duration
+
+    return Track(artist, title, album, additional_info)

api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py

@@ -0,0 +1,18 @@
+from config import plugins
+
+PLUGIN = plugins.get_plugin_config(
+    name="listenbrainz",
+    label="ListenBrainz",
+    description="A plugin that allows you to submit your listens to ListenBrainz.",
+    homepage="https://funkwhale.laidback.moe/docs/users/builtinplugins.html#listenbrainz-plugin",  # noqa
+    version="0.3",
+    user=True,
+    conf=[
+        {
+            "name": "user_token",
+            "type": "text",
+            "label": "Your ListenBrainz user token",
+            "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/",
+        }
+    ],
+)

api/funkwhale_api/contrib/maloja/funkwhale_ready.py

@@ -0,0 +1,56 @@
+import json
+
+from config import plugins
+
+from .funkwhale_startup import PLUGIN
+
+
+class MalojaException(Exception):
+    pass
+
+
+@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
+def submit_listen(listening, conf, **kwargs):
+    server_url = conf["server_url"]
+    api_key = conf["api_key"]
+    if not server_url or not api_key:
+        return
+
+    logger = PLUGIN["logger"]
+    logger.info("Submitting listening to Maloja at %s", server_url)
+    payload = get_payload(listening, api_key, conf)
+    logger.debug("Maloja payload: %r", payload)
+    url = server_url.rstrip("/") + "/apis/mlj_1/newscrobble"
+    session = plugins.get_session()
+    response = session.post(url, json=payload)
+    response.raise_for_status()
+    details = json.loads(response.text)
+    if details["status"] == "success":
+        logger.info("Maloja listening submitted successfully")
+    else:
+        raise MalojaException(response.text)
+
+
+def get_payload(listening, api_key, conf):
+    track = listening.track
+
+    # See https://github.com/krateng/maloja/blob/master/API.md
+    payload = {
+        "key": api_key,
+        "artists": [track.artist.name],
+        "title": track.title,
+        "time": int(listening.creation_date.timestamp()),
+        "nofix": bool(conf.get("nofix")),
+    }
+
+    if track.album:
+        if track.album.title:
+            payload["album"] = track.album.title
+        if track.album.artist:
+            payload["albumartists"] = [track.album.artist.name]
+
+    upload = track.uploads.filter(duration__gte=0).first()
+    if upload:
+        payload["length"] = upload.duration
+
+    return payload

api/funkwhale_api/contrib/maloja/funkwhale_startup.py

@@ -0,0 +1,20 @@
+from config import plugins
+
+PLUGIN = plugins.get_plugin_config(
+    name="maloja",
+    label="Maloja",
+    description="A plugin that allows you to submit your listens to your Maloja server.",
+    homepage="https://funkwhale.laidback.moe/docs/users/builtinplugins.html#maloja-plugin",
+    version="0.2",
+    user=True,
+    conf=[
+        {"name": "server_url", "type": "text", "label": "Maloja server URL"},
+        {"name": "api_key", "type": "text", "label": "Your Maloja API key"},
+        {
+            "name": "nofix",
+            "type": "boolean",
+            "label": "Skip server-side metadata fixing",
+            "default": False,
+        },
+    ],
+)

api/funkwhale_api/contrib/scrobbler/README.rst

@@ -0,0 +1,10 @@
+Scrobbler plugin
+================
+
+A plugin that enables scrobbling to ListenBrainz and Last.fm.
+
+If you're scrobbling to last.fm, you will need to create an `API account <https://www.last.fm/api/account/create>`_
+and add two variables two your .env file:
+
+- ``FUNQUAIL_PLUGIN_SCROBBLER_LASTFM_API_KEY=apikey``
+- ``FUNQUAIL_PLUGIN_SCROBBLER_LASTFM_API_SECRET=apisecret``