novaburst/funquail
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Admin UI for libraries and uploads

Browse source
This commit is contained in:
Eliot Berriot 2019-04-19 12:05:13 +02:00
commit a605bcbe76
27 changed files with 2140 additions and 361 deletions
Download patch file Download diff file Expand all files Collapse all files

38
api/funkwhale_api/common/models.py
View file

@ -5,9 +5,10 @@ from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models, transaction
from django.db import connections, models, transaction
from django.db.models import Lookup
from django.db.models.fields import Field
from django.db.models.sql.compiler import SQLCompiler
from django.utils import timezone
from django.urls import reverse
@ -25,6 +26,41 @@ class NotEqual(Lookup):
return "%s <> %s" % (lhs, rhs), params
class NullsLastSQLCompiler(SQLCompiler):
def get_order_by(self):
result = super().get_order_by()
if result and self.connection.vendor == "postgresql":
return [
(
expr,
(
sql + " NULLS LAST" if not sql.endswith(" NULLS LAST") else sql,
params,
is_ref,
),
)
for (expr, (sql, params, is_ref)) in result
]
return result
class NullsLastQuery(models.sql.query.Query):
"""Use a custom compiler to inject 'NULLS LAST' (for PostgreSQL)."""
def get_compiler(self, using=None, connection=None):
if using is None and connection is None:
raise ValueError("Need either using or connection")
if using:
connection = connections[using]
return NullsLastSQLCompiler(self, connection, using)
class NullsLastQuerySet(models.QuerySet):
def __init__(self, model=None, query=None, using=None, hints=None):
super().__init__(model, query, using, hints)
self.query = query or NullsLastQuery(self.model)
class LocalFromFidQuerySet:
def local(self, include=True):
host = settings.FEDERATION_HOSTNAME

13
api/funkwhale_api/federation/models.py
View file

@ -1,4 +1,5 @@
import tempfile
import urllib.parse
import uuid
from django.conf import settings
@ -43,6 +44,18 @@ class FederationMixin(models.Model):
class Meta:
abstract = True
@property
def is_local(self):
return federation_utils.is_local(self.fid)
@property
def domain_name(self):
if not self.fid:
return
parsed = urllib.parse.urlparse(self.fid)
return parsed.hostname
class ActorQuerySet(models.QuerySet):
def local(self, include=True):

145
api/funkwhale_api/manage/filters.py
View file

@ -1,4 +1,8 @@
from django import forms
from django.db.models import Q
from django.conf import settings
import django_filters
from django_filters import rest_framework as filters
from funkwhale_api.common import fields
@ -11,19 +15,32 @@ from funkwhale_api.music import models as music_models
from funkwhale_api.users import models as users_models
class ManageUploadFilterSet(filters.FilterSet):
q = fields.SearchFilter(
search_fields=[
"track__title",
"track__album__title",
"track__artist__name",
"source",
]
)
class ActorField(forms.CharField):
def clean(self, value):
value = super().clean(value)
if not value:
return value
class Meta:
model = music_models.Upload
fields = ["q", "track__album", "track__artist", "track"]
parts = value.split("@")
return {
"username": parts[0],
"domain": parts[1] if len(parts) > 1 else settings.FEDERATION_HOSTNAME,
}
def get_actor_filter(actor_field):
def handler(v):
if not v:
return Q(**{actor_field: None})
return Q(
**{
"{}__preferred_username__iexact".format(actor_field): v["username"],
"{}__domain__name__iexact".format(actor_field): v["domain"],
}
)
return {"field": ActorField(), "handler": handler}
class ManageArtistFilterSet(filters.FilterSet):
@ -37,7 +54,11 @@ class ManageArtistFilterSet(filters.FilterSet):
filter_fields={
"domain": {
"handler": lambda v: federation_utils.get_domain_query_from_url(v)
}
},
"library_id": {
"to": "tracks__uploads__library_id",
"field": forms.IntegerField(),
},
},
)
)
@ -61,6 +82,10 @@ class ManageAlbumFilterSet(filters.FilterSet):
"domain": {
"handler": lambda v: federation_utils.get_domain_query_from_url(v)
},
"library_id": {
"to": "tracks__uploads__library_id",
"field": forms.IntegerField(),
},
},
)
)
@ -93,6 +118,10 @@ class ManageTrackFilterSet(filters.FilterSet):
"domain": {
"handler": lambda v: federation_utils.get_domain_query_from_url(v)
},
"library_id": {
"to": "uploads__library_id",
"field": forms.IntegerField(),
},
},
)
)
@ -102,6 +131,96 @@ class ManageTrackFilterSet(filters.FilterSet):
fields = ["q", "title", "mbid", "fid", "artist", "album", "license"]
class ManageLibraryFilterSet(filters.FilterSet):
ordering = django_filters.OrderingFilter(
# tuple-mapping retains order
fields=(
("creation_date", "creation_date"),
("_uploads_count", "uploads_count"),
("followers_count", "followers_count"),
)
)
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
"name": {"to": "name"},
"description": {"to": "description"},
"fid": {"to": "fid"},
},
filter_fields={
"artist_id": {
"to": "uploads__track__artist_id",
"field": forms.IntegerField(),
},
"album_id": {
"to": "uploads__track__album_id",
"field": forms.IntegerField(),
},
"track_id": {"to": "uploads__track__id", "field": forms.IntegerField()},
"domain": {"to": "actor__domain_id"},
"account": get_actor_filter("actor"),
"privacy_level": {"to": "privacy_level"},
},
)
)
domain = filters.CharFilter("actor__domain_id")
class Meta:
model = music_models.Library
fields = ["q", "name", "fid", "privacy_level", "domain"]
class ManageUploadFilterSet(filters.FilterSet):
ordering = django_filters.OrderingFilter(
# tuple-mapping retains order
fields=(
("creation_date", "creation_date"),
("modification_date", "modification_date"),
("accessed_date", "accessed_date"),
("size", "size"),
("bitrate", "bitrate"),
("duration", "duration"),
)
)
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
"source": {"to": "source"},
"fid": {"to": "fid"},
"track": {"to": "track__title"},
"album": {"to": "track__album__title"},
"artist": {"to": "track__artist__name"},
},
filter_fields={
"library_id": {"to": "library_id", "field": forms.IntegerField()},
"artist_id": {"to": "track__artist_id", "field": forms.IntegerField()},
"album_id": {"to": "track__album_id", "field": forms.IntegerField()},
"track_id": {"to": "track__id", "field": forms.IntegerField()},
"domain": {"to": "library__actor__domain_id"},
"import_reference": {"to": "import_reference"},
"type": {"to": "mimetype"},
"status": {"to": "import_status"},
"account": get_actor_filter("library__actor"),
"privacy_level": {"to": "library__privacy_level"},
},
)
)
domain = filters.CharFilter("library__actor__domain_id")
privacy_level = filters.CharFilter("library__privacy_level")
class Meta:
model = music_models.Upload
fields = [
"q",
"fid",
"privacy_level",
"domain",
"mimetype",
"import_reference",
"import_status",
]
class ManageDomainFilterSet(filters.FilterSet):
q = fields.SearchFilter(search_fields=["name"])

169
api/funkwhale_api/manage/serializers.py
View file

@ -15,67 +15,6 @@ from funkwhale_api.users import models as users_models
from . import filters
class ManageUploadArtistSerializer(serializers.ModelSerializer):
class Meta:
model = music_models.Artist
fields = ["id", "mbid", "creation_date", "name"]
class ManageUploadAlbumSerializer(serializers.ModelSerializer):
artist = ManageUploadArtistSerializer()
class Meta:
model = music_models.Album
fields = (
"id",
"mbid",
"title",
"artist",
"release_date",
"cover",
"creation_date",
)
class ManageUploadTrackSerializer(serializers.ModelSerializer):
artist = ManageUploadArtistSerializer()
album = ManageUploadAlbumSerializer()
class Meta:
model = music_models.Track
fields = ("id", "mbid", "title", "album", "artist", "creation_date", "position")
class ManageUploadSerializer(serializers.ModelSerializer):
track = ManageUploadTrackSerializer()
class Meta:
model = music_models.Upload
fields = (
"id",
"path",
"source",
"filename",
"mimetype",
"track",
"duration",
"mimetype",
"creation_date",
"bitrate",
"size",
"path",
)
class ManageUploadActionSerializer(common_serializers.ActionSerializer):
actions = [common_serializers.Action("delete", allow_all=False)]
filterset_class = filters.ManageUploadFilterSet
@transaction.atomic
def handle_delete(self, objects):
return objects.delete()
class PermissionsSerializer(serializers.Serializer):
def to_representation(self, o):
return o.get_permissions(defaults=self.context.get("default_permissions"))
@ -493,3 +432,111 @@ class ManageArtistActionSerializer(common_serializers.ActionSerializer):
@transaction.atomic
def handle_delete(self, objects):
return objects.delete()
class ManageLibraryActionSerializer(common_serializers.ActionSerializer):
actions = [common_serializers.Action("delete", allow_all=False)]
filterset_class = filters.ManageLibraryFilterSet
@transaction.atomic
def handle_delete(self, objects):
return objects.delete()
class ManageUploadActionSerializer(common_serializers.ActionSerializer):
actions = [common_serializers.Action("delete", allow_all=False)]
filterset_class = filters.ManageUploadFilterSet
@transaction.atomic
def handle_delete(self, objects):
return objects.delete()
class ManageLibrarySerializer(serializers.ModelSerializer):
domain = serializers.CharField(source="domain_name")
actor = ManageBaseActorSerializer()
uploads_count = serializers.SerializerMethodField()
followers_count = serializers.SerializerMethodField()
class Meta:
model = music_models.Library
fields = [
"id",
"uuid",
"fid",
"url",
"name",
"description",
"domain",
"is_local",
"creation_date",
"privacy_level",
"uploads_count",
"followers_count",
"followers_url",
"actor",
]
def get_uploads_count(self, obj):
return getattr(obj, "_uploads_count", obj.uploads_count)
def get_followers_count(self, obj):
return getattr(obj, "followers_count", None)
class ManageNestedLibrarySerializer(serializers.ModelSerializer):
domain = serializers.CharField(source="domain_name")
actor = ManageBaseActorSerializer()
class Meta:
model = music_models.Library
fields = [
"id",
"uuid",
"fid",
"url",
"name",
"description",
"domain",
"is_local",
"creation_date",
"privacy_level",
"followers_url",
"actor",
]
class ManageUploadSerializer(serializers.ModelSerializer):
track = ManageNestedTrackSerializer()
library = ManageNestedLibrarySerializer()
domain = serializers.CharField(source="domain_name")
class Meta:
model = music_models.Upload
fields = (
"id",
"uuid",
"fid",
"domain",
"is_local",
"audio_file",
"listen_url",
"source",
"filename",
"mimetype",
"duration",
"mimetype",
"bitrate",
"size",
"creation_date",
"accessed_date",
"modification_date",
"metadata",
"import_date",
"import_details",
"import_status",
"import_metadata",
"import_reference",
"track",
"library",
)

5
api/funkwhale_api/manage/urls.py
View file

@ -7,10 +7,11 @@ federation_router = routers.SimpleRouter()
federation_router.register(r"domains", views.ManageDomainViewSet, "domains")
library_router = routers.SimpleRouter()
library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
library_router.register(r"artists", views.ManageArtistViewSet, "artists")
library_router.register(r"albums", views.ManageAlbumViewSet, "albums")
library_router.register(r"artists", views.ManageArtistViewSet, "artists")
library_router.register(r"libraries", views.ManageLibraryViewSet, "libraries")
library_router.register(r"tracks", views.ManageTrackViewSet, "tracks")
library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
moderation_router = routers.SimpleRouter()
moderation_router.register(

131
api/funkwhale_api/manage/views.py
View file

@ -19,38 +19,6 @@ from funkwhale_api.users import models as users_models
from . import filters, serializers
class ManageUploadViewSet(
mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
queryset = (
music_models.Upload.objects.all()
.select_related("track__artist", "track__album__artist")
.order_by("-id")
)
serializer_class = serializers.ManageUploadSerializer
filterset_class = filters.ManageUploadFilterSet
required_scope = "instance:libraries"
ordering_fields = [
"accessed_date",
"modification_date",
"creation_date",
"track__artist__name",
"bitrate",
"size",
"duration",
]
@rest_decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.ManageUploadActionSerializer(
request.data, queryset=queryset
)
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
def get_stats(tracks, target):
data = {}
tracks = list(tracks.values_list("pk", flat=True))
@ -70,6 +38,12 @@ def get_stats(tracks, target):
).count()
data["libraries"] = uploads.values_list("library", flat=True).distinct().count()
data["uploads"] = uploads.count()
data.update(get_media_stats(uploads))
return data
def get_media_stats(uploads):
data = {}
data["media_total_size"] = uploads.aggregate(v=Sum("size"))["v"] or 0
data["media_downloaded_size"] = (
uploads.with_file().aggregate(v=Sum("size"))["v"] or 0
@ -85,6 +59,7 @@ class ManageArtistViewSet(
):
queryset = (
music_models.Artist.objects.all()
.distinct()
.order_by("-id")
.select_related("attributed_to")
.prefetch_related(
@ -130,6 +105,7 @@ class ManageAlbumViewSet(
):
queryset = (
music_models.Album.objects.all()
.distinct()
.order_by("-id")
.select_related("attributed_to", "artist")
.prefetch_related("tracks")
@ -164,6 +140,7 @@ class ManageTrackViewSet(
):
queryset = (
music_models.Track.objects.all()
.distinct()
.order_by("-id")
.select_related("attributed_to", "artist", "album__artist")
.annotate(uploads_count=Count("uploads"))
@ -196,6 +173,96 @@ class ManageTrackViewSet(
return response.Response(result, status=200)
class ManageLibraryViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet,
):
lookup_field = "uuid"
queryset = (
music_models.Library.objects.all()
.distinct()
.order_by("-id")
.select_related("actor")
.annotate(
followers_count=Count("received_follows", distinct=True),
_uploads_count=Count("uploads", distinct=True),
)
)
serializer_class = serializers.ManageLibrarySerializer
filterset_class = filters.ManageLibraryFilterSet
required_scope = "instance:libraries"
@rest_decorators.action(methods=["get"], detail=True)
def stats(self, request, *args, **kwargs):
library = self.get_object()
uploads = library.uploads.all()
tracks = uploads.values_list("track", flat=True).distinct()
albums = (
music_models.Track.objects.filter(pk__in=tracks)
.values_list("album", flat=True)
.distinct()
)
artists = set(
music_models.Album.objects.filter(pk__in=albums).values_list(
"artist", flat=True
)
) | set(
music_models.Track.objects.filter(pk__in=tracks).values_list(
"artist", flat=True
)
)
data = {
"uploads": uploads.count(),
"followers": library.received_follows.count(),
"tracks": tracks.count(),
"albums": albums.count(),
"artists": len(artists),
}
data.update(get_media_stats(uploads.all()))
return response.Response(data, status=200)
@rest_decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.ManageTrackActionSerializer(
request.data, queryset=queryset
)
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
class ManageUploadViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet,
):
lookup_field = "uuid"
queryset = (
music_models.Upload.objects.all()
.distinct()
.order_by("-id")
.select_related("library__actor", "track__artist", "track__album__artist")
)
serializer_class = serializers.ManageUploadSerializer
filterset_class = filters.ManageUploadFilterSet
required_scope = "instance:libraries"
@rest_decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.ManageTrackActionSerializer(
request.data, queryset=queryset
)
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
class ManageUserViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,

14
api/funkwhale_api/music/models.py
View file

@ -649,7 +649,7 @@ class Track(APIModelMixin):
return licenses.LICENSES_BY_ID.get(self.license_id)
class UploadQuerySet(models.QuerySet):
class UploadQuerySet(common_models.NullsLastQuerySet):
def playable_by(self, actor, include=True):
libraries = Library.objects.viewable_by(actor)
@ -746,6 +746,18 @@ class Upload(models.Model):
objects = UploadQuerySet.as_manager()
@property
def is_local(self):
return federation_utils.is_local(self.fid)
@property
def domain_name(self):
if not self.fid:
return
parsed = urllib.parse.urlparse(self.fid)
return parsed.hostname
def download_audio_from_remote(self, actor):
from funkwhale_api.common import session
from funkwhale_api.federation import signing

2
api/funkwhale_api/music/views.py
View file

@ -440,8 +440,6 @@ class UploadViewSet(
"artist__name",
)
fetches = federation_decorators.fetches_route()
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(library__actor=self.request.user.actor)

61
api/tests/manage/test_serializers.py
View file

@ -399,12 +399,73 @@ def test_manage_track_serializer(factories, now):
assert s.data == expected
def test_manage_library_serializer(factories, now):
library = factories["music.Library"]()
setattr(library, "followers_count", 42)
setattr(library, "_uploads_count", 44)
expected = {
"id": library.id,
"fid": library.fid,
"url": library.url,
"uuid": str(library.uuid),
"followers_url": library.followers_url,
"domain": library.domain_name,
"is_local": library.is_local,
"name": library.name,
"description": library.description,
"privacy_level": library.privacy_level,
"creation_date": library.creation_date.isoformat().split("+")[0] + "Z",
"actor": serializers.ManageBaseActorSerializer(library.actor).data,
"uploads_count": 44,
"followers_count": 42,
}
s = serializers.ManageLibrarySerializer(library)
assert s.data == expected
def test_manage_upload_serializer(factories, now):
upload = factories["music.Upload"]()
expected = {
"id": upload.id,
"fid": upload.fid,
"audio_file": upload.audio_file.url,
"listen_url": upload.listen_url,
"uuid": str(upload.uuid),
"domain": upload.domain_name,
"is_local": upload.is_local,
"duration": upload.duration,
"size": upload.size,
"bitrate": upload.bitrate,
"mimetype": upload.mimetype,
"source": upload.source,
"filename": upload.filename,
"metadata": upload.metadata,
"creation_date": upload.creation_date.isoformat().split("+")[0] + "Z",
"modification_date": upload.modification_date.isoformat().split("+")[0] + "Z",
"accessed_date": None,
"import_date": None,
"import_metadata": upload.import_metadata,
"import_status": upload.import_status,
"import_reference": upload.import_reference,
"import_details": upload.import_details,
"library": serializers.ManageNestedLibrarySerializer(upload.library).data,
"track": serializers.ManageNestedTrackSerializer(upload.track).data,
}
s = serializers.ManageUploadSerializer(upload)
assert s.data == expected
@pytest.mark.parametrize(
"factory, serializer_class",
[
("music.Track", serializers.ManageTrackActionSerializer),
("music.Album", serializers.ManageAlbumActionSerializer),
("music.Artist", serializers.ManageArtistActionSerializer),
("music.Library", serializers.ManageLibraryActionSerializer),
("music.Upload", serializers.ManageUploadActionSerializer),
],
)
def test_action_serializer_delete(factory, serializer_class, factories):

95
api/tests/manage/test_views.py
View file

@ -1,4 +1,3 @@
import pytest
from django.urls import reverse
from funkwhale_api.federation import models as federation_models
@ -6,21 +5,6 @@ from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.manage import serializers
@pytest.mark.skip(reason="Refactoring in progress")
def test_upload_view(factories, superuser_api_client):
uploads = factories["music.Upload"].create_batch(size=5)
qs = uploads[0].__class__.objects.order_by("-creation_date")
url = reverse("api:v1:manage:library:uploads-list")
response = superuser_api_client.get(url, {"sort": "-creation_date"})
expected = serializers.ManageUploadSerializer(
qs, many=True, context={"request": response.wsgi_request}
).data
assert response.data["count"] == len(uploads)
assert response.data["results"] == expected
def test_user_view(factories, superuser_api_client, mocker):
mocker.patch("funkwhale_api.users.models.User.record_activity")
users = factories["users.User"].create_batch(size=5) + [superuser_api_client.user]
@ -289,3 +273,82 @@ def test_track_delete(factories, superuser_api_client):
response = superuser_api_client.delete(url)
assert response.status_code == 204
def test_library_list(factories, superuser_api_client, settings):
library = factories["music.Library"]()
url = reverse("api:v1:manage:library:libraries-list")
response = superuser_api_client.get(url)
assert response.status_code == 200
assert response.data["count"] == 1
assert response.data["results"][0]["id"] == library.id
def test_library_detail(factories, superuser_api_client):
library = factories["music.Library"]()
url = reverse(
"api:v1:manage:library:libraries-detail", kwargs={"uuid": library.uuid}
)
response = superuser_api_client.get(url)
assert response.status_code == 200
assert response.data["id"] == library.id
def test_library_detail_stats(factories, superuser_api_client):
library = factories["music.Library"]()
url = reverse(
"api:v1:manage:library:libraries-stats", kwargs={"uuid": library.uuid}
)
response = superuser_api_client.get(url)
expected = {
"uploads": 0,
"followers": 0,
"tracks": 0,
"albums": 0,
"artists": 0,
"media_total_size": 0,
"media_downloaded_size": 0,
}
assert response.status_code == 200
assert response.data == expected
def test_library_delete(factories, superuser_api_client):
library = factories["music.Library"]()
url = reverse(
"api:v1:manage:library:libraries-detail", kwargs={"uuid": library.uuid}
)
response = superuser_api_client.delete(url)
assert response.status_code == 204
def test_upload_list(factories, superuser_api_client, settings):
upload = factories["music.Upload"]()
url = reverse("api:v1:manage:library:uploads-list")
response = superuser_api_client.get(url)
assert response.status_code == 200
assert response.data["count"] == 1
assert response.data["results"][0]["id"] == upload.id
def test_upload_detail(factories, superuser_api_client):
upload = factories["music.Upload"]()
url = reverse("api:v1:manage:library:uploads-detail", kwargs={"uuid": upload.uuid})
response = superuser_api_client.get(url)
assert response.status_code == 200
assert response.data["id"] == upload.id
def test_upload_delete(factories, superuser_api_client):
upload = factories["music.Upload"]()
url = reverse("api:v1:manage:library:uploads-detail", kwargs={"uuid": upload.uuid})
response = superuser_api_client.delete(url)
assert response.status_code == 204