novaburst/funquail
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'develop' into stable

Browse source
This commit is contained in:
Georg Krause 2023-12-12 13:29:29 +01:00
commit 98d4f4ef54
511 changed files with 60771 additions and 41670 deletions
Download patch file Download diff file Expand all files Collapse all files

41
api/Dockerfile
View file

@ -1,14 +1,20 @@
FROM alpine:3.17 as pre-build
FROM alpine:3.17 as requirements
# We need this additional step to avoid having poetrys deps interacting with our
# dependencies. This is only required until alpine 3.16 is released, since this
# allows us to install poetry as package.
RUN apk add --no-cache python3 py3-cryptography py3-pip poetry
COPY pyproject.toml poetry.lock /
RUN poetry export --without-hashes > requirements.txt
RUN poetry export --with dev --without-hashes > dev-requirements.txt
RUN set -eux; \
apk add --no-cache \
poetry \
py3-cryptography \
py3-pip \
python3
COPY pyproject.toml poetry.lock /
RUN set -eux; \
poetry export --without-hashes --extras typesense > requirements.txt; \
poetry export --without-hashes --with dev > dev-requirements.txt;
FROM alpine:3.17 as builder
@ -22,6 +28,7 @@ RUN set -eux; \
cargo \
curl \
gcc \
g++ \
git \
jpeg-dev \
libffi-dev \
@ -41,15 +48,15 @@ RUN set -eux; \
py3-watchfiles=0.18.1-r0 \
python3-dev
# create virtual env for next stage
# Create virtual env
RUN python3 -m venv --system-site-packages /venv
# emulate activation by prefixing PATH
ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv
ENV PATH="/venv/bin:$PATH"
COPY --from=pre-build /requirements.txt /requirements.txt
COPY --from=pre-build /dev-requirements.txt /dev-requirements.txt
COPY --from=requirements /requirements.txt /requirements.txt
COPY --from=requirements /dev-requirements.txt /dev-requirements.txt
RUN set -eux; \
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
pip3 install --upgrade pip; \
pip3 install setuptools wheel; \
# Currently we are unable to relieably build rust-based packages on armv7. This
@ -65,7 +72,8 @@ RUN set -eux; \
watchfiles==0.18.1
ARG install_dev_deps=0
RUN set -eux; \
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
if [ "$install_dev_deps" = "1" ] ; then \
grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /dev-requirements.txt \
| pip3 install -r /dev/stdin \
@ -76,7 +84,7 @@ RUN set -eux; \
watchfiles==0.18.1; \
fi
FROM alpine:3.17 as image
FROM alpine:3.17 as production
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
@ -98,16 +106,17 @@ RUN set -eux; \
py3-pillow=9.3.0-r0 \
py3-psycopg2=2.9.5-r0 \
py3-watchfiles=0.18.1-r0 \
python3
python3 \
tzdata
COPY --from=builder /venv /venv
# emulate activation by prefixing PATH
ENV PATH="/venv/bin:$PATH"
COPY . /app
WORKDIR /app
RUN set -eux; \
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
pip3 install --no-deps --editable .
ENV IS_DOCKER_SETUP=true

14
api/Makefile Normal file
View file

@ -0,0 +1,14 @@
SHELL := bash
CPU_CORES := $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N )))
.PHONY: install lint
install:
poetry install
lint:
poetry run pylint \
--jobs=$(CPU_CORES) \
--output-format=colorized \
--recursive=true \
config funkwhale_api tests

3
api/config/__init__.py
View file

@ -1,3 +0,0 @@
# loads what is required to generate the swagger docs
# https://matrix.to/#/!nNBDNverFlbfNpReEO:matrix.org/$16579878472182UmZUv:tchncs.de?via=tchncs.de&via=matrix.org&via=juniorjpdj.pl
import config.schema # noqa: F401

97
api/config/api_urls.py
View file

@ -1,97 +0,0 @@
from django.conf.urls import include, url
from rest_framework import routers
from rest_framework.urlpatterns import format_suffix_patterns
from funkwhale_api.activity import views as activity_views
from funkwhale_api.audio import views as audio_views
from funkwhale_api.common import routers as common_routers
from funkwhale_api.common import views as common_views
from funkwhale_api.music import views
from funkwhale_api.playlists import views as playlists_views
from funkwhale_api.subsonic.views import SubsonicViewSet
from funkwhale_api.tags import views as tags_views
router = common_routers.OptionalSlashRouter()
router.register(r"activity", activity_views.ActivityViewSet, "activity")
router.register(r"tags", tags_views.TagViewSet, "tags")
router.register(r"plugins", common_views.PluginViewSet, "plugins")
router.register(r"tracks", views.TrackViewSet, "tracks")
router.register(r"uploads", views.UploadViewSet, "uploads")
router.register(r"libraries", views.LibraryViewSet, "libraries")
router.register(r"listen", views.ListenViewSet, "listen")
router.register(r"stream", views.StreamViewSet, "stream")
router.register(r"artists", views.ArtistViewSet, "artists")
router.register(r"channels", audio_views.ChannelViewSet, "channels")
router.register(r"subscriptions", audio_views.SubscriptionsViewSet, "subscriptions")
router.register(r"albums", views.AlbumViewSet, "albums")
router.register(r"licenses", views.LicenseViewSet, "licenses")
router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
router.register(r"mutations", common_views.MutationViewSet, "mutations")
router.register(r"attachments", common_views.AttachmentViewSet, "attachments")
v1_patterns = router.urls
subsonic_router = routers.SimpleRouter(trailing_slash=False)
subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
v1_patterns += [
url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
url(
r"^instance/",
include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
),
url(
r"^manage/",
include(("funkwhale_api.manage.urls", "manage"), namespace="manage"),
),
url(
r"^moderation/",
include(
("funkwhale_api.moderation.urls", "moderation"), namespace="moderation"
),
),
url(
r"^federation/",
include(
("funkwhale_api.federation.api_urls", "federation"), namespace="federation"
),
),
url(
r"^providers/",
include(("funkwhale_api.providers.urls", "providers"), namespace="providers"),
),
url(
r"^favorites/",
include(("funkwhale_api.favorites.urls", "favorites"), namespace="favorites"),
),
url(r"^search$", views.Search.as_view(), name="search"),
url(
r"^radios/",
include(("funkwhale_api.radios.urls", "radios"), namespace="radios"),
),
url(
r"^history/",
include(("funkwhale_api.history.urls", "history"), namespace="history"),
),
url(
r"^",
include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
),
# XXX: remove if Funkwhale 1.1
url(
r"^users/",
include(("funkwhale_api.users.api_urls", "users"), namespace="users-nested"),
),
url(
r"^oauth/",
include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
),
url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
url(
r"^text-preview/?$", common_views.TextPreviewView.as_view(), name="text-preview"
),
]
urlpatterns = [
url(r"^v1/", include((v1_patterns, "v1"), namespace="v1"))
] + format_suffix_patterns(subsonic_router.urls, allowed=["view"])

74
api/config/settings/common.py
View file

@ -13,7 +13,29 @@ APPS_DIR = ROOT_DIR.path("funkwhale_api")
env = environ.Env()
ENV = env
LOGLEVEL = env("LOGLEVEL", default="info").upper()
# If DEBUG is `true`, we automatically set the loglevel to "DEBUG"
# If DEBUG is `false`, we try to read the level from LOGLEVEL environment and default to "INFO"
LOGLEVEL = (
"DEBUG" if env.bool("DEBUG", False) else env("LOGLEVEL", default="info").upper()
)
"""
Default logging level for the Funkwhale processes.
.. note::
The `DEBUG` variable overrides the `LOGLEVEL` if it is set to `TRUE`.
The `LOGLEVEL` value only applies if `DEBUG` is `false` or not present.
Available levels:
- ``debug``
- ``info``
- ``warning``
- ``error``
- ``critical``
"""
IS_DOCKER_SETUP = env.bool("IS_DOCKER_SETUP", False)
@ -35,19 +57,6 @@ if env("FUNKWHALE_SENTRY_DSN", default=None) is not None:
)
sentry_sdk.set_tag("instance", env("FUNKWHALE_HOSTNAME"))
"""
Default logging level for the Funkwhale processes
Available levels:
- ``debug``
- ``info``
- ``warning``
- ``error``
- ``critical``
""" # pylint: disable=W0105
LOGGING_CONFIG = None
logging.config.dictConfig(
{
@ -187,9 +196,7 @@ request errors related to this.
FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
"FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
)
FUNKWHALE_EMBED_URL = env(
"FUNKWHALE_EMBED_URL", default=FUNKWHALE_URL + "/front/embed.html"
)
FUNKWHALE_EMBED_URL = env("FUNKWHALE_EMBED_URL", default=FUNKWHALE_URL + "/embed.html")
FUNKWHALE_SPA_REWRITE_MANIFEST = env.bool(
"FUNKWHALE_SPA_REWRITE_MANIFEST", default=True
)
@ -272,6 +279,7 @@ LOCAL_APPS = (
"funkwhale_api.playlists",
"funkwhale_api.subsonic",
"funkwhale_api.tags",
"funkwhale_api.typesense",
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@ -281,9 +289,9 @@ ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
List of Django apps to load in addition to Funkwhale plugins and apps.
"""
INSTALLED_APPS = (
DJANGO_APPS
LOCAL_APPS
+ DJANGO_APPS
+ THIRD_PARTY_APPS
+ LOCAL_APPS
+ tuple(ADDITIONAL_APPS)
+ tuple(plugins.trigger_filter(plugins.PLUGINS_APPS, [], enabled=True))
)
@ -822,7 +830,7 @@ If you're using password auth (the extra slash is important)
.. note::
If you want to use Redis over unix sockets, you also need to update
:attr:`CELERY_BROKER_URL`, because the scheme differ from the one used by
:attr:`CELERY_BROKER_URL`, because the scheme differs from the one used by
:attr:`CACHE_URL`.
"""
@ -873,7 +881,7 @@ to use a different server or use Redis sockets to connect.
Example:
- ``redis://127.0.0.1:6379/0``
- ``unix://127.0.0.1:6379/0``
- ``redis+socket:///run/redis/redis.sock?virtual_host=0``
"""
@ -934,6 +942,11 @@ CELERY_BEAT_SCHEDULE = {
),
"options": {"expires": 60 * 60},
},
"typesense.build_canonical_index": {
"task": "typesense.build_canonical_index",
"schedule": crontab(day_of_week="*/2", minute="0", hour="3"),
"options": {"expires": 60 * 60 * 24},
},
}
if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True):
@ -1459,3 +1472,22 @@ instead of request header.
HASHING_ALGORITHM = "sha256"
HASHING_CHUNK_SIZE = 1024 * 100
"""
Typenses Settings
"""
TYPESENSE_API_KEY = env("TYPESENSE_API_KEY", default=None)
""" Typesense API key. This need to be defined in the .env file for Typenses to be activated."""
TYPESENSE_PORT = env("TYPESENSE_PORT", default="8108")
"""Typesense listening port"""
TYPESENSE_PROTOCOL = env("TYPESENSE_PROTOCOL", default="http")
"""Typesense listening protocol"""
TYPESENSE_HOST = env(
"TYPESENSE_HOST",
default="typesense" if IS_DOCKER_SETUP else "localhost",
)
"""
Typesense hostname. Defaults to `localhost` on non-Docker deployments and to `typesense` on
Docker deployments.
"""
TYPESENSE_NUM_TYPO = env("TYPESENSE_NUM_TYPO", default=5)

4
api/config/settings/local.py
View file

@ -76,7 +76,7 @@ DEBUG_TOOLBAR_PANELS = [
# django-extensions
# ------------------------------------------------------------------------------
# INSTALLED_APPS += ('django_extensions', )
INSTALLED_APPS += ("django_extensions",)
INSTALLED_APPS += ("drf_spectacular",)
@ -149,3 +149,5 @@ MIDDLEWARE = (
"funkwhale_api.common.middleware.ProfilerMiddleware",
"funkwhale_api.common.middleware.PymallocMiddleware",
) + MIDDLEWARE
TYPESENSE_API_KEY = "apikey"

9
api/config/settings/testing.py Normal file
View file

@ -0,0 +1,9 @@
import os
os.environ.setdefault("FUNKWHALE_URL", "http://funkwhale.dev")
from .common import * # noqa
DEBUG = True
SECRET_KEY = "a_super_secret_key!"
TYPESENSE_API_KEY = "apikey"

6
api/config/urls/api_v2.py
View file

@ -8,7 +8,11 @@ v2_patterns = router.urls
v2_patterns += [
url(
r"^instance/",
include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
include(("funkwhale_api.instance.urls_v2", "instance"), namespace="instance"),
),
url(
r"^radios/",
include(("funkwhale_api.radios.urls_v2", "radios"), namespace="radios"),
),
]

11
api/funkwhale_api/__init__.py
View file

@ -1,7 +1,4 @@
__version__ = "1.3.4"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num
for num in __version__.replace("-", ".", 1).split(".")
]
)
from importlib.metadata import version as get_version
version = get_version("funkwhale_api")
__version__ = version

9
api/funkwhale_api/audio/serializers.py
View file

@ -1,10 +1,10 @@
import datetime
import logging
import sys
import time
import uuid
import feedparser
import pytz
import requests
from django.conf import settings
from django.db import transaction
@ -33,6 +33,11 @@ from funkwhale_api.users import serializers as users_serializers
from . import categories, models
if sys.version_info < (3, 9):
from backports.zoneinfo import ZoneInfo
else:
from zoneinfo import ZoneInfo
logger = logging.getLogger(__name__)
@ -769,7 +774,7 @@ class RssFeedItemSerializer(serializers.Serializer):
if "published_parsed" in validated_data:
track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
time.mktime(validated_data["published_parsed"])
).replace(tzinfo=pytz.utc)
).replace(tzinfo=ZoneInfo("UTC"))
upload_defaults = {
"source": validated_data["links"]["audio"]["source"],

2
api/funkwhale_api/cli/users.py
View file

@ -38,7 +38,7 @@ def handler_create_user(
utils.logger.debug("Creating user…")
user = serializer.save(request=request)
utils.logger.debug("Setting permissions and other attributes…")
user.is_staff = is_staff
user.is_staff = is_staff or is_superuser # Always set staff if superuser is set
user.upload_quota = upload_quota
user.is_superuser = is_superuser
for permission in permissions:

17
api/funkwhale_api/common/filters.py
View file

@ -1,5 +1,6 @@
from django import forms
from django.db.models import Q
from django.db.models.functions import Lower
from django_filters import rest_framework as filters
from django_filters import widgets
from drf_spectacular.utils import extend_schema_field
@ -239,3 +240,19 @@ class ActorScopeFilter(filters.CharFilter):
raise EmptyQuerySet()
return Q(**{self.actor_field: actor})
class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
def filter(self, qs, value):
order_by = []
if value is None:
return qs
for param in value:
if param == "name":
order_by.append(Lower("name"))
else:
order_by.append(self.get_ordering_value(param))
return qs.order_by(*order_by)

22
api/funkwhale_api/common/management/commands/createsuperuser.py Normal file
View file

@ -0,0 +1,22 @@
import os
from django.contrib.auth.management.commands.createsuperuser import (
Command as BaseCommand,
)
from django.core.management.base import CommandError
class Command(BaseCommand):
def handle(self, *apps_label, **options):
"""
Creating Django Superusers would bypass some of our username checks, which can lead to unexpected behaviour.
We therefore prohibit the execution of the command.
"""
if not os.environ.get("FORCE") == "1":
raise CommandError(
"Running createsuperuser on your Funkwhale instance bypasses some of our checks "
"which can lead to unexpected behavior of your instance. We therefore suggest to "
"run `funkwhale-manage fw users create --superuser` instead."
)
return super().handle(*apps_label, **options)

17
api/funkwhale_api/common/management/commands/gitpod.py
View file

@ -36,22 +36,7 @@ class Command(BaseCommand):
self.stdout.write("")
def init(self):
try:
user = User.objects.get(username="gitpod")
except Exception:
call_command(
"createsuperuser",
username="gitpod",
email="gitpod@example.com",
no_input=False,
)
user = User.objects.get(username="gitpod")
user.set_password("gitpod")
if not user.actor:
user.create_actor()
user.save()
user = User.objects.get(username="gitpod")
# Allow anonymous access
preferences.set("common__api_authentication_required", False)

95
api/funkwhale_api/common/management/commands/inplace_to_s3.py Normal file
View file

@ -0,0 +1,95 @@
import pathlib
from argparse import RawTextHelpFormatter
from django.core.management.base import BaseCommand
from django.db import transaction
from funkwhale_api.music import models
class Command(BaseCommand):
help = """
Update the reference for Uploads that have been imported with --in-place and are now moved to s3.
Please note: This does not move any file! Make sure you already moved the files to your s3 bucket.
Specify --source to filter the reference to update to files from a specific in-place directory. If no
--source is given, all in-place imported track references will be updated.
Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is
given, the file is expected to be stored in the same path as before.
Examples:
Music File: /music/Artist/Album/track.ogg
--source: /music
--target unset
All files imported from /music will be updated and expected to be in the same folder structure in the bucket
Music File: /music/Artist/Album/track.ogg
--source: /music
--target: /in_place
The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg
"""
def create_parser(self, *args, **kwargs):
parser = super().create_parser(*args, **kwargs)
parser.formatter_class = RawTextHelpFormatter
return parser
def add_arguments(self, parser):
parser.add_argument(
"--no-dry-run",
action="store_false",
dest="dry_run",
default=True,
help="Disable dry run mode and apply updates for real on the database",
)
parser.add_argument(
"--source",
type=pathlib.Path,
required=True,
help="Specify the path of the directory where the files originally were stored to update their reference.",
)
parser.add_argument(
"--target",
type=pathlib.Path,
help="Specify a subdirectory in the S3 bucket where you moved the files to.",
)
@transaction.atomic
def handle(self, *args, **options):
if options["dry_run"]:
self.stdout.write("Dry-run on, will not touch the database")
else:
self.stdout.write("Dry-run off, *changing the database*")
self.stdout.write("")
prefix = f"file://{options['source']}"
to_change = models.Upload.objects.filter(source__startswith=prefix)
self.stdout.write(f"Found {to_change.count()} uploads to update.")
target = options.get("target")
if target is None:
target = options["source"]
for upl in to_change:
upl.audio_file = str(upl.source).replace(str(prefix), str(target))
upl.source = None
self.stdout.write(f"Upload expected in {upl.audio_file}")
if not options["dry_run"]:
upl.save()
self.stdout.write("")
if options["dry_run"]:
self.stdout.write(
"Nothing was updated, rerun this command with --no-dry-run to apply the changes"
)
else:
self.stdout.write("Updating completed!")
self.stdout.write("")

4
api/funkwhale_api/common/management/commands/script.py
View file

@ -26,7 +26,7 @@ class Command(BaseCommand):
script = available_scripts[name]
except KeyError:
raise CommandError(
"{} is not a valid script. Run funkwhale-manage for a "
"{} is not a valid script. Run funkwhale-manage script for a "
"list of available scripts".format(name)
)
@ -43,7 +43,7 @@ class Command(BaseCommand):
def show_help(self):
self.stdout.write("")
self.stdout.write("Available scripts:")
self.stdout.write("Launch with: funkwhale-manage <script_name>")
self.stdout.write("Launch with: funkwhale-manage script <script_name>")
available_scripts = self.get_scripts()
for name, script in sorted(available_scripts.items()):
self.stdout.write("")

43
api/funkwhale_api/common/management/commands/testdata.py Normal file
View file

@ -0,0 +1,43 @@
from django.core.management.commands.migrate import Command as BaseCommand
from funkwhale_api.federation import factories
from funkwhale_api.federation.models import Actor
class Command(BaseCommand):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.help = "Helper to generate randomized testdata"
self.type_choices = {"notifications": self.handle_notifications}
self.missing_args_message = f"Please specify one of the following sub-commands: { *self.type_choices.keys(), }"
def add_arguments(self, parser):
subparsers = parser.add_subparsers(dest="subcommand")
notification_parser = subparsers.add_parser("notifications")
notification_parser.add_argument(
"username", type=str, help="Username to send the notifications to"
)
notification_parser.add_argument(
"--count", type=int, help="Number of elements to create", default=1
)
def handle(self, *args, **options):
self.type_choices[options["subcommand"]](options)
def handle_notifications(self, options):
self.stdout.write(
f"Create {options['count']} notification(s) for {options['username']}"
)
try:
actor = Actor.objects.get(preferred_username=options["username"])
except Actor.DoesNotExist:
self.stdout.write(
"The user you want to create notifications for does not exist"
)
return
follow_activity = factories.ActivityFactory(type="Follow")
for _ in range(options["count"]):
factories.InboxItemFactory(actor=actor, activity=follow_activity)

4
api/funkwhale_api/common/middleware.py
View file

@ -150,7 +150,9 @@ def get_default_head_tags(path):
{
"tag": "meta",
"property": "og:image",
"content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
"content": utils.join_url(
settings.FUNKWHALE_URL, "/android-chrome-512x512.png"
),
},
{
"tag": "meta",

2
api/funkwhale_api/common/serializers.py
View file

@ -349,7 +349,7 @@ class ScopesSerializer(serializers.Serializer):
class IdentSerializer(serializers.Serializer):
type = serializers.CharField()
id = serializers.IntegerField()
id = serializers.CharField()
class RateLimitSerializer(serializers.Serializer):

2
api/funkwhale_api/common/throttling.py
View file

@ -7,7 +7,7 @@ from rest_framework import throttling as rest_throttling
def get_ident(user, request):
if user and user.is_authenticated:
return {"type": "authenticated", "id": user.pk}
return {"type": "authenticated", "id": f"{user.pk}"}
ident = rest_throttling.BaseThrottle().get_ident(request)
return {"type": "anonymous", "id": ident}

9
api/funkwhale_api/common/utils.py
View file

@ -477,14 +477,13 @@ def monkey_patch_request_build_absolute_uri():
def get_file_hash(file, algo=None, chunk_size=None, full_read=False):
algo = algo or settings.HASHING_ALGORITHM
chunk_size = chunk_size or settings.HASHING_CHUNK_SIZE
handler = getattr(hashlib, algo)
hash = handler()
hasher = hashlib.new(algo)
file.seek(0)
if full_read:
for byte_block in iter(lambda: file.read(chunk_size), b""):
hash.update(byte_block)
hasher.update(byte_block)
else:
# sometimes, it's useful to only hash the beginning of the file, e.g
# to avoid a lot of I/O when crawling large libraries
hash.update(file.read(chunk_size))
return f"{algo}:{hash.hexdigest()}"
hasher.update(file.read(chunk_size))
return f"{algo}:{hasher.hexdigest()}"

19
api/funkwhale_api/contrib/maloja/funkwhale_ready.py
View file

@ -17,31 +17,40 @@ def submit_listen(listening, conf, **kwargs):
return
logger = PLUGIN["logger"]
logger.info("Submitting listening to Majola at %s", server_url)
payload = get_payload(listening, api_key)
logger.debug("Majola payload: %r", payload)
logger.info("Submitting listening to Maloja at %s", server_url)
payload = get_payload(listening, api_key, conf)
logger.debug("Maloja payload: %r", payload)
url = server_url.rstrip("/") + "/apis/mlj_1/newscrobble"
session = plugins.get_session()
response = session.post(url, json=payload)
response.raise_for_status()
details = json.loads(response.text)
if details["status"] == "success":
logger.info("Majola listening submitted successfully")
logger.info("Maloja listening submitted successfully")
else:
raise MalojaException(response.text)
def get_payload(listening, api_key):
def get_payload(listening, api_key, conf):
track = listening.track
# See https://github.com/krateng/maloja/blob/master/API.md
payload = {
"key": api_key,
"artists": [track.artist.name],
"title": track.title,
"time": int(listening.creation_date.timestamp()),
"nofix": bool(conf.get("nofix")),
}
if track.album:
if track.album.title:
payload["album"] = track.album.title
if track.album.artist:
payload["albumartists"] = [track.album.artist.name]
upload = track.uploads.filter(duration__gte=0).first()
if upload:
payload["length"] = upload.duration
return payload

8
api/funkwhale_api/contrib/maloja/funkwhale_startup.py
View file

@ -5,10 +5,16 @@ PLUGIN = plugins.get_plugin_config(
label="Maloja",
description="A plugin that allows you to submit your listens to your Maloja server.",
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#maloja-plugin",
version="0.1.1",
version="0.2",
user=True,
conf=[
{"name": "server_url", "type": "text", "label": "Maloja server URL"},
{"name": "api_key", "type": "text", "label": "Your Maloja API key"},
{
"name": "nofix",
"type": "boolean",
"label": "Skip server-side metadata fixing",
"default": False,
},
],
)

9
api/funkwhale_api/federation/signing.py
View file

@ -1,8 +1,8 @@
import datetime
import logging
import sys
import cryptography.exceptions
import pytz
import requests
import requests_http_message_signatures
from django import forms
@ -11,6 +11,11 @@ from django.utils.http import parse_http_date
from . import exceptions, utils
if sys.version_info < (3, 9):
from backports.zoneinfo import ZoneInfo
else:
from zoneinfo import ZoneInfo
logger = logging.getLogger(__name__)
# the request Date should be between now - 30s and now + 30s
@ -26,7 +31,7 @@ def verify_date(raw_date):
except ValueError as e:
raise forms.ValidationError(str(e))
dt = datetime.datetime.utcfromtimestamp(ts)
dt = dt.replace(tzinfo=pytz.utc)
dt = dt.replace(tzinfo=ZoneInfo("UTC"))
delta = datetime.timedelta(seconds=DATE_HEADER_VALID_FOR)
now = timezone.now()
if dt < now - delta or dt > now + delta:

16
api/funkwhale_api/instance/dynamic_preferences_registry.py
View file

@ -1,3 +1,4 @@
import pycountry
from django.core.validators import FileExtensionValidator
from django.forms import widgets
from dynamic_preferences import types
@ -170,3 +171,18 @@ class Banner(ImagePreference):
default = None
help_text = "This banner will be displayed on your pod's landing and about page. At least 600x100px recommended."
field_kwargs = {"required": False}
@global_preferences_registry.register
class Location(types.ChoicePreference):
show_in_api = True
section = instance
name = "location"
verbose_name = "Server Location"
default = ""
choices = [(country.alpha_2, country.name) for country in pycountry.countries]
help_text = (
"The country or territory in which your server is located. This is displayed in the server's Nodeinfo "
"endpoint."
)
field_kwargs = {"choices": choices, "required": False}

120
api/funkwhale_api/instance/serializers.py
View file

@ -12,6 +12,17 @@ class SoftwareSerializer(serializers.Serializer):
return "funkwhale"
class SoftwareSerializer_v2(SoftwareSerializer):
repository = serializers.SerializerMethodField()
homepage = serializers.SerializerMethodField()
def get_repository(self, obj):
return "https://dev.funkwhale.audio/funkwhale/funkwhale"
def get_homepage(self, obj):
return "https://funkwhale.audio"
class ServicesSerializer(serializers.Serializer):
inbound = serializers.ListField(child=serializers.CharField(), default=[])
outbound = serializers.ListField(child=serializers.CharField(), default=[])
@ -31,6 +42,8 @@ class UsersUsageSerializer(serializers.Serializer):
class UsageSerializer(serializers.Serializer):
users = UsersUsageSerializer()
localPosts = serializers.IntegerField(required=False)
localComments = serializers.IntegerField(required=False)
class TotalCountSerializer(serializers.Serializer):
@ -92,19 +105,14 @@ class MetadataSerializer(serializers.Serializer):
private = serializers.SerializerMethodField()
shortDescription = serializers.SerializerMethodField()
longDescription = serializers.SerializerMethodField()
rules = serializers.SerializerMethodField()
contactEmail = serializers.SerializerMethodField()
terms = serializers.SerializerMethodField()
nodeName = serializers.SerializerMethodField()
banner = serializers.SerializerMethodField()
defaultUploadQuota = serializers.SerializerMethodField()
library = serializers.SerializerMethodField()
supportedUploadExtensions = serializers.ListField(child=serializers.CharField())
allowList = serializers.SerializerMethodField()
reportTypes = ReportTypeSerializer(source="report_types", many=True)
funkwhaleSupportMessageEnabled = serializers.SerializerMethodField()
instanceSupportMessage = serializers.SerializerMethodField()
endpoints = EndpointsSerializer()
usage = MetadataUsageSerializer(source="stats", required=False)
def get_private(self, obj) -> bool:
@ -116,15 +124,9 @@ class MetadataSerializer(serializers.Serializer):
def get_longDescription(self, obj) -> str:
return obj["preferences"].get("instance__long_description")
def get_rules(self, obj) -> str:
return obj["preferences"].get("instance__rules")
def get_contactEmail(self, obj) -> str:
return obj["preferences"].get("instance__contact_email")
def get_terms(self, obj) -> str:
return obj["preferences"].get("instance__terms")
def get_nodeName(self, obj) -> str:
return obj["preferences"].get("instance__name")
@ -137,15 +139,6 @@ class MetadataSerializer(serializers.Serializer):
def get_defaultUploadQuota(self, obj) -> int:
return obj["preferences"].get("users__upload_quota")
@extend_schema_field(NodeInfoLibrarySerializer)
def get_library(self, obj):
data = obj["stats"] or {}
data["federationEnabled"] = obj["preferences"].get("federation__enabled")
data["anonymousCanListen"] = not obj["preferences"].get(
"common__api_authentication_required"
)
return NodeInfoLibrarySerializer(data).data
@extend_schema_field(AllowListStatSerializer)
def get_allowList(self, obj):
return AllowListStatSerializer(
@ -166,6 +159,62 @@ class MetadataSerializer(serializers.Serializer):
return MetadataUsageSerializer(obj["stats"]).data
class Metadata20Serializer(MetadataSerializer):
library = serializers.SerializerMethodField()
reportTypes = ReportTypeSerializer(source="report_types", many=True)
endpoints = EndpointsSerializer()
rules = serializers.SerializerMethodField()
terms = serializers.SerializerMethodField()
def get_rules(self, obj) -> str:
return obj["preferences"].get("instance__rules")
def get_terms(self, obj) -> str:
return obj["preferences"].get("instance__terms")
@extend_schema_field(NodeInfoLibrarySerializer)
def get_library(self, obj):
data = obj["stats"] or {}
data["federationEnabled"] = obj["preferences"].get("federation__enabled")
data["anonymousCanListen"] = not obj["preferences"].get(
"common__api_authentication_required"
)
return NodeInfoLibrarySerializer(data).data
class MetadataContentLocalSerializer(serializers.Serializer):
artists = serializers.IntegerField()
releases = serializers.IntegerField()
recordings = serializers.IntegerField()
hoursOfContent = serializers.IntegerField()
class MetadataContentCategorySerializer(serializers.Serializer):
name = serializers.CharField()
count = serializers.IntegerField()
class MetadataContentSerializer(serializers.Serializer):
local = MetadataContentLocalSerializer()
topMusicCategories = MetadataContentCategorySerializer(many=True)
topPodcastCategories = MetadataContentCategorySerializer(many=True)
class Metadata21Serializer(MetadataSerializer):
languages = serializers.ListField(child=serializers.CharField())
location = serializers.CharField()
content = MetadataContentSerializer()
features = serializers.ListField(child=serializers.CharField())
codeOfConduct = serializers.SerializerMethodField()
def get_codeOfConduct(self, obj) -> str:
return (
full_url("/about/pod#rules")
if obj["preferences"].get("instance__rules")
else ""
)
class NodeInfo20Serializer(serializers.Serializer):
version = serializers.SerializerMethodField()
software = SoftwareSerializer()
@ -196,9 +245,36 @@ class NodeInfo20Serializer(serializers.Serializer):
usage = {"users": {"total": 0, "activeMonth": 0, "activeHalfyear": 0}}
return UsageSerializer(usage).data
@extend_schema_field(MetadataSerializer)
@extend_schema_field(Metadata20Serializer)
def get_metadata(self, obj):
return MetadataSerializer(obj).data
return Metadata20Serializer(obj).data
class NodeInfo21Serializer(NodeInfo20Serializer):
version = serializers.SerializerMethodField()
software = SoftwareSerializer_v2()
def get_version(self, obj) -> str:
return "2.1"
@extend_schema_field(UsageSerializer)
def get_usage(self, obj):
usage = None
if obj["preferences"]["instance__nodeinfo_stats_enabled"]:
usage = obj["stats"]
usage["localPosts"] = 0
usage["localComments"] = 0
else:
usage = {
"users": {"total": 0, "activeMonth": 0, "activeHalfyear": 0},
"localPosts": 0,
"localComments": 0,
}
return UsageSerializer(usage).data
@extend_schema_field(Metadata21Serializer)
def get_metadata(self, obj):
return Metadata21Serializer(obj).data
class SpaManifestIconSerializer(serializers.Serializer):

35
api/funkwhale_api/instance/stats.py
View file

@ -1,6 +1,6 @@
import datetime
from django.db.models import Sum
from django.db.models import Count, F, Sum
from django.utils import timezone
from funkwhale_api.favorites.models import TrackFavorite
@ -22,6 +22,39 @@ def get():
}
def get_content():
return {
"local": {
"artists": get_artists(),
"releases": get_albums(),
"recordings": get_tracks(),
"hoursOfContent": get_music_duration(),
},
"topMusicCategories": get_top_music_categories(),
"topPodcastCategories": get_top_podcast_categories(),
}
def get_top_music_categories():
return (
models.Track.objects.filter(artist__content_category="music")
.exclude(tagged_items__tag_id=None)
.values(name=F("tagged_items__tag__name"))
.annotate(count=Count("name"))
.order_by("-count")[:3]
)
def get_top_podcast_categories():
return (
models.Track.objects.filter(artist__content_category="podcast")
.exclude(tagged_items__tag_id=None)
.values(name=F("tagged_items__tag__name"))
.annotate(count=Count("name"))
.order_by("-count")[:3]
)
def get_users():
qs = User.objects.filter(is_active=True)
now = timezone.now()

2
api/funkwhale_api/instance/urls.py
View file

@ -8,7 +8,7 @@ admin_router = routers.OptionalSlashRouter()
admin_router.register(r"admin/settings", views.AdminSettings, "admin-settings")
urlpatterns = [
url(r"^nodeinfo/2.0/?$", views.NodeInfo.as_view(), name="nodeinfo-2.0"),
url(r"^nodeinfo/2.0/?$", views.NodeInfo20.as_view(), name="nodeinfo-2.0"),
url(r"^settings/?$", views.InstanceSettings.as_view(), name="settings"),
url(r"^spa-manifest.json", views.SpaManifest.as_view(), name="spa-manifest"),
] + admin_router.urls

7
api/funkwhale_api/instance/urls_v2.py Normal file
View file

@ -0,0 +1,7 @@
from django.conf.urls import url
from . import views
urlpatterns = [
url(r"^nodeinfo/2.1/?$", views.NodeInfo21.as_view(), name="nodeinfo-2.1"),
]

63
api/funkwhale_api/instance/views.py
View file

@ -11,6 +11,7 @@ from dynamic_preferences.api import viewsets as preferences_viewsets
from dynamic_preferences.api.serializers import GlobalPreferenceSerializer
from dynamic_preferences.registries import global_preferences_registry
from rest_framework import generics, views
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from funkwhale_api import __version__ as funkwhale_version
@ -58,9 +59,11 @@ class InstanceSettings(generics.GenericAPIView):
@method_decorator(ensure_csrf_cookie, name="dispatch")
class NodeInfo(views.APIView):
class NodeInfo20(views.APIView):
permission_classes = []
authentication_classes = []
serializer_class = serializers.NodeInfo20Serializer
renderer_classes = (JSONRenderer,)
@extend_schema(
responses=serializers.NodeInfo20Serializer, operation_id="getNodeInfo20"
@ -81,6 +84,7 @@ class NodeInfo(views.APIView):
data = {
"software": {"version": funkwhale_version},
"services": {"inbound": ["atom1.0"], "outbound": ["atom1.0"]},
"preferences": pref,
"stats": cache_memoize(600, prefix="memoize:instance:stats")(stats.get)()
if pref["instance__nodeinfo_stats_enabled"]
@ -112,7 +116,62 @@ class NodeInfo(views.APIView):
data["endpoints"]["channels"] = reverse(
"federation:index:index-channels"
)
serializer = serializers.NodeInfo20Serializer(data)
serializer = self.serializer_class(data)
return Response(
serializer.data, status=200, content_type=NODEINFO_2_CONTENT_TYPE
)
class NodeInfo21(NodeInfo20):
serializer_class = serializers.NodeInfo21Serializer
@extend_schema(
responses=serializers.NodeInfo20Serializer, operation_id="getNodeInfo20"
)
def get(self, request):
pref = preferences.all()
if (
pref["moderation__allow_list_public"]
and pref["moderation__allow_list_enabled"]
):
allowed_domains = list(
Domain.objects.filter(allowed=True)
.order_by("name")
.values_list("name", flat=True)
)
else:
allowed_domains = None
data = {
"software": {"version": funkwhale_version},
"services": {"inbound": ["atom1.0"], "outbound": ["atom1.0"]},
"preferences": pref,
"stats": cache_memoize(600, prefix="memoize:instance:stats")(stats.get)()
if pref["instance__nodeinfo_stats_enabled"]
else None,
"actorId": get_service_actor().fid,
"supportedUploadExtensions": SUPPORTED_EXTENSIONS,
"allowed_domains": allowed_domains,
"languages": pref.get("moderation__languages"),
"location": pref.get("instance__location"),
"content": cache_memoize(600, prefix="memoize:instance:content")(
stats.get_content
)()
if pref["instance__nodeinfo_stats_enabled"]
else None,
"features": [
"channels",
"podcasts",
],
}
if not pref.get("common__api_authentication_required"):
data["features"].append("anonymousCanListen")
if pref.get("federation__enabled"):
data["features"].append("federation")
serializer = self.serializer_class(data)
return Response(
serializer.data, status=200, content_type=NODEINFO_2_CONTENT_TYPE
)

16
api/funkwhale_api/moderation/dynamic_preferences_registry.py
View file

@ -1,3 +1,4 @@
import pycountry
from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry
from rest_framework import serializers
@ -92,3 +93,18 @@ class SignupFormCustomization(common_preferences.SerializedPreference):
required = False
default = {}
data_serializer_class = CustomFormSerializer
@global_preferences_registry.register
class Languages(common_preferences.StringListPreference):
show_in_api = True
section = moderation
name = "languages"
default = ["en"]
verbose_name = "Moderation languages"
help_text = (
"The language(s) spoken by the server moderator(s). Set this to inform users "
"what languages they should write reports and requests in."
)
choices = [(lang.alpha_3, lang.name) for lang in pycountry.languages]
field_kwargs = {"choices": choices, "required": False}

15
api/funkwhale_api/music/dynamic_preferences_registry.py
View file

@ -32,3 +32,18 @@ class MusicCacheDuration(types.IntPreference):
"will be erased and retranscoded on the next listening."
)
field_kwargs = {"required": False}
@global_preferences_registry.register
class MbidTaggedContent(types.BooleanPreference):
show_in_api = True
section = music
name = "only_allow_musicbrainz_tagged_files"
verbose_name = "Only allow Musicbrainz tagged files"
help_text = (
"Requires uploaded files to be tagged with a MusicBrainz ID. "
"Enabling this setting has no impact on previously uploaded files. "
"You can use the CLI to clear files that don't contain an MBID or "
"or enable quality filtering to hide untagged content from API calls. "
)
default = False

7
api/funkwhale_api/music/factories.py
View file

@ -151,8 +151,9 @@ class TrackFactory(
if created:
self.save()
@factory.post_generation
def license(self, created, extracted, **kwargs):
# The @factory.post_generation is not used because we must
# not redefine the builtin `license` function.
def _license_post_generation(self, created, extracted, **kwargs):
if not created:
return
@ -160,6 +161,8 @@ class TrackFactory(
self.license = LicenseFactory(code=extracted)
self.save()
license = factory.PostGeneration(_license_post_generation)
@registry.register
class UploadFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):

5
api/funkwhale_api/music/fake_data.py
View file

@ -15,7 +15,10 @@ def create_data(count=25):
)
for album in albums:
factories.UploadFactory.create_batch(
track__album=album, size=random.randint(3, 18)
track__album=album,
size=random.randint(3, 18),
playable=True,
in_place=True,
)

2
api/funkwhale_api/music/filters.py
View file

@ -104,7 +104,7 @@ class ArtistFilter(
distinct=True,
library_field="tracks__uploads__library",
)
ordering = django_filters.OrderingFilter(
ordering = common_filters.CaseInsensitiveNameOrderingFilter(
fields=(
("id", "id"),
("name", "name"),

16
api/funkwhale_api/music/licenses.py
View file

@ -28,7 +28,7 @@ def load(data):
for row in data:
try:
license = existing_by_code[row["code"]]
license_ = existing_by_code[row["code"]]
except KeyError:
logger.debug("Loading new license: {}".format(row["code"]))
to_create.append(
@ -36,15 +36,15 @@ def load(data):
)
else:
logger.debug("Updating license: {}".format(row["code"]))
stored = [getattr(license, f) for f in MODEL_FIELDS]
stored = [getattr(license_, f) for f in MODEL_FIELDS]
wanted = [row[f] for f in MODEL_FIELDS]
if wanted == stored:
continue
# the object in database needs an update
for f in MODEL_FIELDS:
setattr(license, f, row[f])
setattr(license_, f, row[f])
license.save()
license_.save()
models.License.objects.bulk_create(to_create)
return sorted(models.License.objects.all(), key=lambda o: o.code)
@ -78,12 +78,12 @@ def match(*values):
else:
existing = load(LICENSES)
_cache = existing
for license in existing:
if license.conf is None:
for license_ in existing:
if license_.conf is None:
continue
for i in license.conf["identifiers"]:
for i in license_.conf["identifiers"]:
if match_urls(url, i):
return license
return license_
def match_urls(*urls):

13
api/funkwhale_api/music/management/commands/generate_typesense_index.py Normal file
View file

@ -0,0 +1,13 @@
from django.core.management.base import BaseCommand
from funkwhale_api.typesense import tasks
class Command(BaseCommand):
help = """
Trigger the generation of a new typesense index for canonical Funkwhale tracks metadata.
This is use to resolve Funkwhale tracks to MusicBrainz ids"""
def handle(self, *args, **kwargs):
tasks.build_canonical_index.delay()
self.stdout.write("Tasks launched in celery worker.")

7
api/funkwhale_api/music/tasks.py
View file

@ -247,6 +247,13 @@ def process_upload(upload, update_denormalization=True):
return fail_import(
upload, "invalid_metadata", detail=detail, file_metadata=metadata_dump
)
check_mbid = preferences.get("music__only_allow_musicbrainz_tagged_files")
if check_mbid and not serializer.validated_data.get("mbid"):
return fail_import(
upload,
"Only content tagged with a MusicBrainz ID is permitted on this pod.",
detail="You can tag your files with MusicBrainz Picard",
)
final_metadata = collections.ChainMap(
additional_data, serializer.validated_data, internal_config

2
api/funkwhale_api/music/views.py
View file

@ -583,7 +583,7 @@ def handle_serve(
try:
f.download_audio_from_remote(actor=actor)
except requests.exceptions.RequestException:
return Response({"detail": "Remove track is unavailable"}, status=503)
return Response({"detail": "Remote track is unavailable"}, status=503)
data = f.get_audio_data()
if data:
f.duration = data["duration"]

148
api/funkwhale_api/radios/lb_recommendations.py Normal file
View file

@ -0,0 +1,148 @@
import logging
import time
import troi
import troi.core
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.db.models import Q
from requests.exceptions import ConnectTimeout
from funkwhale_api.music import models as music_models
from funkwhale_api.typesense import utils
logger = logging.getLogger(__name__)
patches = troi.utils.discover_patches()
SUPPORTED_PATCHES = patches.keys()
def run(config, **kwargs):
"""Validate the received config and run the queryset generation"""
candidates = kwargs.pop("candidates", music_models.Track.objects.all())
validate(config)
return TroiPatch().get_queryset(config, candidates)
def validate(config):
patch = config.get("patch")
if patch not in SUPPORTED_PATCHES:
raise ValidationError(
'Invalid patch "{}". Supported patches: {}'.format(
config["patch"], SUPPORTED_PATCHES
)
)
return True
def build_radio_queryset(patch, config, radio_qs):
"""Take a troi patch and its arg, match the missing mbid and then build a radio queryset"""
logger.info("Config used for troi radio generation is " + str(config))
start_time = time.time()
try:
recommendations = troi.core.generate_playlist(patch, config)
except ConnectTimeout:
raise ValueError(
"Timed out while connecting to ListenBrainz. No candidates could be retrieved for the radio."
)
end_time_rec = time.time()
logger.info("Troi fetch took :" + str(end_time_rec - start_time))
if not recommendations:
raise ValueError("No candidates found by troi")
recommended_mbids = [
recommended_recording.mbid
for recommended_recording in recommendations.playlists[0].recordings
]
logger.info("Searching for MusicBrainz ID in Funkwhale database")
qs_recommended = (
music_models.Track.objects.all()
.filter(mbid__in=recommended_mbids)
.order_by("mbid", "pk")
.distinct("mbid")
)
qs_recommended_mbid = [str(i.mbid) for i in qs_recommended]
recommended_mbids_not_qs = [
mbid for mbid in recommended_mbids if mbid not in qs_recommended_mbid
]
cached_match = cache.get_many(recommended_mbids_not_qs)
cached_match_mbid = [str(i) for i in cached_match.keys()]
if qs_recommended and cached_match_mbid:
logger.info("MusicBrainz IDs found in Funkwhale database and redis")
qs_recommended_mbid.extend(cached_match_mbid)
mbids_found = qs_recommended_mbid
elif qs_recommended and not cached_match_mbid:
logger.info("MusicBrainz IDs found in Funkwhale database")
mbids_found = qs_recommended_mbid
elif not qs_recommended and cached_match_mbid:
logger.info("MusicBrainz IDs found in redis cache")
mbids_found = cached_match_mbid
else:
logger.info(
"Couldn't find any matches in Funkwhale database. Trying to match all"
)
mbids_found = []
recommended_recordings_not_found = [
i for i in recommendations.playlists[0].recordings if i.mbid not in mbids_found
]
logger.info("Matching missing MusicBrainz ID to Funkwhale track")
start_time_resolv = time.time()
utils.resolve_recordings_to_fw_track(recommended_recordings_not_found)
end_time_resolv = time.time()
logger.info(
"Resolving "
+ str(len(recommended_recordings_not_found))
+ " tracks in "
+ str(end_time_resolv - start_time_resolv)
)
cached_match = cache.get_many(recommended_mbids)
if not mbids_found and not cached_match:
raise ValueError("No candidates found for troi radio")
mbids_found_pks = list(
music_models.Track.objects.all()
.filter(mbid__in=mbids_found)
.order_by("mbid", "pk")
.distinct("mbid")
.values_list("pk", flat=True)
)
mbids_found_pks_unique = [
i for i in mbids_found_pks if i not in cached_match.keys()
]
if mbids_found and cached_match:
return radio_qs.filter(
Q(pk__in=mbids_found_pks_unique) | Q(pk__in=cached_match.values())
)
if mbids_found and not cached_match:
return radio_qs.filter(pk__in=mbids_found_pks_unique)
if not mbids_found and cached_match:
return radio_qs.filter(pk__in=cached_match.values())
class TroiPatch:
code = "troi-patch"
label = "Troi Patch"
def get_queryset(self, config, qs):
patch_string = config.pop("patch")
patch = patches[patch_string]
return build_radio_queryset(patch(), config, qs)

28
api/funkwhale_api/radios/models.py
View file

@ -54,10 +54,6 @@ class RadioSession(models.Model):
CONFIG_VERSION = 0
config = JSONField(encoder=DjangoJSONEncoder, blank=True, null=True)
def save(self, **kwargs):
self.radio.clean(self)
super().save(**kwargs)
@property
def next_position(self):
next_position = 1
@ -68,16 +64,24 @@ class RadioSession(models.Model):
return next_position
def add(self, track):
new_session_track = RadioSessionTrack.objects.create(
track=track, session=self, position=self.next_position
)
def add(self, tracks):
next_position = self.next_position
radio_session_tracks = []
for i, track in enumerate(tracks):
radio_session_track = RadioSessionTrack(
track=track, session=self, position=next_position + i
)
radio_session_tracks.append(radio_session_track)
return new_session_track
new_session_tracks = RadioSessionTrack.objects.bulk_create(radio_session_tracks)
@property
def radio(self):
from .registries import registry
return new_session_tracks
def radio(self, api_version):
if api_version == 2:
from .registries_v2 import registry
else:
from .registries import registry
return registry[self.radio_type](session=self)

99
api/funkwhale_api/radios/radios.py
View file

@ -1,4 +1,5 @@
import datetime
import json
import logging
import random
from typing import List, Optional, Tuple
@ -14,7 +15,7 @@ from funkwhale_api.moderation import filters as moderation_filters
from funkwhale_api.music.models import Artist, Library, Track, Upload
from funkwhale_api.tags.models import Tag
from . import filters, models
from . import filters, lb_recommendations, models
from .registries import registry
logger = logging.getLogger(__name__)
@ -61,11 +62,19 @@ class SessionRadio(SimpleRadio):
return self.session
def get_queryset(self, **kwargs):
qs = Track.objects.all()
if not self.session:
return qs
if not self.session.user:
return qs
if not self.session or not self.session.user:
return (
Track.objects.all()
.with_playable_uploads(actor=None)
.select_related("artist", "album__artist", "attributed_to")
)
else:
qs = (
Track.objects.all()
.with_playable_uploads(self.session.user.actor)
.select_related("artist", "album__artist", "attributed_to")
)
query = moderation_filters.get_filtered_content_query(
config=moderation_filters.USER_FILTER_CONFIG["TRACK"],
user=self.session.user,
@ -75,6 +84,16 @@ class SessionRadio(SimpleRadio):
def get_queryset_kwargs(self):
return {}
def filter_queryset(self, queryset):
return queryset
def filter_from_session(self, queryset):
already_played = self.session.session_tracks.all().values_list(
"track", flat=True
)
queryset = queryset.exclude(pk__in=already_played)
return queryset
def get_choices(self, **kwargs):
kwargs.update(self.get_queryset_kwargs())
queryset = self.get_queryset(**kwargs)
@ -87,16 +106,6 @@ class SessionRadio(SimpleRadio):
queryset = self.filter_queryset(queryset)
return queryset
def filter_queryset(self, queryset):
return queryset
def filter_from_session(self, queryset):
already_played = self.session.session_tracks.all().values_list(
"track", flat=True
)
queryset = queryset.exclude(pk__in=already_played)
return queryset
def pick(self, **kwargs):
return self.pick_many(quantity=1, **kwargs)[0]
@ -104,8 +113,7 @@ class SessionRadio(SimpleRadio):
choices = self.get_choices(**kwargs)
picked_choices = super().pick_many(choices=choices, quantity=quantity)
if self.session:
for choice in picked_choices:
self.session.add(choice)
self.session.add(picked_choices)
return picked_choices
def validate_session(self, data, **context):
@ -405,3 +413,58 @@ class RecentlyAdded(SessionRadio):
Q(artist__content_category="music"),
Q(creation_date__gt=date),
)
# Use this to experiment on the custom multiple radio with troi
@registry.register(name="troi")
class Troi(SessionRadio):
"""
Receive a vuejs generated config and use it to launch a troi radio session.
The config data should follow :
{"patch": "troi_patch_name", "troi_arg1":"troi_arg_1", "troi_arg2": ...}
Validation of the config (args) is done by troi during track fetch.
Funkwhale only checks if the patch is implemented
"""
config = serializers.JSONField(required=True)
def append_lb_config(self, data):
if self.session.user.settings is None:
logger.warning(
"No lb_user_name set in user settings. Some troi patches will fail"
)
return data
elif self.session.user.settings.get("lb_user_name") is None:
logger.warning(
"No lb_user_name set in user settings. Some troi patches will fail"
)
else:
data["user_name"] = self.session.user.settings["lb_user_name"]
if self.session.user.settings.get("lb_user_token") is None:
logger.warning(
"No lb_user_token set in user settings. Some troi patch will fail"
)
else:
data["user_token"] = self.session.user.settings["lb_user_token"]
return data
def get_queryset_kwargs(self):
kwargs = super().get_queryset_kwargs()
kwargs["config"] = self.session.config
return kwargs
def validate_session(self, data, **context):
data = super().validate_session(data, **context)
if data.get("config") is None:
raise serializers.ValidationError(
"You must provide a configuration for this radio"
)
return data
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
config = self.append_lb_config(json.loads(kwargs["config"]))
return lb_recommendations.run(config, candidates=qs)

510
api/funkwhale_api/radios/radios_v2.py Normal file
View file

@ -0,0 +1,510 @@
import datetime
import json
import logging
import pickle
import random
from typing import List, Optional, Tuple
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.db import connection
from django.db.models import Q
from rest_framework import serializers
from funkwhale_api.federation import fields as federation_fields
from funkwhale_api.federation import models as federation_models
from funkwhale_api.moderation import filters as moderation_filters
from funkwhale_api.music.models import Artist, Library, Track, Upload
from funkwhale_api.tags.models import Tag
from . import filters, lb_recommendations, models
from .registries_v2 import registry
logger = logging.getLogger(__name__)
class SimpleRadio:
related_object_field = None
def clean(self, instance):
return
def weighted_pick(
self,
choices: List[Tuple[int, int]],
previous_choices: Optional[List[int]] = None,
) -> int:
total = sum(weight for c, weight in choices)
r = random.uniform(0, total)
upto = 0
for choice, weight in choices:
if upto + weight >= r:
return choice
upto += weight
class SessionRadio(SimpleRadio):
def __init__(self, session=None):
self.session = session
def start_session(self, user, **kwargs):
self.session = models.RadioSession.objects.create(
user=user, radio_type=self.radio_type, **kwargs
)
return self.session
def get_queryset(self, **kwargs):
actor = None
try:
actor = self.session.user.actor
except KeyError:
pass # Maybe logging would be helpful
qs = (
Track.objects.all()
.with_playable_uploads(actor=actor)
.select_related("artist", "album__artist", "attributed_to")
)
query = moderation_filters.get_filtered_content_query(
config=moderation_filters.USER_FILTER_CONFIG["TRACK"],
user=self.session.user,
)
return qs.exclude(query)
def get_queryset_kwargs(self):
return {}
def filter_queryset(self, queryset):
return queryset
def filter_from_session(self, queryset):
already_played = self.session.session_tracks.all().values_list(
"track", flat=True
)
queryset = queryset.exclude(pk__in=already_played)
return queryset
def cache_batch_radio_track(self, **kwargs):
BATCH_SIZE = 100
# get cached RadioTracks if any
try:
cached_evaluated_radio_tracks = pickle.loads(
cache.get(f"radiotracks{self.session.id}")
)
except TypeError:
cached_evaluated_radio_tracks = None
# get the queryset and apply filters
kwargs.update(self.get_queryset_kwargs())
queryset = self.get_queryset(**kwargs)
queryset = self.filter_from_session(queryset)
if kwargs["filter_playable"] is True:
queryset = queryset.playable_by(
self.session.user.actor if self.session.user else None
)
queryset = self.filter_queryset(queryset)
# select a random batch of the qs
sliced_queryset = queryset.order_by("?")[:BATCH_SIZE]
if len(sliced_queryset) <= 0 and not cached_evaluated_radio_tracks:
raise ValueError("No more radio candidates")
# create the radio session tracks into db in bulk
self.session.add(sliced_queryset)
# evaluate the queryset to save it in cache
radio_tracks = list(sliced_queryset)
if cached_evaluated_radio_tracks is not None:
radio_tracks.extend(cached_evaluated_radio_tracks)
logger.info(
f"Setting redis cache for radio generation with radio id {self.session.id}"
)
cache.set(f"radiotracks{self.session.id}", pickle.dumps(radio_tracks), 3600)
cache.set(f"radioqueryset{self.session.id}", sliced_queryset, 3600)
return sliced_queryset
def get_choices(self, quantity, **kwargs):
if cache.get(f"radiotracks{self.session.id}"):
cached_radio_tracks = pickle.loads(
cache.get(f"radiotracks{self.session.id}")
)
logger.info("Using redis cache for radio generation")
radio_tracks = cached_radio_tracks
if len(radio_tracks) < quantity:
logger.info(
"Not enough radio tracks in cache. Trying to generate new cache"
)
sliced_queryset = self.cache_batch_radio_track(**kwargs)
sliced_queryset = cache.get(f"radioqueryset{self.session.id}")
else:
sliced_queryset = self.cache_batch_radio_track(**kwargs)
return sliced_queryset[:quantity]
def pick_many(self, quantity, **kwargs):
if self.session:
sliced_queryset = self.get_choices(quantity=quantity, **kwargs)
else:
logger.info(
"No radio session. Can't track user playback. Won't cache queryset results"
)
sliced_queryset = self.get_choices(quantity=quantity, **kwargs)
return sliced_queryset
def validate_session(self, data, **context):
return data
@registry.register(name="random")
class RandomRadio(SessionRadio):
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return qs.filter(artist__content_category="music").order_by("?")
@registry.register(name="random_library")
class RandomLibraryRadio(SessionRadio):
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
tracks_ids = self.session.user.actor.attributed_tracks.all().values_list(
"id", flat=True
)
query = Q(artist__content_category="music") & Q(pk__in=tracks_ids)
return qs.filter(query).order_by("?")
@registry.register(name="favorites")
class FavoritesRadio(SessionRadio):
def get_queryset_kwargs(self):
kwargs = super().get_queryset_kwargs()
if self.session:
kwargs["user"] = self.session.user
return kwargs
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
track_ids = kwargs["user"].track_favorites.all().values_list("track", flat=True)
return qs.filter(pk__in=track_ids, artist__content_category="music")
@registry.register(name="custom")
class CustomRadio(SessionRadio):
def get_queryset_kwargs(self):
kwargs = super().get_queryset_kwargs()
kwargs["user"] = self.session.user
kwargs["custom_radio"] = self.session.custom_radio
return kwargs
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return filters.run(kwargs["custom_radio"].config, candidates=qs)
def validate_session(self, data, **context):
data = super().validate_session(data, **context)
try:
user = data["user"]
except KeyError:
user = context.get("user")
try:
assert data["custom_radio"].user == user or data["custom_radio"].is_public
except KeyError:
raise serializers.ValidationError("You must provide a custom radio")
except AssertionError:
raise serializers.ValidationError("You don't have access to this radio")
return data
@registry.register(name="custom_multiple")
class CustomMultiple(SessionRadio):
"""
Receive a vuejs generated config and use it to launch a radio session
"""
config = serializers.JSONField(required=True)
def get_config(self, data):
return data["config"]
def get_queryset_kwargs(self):
kwargs = super().get_queryset_kwargs()
kwargs["config"] = self.session.config
return kwargs
def validate_session(self, data, **context):
data = super().validate_session(data, **context)
try:
data["config"] is not None
except KeyError:
raise serializers.ValidationError(
"You must provide a configuration for this radio"
)
return data
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return filters.run([kwargs["config"]], candidates=qs)
class RelatedObjectRadio(SessionRadio):
"""Abstract radio related to an object (tag, artist, user...)"""
related_object_field = serializers.IntegerField(required=True)
def clean(self, instance):
super().clean(instance)
if not instance.related_object:
raise ValidationError(
"Cannot start RelatedObjectRadio without related object"
)
if not isinstance(instance.related_object, self.model):
raise ValidationError("Trying to start radio with bad related object")
def get_related_object(self, pk):
return self.model.objects.get(pk=pk)
@registry.register(name="tag")
class TagRadio(RelatedObjectRadio):
model = Tag
related_object_field = serializers.CharField(required=True)
def get_related_object(self, name):
return self.model.objects.get(name=name)
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
query = (
Q(tagged_items__tag=self.session.related_object)
| Q(artist__tagged_items__tag=self.session.related_object)
| Q(album__tagged_items__tag=self.session.related_object)
)
return qs.filter(query)
def get_related_object_id_repr(self, obj):
return obj.name
def weighted_choice(choices):
total = sum(w for c, w in choices)
r = random.uniform(0, total)
upto = 0
for c, w in choices:
if upto + w >= r:
return c
upto += w
assert False, "Shouldn't get here"
class NextNotFound(Exception):
pass
@registry.register(name="similar")
class SimilarRadio(RelatedObjectRadio):
model = Track
def filter_queryset(self, queryset):
queryset = super().filter_queryset(queryset)
seeds = list(
self.session.session_tracks.all()
.values_list("track_id", flat=True)
.order_by("-id")[:3]
) + [self.session.related_object.pk]
for seed in seeds:
try:
return queryset.filter(pk=self.find_next_id(queryset, seed))
except NextNotFound:
continue
return queryset.none()
def find_next_id(self, queryset, seed):
with connection.cursor() as cursor:
query = """
SELECT next, count(next) AS c
FROM (
SELECT
track_id,
creation_date,
LEAD(track_id) OVER (
PARTITION by user_id order by creation_date asc
) AS next
FROM history_listening
INNER JOIN users_user ON (users_user.id = user_id)
WHERE users_user.privacy_level = 'instance' OR users_user.privacy_level = 'everyone' OR user_id = %s
ORDER BY creation_date ASC
) t WHERE track_id = %s AND next != %s GROUP BY next ORDER BY c DESC;
"""
cursor.execute(query, [self.session.user_id, seed, seed])
next_candidates = list(cursor.fetchall())
if not next_candidates:
raise NextNotFound()
matching_tracks = list(
queryset.filter(pk__in=[c[0] for c in next_candidates]).values_list(
"id", flat=True
)
)
next_candidates = [n for n in next_candidates if n[0] in matching_tracks]
if not next_candidates:
raise NextNotFound()
return random.choice([c[0] for c in next_candidates])
@registry.register(name="artist")
class ArtistRadio(RelatedObjectRadio):
model = Artist
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return qs.filter(artist=self.session.related_object)
@registry.register(name="less-listened")
class LessListenedRadio(SessionRadio):
def clean(self, instance):
instance.related_object = instance.user
super().clean(instance)
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
listened = self.session.user.listenings.all().values_list("track", flat=True)
return (
qs.filter(artist__content_category="music")
.exclude(pk__in=listened)
.order_by("?")
)
@registry.register(name="less-listened_library")
class LessListenedLibraryRadio(SessionRadio):
def clean(self, instance):
instance.related_object = instance.user
super().clean(instance)
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
listened = self.session.user.listenings.all().values_list("track", flat=True)
tracks_ids = self.session.user.actor.attributed_tracks.all().values_list(
"id", flat=True
)
query = Q(artist__content_category="music") & Q(pk__in=tracks_ids)
return qs.filter(query).exclude(pk__in=listened).order_by("?")
@registry.register(name="actor-content")
class ActorContentRadio(RelatedObjectRadio):
"""
Play content from given actor libraries
"""
model = federation_models.Actor
related_object_field = federation_fields.ActorRelatedField(required=True)
def get_related_object(self, value):
return value
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
actor_uploads = Upload.objects.filter(
library__actor=self.session.related_object,
)
return qs.filter(pk__in=actor_uploads.values("track"))
def get_related_object_id_repr(self, obj):
return obj.full_username
@registry.register(name="library")
class LibraryRadio(RelatedObjectRadio):
"""
Play content from a given library
"""
model = Library
related_object_field = serializers.UUIDField(required=True)
def get_related_object(self, value):
return Library.objects.get(uuid=value)
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
actor_uploads = Upload.objects.filter(
library=self.session.related_object,
)
return qs.filter(pk__in=actor_uploads.values("track"))
def get_related_object_id_repr(self, obj):
return obj.uuid
@registry.register(name="recently-added")
class RecentlyAdded(SessionRadio):
def get_queryset(self, **kwargs):
date = datetime.date.today() - datetime.timedelta(days=30)
qs = super().get_queryset(**kwargs)
return qs.filter(
Q(artist__content_category="music"),
Q(creation_date__gt=date),
)
# Use this to experiment on the custom multiple radio with troi
@registry.register(name="troi")
class Troi(SessionRadio):
"""
Receive a vuejs generated config and use it to launch a troi radio session.
The config data should follow :
{"patch": "troi_patch_name", "troi_arg1":"troi_arg_1", "troi_arg2": ...}
Validation of the config (args) is done by troi during track fetch.
Funkwhale only checks if the patch is implemented
"""
config = serializers.JSONField(required=True)
def append_lb_config(self, data):
if self.session.user.settings is None:
logger.warning(
"No lb_user_name set in user settings. Some troi patches will fail"
)
return data
elif self.session.user.settings.get("lb_user_name") is None:
logger.warning(
"No lb_user_name set in user settings. Some troi patches will fail"
)
else:
data["user_name"] = self.session.user.settings["lb_user_name"]
if self.session.user.settings.get("lb_user_token") is None:
logger.warning(
"No lb_user_token set in user settings. Some troi patch will fail"
)
else:
data["user_token"] = self.session.user.settings["lb_user_token"]
return data
def get_queryset_kwargs(self):
kwargs = super().get_queryset_kwargs()
kwargs["config"] = self.session.config
return kwargs
def validate_session(self, data, **context):
data = super().validate_session(data, **context)
if data.get("config") is None:
raise serializers.ValidationError(
"You must provide a configuration for this radio"
)
return data
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
config = self.append_lb_config(json.loads(kwargs["config"]))
return lb_recommendations.run(config, candidates=qs)

10
api/funkwhale_api/radios/registries_v2.py Normal file
View file

@ -0,0 +1,10 @@
import persisting_theory
class RadioRegistry_v2(persisting_theory.Registry):
def prepare_name(self, data, name=None):
setattr(data, "radio_type", name)
return name
registry = RadioRegistry_v2()

4
api/funkwhale_api/radios/serializers.py
View file

@ -40,9 +40,11 @@ class RadioSerializer(serializers.ModelSerializer):
class RadioSessionTrackSerializerCreate(serializers.ModelSerializer):
count = serializers.IntegerField(required=False, allow_null=True)
class Meta:
model = models.RadioSessionTrack
fields = ("session",)
fields = ("session", "count")
class RadioSessionTrackSerializer(serializers.ModelSerializer):

2
api/funkwhale_api/radios/urls.py
View file

@ -5,7 +5,7 @@ from . import views
router = routers.OptionalSlashRouter()
router.register(r"sessions", views.RadioSessionViewSet, "sessions")
router.register(r"radios", views.RadioViewSet, "radios")
router.register(r"tracks", views.RadioSessionTrackViewSet, "tracks")
router.register(r"tracks", views.V1_RadioSessionTrackViewSet, "tracks")
urlpatterns = router.urls

10
api/funkwhale_api/radios/urls_v2.py Normal file
View file

@ -0,0 +1,10 @@
from funkwhale_api.common import routers
from . import views
router = routers.OptionalSlashRouter()
router.register(r"sessions", views.V2_RadioSessionViewSet, "sessions")
urlpatterns = router.urls

115
api/funkwhale_api/radios/views.py
View file

@ -1,3 +1,6 @@
import pickle
from django.core.cache import cache
from django.db.models import Q
from drf_spectacular.utils import extend_schema
from rest_framework import mixins, status, viewsets
@ -121,7 +124,7 @@ class RadioSessionViewSet(
return context
class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
class V1_RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
serializer_class = serializers.RadioSessionTrackSerializer
queryset = models.RadioSessionTrack.objects.all()
permission_classes = []
@ -133,21 +136,19 @@ class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet)
session = serializer.validated_data["session"]
if not request.user.is_authenticated and not request.session.session_key:
self.request.session.create()
try:
assert (request.user == session.user) or (
request.session.session_key == session.session_key
and session.session_key
)
except AssertionError:
if not request.user == session.user or (
not request.session.session_key == session.session_key
and not session.session_key
):
return Response(status=status.HTTP_403_FORBIDDEN)
try:
session.radio.pick()
session.radio(api_version=1).pick()
except ValueError:
return Response(
"Radio doesn't have more candidates", status=status.HTTP_404_NOT_FOUND
)
session_track = session.session_tracks.all().latest("id")
# self.perform_create(serializer)
# dirty override here, since we use a different serializer for creation and detail
serializer = self.serializer_class(
instance=session_track, context=self.get_serializer_context()
@ -161,3 +162,99 @@ class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet)
if self.action == "create":
return serializers.RadioSessionTrackSerializerCreate
return super().get_serializer_class(*args, **kwargs)
class V2_RadioSessionViewSet(
mixins.CreateModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
"""Returns a list of RadioSessions"""
serializer_class = serializers.RadioSessionSerializer
queryset = models.RadioSession.objects.all()
permission_classes = []
@action(detail=True, serializer_class=serializers.RadioSessionTrackSerializerCreate)
def tracks(self, request, pk, *args, **kwargs):
data = {"session": pk}
data["count"] = (
request.query_params["count"]
if "count" in request.query_params.keys()
else 1
)
serializer = serializers.RadioSessionTrackSerializerCreate(data=data)
serializer.is_valid(raise_exception=True)
session = serializer.validated_data["session"]
count = int(data["count"])
# this is used for test purpose.
filter_playable = (
request.query_params["filter_playable"]
if "filter_playable" in request.query_params.keys()
else True
)
if not request.user.is_authenticated and not request.session.session_key:
self.request.session.create()
if not request.user == session.user or (
not request.session.session_key == session.session_key
and not session.session_key
):
return Response(status=status.HTTP_403_FORBIDDEN)
try:
from . import radios_v2 # noqa
session.radio(api_version=2).pick_many(
count, filter_playable=filter_playable
)
except ValueError:
return Response(
"Radio doesn't have more candidates", status=status.HTTP_404_NOT_FOUND
)
# dirty override here, since we use a different serializer for creation and detail
evaluated_radio_tracks = pickle.loads(cache.get(f"radiotracks{session.id}"))
batch = evaluated_radio_tracks[:count]
serializer = TrackSerializer(
data=batch,
many="true",
)
serializer.is_valid()
# delete the tracks we sent from the cache
new_cached_radiotracks = evaluated_radio_tracks[count:]
cache.set(f"radiotracks{session.id}", pickle.dumps(new_cached_radiotracks))
return Response(
serializer.data,
status=status.HTTP_201_CREATED,
)
def get_queryset(self):
queryset = super().get_queryset()
if self.request.user.is_authenticated:
return queryset.filter(
Q(user=self.request.user)
| Q(session_key=self.request.session.session_key)
)
return queryset.filter(session_key=self.request.session.session_key).exclude(
session_key=None
)
def perform_create(self, serializer):
if (
not self.request.user.is_authenticated
and not self.request.session.session_key
):
self.request.session.create()
return serializer.save(
user=self.request.user if self.request.user.is_authenticated else None,
session_key=self.request.session.session_key,
)
def get_serializer_context(self):
context = super().get_serializer_context()
context["user"] = (
self.request.user if self.request.user.is_authenticated else None
)
return context

0
api/funkwhale_api/typesense/__init__.py Normal file
View file

111
api/funkwhale_api/typesense/factories.py Normal file
View file

@ -0,0 +1,111 @@
from troi import Artist, Element, Playlist, Recording
from troi.patch import Patch
recording_list = [
Recording(
name="I Want It That Way",
mbid="87dfa566-21c3-45ed-bc42-1d345b8563fa",
artist=Artist(name="artist_name"),
),
Recording(name="Untouchable", artist=Artist(name="Another lol")),
Recording(
name="The Perfect Kiss",
mbid="ec0da94e-fbfe-4eb0-968e-024d4c32d1d0",
artist=Artist(name="artist_name2"),
),
Recording(
name="Love Your Voice",
mbid="93726547-f8c0-4efd-8e16-d2dee76500f6",
artist=Artist(name="artist_name"),
),
Recording(
name="Hall of Fame",
mbid="395bd5a1-79cc-4e04-8869-ca9eabc78d09",
artist=Artist(name="artist_name_3"),
),
]
class DummyElement(Element):
"""Dummy element that returns a fixed playlist for testing"""
@staticmethod
def outputs():
return [Playlist]
def read(self, sources):
recordings = recording_list
return [
Playlist(
name="Test Export Playlist",
description="A playlist to test exporting playlists to spotify",
recordings=recordings,
)
]
class DummyPatch(Patch):
"""Dummy patch that always returns a fixed set of recordings for testing"""
@staticmethod
def slug():
return "test-patch"
def create(self, inputs):
return DummyElement()
@staticmethod
def outputs():
return [Recording]
recommended_recording_mbids = [
"87dfa566-21c3-45ed-bc42-1d345b8563fa",
"ec0da94e-fbfe-4eb0-968e-024d4c32d1d0",
"93726547-f8c0-4efd-8e16-d2dee76500f6",
"395bd5a1-79cc-4e04-8869-ca9eabc78d09",
]
typesense_search_result = {
"facet_counts": [],
"found": 1,
"out_of": 1,
"page": 1,
"request_params": {
"collection_name": "canonical_fw_data",
"per_page": 10,
"q": "artist_nameiwantitthatway",
},
"search_time_ms": 1,
"hits": [
{
"highlights": [
{
"field": "combined",
"snippet": "string",
"matched_tokens": ["string"],
}
],
"document": {
"pk": "1",
"combined": "artist_nameiwantitthatway",
},
"text_match": 130916,
},
{
"highlights": [
{
"field": "combined",
"snippet": "string",
"matched_tokens": ["string"],
}
],
"document": {
"pk": "2",
"combined": "artist_nameiwantitthatway",
},
"text_match": 130916,
},
],
}

108
api/funkwhale_api/typesense/tasks.py Normal file
View file

@ -0,0 +1,108 @@
import logging
from django.conf import settings
from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
from . import utils
logger = logging.getLogger(__name__)
class TypesenseNotActivate(Exception):
pass
if not settings.TYPESENSE_API_KEY:
logger.info(
"Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
)
else:
import typesense
from typesense.exceptions import ObjectAlreadyExists
api_key = settings.TYPESENSE_API_KEY
host = settings.TYPESENSE_HOST
port = settings.TYPESENSE_PORT
protocol = settings.TYPESENSE_PROTOCOL
collection_name = "canonical_fw_data"
BATCH_SIZE = 10000
@celery.app.task(name="typesense.add_tracks_to_index")
def add_tracks_to_index(tracks_pk):
"""
This will add fw tracks data to the typesense index. It will concatenate the artist name
and the track title into one string.
"""
client = typesense.Client(
{
"api_key": api_key,
"nodes": [{"host": host, "port": port, "protocol": protocol}],
"connection_timeout_seconds": 2,
}
)
try:
logger.info(f"Updating index {collection_name}")
tracks = music_models.Track.objects.all().filter(pk__in=tracks_pk)
documents = []
for track in tracks:
document = dict()
document["pk"] = track.pk
document["combined"] = utils.delete_non_alnum_characters(
track.artist.name + track.title
)
documents.append(document)
client.collections[collection_name].documents.import_(
documents, {"action": "upsert"}
)
except typesense.exceptions.TypesenseClientError as err:
logger.error(f"Can't build index: {str(err)}")
@celery.app.task(name="typesense.build_canonical_index")
def build_canonical_index():
if not settings.TYPESENSE_API_KEY:
raise TypesenseNotActivate(
"Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
)
schema = {
"name": collection_name,
"fields": [
{"name": "combined", "type": "string"},
{"name": "pk", "type": "int32"},
],
"default_sorting_field": "pk",
}
client = typesense.Client(
{
"api_key": api_key,
"nodes": [{"host": host, "port": port, "protocol": protocol}],
"connection_timeout_seconds": 2,
}
)
try:
client.collections.create(schema)
except ObjectAlreadyExists:
pass
tracks = music_models.Track.objects.all().values_list("pk", flat=True)
total_tracks = tracks.count()
total_batches = (total_tracks - 1) // BATCH_SIZE + 1
for i in range(total_batches):
start_index = i * BATCH_SIZE
end_index = (i + 1) * (BATCH_SIZE - 1)
batch_tracks = tracks[start_index:end_index]
logger.info(
f"Launching async task to add {str(batch_tracks)} tracks pks to index"
)
add_tracks_to_index.delay(list(batch_tracks))

92
api/funkwhale_api/typesense/utils.py Normal file
View file

@ -0,0 +1,92 @@
import logging
import re
import unidecode
from django.conf import settings
from django.core.cache import cache
from lb_matching_tools.cleaner import MetadataCleaner
from funkwhale_api.music import models as music_models
logger = logging.getLogger(__name__)
api_key = settings.TYPESENSE_API_KEY
host = settings.TYPESENSE_HOST
port = settings.TYPESENSE_PORT
protocol = settings.TYPESENSE_PROTOCOL
TYPESENSE_NUM_TYPO = settings.TYPESENSE_NUM_TYPO
class TypesenseNotActivate(Exception):
pass
if not settings.TYPESENSE_API_KEY:
logger.info(
"Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
)
else:
import typesense
def delete_non_alnum_characters(text):
return unidecode.unidecode(re.sub(r"[^\w]+", "", text).lower())
def resolve_recordings_to_fw_track(recordings):
"""
Tries to match a troi recording entity to a fw track using the typesense index.
It will save the results in the match_mbid attribute of the Track table.
For test purposes : if multiple fw tracks are returned, we log the information
but only keep the best result in db to avoid duplicates.
"""
if not settings.TYPESENSE_API_KEY:
raise TypesenseNotActivate(
"Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
)
client = typesense.Client(
{
"api_key": api_key,
"nodes": [{"host": host, "port": port, "protocol": protocol}],
"connection_timeout_seconds": 2,
}
)
mc = MetadataCleaner()
for recording in recordings:
rec = mc.clean_recording(recording.name)
artist = mc.clean_artist(recording.artist.name)
canonical_name_for_track = delete_non_alnum_characters(artist + rec)
logger.debug(f"Trying to resolve : {canonical_name_for_track}")
search_parameters = {
"q": canonical_name_for_track,
"query_by": "combined",
"num_typos": TYPESENSE_NUM_TYPO,
"drop_tokens_threshold": 0,
}
matches = client.collections["canonical_fw_data"].documents.search(
search_parameters
)
if matches["hits"]:
hit = matches["hits"][0]
pk = hit["document"]["pk"]
logger.debug(f"Saving match for track with primary key {pk}")
cache.set(recording.mbid, pk)
if settings.DEBUG and matches["hits"][1]:
for hit in matches["hits"][1:]:
pk = hit["document"]["pk"]
fw_track = music_models.Track.objects.get(pk=pk)
logger.info(
f"Duplicate match found for {fw_track.artist.name} {fw_track.title} \
and primary key {pk}. Skipping because of better match."
)
else:
logger.debug("No match found in fw db")
return cache.get_many([rec.mbid for rec in recordings])

10
api/funkwhale_api/users/factories.py
View file

@ -1,5 +1,6 @@
import sys
import factory
import pytz
from django.contrib.auth.models import Permission
from django.utils import timezone
@ -7,6 +8,11 @@ from funkwhale_api.factories import ManyToManyFromList, NoUpdateOnCreate, regist
from . import models
if sys.version_info < (3, 9):
from backports.zoneinfo import ZoneInfo
else:
from zoneinfo import ZoneInfo
@registry.register
class GroupFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
@ -159,7 +165,7 @@ class GrantFactory(factory.django.DjangoModelFactory):
class AccessTokenFactory(factory.django.DjangoModelFactory):
application = factory.SubFactory(ApplicationFactory)
user = factory.SubFactory(UserFactory)
expires = factory.Faker("future_datetime", tzinfo=pytz.UTC)
expires = factory.Faker("future_datetime", tzinfo=ZoneInfo("UTC"))
token = factory.Faker("uuid4")
scope = "read"

2221
api/poetry.lock generated
View file

File diff suppressed because it is too large Load diff

45
api/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "funkwhale-api"
version = "1.2.9"
version = "1.4.0"
description = "Funkwhale API"
authors = ["Funkwhale Collective"]
@ -25,7 +25,7 @@ exclude = ["tests"]
funkwhale-manage = 'funkwhale_api.main:main'
[tool.poetry.dependencies]
python = "^3.7"
python = "^3.8,<3.12"
# Django
dj-rest-auth = { extras = ["with_social"], version = "2.2.8" }
@ -43,7 +43,7 @@ django-redis = "==5.2.0"
django-storages = "==1.13.2"
django-versatileimagefield = "==2.2"
djangorestframework = "==3.14.0"
drf-spectacular = "==0.25.1"
drf-spectacular = "==0.26.5"
markdown = "==3.4.4"
persisting-theory = "==1.0"
psycopg2 = "==2.9.9"
@ -51,7 +51,7 @@ redis = "==4.5.5"
# Django LDAP
django-auth-ldap = "==4.1.0"
python-ldap = "==3.4.3"
python-ldap = "==3.4.4"
# Channels
channels = { extras = ["daphne"], version = "==4.0.0" }
@ -68,6 +68,7 @@ uvicorn = { version = "==0.20.0", extras = ["standard"] }
# Libs
aiohttp = "==3.8.6"
arrow = "==1.2.3"
backports-zoneinfo = { version = "==0.2.1", python = "<3.9" }
bleach = "==5.0.1"
boto3 = "==1.26.161"
click = "==8.1.7"
@ -79,23 +80,28 @@ pillow = "==9.3.0"
pydub = "==0.25.1"
pyld = "==2.0.3"
python-magic = "==0.4.27"
pytz = "==2022.7.1"
requests = "==2.28.2"
requests-http-message-signatures = "==0.3.1"
sentry-sdk = "==1.12.1"
sentry-sdk = "==1.19.1"
watchdog = "==2.2.1"
troi = { git = "https://github.com/metabrainz/troi-recommendation-playground.git", tag = "v-2023-10-30.0"}
lb-matching-tools = { git = "https://github.com/metabrainz/listenbrainz-matching-tools.git", branch = "main"}
unidecode = "==1.3.7"
pycountry = "22.3.5"
# Typesense
typesense = { version = "==0.15.1", optional = true }
# Dependencies pinning
importlib-metadata = { version = "==4.13.0", python = "^3.7" }
ipython = "==7.34.0"
pluralizer = "==1.2.0"
service-identity = "==21.1.0"
unicode-slugify = "==0.1.5"
[tool.poetry.dev-dependencies]
[tool.poetry.group.dev.dependencies]
aioresponses = "==0.7.6"
asynctest = "==0.13.0"
black = "==22.12.0"
black = "==23.3.0"
coverage = { version = "==6.5.0", extras = ["toml"] }
debugpy = "==1.6.7.post1"
django-coverage-plugin = "==3.0.0"
@ -104,27 +110,30 @@ factory-boy = "==3.2.1"
faker = "==15.3.4"
flake8 = "==3.9.2"
ipdb = "==0.13.13"
pytest = "==7.4.3"
pytest-asyncio = "==0.21.0"
prompt-toolkit = "==3.0.41"
pytest = "==7.2.1"
pytest-asyncio = "==0.20.3"
pytest-cov = "==4.0.0"
pytest-django = "==4.5.2"
pytest-env = "==0.8.1"
pytest-env = "==0.8.2"
pytest-mock = "==3.10.0"
pytest-randomly = "==3.12.0"
pytest-sugar = "==0.9.7"
requests-mock = "==1.10.0"
pylint = "==2.17.7"
pylint-django = "==2.5.5"
django-extensions = "==3.2.3"
[tool.poetry.extras]
typesense = ["typesense"]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.pylint.master]
load-plugins = [
"pylint_common",
"pylint_django",
"pylint_celery",
]
load-plugins = ["pylint_django"]
django-settings-module = "config.settings.testing"
[tool.pylint.messages_control]
disable = [
@ -175,7 +184,7 @@ env = [
[tool.coverage.run]
plugins = ["django_coverage_plugin"]
include = ["funkwhale_api/*"]
source = ["funkwhale_api"]
omit = [
"*migrations*",
"*tests*",

13
api/tests/audio/test_serializers.py
View file

@ -1,9 +1,9 @@
import datetime
import sys
import uuid
import feedparser
import pytest
import pytz
from django.templatetags.static import static
from django.urls import reverse
@ -14,6 +14,11 @@ from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.federation import utils as federation_utils
if sys.version_info < (3, 9):
from backports.zoneinfo import ZoneInfo
else:
from zoneinfo import ZoneInfo
def test_channel_serializer_create(factories, mocker):
attributed_to = factories["federation.Actor"](local=True)
@ -456,7 +461,7 @@ def test_rss_duration(seconds, expected):
"dt, expected",
[
(
datetime.datetime(2020, 1, 30, 6, 0, 49, tzinfo=pytz.UTC),
datetime.datetime(2020, 1, 30, 6, 0, 49, tzinfo=ZoneInfo("UTC")),
"Thu, 30 Jan 2020 06:00:49 +0000",
),
],
@ -678,7 +683,7 @@ def test_rss_feed_item_serializer_create(factories):
assert upload.track.position == 33
assert upload.track.disc_number == 2
assert upload.track.creation_date == datetime.datetime(2020, 3, 11, 16).replace(
tzinfo=pytz.utc
tzinfo=ZoneInfo("UTC")
)
assert upload.track.get_tags() == ["pop", "rock"]
assert upload.track.attachment_cover.url == "https://image.url/"
@ -748,7 +753,7 @@ def test_rss_feed_item_serializer_update(factories):
assert upload.track.position == 33
assert upload.track.disc_number == 2
assert upload.track.creation_date == datetime.datetime(2020, 3, 11, 16).replace(
tzinfo=pytz.utc
tzinfo=ZoneInfo("UTC")
)
assert upload.track.get_tags() == ["pop", "rock"]
assert upload.track.attachment_cover.url == "https://image.url/"

16
api/tests/cli/test_users.py
View file

@ -32,6 +32,22 @@ def test_user_create_handler(factories, mocker, now):
assert user.all_permissions == expected_permissions
def test_user_implicit_staff():
kwargs = {
"username": "helloworld",
"password": "securepassword",
"is_superuser": True,
"email": "hello@world.email",
"upload_quota": 35,
"permissions": ["moderation"],
}
user = users.handler_create_user(**kwargs)
assert user.username == kwargs["username"]
assert user.is_superuser == kwargs["is_superuser"]
assert user.is_staff is True
def test_user_delete_handler_soft(factories, mocker, now):
user1 = factories["federation.Actor"](local=True).user
actor1 = user1.actor

31
api/tests/common/test_commands.py
View file

@ -1,5 +1,8 @@
import os
import pytest
from django.core.management import call_command
from django.core.management.base import CommandError
from funkwhale_api.federation import models as federation_models
from funkwhale_api.music import models as music_models
@ -97,3 +100,31 @@ def test_load_test_data_skip_dependencies(factories):
assert music_models.Artist.objects.count() == 5
assert music_models.Album.objects.count() == 10
commands = ["createsuperuser", "makemigrations"]
@pytest.mark.parametrize("command", commands)
def test_blocked_commands(command):
with pytest.raises(CommandError):
call_command(command)
@pytest.mark.parametrize("command", commands)
def test_unblocked_commands(command, mocker):
mocker.patch.dict(os.environ, {"FORCE": "1"})
call_command(command)
def test_inplace_to_s3_without_source():
with pytest.raises(CommandError):
call_command("inplace_to_s3")
def test_inplace_to_s3_dryrun(factories):
upload = factories["music.Upload"](in_place=True, source="file:///music/music.mp3")
call_command("inplace_to_s3", "--source", "/music")
assert upload.source == "file:///music/music.mp3"
assert not upload.audio_file

2
api/tests/common/test_middleware.py
View file

@ -108,7 +108,7 @@ def test_get_default_head_tags(preferences, settings):
{
"tag": "meta",
"property": "og:image",
"content": settings.FUNKWHALE_URL + "/front/favicon.png",
"content": settings.FUNKWHALE_URL + "/android-chrome-512x512.png",
},
{"tag": "meta", "property": "og:url", "content": settings.FUNKWHALE_URL + "/"},
]

4
api/tests/common/test_throttling.py
View file

@ -17,7 +17,7 @@ def test_get_ident_anonymous(api_request):
def test_get_ident_authenticated(api_request, factories):
user = factories["users.User"]()
request = api_request.get("/")
expected = {"id": user.pk, "type": "authenticated"}
expected = {"id": f"{user.pk}", "type": "authenticated"}
assert throttling.get_ident(user, request) == expected
@ -26,7 +26,7 @@ def test_get_ident_authenticated(api_request, factories):
[
(
"create",
{"id": 42, "type": "authenticated"},
{"id": "42", "type": "authenticated"},
"throttling:create:authenticated:42",
),
(

2
api/tests/common/test_views.py
View file

@ -160,7 +160,7 @@ def test_cannot_approve_reject_without_perm(
def test_rate_limit(logged_in_api_client, now_time, settings, mocker):
expected_ident = {"type": "authenticated", "id": logged_in_api_client.user.pk}
expected_ident = {"type": "authenticated", "id": f"{logged_in_api_client.user.pk}"}
expected = {
"ident": expected_ident,

75
api/tests/instance/test_nodeinfo.py
View file

@ -6,7 +6,7 @@ from funkwhale_api import __version__ as api_version
from funkwhale_api.music.utils import SUPPORTED_EXTENSIONS
def test_nodeinfo_default(api_client):
def test_nodeinfo_20(api_client):
url = reverse("api:v1:instance:nodeinfo-2.0")
response = api_client.get(url)
@ -14,7 +14,7 @@ def test_nodeinfo_default(api_client):
"version": "2.0",
"software": OrderedDict([("name", "funkwhale"), ("version", api_version)]),
"protocols": ["activitypub"],
"services": OrderedDict([("inbound", []), ("outbound", [])]),
"services": OrderedDict([("inbound", ["atom1.0"]), ("outbound", ["atom1.0"])]),
"openRegistrations": False,
"usage": {
"users": OrderedDict(
@ -89,3 +89,74 @@ def test_nodeinfo_default(api_client):
}
assert response.data == expected
def test_nodeinfo_21(api_client):
url = reverse("api:v2:instance:nodeinfo-2.1")
response = api_client.get(url)
expected = {
"version": "2.1",
"software": OrderedDict(
[
("name", "funkwhale"),
("version", api_version),
("repository", "https://dev.funkwhale.audio/funkwhale/funkwhale"),
("homepage", "https://funkwhale.audio"),
]
),
"protocols": ["activitypub"],
"services": OrderedDict([("inbound", ["atom1.0"]), ("outbound", ["atom1.0"])]),
"openRegistrations": False,
"usage": {
"users": OrderedDict(
[("total", 0), ("activeHalfyear", 0), ("activeMonth", 0)]
),
"localPosts": 0,
"localComments": 0,
},
"metadata": {
"actorId": "https://test.federation/federation/actors/service",
"private": False,
"shortDescription": "",
"longDescription": "",
"contactEmail": "",
"nodeName": "",
"banner": None,
"defaultUploadQuota": 1000,
"supportedUploadExtensions": SUPPORTED_EXTENSIONS,
"allowList": {"enabled": False, "domains": None},
"funkwhaleSupportMessageEnabled": True,
"instanceSupportMessage": "",
"usage": OrderedDict(
[
("favorites", OrderedDict([("tracks", {"total": 0})])),
("listenings", OrderedDict([("total", 0)])),
("downloads", OrderedDict([("total", 0)])),
]
),
"location": "",
"languages": ["en"],
"features": ["channels", "podcasts", "federation"],
"content": OrderedDict(
[
(
"local",
OrderedDict(
[
("artists", 0),
("releases", 0),
("recordings", 0),
("hoursOfContent", 0),
]
),
),
("topMusicCategories", []),
("topPodcastCategories", []),
]
),
"codeOfConduct": "",
},
}
assert response.data == expected

23
api/tests/music/test_filters.py
View file

@ -3,9 +3,32 @@ import pytest
from funkwhale_api.music import filters, models
def test_artist_filter_ordering(factories, mocker):
# Lista de prueba
artist1 = factories["music.Artist"](name="Anita Muller")
artist2 = factories["music.Artist"](name="Jane Smith")
artist3 = factories["music.Artist"](name="Adam Johnson")
artist4 = factories["music.Artist"](name="anita iux")
qs = models.Artist.objects.all()
cf = factories["moderation.UserFilter"](for_artist=True)
# Request con ordenamiento
filterset = filters.ArtistFilter(
{"ordering": "name"}, request=mocker.Mock(user=cf.user), queryset=qs
)
expected_order = [artist3.name, artist4.name, artist1.name, artist2.name]
actual_order = list(filterset.qs.values_list("name", flat=True))
assert actual_order == expected_order
def test_album_filter_hidden(factories, mocker, queryset_equal_list):
factories["music.Album"]()
cf = factories["moderation.UserFilter"](for_artist=True)
hidden_album = factories["music.Album"](artist=cf.target_artist)
qs = models.Album.objects.all()

50
api/tests/music/test_tasks.py
View file

@ -1400,3 +1400,53 @@ def test_fs_import(factories, cache, mocker, settings):
}
assert cache.get("fs-import:status") == "finished"
assert "Pruning dangling tracks" in cache.get("fs-import:logs")[-1]
def test_upload_checks_mbid_tag(temp_signal, factories, mocker, preferences):
preferences["music__only_allow_musicbrainz_tagged_files"] = True
mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
mocker.patch("funkwhale_api.music.tasks.populate_album_cover")
mocker.patch("funkwhale_api.music.metadata.Metadata.get_picture")
track = factories["music.Track"](album__attachment_cover=None, mbid=None)
path = os.path.join(DATA_DIR, "with_cover.opus")
upload = factories["music.Upload"](
track=None,
audio_file__from_path=path,
import_metadata={"funkwhale": {"track": {"uuid": str(track.uuid)}}},
)
mocker.patch("funkwhale_api.music.models.TrackActor.create_entries")
with temp_signal(signals.upload_import_status_updated):
tasks.process_upload(upload_id=upload.pk)
upload.refresh_from_db()
assert upload.import_status == "errored"
assert upload.import_details == {
"error_code": "Only content tagged with a MusicBrainz ID is permitted on this pod.",
"detail": "You can tag your files with MusicBrainz Picard",
}
def test_upload_checks_mbid_tag_pass(temp_signal, factories, mocker, preferences):
preferences["music__only_allow_musicbrainz_tagged_files"] = True
mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
mocker.patch("funkwhale_api.music.tasks.populate_album_cover")
mocker.patch("funkwhale_api.music.metadata.Metadata.get_picture")
track = factories["music.Track"](album__attachment_cover=None, mbid=None)
path = os.path.join(DATA_DIR, "test.mp3")
upload = factories["music.Upload"](
track=None,
audio_file__from_path=path,
import_metadata={"funkwhale": {"track": {"uuid": str(track.uuid)}}},
)
mocker.patch("funkwhale_api.music.models.TrackActor.create_entries")
with temp_signal(signals.upload_import_status_updated):
tasks.process_upload(upload_id=upload.pk)
upload.refresh_from_db()
assert upload.import_status == "finished"

116
api/tests/radios/test_lb_recommendations.py Normal file
View file

@ -0,0 +1,116 @@
import pytest
import troi.core
from django.core.cache import cache
from django.db.models import Q
from requests.exceptions import ConnectTimeout
from funkwhale_api.music.models import Track
from funkwhale_api.radios import lb_recommendations
from funkwhale_api.typesense import factories as custom_factories
from funkwhale_api.typesense import utils
def test_can_build_radio_queryset_with_fw_db(factories, mocker):
factories["music.Track"](
title="I Want It That Way", mbid="87dfa566-21c3-45ed-bc42-1d345b8563fa"
)
factories["music.Track"](
title="The Perfect Kiss", mbid="ec0da94e-fbfe-4eb0-968e-024d4c32d1d0"
)
factories["music.Track"]()
qs = Track.objects.all()
mocker.patch("funkwhale_api.typesense.utils.resolve_recordings_to_fw_track")
radio_qs = lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)
recommended_recording_mbids = [
"87dfa566-21c3-45ed-bc42-1d345b8563fa",
"ec0da94e-fbfe-4eb0-968e-024d4c32d1d0",
]
assert list(
Track.objects.all().filter(Q(mbid__in=recommended_recording_mbids))
) == list(radio_qs)
def test_build_radio_queryset_without_fw_db(mocker):
resolve_recordings_to_fw_track = mocker.patch.object(
utils, "resolve_recordings_to_fw_track", return_value=None
)
# mocker.patch.object(cache, "get_many", return_value=None)
qs = Track.objects.all()
with pytest.raises(ValueError):
lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)
assert resolve_recordings_to_fw_track.called_once_with(
custom_factories.recommended_recording_mbids
)
def test_build_radio_queryset_with_redis_and_fw_db(factories, mocker):
factories["music.Track"](
pk="1", title="I Want It That Way", mbid="87dfa566-21c3-45ed-bc42-1d345b8563fa"
)
mocker.patch.object(utils, "resolve_recordings_to_fw_track", return_value=None)
redis_cache = {}
redis_cache["ec0da94e-fbfe-4eb0-968e-024d4c32d1d0"] = 2
mocker.patch.object(cache, "get_many", return_value=redis_cache)
qs = Track.objects.all()
assert list(
lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)
) == list(Track.objects.all().filter(pk__in=[1, 2]))
def test_build_radio_queryset_with_redis_and_without_fw_db(factories, mocker):
factories["music.Track"](
pk="1", title="Super title", mbid="87dfaaaa-2aaa-45ed-bc42-1d34aaaaaaaa"
)
mocker.patch.object(utils, "resolve_recordings_to_fw_track", return_value=None)
redis_cache = {}
redis_cache["87dfa566-21c3-45ed-bc42-1d345b8563fa"] = 1
mocker.patch.object(cache, "get_many", return_value=redis_cache)
qs = Track.objects.all()
assert list(
lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)
) == list(Track.objects.all().filter(pk=1))
def test_build_radio_queryset_catch_troi_ConnectTimeout(mocker):
mocker.patch.object(
troi.core,
"generate_playlist",
side_effect=ConnectTimeout,
)
qs = Track.objects.all()
with pytest.raises(ValueError):
lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)
def test_build_radio_queryset_catch_troi_no_candidates(mocker):
mocker.patch.object(
troi.core,
"generate_playlist",
)
qs = Track.objects.all()
with pytest.raises(ValueError):
lb_recommendations.build_radio_queryset(
custom_factories.DummyPatch(), {"min_recordings": 1}, qs
)

19
api/tests/radios/test_radios.py
View file

@ -2,8 +2,8 @@ import json
import random
import pytest
from django.core.exceptions import ValidationError
from django.urls import reverse
from rest_framework.exceptions import ValidationError
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.radios import models, radios, serializers
@ -98,7 +98,7 @@ def test_can_get_choices_for_custom_radio(factories):
session = factories["radios.CustomRadioSession"](
custom_radio__config=[{"type": "artist", "ids": [artist.pk]}]
)
choices = session.radio.get_choices(filter_playable=False)
choices = session.radio(api_version=1).get_choices(filter_playable=False)
expected = [t.pk for t in tracks]
assert list(choices.values_list("id", flat=True)) == expected
@ -191,16 +191,17 @@ def test_can_get_track_for_session_from_api(factories, logged_in_api_client):
def test_related_object_radio_validate_related_object(factories):
user = factories["users.User"]()
# cannot start without related object
radio = radios.ArtistRadio()
radio = {"radio_type": "tag"}
serializer = serializers.RadioSessionSerializer()
with pytest.raises(ValidationError):
radio.start_session(user)
serializer.validate(data=radio)
# cannot start with bad related object type
radio = radios.ArtistRadio()
radio = {"radio_type": "tag", "related_object": "whatever"}
serializer = serializers.RadioSessionSerializer()
with pytest.raises(ValidationError):
radio.start_session(user, related_object=user)
serializer.validate(data=radio)
def test_can_start_artist_radio(factories):
@ -391,7 +392,7 @@ def test_get_choices_for_custom_radio_exclude_artist(factories):
{"type": "artist", "ids": [excluded_artist.pk], "not": True},
]
)
choices = session.radio.get_choices(filter_playable=False)
choices = session.radio(api_version=1).get_choices(filter_playable=False)
expected = [u.track.pk for u in included_uploads]
assert list(choices.values_list("id", flat=True)) == expected
@ -409,7 +410,7 @@ def test_get_choices_for_custom_radio_exclude_tag(factories):
{"type": "tag", "names": ["rock"], "not": True},
]
)
choices = session.radio.get_choices(filter_playable=False)
choices = session.radio(api_version=1).get_choices(filter_playable=False)
expected = [u.track.pk for u in included_uploads]
assert list(choices.values_list("id", flat=True)) == expected

144
api/tests/radios/test_radios_v2.py Normal file
View file

@ -0,0 +1,144 @@
import json
import logging
import pickle
import random
from django.core.cache import cache
from django.urls import reverse
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.radios import models, radios_v2
def test_can_get_track_for_session_from_api_v2(factories, logged_in_api_client):
actor = logged_in_api_client.user.create_actor()
track = factories["music.Upload"](
library__actor=actor, import_status="finished"
).track
url = reverse("api:v2:radios:sessions-list")
response = logged_in_api_client.post(url, {"radio_type": "random"})
session = models.RadioSession.objects.latest("id")
url = reverse("api:v2:radios:sessions-tracks", kwargs={"pk": session.pk})
response = logged_in_api_client.get(url, {"session": session.pk})
data = json.loads(response.content.decode("utf-8"))
assert data[0]["id"] == track.pk
next_track = factories["music.Upload"](
library__actor=actor, import_status="finished"
).track
response = logged_in_api_client.get(url, {"session": session.pk})
data = json.loads(response.content.decode("utf-8"))
assert data[0]["id"] == next_track.id
def test_can_use_radio_session_to_filter_choices_v2(factories):
factories["music.Upload"].create_batch(10)
user = factories["users.User"]()
radio = radios_v2.RandomRadio()
session = radio.start_session(user)
radio.pick_many(quantity=10, filter_playable=False)
# ensure 10 different tracks have been suggested
tracks_id = [
session_track.track.pk for session_track in session.session_tracks.all()
]
assert len(set(tracks_id)) == 10
def test_session_radio_excludes_previous_picks_v2(factories, logged_in_api_client):
tracks = factories["music.Track"].create_batch(5)
url = reverse("api:v2:radios:sessions-list")
response = logged_in_api_client.post(url, {"radio_type": "random"})
session = models.RadioSession.objects.latest("id")
url = reverse("api:v2:radios:sessions-tracks", kwargs={"pk": session.pk})
previous_choices = []
for i in range(5):
response = logged_in_api_client.get(
url, {"session": session.pk, "filter_playable": False}
)
pick = json.loads(response.content.decode("utf-8"))
assert pick[0]["title"] not in previous_choices
assert pick[0]["title"] in [t.title for t in tracks]
previous_choices.append(pick[0]["title"])
response = logged_in_api_client.get(url, {"session": session.pk})
assert (
json.loads(response.content.decode("utf-8"))
== "Radio doesn't have more candidates"
)
def test_can_get_choices_for_favorites_radio_v2(factories):
files = factories["music.Upload"].create_batch(10)
tracks = [f.track for f in files]
user = factories["users.User"]()
for i in range(5):
TrackFavorite.add(track=random.choice(tracks), user=user)
radio = radios_v2.FavoritesRadio()
session = radio.start_session(user=user)
choices = session.radio(api_version=2).get_choices(
quantity=100, filter_playable=False
)
assert len(choices) == user.track_favorites.all().count()
for favorite in user.track_favorites.all():
assert favorite.track in choices
def test_can_get_choices_for_custom_radio_v2(factories):
artist = factories["music.Artist"]()
files = factories["music.Upload"].create_batch(5, track__artist=artist)
tracks = [f.track for f in files]
factories["music.Upload"].create_batch(5)
session = factories["radios.CustomRadioSession"](
custom_radio__config=[{"type": "artist", "ids": [artist.pk]}]
)
choices = session.radio(api_version=2).get_choices(
quantity=1, filter_playable=False
)
expected = [t.pk for t in tracks]
for t in choices:
assert t.id in expected
def test_can_cache_radio_track(factories):
uploads = factories["music.Track"].create_batch(10)
user = factories["users.User"]()
radio = radios_v2.RandomRadio()
session = radio.start_session(user)
picked = session.radio(api_version=2).pick_many(quantity=1, filter_playable=False)
assert len(picked) == 1
for t in pickle.loads(cache.get(f"radiotracks{session.id}")):
assert t in uploads
def test_regenerate_cache_if_not_enought_tracks_in_it(
factories, caplog, logged_in_api_client
):
logger = logging.getLogger("funkwhale_api.radios.radios_v2")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
factories["music.Track"].create_batch(10)
factories["users.User"]()
url = reverse("api:v2:radios:sessions-list")
response = logged_in_api_client.post(url, {"radio_type": "random"})
session = models.RadioSession.objects.latest("id")
url = reverse("api:v2:radios:sessions-tracks", kwargs={"pk": session.pk})
logged_in_api_client.get(url, {"count": 9, "filter_playable": False})
response = logged_in_api_client.get(url, {"count": 10, "filter_playable": False})
pick = json.loads(response.content.decode("utf-8"))
assert (
"Not enough radio tracks in cache. Trying to generate new cache" in caplog.text
)
assert len(pick) == 1

5
api/tests/test_init.py Normal file
View file

@ -0,0 +1,5 @@
def test_version():
from funkwhale_api import __version__, version
assert isinstance(version, str)
assert version == __version__

4
api/tests/test_urls.py
View file

@ -12,5 +12,5 @@ def test_can_resolve_subsonic():
def test_can_resolve_v2():
path = reverse("api:v2:instance:nodeinfo-2.0")
assert path == "/api/v2/instance/nodeinfo/2.0"
path = reverse("api:v2:instance:nodeinfo-2.1")
assert path == "/api/v2/instance/nodeinfo/2.1"

58
api/tests/typesense/test_tasks.py Normal file
View file

@ -0,0 +1,58 @@
import logging
import requests_mock
import typesense
from funkwhale_api.typesense import tasks
def test_add_tracks_to_index_fails(mocker, caplog):
logger = logging.getLogger("funkwhale_api.typesense.tasks")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
client = typesense.Client(
{
"api_key": "api_key",
"nodes": [{"host": "host", "port": "port", "protocol": "protocol"}],
"connection_timeout_seconds": 2,
}
)
with requests_mock.Mocker() as r_mocker:
r_mocker.post(
"protocol://host:port/collections/canonical_fw_data/documents/import",
json=[{"name": "data"}],
)
mocker.patch.object(typesense, "Client", return_value=client)
mocker.patch.object(
typesense.client.ApiCall,
"post",
side_effect=typesense.exceptions.TypesenseClientError("Hello"),
)
tasks.add_tracks_to_index([1, 2, 3])
assert "Can't build index" in caplog.text
def test_build_canonical_index_success(mocker, caplog, factories):
logger = logging.getLogger("funkwhale_api.typesense.tasks")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
client = typesense.Client(
{
"api_key": "api_key",
"nodes": [{"host": "host", "port": "port", "protocol": "protocol"}],
"connection_timeout_seconds": 2,
}
)
factories["music.Track"].create_batch(size=5)
with requests_mock.Mocker() as r_mocker:
mocker.patch.object(typesense, "Client", return_value=client)
r_mocker.post("protocol://host:port/collections", json={"name": "data"})
tasks.build_canonical_index()
assert "Launching async task to add " in caplog.text

43
api/tests/typesense/test_utils.py Normal file
View file

@ -0,0 +1,43 @@
import requests_mock
import typesense
from django.core.cache import cache
from funkwhale_api.typesense import factories as custom_factories
from funkwhale_api.typesense import utils
def test_resolve_recordings_to_fw_track(mocker, factories):
artist = factories["music.Artist"](name="artist_name")
factories["music.Track"](
pk=1,
title="I Want It That Way",
artist=artist,
mbid="87dfa566-21c3-45ed-bc42-1d345b8563fa",
)
factories["music.Track"](
pk=2,
title="I Want It That Way",
artist=artist,
)
client = typesense.Client(
{
"api_key": "api_key",
"nodes": [{"host": "host", "port": "port", "protocol": "protocol"}],
"connection_timeout_seconds": 2,
}
)
with requests_mock.Mocker() as r_mocker:
mocker.patch.object(typesense, "Client", return_value=client)
mocker.patch.object(
typesense.client.ApiCall,
"post",
return_value=custom_factories.typesense_search_result,
)
r_mocker.get(
"protocol://host:port/collections/canonical_fw_data/documents/search",
json=custom_factories.typesense_search_result,
)
utils.resolve_recordings_to_fw_track(custom_factories.recording_list)
assert cache.get("87dfa566-21c3-45ed-bc42-1d345b8563fa") == "1"