0.17 release documentation initial draft and migration script

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

@@ -1,64 +1,155 @@
 """
 Mirate instance files to a library #463. For each user that imported music on an
 instance, we will create a "default" library with related files and an instance-level
-visibility.
+visibility (unless instance has common__api_authentication_required set to False,
+in which case the libraries will be public).
 
 Files without any import job will be bounded to a "default" library on the first
 superuser account found. This should now happen though.
 
-XXX TODO:
+This command will also generate federation ids for existing resources.
 
-- add followers url on actor
-- shared inbox url on actor
-- compute hash from files
 """
 
+from django.conf import settings
+from django.db.models import functions, CharField, F, Value
+
 from funkwhale_api.music import models
 from funkwhale_api.users.models import User
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.common import preferences
+
+
+def create_libraries(open_api, stdout):
+    local_actors = federation_models.Actor.objects.exclude(user=None).only("pk", "user")
+    privacy_level = "everyone" if open_api else "instance"
+    stdout.write(
+        "* Creating {} libraries with {} visibility".format(
+            len(local_actors), privacy_level
+        )
+    )
+    libraries_by_user = {}
+
+    for a in local_actors:
+        library, created = models.Library.objects.get_or_create(
+            name="default", actor=a, defaults={"privacy_level": privacy_level}
+        )
+        libraries_by_user[library.actor.user.pk] = library.pk
+        if created:
+            stdout.write(
+                "  * Created library {} for user {}".format(library.pk, a.user.pk)
+            )
+        else:
+            stdout.write(
+                "  * Found existing library {} for user {}".format(
+                    library.pk, a.user.pk
+                )
+            )
+
+    return libraries_by_user
+
+
+def update_uploads(libraries_by_user, stdout):
+    stdout.write("* Updating uploads with proper libraries...")
+    for user_id, library_id in libraries_by_user.items():
+        jobs = models.ImportJob.objects.filter(
+            upload__library=None, batch__submitted_by=user_id
+        )
+        candidates = models.Upload.objects.filter(
+            pk__in=jobs.values_list("upload", flat=True)
+        )
+        total = candidates.update(library=library_id, import_status="finished")
+        if total:
+            stdout.write(
+                "  * Assigned {} uploads to user {}'s library".format(total, user_id)
+            )
+        else:
+            stdout.write(
+                "  * No uploads to assign to user {}'s library".format(user_id)
+            )
+
+
+def update_orphan_uploads(open_api, stdout):
+    privacy_level = "everyone" if open_api else "instance"
+    first_superuser = User.objects.filter(is_superuser=True).order_by("pk").first()
+    library, _ = models.Library.objects.get_or_create(
+        name="default",
+        actor=first_superuser.actor,
+        defaults={"privacy_level": privacy_level},
+    )
+    candidates = (
+        models.Upload.objects.filter(library=None, jobs__isnull=True)
+        .exclude(audio_file=None)
+        .exclude(audio_file="")
+    )
+
+    total = candidates.update(library=library, import_status="finished")
+    if total:
+        stdout.write(
+            "* Assigned {} orphaned uploads to superuser {}".format(
+                total, first_superuser.pk
+            )
+        )
+    else:
+        stdout.write("* No orphaned uploads found")
+
+
+def set_fid(queryset, path, stdout):
+    model = queryset.model._meta.label
+    qs = queryset.filter(fid=None)
+    base_url = "{}{}".format(settings.FUNKWHALE_URL, path)
+    stdout.write(
+        "* Assigning federation ids to {} entries (path: {})".format(model, base_url)
+    )
+    new_fid = functions.Concat(Value(base_url), F("uuid"), output_field=CharField())
+    total = qs.update(fid=new_fid)
+
+    stdout.write("  * {} entries updated".format(total))
+
+
+def update_shared_inbox_url(stdout):
+    stdout.write("* Update shared inbox url for local actors...")
+    candidates = federation_models.Actor.objects.local().filter(shared_inbox_url=None)
+    url = federation_models.get_shared_inbox_url()
+    candidates.update(shared_inbox_url=url)
+
+
+def generate_actor_urls(part, stdout):
+    field = "{}_url".format(part)
+    stdout.write("* Update {} for local actors...".format(field))
+
+    queryset = federation_models.Actor.objects.local().filter(**{field: None})
+    base_url = "{}/federation/actors/".format(settings.FUNKWHALE_URL)
+
+    new_field = functions.Concat(
+        Value(base_url),
+        F("preferred_username"),
+        Value("/{}".format(part)),
+        output_field=CharField(),
+    )
+
+    queryset.update(**{field: new_field})
 
 
 def main(command, **kwargs):
-    importer_ids = set(
-        models.ImportBatch.objects.values_list("submitted_by", flat=True)
-    )
-    importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
-    command.stdout.write(
-        "* {} users imported music on this instance".format(len(importers))
-    )
-    files = models.Upload.objects.filter(
-        library__isnull=True, jobs__isnull=False
-    ).distinct()
-    command.stdout.write(
-        "* Reassigning {} files to importers libraries...".format(files.count())
-    )
-    for user in importers:
-        command.stdout.write(
-            "  * Setting up @{}'s 'default' library".format(user.username)
-        )
-        library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
-            0
-        ]
-        user_files = files.filter(jobs__batch__submitted_by=user)
-        total = user_files.count()
-        command.stdout.write(
-            "    * Reassigning {} files to the user library...".format(total)
-        )
-        user_files.update(library=library)
+    open_api = not preferences.get("common__api_authentication_required")
+    libraries_by_user = create_libraries(open_api, command.stdout)
+    update_uploads(libraries_by_user, command.stdout)
+    update_orphan_uploads(open_api, command.stdout)
 
-    files = models.Upload.objects.filter(
-        library__isnull=True, jobs__isnull=True
-    ).distinct()
-    command.stdout.write(
-        "* Handling {} files with no import jobs...".format(files.count())
-    )
+    set_fid_params = [
+        (
+            models.Upload.objects.exclude(library__actor__user=None),
+            "/federation/music/uploads/",
+        ),
+        (models.Artist.objects.all(), "/federation/music/artists/"),
+        (models.Album.objects.all(), "/federation/music/albums/"),
+        (models.Track.objects.all(), "/federation/music/tracks/"),
+    ]
+    for qs, path in set_fid_params:
+        set_fid(qs, path, command.stdout)
 
-    user = User.objects.order_by("id").filter(is_superuser=True).first()
+    update_shared_inbox_url(command.stdout)
 
-    command.stdout.write("  * Setting up @{}'s 'default' library".format(user.username))
-    library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
-    total = files.count()
-    command.stdout.write(
-        "    * Reassigning {} files to the user library...".format(total)
-    )
-    files.update(library=library)
-    command.stdout.write(" * Done!")
+    for part in ["followers", "following"]:
+        generate_actor_urls(part, command.stdout)

api/funkwhale_api/federation/models.py

@@ -9,6 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
+from django.urls import reverse
 
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
@@ -29,6 +30,10 @@ def empty_dict():
     return {}
 
 
+def get_shared_inbox_url():
+    return federation_utils.full_url(reverse("federation:shared-inbox"))
+
+
 class FederationMixin(models.Model):
     # federation id/url
     fid = models.URLField(unique=True, max_length=500, db_index=True)

api/funkwhale_api/music/factories.py

@@ -4,6 +4,7 @@ import factory
 
 from funkwhale_api.factories import ManyToManyFromList, registry
 from funkwhale_api.federation import factories as federation_factories
+from funkwhale_api.users import factories as users_factories
 
 
 SAMPLES_PATH = os.path.join(
@@ -100,3 +101,24 @@ class TagFactory(factory.django.DjangoModelFactory):
 
     class Meta:
         model = "taggit.Tag"
+
+
+# XXX To remove
+
+
+class ImportBatchFactory(factory.django.DjangoModelFactory):
+    submitted_by = factory.SubFactory(users_factories.UserFactory)
+
+    class Meta:
+        model = "music.ImportBatch"
+
+
+@registry.register
+class ImportJobFactory(factory.django.DjangoModelFactory):
+    batch = factory.SubFactory(ImportBatchFactory)
+    source = factory.Faker("url")
+    mbid = factory.Faker("uuid4")
+    replace_if_duplicate = False
+
+    class Meta:
+        model = "music.ImportJob"

api/funkwhale_api/music/models.py

@@ -557,8 +557,8 @@ class UploadQuerySet(models.QuerySet):
         libraries = Library.objects.viewable_by(actor)
 
         if include:
-            return self.filter(library__in=libraries)
-        return self.exclude(library__in=libraries)
+            return self.filter(library__in=libraries, import_status="finished")
+        return self.exclude(library__in=libraries, import_status="finished")
 
     def local(self, include=True):
         return self.exclude(library__actor__user__isnull=include)
@@ -899,7 +899,7 @@ class Library(federation_models.FederationMixin):
         )
 
     def save(self, **kwargs):
-        if not self.pk and not self.fid and self.actor.is_local:
+        if not self.pk and not self.fid and self.actor.get_user():
             self.fid = self.get_federation_id()
             self.followers_url = self.fid + "/followers"
 

api/funkwhale_api/users/models.py

@@ -248,10 +248,9 @@ class Invitation(models.Model):
         return super().save(**kwargs)
 
 
-def create_actor(user):
+def get_actor_data(user):
     username = federation_utils.slugify_username(user.username)
-    private, public = keys.get_key_pair()
-    args = {
+    return {
         "preferred_username": username,
         "domain": settings.FEDERATION_HOSTNAME,
         "type": "Person",
@@ -260,9 +259,7 @@ def create_actor(user):
         "fid": federation_utils.full_url(
             reverse("federation:actors-detail", kwargs={"preferred_username": username})
         ),
-        "shared_inbox_url": federation_utils.full_url(
-            reverse("federation:shared-inbox")
-        ),
+        "shared_inbox_url": federation_models.get_shared_inbox_url(),
         "inbox_url": federation_utils.full_url(
             reverse("federation:actors-inbox", kwargs={"preferred_username": username})
         ),
@@ -280,6 +277,11 @@ def create_actor(user):
             )
         ),
     }
+
+
+def create_actor(user):
+    args = get_actor_data(user)
+    private, public = keys.get_key_pair()
     args["private_key"] = private.decode("utf-8")
     args["public_key"] = public.decode("utf-8")