novaburst/funquail
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

See #170: fetching remote objects

Browse source
This commit is contained in:
Eliot Berriot 2020-03-02 17:23:03 +01:00
commit c2eeee5eb1
30 changed files with 1175 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

62
api/tests/federation/test_api_serializers.py
View file

@ -141,3 +141,65 @@ def test_api_full_actor_serializer(factories, to_api_date):
serializer = api_serializers.FullActorSerializer(actor)
assert serializer.data == expected
def test_fetch_serializer_no_obj(factories, to_api_date):
fetch = factories["federation.Fetch"]()
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": None,
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected
@pytest.mark.parametrize(
"object_factory, expected_type, expected_id",
[
("music.Album", "album", "id"),
("music.Artist", "artist", "id"),
("music.Track", "track", "id"),
("music.Library", "library", "uuid"),
("music.Upload", "upload", "uuid"),
("federation.Actor", "account", "full_username"),
],
)
def test_fetch_serializer_with_object(
object_factory, expected_type, expected_id, factories, to_api_date
):
obj = factories[object_factory]()
fetch = factories["federation.Fetch"](object=obj)
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": {"type": expected_type, expected_id: getattr(obj, expected_id)},
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected
def test_fetch_serializer_unhandled_obj(factories, to_api_date):
fetch = factories["federation.Fetch"](object=factories["users.User"]())
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": None,
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected

79
api/tests/federation/test_api_views.py
View file

@ -1,9 +1,12 @@
import datetime
import pytest
from django.urls import reverse
from funkwhale_api.federation import api_serializers
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
from funkwhale_api.federation import views
@ -170,7 +173,8 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie
def test_can_detail_fetch(logged_in_api_client, factories):
fetch = factories["federation.Fetch"](url="http://test.object")
actor = logged_in_api_client.user.create_actor()
fetch = factories["federation.Fetch"](url="http://test.object", actor=actor)
url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk})
response = logged_in_api_client.get(url)
@ -209,3 +213,76 @@ def test_can_retrieve_actor(factories, api_client, preferences):
expected = api_serializers.FullActorSerializer(actor).data
assert response.data == expected
@pytest.mark.parametrize(
"object_id, expected_url",
[
("https://fetch.url", "https://fetch.url"),
("name@domain.tld", "webfinger://name@domain.tld"),
("@name@domain.tld", "webfinger://name@domain.tld"),
],
)
def test_can_fetch_using_url_synchronous(
object_id, expected_url, factories, logged_in_api_client, mocker, settings
):
settings.FEDERATION_SYNCHRONOUS_FETCH = True
actor = logged_in_api_client.user.create_actor()
def fake_task(fetch_id):
actor.fetches.filter(id=fetch_id).update(status="finished")
fetch_task = mocker.patch.object(tasks, "fetch", side_effect=fake_task)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id}
response = logged_in_api_client.post(url, data)
assert response.status_code == 201
fetch = actor.fetches.latest("id")
assert fetch.status == "finished"
assert fetch.url == expected_url
assert response.data == api_serializers.FetchSerializer(fetch).data
fetch_task.assert_called_once_with(fetch_id=fetch.pk)
def test_fetch_duplicate(factories, logged_in_api_client, settings, now):
object_id = "http://example.test"
settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
actor = logged_in_api_client.user.create_actor()
duplicate = factories["federation.Fetch"](
actor=actor,
status="finished",
url=object_id,
creation_date=now - datetime.timedelta(seconds=59),
)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id}
response = logged_in_api_client.post(url, data)
assert response.status_code == 201
assert response.data == api_serializers.FetchSerializer(duplicate).data
def test_fetch_duplicate_bypass_with_force(
factories, logged_in_api_client, mocker, settings, now
):
fetch_task = mocker.patch.object(tasks, "fetch")
object_id = "http://example.test"
settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
actor = logged_in_api_client.user.create_actor()
duplicate = factories["federation.Fetch"](
actor=actor,
status="finished",
url=object_id,
creation_date=now - datetime.timedelta(seconds=59),
)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id, "force": True}
response = logged_in_api_client.post(url, data)
fetch = actor.fetches.latest("id")
assert fetch != duplicate
assert response.status_code == 201
assert response.data == api_serializers.FetchSerializer(fetch).data
fetch_task.assert_called_once_with(fetch_id=fetch.pk)

195
api/tests/federation/test_serializers.py
View file

@ -580,6 +580,37 @@ def test_music_library_serializer_from_private(factories, mocker):
)
def test_music_library_serializer_from_ap_update(factories, mocker):
actor = factories["federation.Actor"]()
library = factories["music.Library"]()
data = {
"@context": jsonld.get_default_context(),
"audience": "https://www.w3.org/ns/activitystreams#Public",
"name": "Hello",
"summary": "World",
"type": "Library",
"id": library.fid,
"followers": "https://library.id/followers",
"attributedTo": actor.fid,
"totalItems": 12,
"first": "https://library.id?page=1",
"last": "https://library.id?page=2",
}
serializer = serializers.LibrarySerializer(library, data=data)
assert serializer.is_valid(raise_exception=True)
serializer.save()
library.refresh_from_db()
assert library.uploads_count == data["totalItems"]
assert library.privacy_level == "everyone"
assert library.name == "Hello"
assert library.description == "World"
assert library.followers_url == data["followers"]
def test_activity_pub_artist_serializer_to_ap(factories):
content = factories["common.Content"]()
artist = factories["music.Artist"](
@ -610,6 +641,86 @@ def test_activity_pub_artist_serializer_to_ap(factories):
assert serializer.data == expected
def test_activity_pub_artist_serializer_from_ap_create(factories, faker, now, mocker):
actor = factories["federation.Actor"]()
mocker.patch(
"funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
)
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": "https://test.artist",
"name": "Art",
"musicbrainzId": faker.uuid4(),
"published": now.isoformat(),
"attributedTo": actor.fid,
"content": "Summary",
"image": {
"type": "Image",
"mediaType": "image/jpeg",
"url": "https://attachment.file",
},
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(data=payload)
assert serializer.is_valid(raise_exception=True) is True
artist = serializer.save()
assert artist.fid == payload["id"]
assert artist.attributed_to == actor
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.get_tags() == ["Punk", "Rock"]
def test_activity_pub_artist_serializer_from_ap_update(factories, faker, now, mocker):
artist = factories["music.Artist"]()
actor = factories["federation.Actor"]()
mocker.patch(
"funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
)
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": artist.fid,
"name": "Art",
"musicbrainzId": faker.uuid4(),
"published": now.isoformat(),
"attributedTo": actor.fid,
"content": "Summary",
"image": {
"type": "Image",
"mediaType": "image/jpeg",
"url": "https://attachment.file",
},
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(artist, data=payload)
assert serializer.is_valid(raise_exception=True) is True
serializer.save()
artist.refresh_from_db()
assert artist.attributed_to == actor
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.get_tags() == ["Punk", "Rock"]
def test_activity_pub_album_serializer_to_ap(factories):
content = factories["common.Content"]()
album = factories["music.Album"](
@ -652,39 +763,42 @@ def test_activity_pub_album_serializer_to_ap(factories):
assert serializer.data == expected
def test_activity_pub_artist_serializer_from_ap_update(factories, faker):
artist = factories["music.Artist"](attributed=True)
def test_activity_pub_album_serializer_from_ap_create(factories, faker, now):
actor = factories["federation.Actor"]()
artist = factories["music.Artist"]()
released = faker.date_object()
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": artist.fid,
"type": "Album",
"id": "https://album.example",
"name": faker.sentence(),
"cover": {"type": "Link", "mediaType": "image/jpeg", "href": faker.url()},
"musicbrainzId": faker.uuid4(),
"published": artist.creation_date.isoformat(),
"attributedTo": artist.attributed_to.fid,
"mediaType": "text/html",
"content": common_utils.render_html(faker.sentence(), "text/html"),
"image": {"type": "Image", "mediaType": "image/jpeg", "url": faker.url()},
"published": now.isoformat(),
"released": released.isoformat(),
"artists": [
serializers.ArtistSerializer(
artist, context={"include_ap_context": False}
).data
],
"attributedTo": actor.fid,
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(artist, data=payload)
serializer = serializers.AlbumSerializer(data=payload)
assert serializer.is_valid(raise_exception=True) is True
serializer.save()
album = serializer.save()
artist.refresh_from_db()
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert sorted(artist.tagged_items.values_list("tag__name", flat=True)) == [
assert album.title == payload["name"]
assert str(album.mbid) == payload["musicbrainzId"]
assert album.release_date == released
assert album.artist == artist
assert album.attachment_cover.url == payload["cover"]["href"]
assert album.attachment_cover.mimetype == payload["cover"]["mediaType"]
assert sorted(album.tagged_items.values_list("tag__name", flat=True)) == [
"Punk",
"Rock",
]
@ -1062,6 +1176,43 @@ def test_activity_pub_upload_serializer_from_ap(factories, mocker, r_mock):
assert upload.modification_date == updated
def test_activity_pub_upload_serializer_from_ap_update(factories, mocker, now, r_mock):
library = factories["music.Library"]()
upload = factories["music.Upload"](library=library)
data = {
"@context": jsonld.get_default_context(),
"type": "Audio",
"id": upload.fid,
"name": "Ignored",
"published": now.isoformat(),
"updated": now.isoformat(),
"duration": 42,
"bitrate": 42,
"size": 66,
"url": {
"href": "https://audio.file/url",
"type": "Link",
"mediaType": "audio/mp3",
},
"library": library.fid,
"track": serializers.TrackSerializer(upload.track).data,
}
r_mock.get(data["track"]["album"]["cover"]["href"], body=io.BytesIO(b"coucou"))
serializer = serializers.UploadSerializer(upload, data=data)
assert serializer.is_valid(raise_exception=True)
serializer.save()
upload.refresh_from_db()
assert upload.fid == data["id"]
assert upload.duration == data["duration"]
assert upload.size == data["size"]
assert upload.bitrate == data["bitrate"]
assert upload.source == data["url"]["href"]
assert upload.mimetype == data["url"]["mediaType"]
def test_activity_pub_upload_serializer_validtes_library_actor(factories, mocker):
library = factories["music.Library"]()
usurpator = factories["federation.Actor"]()
@ -1201,7 +1352,7 @@ def test_track_serializer_update_license(factories):
obj = factories["music.Track"](license=None)
serializer = serializers.TrackSerializer()
serializer = serializers.TrackSerializer(obj)
serializer.update(obj, {"license": "http://creativecommons.org/licenses/by/2.0/"})
obj.refresh_from_db()

153
api/tests/federation/test_tasks.py
View file

@ -395,3 +395,156 @@ def test_fetch_success(factories, r_mock, mocker):
assert init.call_args[0][1] == artist
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_webfinger(factories, r_mock, mocker):
actor = factories["federation.Actor"]()
fetch = factories["federation.Fetch"](
url="webfinger://{}".format(actor.full_username)
)
payload = serializers.ActorSerializer(actor).data
init = mocker.spy(serializers.ActorSerializer, "__init__")
save = mocker.spy(serializers.ActorSerializer, "save")
webfinger_payload = {
"subject": "acct:{}".format(actor.full_username),
"aliases": ["https://test.webfinger"],
"links": [
{"rel": "self", "type": "application/activity+json", "href": actor.fid}
],
}
webfinger_url = "https://{}/.well-known/webfinger?resource={}".format(
actor.domain_id, webfinger_payload["subject"]
)
r_mock.get(actor.fid, json=payload)
r_mock.get(webfinger_url, json=webfinger_payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == actor
assert init.call_count == 1
assert init.call_args[0][1] == actor
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_rel_alternate(factories, r_mock, mocker):
actor = factories["federation.Actor"]()
fetch = factories["federation.Fetch"](url="http://example.page")
html_text = """
<html>
<head>
<link rel="alternate" type="application/activity+json" href="{}" />
</head>
</html>
""".format(
actor.fid
)
ap_payload = serializers.ActorSerializer(actor).data
init = mocker.spy(serializers.ActorSerializer, "__init__")
save = mocker.spy(serializers.ActorSerializer, "save")
r_mock.get(fetch.url, text=html_text)
r_mock.get(actor.fid, json=ap_payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
ap_payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == actor
assert init.call_count == 1
assert init.call_args[0][1] == actor
assert init.call_args[1]["data"] == ap_payload
assert save.call_count == 1
@pytest.mark.parametrize(
"factory_name, serializer_class",
[
("federation.Actor", serializers.ActorSerializer),
("music.Library", serializers.LibrarySerializer),
("music.Artist", serializers.ArtistSerializer),
("music.Album", serializers.AlbumSerializer),
("music.Track", serializers.TrackSerializer),
],
)
def test_fetch_url(factory_name, serializer_class, factories, r_mock, mocker):
obj = factories[factory_name]()
fetch = factories["federation.Fetch"](url=obj.fid)
payload = serializer_class(obj).data
init = mocker.spy(serializer_class, "__init__")
save = mocker.spy(serializer_class, "save")
r_mock.get(obj.fid, json=payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == obj
assert init.call_count == 1
assert init.call_args[0][1] == obj
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_honor_instance_policy_domain(factories):
domain = factories["moderation.InstancePolicy"](
block_all=True, for_domain=True
).target_domain
fid = "https://{}/test".format(domain.name)
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
def test_fetch_honor_mrf_inbox_before_http(mrf_inbox_registry, factories, mocker):
apply = mocker.patch.object(mrf_inbox_registry, "apply", return_value=(None, False))
fid = "http://domain/test"
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
apply.assert_called_once_with({"id": fid})
def test_fetch_honor_mrf_inbox_after_http(
r_mock, mrf_inbox_registry, factories, mocker
):
apply = mocker.patch.object(
mrf_inbox_registry, "apply", side_effect=[(True, False), (None, False)]
)
payload = {"id": "http://domain/test", "actor": "hello"}
r_mock.get(payload["id"], json=payload)
fetch = factories["federation.Fetch"](url=payload["id"])
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
apply.assert_any_call({"id": payload["id"]})
apply.assert_any_call(payload)
def test_fetch_honor_instance_policy_different_url_and_id(r_mock, factories):
domain = factories["moderation.InstancePolicy"](
block_all=True, for_domain=True
).target_domain
fid = "https://ok/test"
r_mock.get(fid, json={"id": "http://{}/test".format(domain.name)})
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"