novaburst/funquail
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Importer updates: watch directories, handle metadata updates

Browse source
This commit is contained in:
Agate 2020-05-07 09:55:29 +02:00
commit 752c993e8e
No known key found for this signature in database
GPG key ID: 6B501DFD73514E14
16 changed files with 1005 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

9
api/tests/common/test_utils.py
View file

@ -258,3 +258,12 @@ def test_monkey_patch_request_build_absolute_uri(
request = fake_request.get("/", **meta)
assert request.build_absolute_uri(path) == expected
def test_get_file_hash(tmpfile, settings):
settings.HASHING_ALGORITHM = "sha256"
content = b"hello"
tmpfile.write(content)
# echo -n "hello" | sha256sum
expected = "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
assert utils.get_file_hash(tmpfile) == expected

26
api/tests/music/test_commands.py
View file

@ -1,6 +1,7 @@
import os
import pytest
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music.management.commands import check_inplace_files
from funkwhale_api.music.management.commands import fix_uploads
from funkwhale_api.music.management.commands import prune_library
@ -18,7 +19,7 @@ def test_fix_uploads_bitrate_length(factories, mocker):
return_value={"bitrate": 42, "length": 43},
)
c.fix_file_data(dry_run=False)
c.fix_file_data(dry_run=False, batch_size=100)
upload1.refresh_from_db()
upload2.refresh_from_db()
@ -41,7 +42,7 @@ def test_fix_uploads_size(factories, mocker):
mocker.patch("funkwhale_api.music.models.Upload.get_file_size", return_value=2)
c.fix_file_size(dry_run=False)
c.fix_file_size(dry_run=False, batch_size=100)
upload1.refresh_from_db()
upload2.refresh_from_db()
@ -69,7 +70,7 @@ def test_fix_uploads_mimetype(factories, mocker):
mimetype="audio/something",
)
c = fix_uploads.Command()
c.fix_mimetypes(dry_run=False)
c.fix_mimetypes(dry_run=False, batch_size=100)
upload1.refresh_from_db()
upload2.refresh_from_db()
@ -78,6 +79,25 @@ def test_fix_uploads_mimetype(factories, mocker):
assert upload2.mimetype == "audio/something"
def test_fix_uploads_checksum(factories, mocker):
upload1 = factories["music.Upload"]()
upload2 = factories["music.Upload"]()
upload1.__class__.objects.filter(pk=upload1.pk).update(checksum="test")
upload2.__class__.objects.filter(pk=upload2.pk).update(checksum=None)
c = fix_uploads.Command()
c.fix_file_checksum(dry_run=False, batch_size=100)
upload1.refresh_from_db()
upload2.refresh_from_db()
# not updated
assert upload1.checksum == "test"
# updated
assert upload2.checksum == common_utils.get_file_hash(upload2.audio_file)
def test_prune_library_dry_run(factories):
prunable = factories["music.Track"]()
not_prunable = factories["music.Track"]()

12
api/tests/music/test_models.py
View file

@ -5,6 +5,7 @@ import pytest
from django.utils import timezone
from django.urls import reverse
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music import importers, models, tasks
from funkwhale_api.federation import utils as federation_utils
@ -164,6 +165,17 @@ def test_audio_track_mime_type(extention, mimetype, factories):
assert upload.mimetype == mimetype
@pytest.mark.parametrize("name", ["test.ogg", "test.mp3"])
def test_audio_track_checksum(name, factories):
path = os.path.join(DATA_DIR, name)
upload = factories["music.Upload"](audio_file__from_path=path, mimetype=None)
with open(path, "rb") as f:
expected = common_utils.get_file_hash(f)
assert upload.checksum == expected
def test_upload_file_name(factories):
name = "test.mp3"
path = os.path.join(DATA_DIR, name)

37
api/tests/music/test_tasks.py
View file

@ -1329,3 +1329,40 @@ def test_can_import_track_with_same_position_in_same_discs_skipped(factories, mo
new_upload.refresh_from_db()
assert new_upload.import_status == "skipped"
def test_update_track_metadata(factories):
track = factories["music.Track"]()
data = {
"title": "Peer Gynt Suite no. 1, op. 46: I. Morning",
"artist": "Edvard Grieg",
"album_artist": "Edvard Grieg; Musopen Symphony Orchestra",
"album": "Peer Gynt Suite no. 1, op. 46",
"date": "2012-08-15",
"position": "4",
"disc_number": "2",
"musicbrainz_albumid": "a766da8b-8336-47aa-a3ee-371cc41ccc75",
"mbid": "bd21ac48-46d8-4e78-925f-d9cc2a294656",
"musicbrainz_artistid": "013c8e5b-d72a-4cd3-8dee-6c64d6125823",
"musicbrainz_albumartistid": "013c8e5b-d72a-4cd3-8dee-6c64d6125823;5b4d7d2d-36df-4b38-95e3-a964234f520f",
"license": "Dummy license: http://creativecommons.org/licenses/by-sa/4.0/",
"copyright": "Someone",
"comment": "hello there",
}
tasks.update_track_metadata(metadata.FakeMetadata(data), track)
track.refresh_from_db()
assert track.title == data["title"]
assert track.position == int(data["position"])
assert track.disc_number == int(data["disc_number"])
assert track.license.code == "cc-by-sa-4.0"
assert track.copyright == data["copyright"]
assert str(track.mbid) == data["mbid"]
assert track.album.title == data["album"]
assert track.album.release_date == datetime.date(2012, 8, 15)
assert str(track.album.mbid) == data["musicbrainz_albumid"]
assert track.artist.name == data["artist"]
assert str(track.artist.mbid) == data["musicbrainz_artistid"]
assert track.album.artist.name == "Edvard Grieg"
assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823"

193
api/tests/test_import_audio_file.py
View file

@ -4,6 +4,8 @@ import pytest
from django.core.management import call_command
from django.core.management.base import CommandError
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music.management.commands import import_files
DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
@ -159,3 +161,194 @@ def test_import_files_in_place(factories, mocker, settings):
def test_storage_rename_utf_8_files(factories):
upload = factories["music.Upload"](audio_file__filename="été.ogg")
assert upload.audio_file.name.endswith("ete.ogg")
@pytest.mark.parametrize("name", ["modified", "moved", "created", "deleted"])
def test_handle_event(name, mocker):
handler = mocker.patch.object(import_files, "handle_{}".format(name))
event = {"type": name}
stdout = mocker.Mock()
kwargs = {"hello": "world"}
import_files.handle_event(event, stdout, **kwargs)
handler.assert_called_once_with(event=event, stdout=stdout, **kwargs)
def test_handle_created(mocker):
handle_modified = mocker.patch.object(import_files, "handle_modified")
event = mocker.Mock()
stdout = mocker.Mock()
kwargs = {"hello": "world"}
import_files.handle_created(event, stdout, **kwargs)
handle_modified.assert_called_once_with(event, stdout, **kwargs)
def test_handle_deleted(factories, mocker):
stdout = mocker.Mock()
event = {
"path": "/path.mp3",
}
library = factories["music.Library"]()
deleted = factories["music.Upload"](
library=library,
source="file://{}".format(event["path"]),
import_status="finished",
audio_file=None,
)
kept = [
factories["music.Upload"](
library=library,
source="file://{}".format(event["path"]),
import_status="finished",
),
factories["music.Upload"](
source="file://{}".format(event["path"]),
import_status="finished",
audio_file=None,
),
]
import_files.handle_deleted(
event=event, stdout=stdout, library=library, in_place=True
)
with pytest.raises(deleted.DoesNotExist):
deleted.refresh_from_db()
for upload in kept:
upload.refresh_from_db()
def test_handle_moved(factories, mocker):
stdout = mocker.Mock()
event = {
"src_path": "/path.mp3",
"dest_path": "/new_path.mp3",
}
library = factories["music.Library"]()
updated = factories["music.Upload"](
library=library,
source="file://{}".format(event["src_path"]),
import_status="finished",
audio_file=None,
)
untouched = [
factories["music.Upload"](
library=library,
source="file://{}".format(event["src_path"]),
import_status="finished",
),
factories["music.Upload"](
source="file://{}".format(event["src_path"]),
import_status="finished",
audio_file=None,
),
]
import_files.handle_moved(
event=event, stdout=stdout, library=library, in_place=True
)
updated.refresh_from_db()
assert updated.source == "file://{}".format(event["dest_path"])
for upload in untouched:
source = upload.source
upload.refresh_from_db()
assert source == upload.source
def test_handle_modified_creates_upload(tmpfile, factories, mocker):
stdout = mocker.Mock()
event = {
"path": tmpfile.name,
}
process_upload = mocker.patch("funkwhale_api.music.tasks.process_upload")
library = factories["music.Library"]()
import_files.handle_modified(
event=event,
stdout=stdout,
library=library,
in_place=True,
reference="hello",
replace=False,
dispatch_outbox=False,
broadcast=False,
)
upload = library.uploads.latest("id")
assert upload.source == "file://{}".format(event["path"])
process_upload.assert_called_once_with(upload_id=upload.pk)
def test_handle_modified_skips_existing_checksum(tmpfile, factories, mocker):
stdout = mocker.Mock()
event = {
"path": tmpfile.name,
}
tmpfile.write(b"hello")
library = factories["music.Library"]()
factories["music.Upload"](
checksum=common_utils.get_file_hash(tmpfile),
library=library,
import_status="finished",
)
import_files.handle_modified(
event=event, stdout=stdout, library=library, in_place=True,
)
assert library.uploads.count() == 1
def test_handle_modified_update_existing_path_if_found(tmpfile, factories, mocker):
stdout = mocker.Mock()
event = {
"path": tmpfile.name,
}
update_track_metadata = mocker.patch(
"funkwhale_api.music.tasks.update_track_metadata"
)
get_metadata = mocker.patch("funkwhale_api.music.models.Upload.get_metadata")
library = factories["music.Library"]()
track = factories["music.Track"](attributed_to=library.actor)
upload = factories["music.Upload"](
source="file://{}".format(event["path"]),
track=track,
checksum="old",
library=library,
import_status="finished",
audio_file=None,
)
import_files.handle_modified(
event=event, stdout=stdout, library=library, in_place=True,
)
update_track_metadata.assert_called_once_with(
get_metadata.return_value, upload.track,
)
def test_handle_modified_update_existing_path_if_found_and_attributed_to(
tmpfile, factories, mocker
):
stdout = mocker.Mock()
event = {
"path": tmpfile.name,
}
update_track_metadata = mocker.patch(
"funkwhale_api.music.tasks.update_track_metadata"
)
library = factories["music.Library"]()
factories["music.Upload"](
source="file://{}".format(event["path"]),
checksum="old",
library=library,
track__attributed_to=factories["federation.Actor"](),
import_status="finished",
audio_file=None,
)
import_files.handle_modified(
event=event, stdout=stdout, library=library, in_place=True,
)
update_track_metadata.assert_not_called()