diff --git a/Pipfile b/Pipfile index f68bd2b24..19647f4e0 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ ruff = "*" [packages] django = "~=6.0.0" django-huey = "*" +django-tasks-db = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish new file mode 100755 index 000000000..8dd0823dd --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish @@ -0,0 +1,12 @@ +#!/command/with-contenv bash + +set -eu + +scan_directory='/run/s6-rc/scandir' +service_dirs_directory='/run/s6-rc/servicedirs' + +service_name='django-tasks-default-workers' +service_directory="${service_dirs_directory}/${service_name}" + +/command/s6-svc -wD -d "${service_directory}" +/command/s6-svunlink "${scan_directory}/" "${service_name}" diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run new file mode 100755 index 000000000..6e6edea06 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run @@ -0,0 +1,65 @@ +#!/command/with-contenv bash + +set -eu + +scan_directory='/run/s6-rc/scandir' +service_dirs_directory='/run/s6-rc/servicedirs' + +service_name='django-tasks-default-workers' +staging_directory="/run/servicedirs-${service_name}" +service_directory="${service_dirs_directory}/${service_name}" +worker_template='/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@' + +[[ -d /run/ ]] + +# Stage the new service +rm -rf "${staging_directory}" +if [[ -e "${service_directory}" ]] ; then + # Attempt to clean up old instance processes + /command/s6-svc -wD -d "${service_directory}" || true + /command/s6-svunlink "${scan_directory}/" "${service_name}" + mkdir -p "${staging_directory}" + mv -t "${staging_directory}" "${service_directory}" + rm -rf "${staging_directory}" +fi +mkdir -p "${staging_directory}" +/command/s6-instance-maker "${worker_template}" "${staging_directory}/${service_name}" + +# Install the new service +mv "${staging_directory}/${service_name}" "${service_directory}" +rmdir "${staging_directory}" +/command/s6-svlink -t 1000 "${scan_directory}/" "${service_directory}" + +create_workers() { + local i=0 timeout=5000 total="${1}" + while (( ++i <= total )); do + /command/s6-instance-create -t "${timeout}" "${service_directory}" "${i}" || return + done +} + +determine_cores() { + local minimum="${1:-1}" nproc_cores usable + + nproc_cores="$(nproc --ignore=2 || echo 0)" + usable="$(python3 -c 'n=int("'"${nproc_cores}"'"); from common.utils import get_usable_cpu_count; print(min(n, get_usable_cpu_count()));' || echo 0)" + + if (( "${usable}" < "${minimum}" )) ; then + printf -- '%d\n' "${minimum}" + else + printf -- '%d\n' "${usable}" + fi +} + +cores="$(determine_cores 2 || echo 2)" +WORKERS="${TUBESYNC_DJANGO_TASKS_WORKERS:-${cores}}" +create_workers "${WORKERS}" || true + +if [[ -d "${service_directory}/instances/1/supervise" ]]; then + exec /command/s6-svwait -D -a "${service_directory}"/instance/* +fi + +# When the first worker was not created, run at least one. +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name '*' --exclude-queues limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/timeout-finish b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/timeout-finish new file mode 100644 index 000000000..b612371b0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/timeout-finish @@ -0,0 +1 @@ +900000 diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal new file mode 100644 index 000000000..d751378e1 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish new file mode 100755 index 000000000..fa7b6e379 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py prune_db_task_results \ + --queue-name '*' diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run new file mode 100755 index 000000000..128259fea --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run @@ -0,0 +1,6 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name '*' --exclude-queues limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run new file mode 100755 index 000000000..53a4e64fc --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run @@ -0,0 +1,6 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py run_huey \ + --disable-autoload --no-periodic \ + --graceful-signal TERM --shutdown-timeout 60 diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal new file mode 100644 index 000000000..d751378e1 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run new file mode 100755 index 000000000..71cf3a8f4 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run @@ -0,0 +1,6 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-default b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-default new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-huey b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-huey new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-net-limited b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-net-limited new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type new file mode 100644 index 000000000..757b42211 --- /dev/null +++ b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type @@ -0,0 +1 @@ +bundle diff --git a/config/root/etc/s6-overlay/user-bundles.d/user/contents.d/django-tasks-consumers b/config/root/etc/s6-overlay/user-bundles.d/user/contents.d/django-tasks-consumers new file mode 100644 index 000000000..e69de29bb diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index 200d75a66..7fda5d4f7 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -1,7 +1,9 @@ from __future__ import annotations +import concurrent.futures import datetime import subprocess +import threading import time import uuid from functools import partial @@ -507,21 +509,13 @@ def close_db(task, task_value=None, exception=None, /, *, huey=None): return db.close_old_connections() - for qn in DJANGO_HUEY.get('queues', dict()): + def prune_queue_storage(qn): + # clean up old history and results from storage q = get_queue(qn) - # hooks to clean up database connections at task boundaries - pre_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) - post_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) - - # signals for tracking / maintenance of tasks - signal(signals.SIGNAL_INTERRUPTED, queue=qn)(on_interrupted) - signal(queue=qn)(historical_task) - signal(signals.SIGNAL_EXECUTING, queue=qn)(on_executing_remove_duplicates) - - # clean up old history and results from storage now_time = time.monotonic() now_dt = datetime.datetime.now(datetime.timezone.utc) + # ruff: ignore[SIM118] for key in q.all_results().keys(): if not key.startswith(storage_key_prefix): @@ -547,6 +541,45 @@ def close_db(task, task_value=None, exception=None, /, *, huey=None): q.get(peek=False, key=result_key) q.get(peek=False, key=key) + def _run_cleanup(qn): + def _target(): + try: + prune_queue_storage(qn) + # ruff: ignore[BLE001] + except Exception: + from common.logger import log + log.exception(f'Cleanup of {qn=} failed.') + # The pruning is happening opportunistically; + # short running tasks won't wait for these threads. + worker = threading.Thread(target=_target, daemon=True) + worker.queue_name = qn + threads.add(worker) + worker.start() + + queues = { qn: get_queue(qn) for qn in DJANGO_HUEY.get('queues', dict()) } + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=get_usable_cpu_count(), + ) + threads = set() + for qn, q in queues.items(): + # hooks to clean up database connections at task boundaries + pre_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) + post_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) + + # signals for tracking / maintenance of tasks + signal(signals.SIGNAL_INTERRUPTED, queue=qn)(on_interrupted) + signal(queue=qn)(historical_task) + signal(signals.SIGNAL_EXECUTING, queue=qn)(on_executing_remove_duplicates) + + # prune the queue storage + executor.submit(_run_cleanup, qn) + + # wait for the worker threads to start, + # not for the storage pruning tasks to finish + executor.shutdown(wait=True) + + return threads + class BackoffAlgorithm: # ruff: ignore[RUF012] diff --git a/tubesync/sync/apps.py b/tubesync/sync/apps.py index 0d95b4856..be57c2810 100644 --- a/tubesync/sync/apps.py +++ b/tubesync/sync/apps.py @@ -3,3 +3,7 @@ class SyncConfig(AppConfig): name = 'sync' + + def ready(self): + from .tasks import cleanup_huey_storage + self.cleanup_huey_storage_result = cleanup_huey_storage.enqueue() diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index c0eb2b74b..6beab3710 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -5,7 +5,7 @@ from common.huey import h_q_reset_maint_func, h_q_reset_tasks from common.logger import log from sync.models import Source -from sync.tasks import check_source_directory_exists +from sync.tasks import make_source_directory class Command(BaseCommand): @@ -19,7 +19,7 @@ def handle(self, *args, **options): # Iter all sources, creating new tasks for source in Source.objects.all(): log.info(f'Resetting tasks for source: {source}') - check_source_directory_exists(str(source.pk)) + make_source_directory.enqueue(str(source.pk)) # This also chains down to call each Media objects .save() as well source.save() diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 4b1d53590..22160c9ef 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -14,7 +14,7 @@ from .tasks import ( get_media_download_task, get_media_metadata_task, get_media_thumbnail_task, delete_all_media_for_source, save_all_media_for_source, - check_source_directory_exists, download_source_images, index_source, + make_source_directory, download_source_images, index_source, download_media_file, download_media_metadata, download_media_image, ) from .utils import delete_file @@ -23,7 +23,9 @@ @receiver(pre_save, sender=Source) def source_pre_save(sender, instance, **kwargs): - source = instance # noqa: F841 + args = ( str(instance.pk), ) + # ruff: ignore[F841] + source = instance # Triggered before a source is saved, if the schedule has been updated recreate # its indexing task try: @@ -31,14 +33,15 @@ def source_pre_save(sender, instance, **kwargs): except Source.DoesNotExist: log.debug(f'source_pre_save signal: no existing source: {sender} - {instance}') return + else: + if not existing_source.directory_exists(): + existing_source.make_directory() - args = ( str(instance.pk), ) - check_source_directory_exists.call_local(*args) existing_copy_channel_images = existing_source.copy_channel_images new_copy_channel_images = instance.copy_channel_images if new_copy_channel_images and not existing_copy_channel_images: download_source_images( - str(instance.pk), + *args, delay=download_source_images.settings.get('delay'), ) existing_dirpath = existing_source.directory_path.resolve(strict=True) @@ -55,7 +58,7 @@ def source_pre_save(sender, instance, **kwargs): work_directory = existing_dirpath for _count in range(parents_count, 0, -1): work_directory = work_directory.parent - if not Path(work_directory).resolve(strict=True).is_relative_to(Path(settings.DOWNLOAD_ROOT)): + if not Path(work_directory).resolve(strict=True).is_relative_to(Path(settings.DOWNLOAD_ROOT).resolve(strict=True)): work_directory = Path(settings.DOWNLOAD_ROOT) with TemporaryDirectory(suffix=('.'+new_dirpath.name), prefix='.tmp.', dir=work_directory) as tmp_dir: tmp_dirpath = Path(tmp_dir) @@ -102,7 +105,7 @@ def source_pre_save(sender, instance, **kwargs): # Indexing schedule has changed, recreate the indexing task TaskHistory.schedule( index_source, - str(instance.pk), + *args, eta=instance.task_run_at_dt, remove_duplicates=True, vn_fmt=_('Index media from source "{}"'), @@ -115,7 +118,7 @@ def source_post_save(sender, instance, created, **kwargs): source = instance # Check directory exists and create an indexing task for newly created sources if created: - check_source_directory_exists(str(source.pk)) + make_source_directory.enqueue(str(source.pk)) if source.copy_channel_images: download_source_images( str(source.pk), diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 5721362e4..6f46b16f4 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -4,6 +4,7 @@ ''' +import contextlib import os import random import requests @@ -18,6 +19,7 @@ from django import db from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile +from django.tasks import task as django_task from django.utils import timezone from django.utils.translation import gettext_lazy as _ from django_huey import lock_task as huey_lock_task, task as huey_task @@ -45,7 +47,7 @@ from .youtube import YouTubeError atomic = db.transaction.atomic -register_huey_signals() +cleanup_threads = register_huey_signals() def get_task_map(): @@ -378,6 +380,22 @@ def wait_for_errors(model, /, *, queue_name=None, task_name=None): raise HueyConsumerError(_('queue consumer stopped')) +@django_task(backend='huey') +def cleanup_huey_storage(): + workers = cleanup_threads + if not all( + (callable(getattr(workers, 'clear', None)),) + + tuple(callable(getattr(w, 'join', None)) for w in workers) + + tuple(callable(getattr(w, 'start', None)) for w in workers), + ): + return + while workers: + worker = workers.pop() + with contextlib.suppress(RuntimeError): + worker.start() + worker.join() + + @db_task(priority=90, retries=2, retry_delay=150, queue=Val(TaskQueue.FS)) @huey_lock_task('sync.tasks.cleanup_old_media', queue=Val(TaskQueue.FS)) def cleanup_old_media(durable=True): @@ -708,6 +726,15 @@ def index_source(source_id): return True +@django_task(priority=100, queue_name=Val(TaskQueue.FS)) +def make_source_directory(source_id): + with contextlib.suppress(Source.DoesNotExist): + source = Source.objects.get(pk=source_id) + if not source.directory_exists(): + log.info(f'Creating directory: {source.directory_path}') + source.make_directory() + + @db_task(priority=100, retries=15, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.FS)) def check_source_directory_exists(source_id): ''' @@ -723,8 +750,8 @@ def check_source_directory_exists(source_id): # Check the source output directory exists if not source.directory_exists(): # Try to create it - log.info(f'Creating directory: {source.directory_path}') source.make_directory() + log.info(f'Created directory: {source.directory_path}') @db_task(delay=10, priority=90, retries=15, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.LIMIT)) diff --git a/tubesync/sync/tests/test_frontend.py b/tubesync/sync/tests/test_frontend.py index 3e8a24cd0..7038f1b96 100644 --- a/tubesync/sync/tests/test_frontend.py +++ b/tubesync/sync/tests/test_frontend.py @@ -11,6 +11,7 @@ from sync.tasks import ( check_source_directory_exists, get_media_download_task, get_media_thumbnail_task, + make_source_directory, ) from sync.choices import ( Val, Fallback, FilterSeconds, IndexSchedule, SourceResolution, @@ -206,6 +207,8 @@ def test_source(self): expected_categories) # Run the check_source_directory_exists task check_source_directory_exists.call_local(source_uuid) + # Run the make_source_directory task + make_source_directory.call(source_uuid) # Check the source is now on the source overview page response = c.get('/sources') self.assertEqual(response.status_code, 200) diff --git a/tubesync/sync/views/tasks.py b/tubesync/sync/views/tasks.py index f3197494e..9adefdc0d 100644 --- a/tubesync/sync/views/tasks.py +++ b/tubesync/sync/views/tasks.py @@ -24,7 +24,7 @@ from ..forms import ScheduleTaskForm from ..tasks import ( get_task_map, map_task_to_instance, get_error_message, - get_running_tasks, check_source_directory_exists, + get_running_tasks, make_source_directory, ) @@ -295,7 +295,7 @@ def form_valid(self, form): h_q_reset_tasks(queue_name, maint_func=h_q_reset_maint_func) # Iter all tasks for source in Source.objects.all(): - check_source_directory_exists(str(source.pk)) + make_source_directory.enqueue(str(source.pk)) # This also chains down to call each Media objects .save() as well source.save() return super().form_valid(form) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a85dcb188..7eec49827 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -28,6 +28,8 @@ 'django.contrib.humanize', 'sass_processor', 'django_huey', + 'django_tasks_db', + 'huey.contrib.djhuey', 'common', 'sync', ] @@ -51,6 +53,23 @@ FORCE_SCRIPT_NAME = None +TASKS = { + 'default': { + 'BACKEND': 'django_tasks_db.DatabaseBackend', + 'QUEUES': TaskQueue.values + ['default'], + }, + 'huey': { + 'BACKEND': 'huey.contrib.djhuey.tasks_backend.HueyBackend', + 'OPTIONS': { + 'enqueue_on_commit': True, + }, + }, +} + +# Used by TASKS 'huey' backend +HUEY = sqlite_tasks('tasks', thread=True, workers=0) +HUEY['consumer']['verbose'] = True if DEBUG else None + DJANGO_HUEY = { 'default': TaskQueue.LIMIT.value, 'queues': dict(),