From 1ff9fd4da9c8d38f8dc3b5f0c18da2bfacdc62ef Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 Oct 2026 23:49:07 -0400 Subject: [PATCH 01/11] chore: add django-tasks-db package to Pipfile --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index f68bd2b24..19647f4e0 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ ruff = "*" [packages] django = "~=6.0.0" django-huey = "*" +django-tasks-db = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" From 6bebd443181f498084f81b42519895a89f1e5c2f Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 00:12:16 -0400 Subject: [PATCH 02/11] chore: add django_tasks_db to settings Added django_tasks_db to installed apps and configured TASKS settings. --- tubesync/tubesync/settings.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a85dcb188..7e973a042 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -28,6 +28,7 @@ 'django.contrib.humanize', 'sass_processor', 'django_huey', + 'django_tasks_db', 'common', 'sync', ] @@ -51,6 +52,13 @@ FORCE_SCRIPT_NAME = None +TASKS = { + 'default': { + 'BACKEND': 'django_tasks_db.DatabaseBackend', + 'QUEUES': TaskQueue.values + ['default'], + }, +} + DJANGO_HUEY = { 'default': TaskQueue.LIMIT.value, 'queues': dict(), From fc88655e16e5869cf9d1553f33ea4713c4850e78 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 00:12:22 -0400 Subject: [PATCH 03/11] chore: add Huey as a backend --- tubesync/tubesync/settings.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 7e973a042..6170e6650 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -29,6 +29,7 @@ 'sass_processor', 'django_huey', 'django_tasks_db', + 'huey.contrib.djhuey', 'common', 'sync', ] @@ -57,8 +58,16 @@ 'BACKEND': 'django_tasks_db.DatabaseBackend', 'QUEUES': TaskQueue.values + ['default'], }, + 'huey': { + 'BACKEND': 'huey.contrib.djhuey.tasks_backend.HueyBackend', + 'ENQUEUE_ON_COMMIT': True, + }, } +# Used by TASKS 'huey' backend +HUEY = sqlite_tasks('tasks', thread=True, workers=0) +HUEY['consumer']['verbose'] = True if DEBUG else None + DJANGO_HUEY = { 'default': TaskQueue.LIMIT.value, 'queues': dict(), From 807c8ed514534d9cc674e51ae2fbbb63ad3c79cd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 00:12:46 -0400 Subject: [PATCH 04/11] feat: background queue cleanup threads Introduce threading for cleanup tasks and utilize `ThreadPoolExecutor` for concurrent execution. --- tubesync/common/huey.py | 55 ++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index 396498422..6958e1598 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -1,7 +1,9 @@ from __future__ import annotations +import concurrent.futures import datetime import subprocess +import threading import time import uuid from functools import partial @@ -507,21 +509,13 @@ def close_db(task, task_value=None, exception=None, /, *, huey=None): return db.close_old_connections() - for qn in DJANGO_HUEY.get('queues', dict()): + def prune_queue_storage(qn): + # clean up old history and results from storage q = get_queue(qn) - # hooks to clean up database connections at task boundaries - pre_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) - post_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) - - # signals for tracking / maintenance of tasks - signal(signals.SIGNAL_INTERRUPTED, queue=qn)(on_interrupted) - signal(queue=qn)(historical_task) - signal(signals.SIGNAL_EXECUTING, queue=qn)(on_executing_remove_duplicates) - - # clean up old history and results from storage now_time = time.monotonic() now_dt = datetime.datetime.now(datetime.timezone.utc) + # ruff: ignore[SIM118] for key in q.all_results().keys(): if not key.startswith(storage_key_prefix): @@ -547,6 +541,45 @@ def close_db(task, task_value=None, exception=None, /, *, huey=None): q.get(peek=False, key=result_key) q.get(peek=False, key=key) + def _run_cleanup(qn): + def _target(): + try: + prune_queue_storage(qn) + # ruff: ignore[BLE001] + except Exception: + from common.logger import log + log.exception(f'Cleanup of {qn=} failed.') + # The pruning is happening opportunistically; + # short running tasks won't wait for these threads. + worker = threading.Thread(target=_target, daemon=True) + worker.queue_name = qn + threads.add(worker) + worker.start() + + queues = { qn: get_queue(qn) for qn in DJANGO_HUEY.get('queues', dict()) } + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=get_usable_cpu_count(), + ) + threads = set() + for qn, q in queues.items(): + # hooks to clean up database connections at task boundaries + pre_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) + post_execute(queue=qn, name='close_db')(partial(close_db, huey=q)) + + # signals for tracking / maintenance of tasks + signal(signals.SIGNAL_INTERRUPTED, queue=qn)(on_interrupted) + signal(queue=qn)(historical_task) + signal(signals.SIGNAL_EXECUTING, queue=qn)(on_executing_remove_duplicates) + + # prune the queue storage + executor.submit(_run_cleanup, qn) + + # wait for the worker threads to start, + # not for the storage pruning tasks to finish + executor.shutdown(wait=True) + + return threads + class BackoffAlgorithm: # ruff: ignore[RUF012] From aafcd4ca47d190c0ff678f397a0693d59ebe8be8 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 06:16:22 -0400 Subject: [PATCH 05/11] chore: move existing bundles to the new directory --- .../huey-consumers/contents.d/huey-database | 0 .../huey-consumers/contents.d/huey-filesystem | 0 .../huey-consumers/contents.d/huey-net-limited | 0 .../huey-consumers/contents.d/huey-network | 0 .../s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/type | 0 .../tubesync-init/contents.d/tubesync-app-init | 0 .../tubesync-init/contents.d/tubesync-config-init | 0 .../tubesync-init/contents.d/tubesync-downloads-init | 0 .../tubesync-init/contents.d/tubesync-migrations-init | 0 .../tubesync-init/contents.d/tubesync-services-init | 0 .../etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/type | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/contents.d/huey-database (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/contents.d/huey-filesystem (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/contents.d/huey-net-limited (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/contents.d/huey-network (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/huey-consumers/type (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/contents.d/tubesync-app-init (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/contents.d/tubesync-config-init (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/contents.d/tubesync-downloads-init (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/contents.d/tubesync-migrations-init (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/contents.d/tubesync-services-init (100%) rename config/root/etc/s6-overlay/{s6-rc.d => user-bundles.d}/tubesync-init/type (100%) diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database b/config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-database similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database rename to config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-database diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem b/config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-filesystem similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem rename to config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-filesystem diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited b/config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-net-limited similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited rename to config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-net-limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network b/config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-network similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network rename to config/root/etc/s6-overlay/user-bundles.d/huey-consumers/contents.d/huey-network diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type b/config/root/etc/s6-overlay/user-bundles.d/huey-consumers/type similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type rename to config/root/etc/s6-overlay/user-bundles.d/huey-consumers/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-app-init b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-app-init similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-app-init rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-app-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-config-init b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-config-init similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-config-init rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-config-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-downloads-init b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-downloads-init similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-downloads-init rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-downloads-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-migrations-init b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-migrations-init similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-migrations-init rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-migrations-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-services-init b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-services-init similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/contents.d/tubesync-services-init rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/contents.d/tubesync-services-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type b/config/root/etc/s6-overlay/user-bundles.d/tubesync-init/type similarity index 100% rename from config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type rename to config/root/etc/s6-overlay/user-bundles.d/tubesync-init/type From fc02c6f1ca2e0dd91afa95d1dc5c4dc97fcc162e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 06:16:28 -0400 Subject: [PATCH 06/11] chore: add an empty django-tasks-consumers bundle --- .../etc/s6-overlay/user-bundles.d/django-tasks-consumers/type | 1 + .../user-bundles.d/user/contents.d/django-tasks-consumers | 0 2 files changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type create mode 100644 config/root/etc/s6-overlay/user-bundles.d/user/contents.d/django-tasks-consumers diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type new file mode 100644 index 000000000..757b42211 --- /dev/null +++ b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/type @@ -0,0 +1 @@ +bundle diff --git a/config/root/etc/s6-overlay/user-bundles.d/user/contents.d/django-tasks-consumers b/config/root/etc/s6-overlay/user-bundles.d/user/contents.d/django-tasks-consumers new file mode 100644 index 000000000..e69de29bb From 1899d60351ddba23123b668ec4d9e8ba0d3b8982 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 06:16:34 -0400 Subject: [PATCH 07/11] chore: add the django-tasks-huey service --- .../s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base | 0 .../s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd | 0 .../django-tasks-huey/dependencies.d/tubesync-downloads-init | 0 .../dependencies.d/tubesync-migrations-init | 0 .../etc/s6-overlay/s6-rc.d/django-tasks-huey/down-signal | 1 + config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run | 5 +++++ config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type | 1 + .../django-tasks-consumers/contents.d/django-tasks-huey | 0 8 files changed, 7 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-downloads-init create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-migrations-init create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/down-signal create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type create mode 100644 config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-huey diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/down-signal b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/down-signal new file mode 100644 index 000000000..d751378e1 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run new file mode 100755 index 000000000..ced8a371b --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py run_huey \ + --shutdown-timeout 60 diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-huey/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-huey b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-huey new file mode 100644 index 000000000..e69de29bb From 0ead703336837772592143afca7683ed1046892a Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 06:16:40 -0400 Subject: [PATCH 08/11] chore: add the django-tasks-net-limited service --- .../django-tasks-net-limited/dependencies.d/base | 0 .../dependencies.d/busybox-syslogd | 0 .../dependencies.d/tubesync-downloads-init | 0 .../dependencies.d/tubesync-migrations-init | 0 .../s6-rc.d/django-tasks-net-limited/down-signal | 1 + .../s6-overlay/s6-rc.d/django-tasks-net-limited/run | 10 ++++++++++ .../s6-overlay/s6-rc.d/django-tasks-net-limited/type | 1 + .../contents.d/django-tasks-net-limited | 0 8 files changed, 12 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/base create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/busybox-syslogd create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-downloads-init create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-migrations-init create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type create mode 100644 config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-net-limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal new file mode 100644 index 000000000..d751378e1 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run new file mode 100755 index 000000000..5b33f8c6d --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run @@ -0,0 +1,10 @@ +#!/command/with-contenv bash + +nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py prune_db_task_results \ + --queue-name limited + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-net-limited b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-net-limited new file mode 100644 index 000000000..e69de29bb From 1af63794390afc851284409c2bcf29b99870c6a7 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 08:33:25 -0400 Subject: [PATCH 09/11] chore: add cleanup_huey_storage task Added a new Django task to wait for the clean up of Huey storage by worker threads. --- tubesync/sync/apps.py | 4 ++++ tubesync/sync/tasks.py | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/apps.py b/tubesync/sync/apps.py index 0d95b4856..be57c2810 100644 --- a/tubesync/sync/apps.py +++ b/tubesync/sync/apps.py @@ -3,3 +3,7 @@ class SyncConfig(AppConfig): name = 'sync' + + def ready(self): + from .tasks import cleanup_huey_storage + self.cleanup_huey_storage_result = cleanup_huey_storage.enqueue() diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 5721362e4..4cc15b3ad 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -18,6 +18,7 @@ from django import db from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile +from django.tasks import task as django_task from django.utils import timezone from django.utils.translation import gettext_lazy as _ from django_huey import lock_task as huey_lock_task, task as huey_task @@ -45,7 +46,7 @@ from .youtube import YouTubeError atomic = db.transaction.atomic -register_huey_signals() +cleanup_threads = register_huey_signals() def get_task_map(): @@ -378,6 +379,24 @@ def wait_for_errors(model, /, *, queue_name=None, task_name=None): raise HueyConsumerError(_('queue consumer stopped')) +@django_task(backend='huey') +def cleanup_huey_storage(): + workers = cleanup_threads + if not all( + (callable(getattr(workers, 'clear', None)),) + + tuple(callable(getattr(w, 'join', None)) for w in workers) + + tuple(callable(getattr(w, 'start', None)) for w in workers), + ): + return + for worker in workers: + try: + worker.start() + except RuntimeError: + pass + worker.join() + workers.clear() + + @db_task(priority=90, retries=2, retry_delay=150, queue=Val(TaskQueue.FS)) @huey_lock_task('sync.tasks.cleanup_old_media', queue=Val(TaskQueue.FS)) def cleanup_old_media(durable=True): From b1d6179103a132c27007a73fc168ff2ed1ed0e95 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 09:12:14 -0400 Subject: [PATCH 10/11] chore: add the django-tasks-default service --- .../django-tasks-default/dependencies.d/base | 0 .../dependencies.d/busybox-syslogd | 0 .../dependencies.d/tubesync-downloads-init | 0 .../dependencies.d/tubesync-migrations-init | 0 .../s6-rc.d/django-tasks-default/finish | 24 ++++++++++ .../s6-rc.d/django-tasks-default/run | 48 +++++++++++++++++++ .../s6-rc.d/django-tasks-default/type | 1 + .../django-tasks-default/worker@/down-signal | 1 + .../django-tasks-default/worker@/finish | 5 ++ .../s6-rc.d/django-tasks-default/worker@/run | 6 +++ .../s6-rc.d/django-tasks-default/worker@/type | 1 + .../s6-rc.d/django-tasks-net-limited/run | 4 -- .../contents.d/django-tasks-default | 0 13 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/base create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/busybox-syslogd create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-downloads-init create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-migrations-init create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish create mode 100755 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type create mode 100644 config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-default diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/busybox-syslogd b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/busybox-syslogd new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-downloads-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-downloads-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-migrations-init b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/dependencies.d/tubesync-migrations-init new file mode 100644 index 000000000..e69de29bb diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish new file mode 100755 index 000000000..bb3a57925 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/finish @@ -0,0 +1,24 @@ +#!/command/with-contenv bash + +set -e + +scan_directory='/run/s6-rc/scandir' +service_dirs_directory='/run/s6-rc/servicedirs' + +service_name='django-tasks-default-workers' +service_directory="${service_dirs_directory}/${service_name}" + +stop_workers() ( + local i + cd "${service_directory}/instances" + for i in * ; do + /command/s6-instance-control -wD -d "${service_directory}" "${i}" + done +) +stop_workers + +/command/s6-svunlink "${scan_directory}/" "${service_name}" + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py prune_db_task_results \ + --queue-name '*' diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run new file mode 100755 index 000000000..3d26aeee4 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/run @@ -0,0 +1,48 @@ +#!/command/with-contenv bash + +set -e + +scan_directory='/run/s6-rc/scandir' +service_dirs_directory='/run/s6-rc/servicedirs' + +service_name='django-tasks-default-workers' +instance_directory="/run/servicedirs-${service_name}" +service_directory="${instance_directory}/${service_name}" +worker_template='/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@' + +[[ -d /run/ ]] +rm -rf "${instance_directory}" +if [[ -e "${service_dirs_directory}/${service_name}" ]] ; then + mkdir -p "${instance_directory}" + mv -t "${instance_directory}" "${service_dirs_directory}/${service_name}" + rm -rf "${instance_directory}" +fi +mkdir -p "${instance_directory}" + +/command/s6-instance-maker "${worker_template}" "${service_directory}" + +mv "${service_directory}" "${service_dirs_directory}/${service_name}" +service_directory="${service_dirs_directory}/${service_name}" +rmdir "${instance_directory}" + +/command/s6-svlink -t 1000 "${scan_directory}/" "${service_directory}" + +create_workers() { + local i=0 + while (( ++i <= $1 )); do + /command/s6-instance-create -t 5000 "${service_directory}" "${i}" || return + done +} + +cores="$(nproc --ignore=2 || echo 2)" +WORKERS="${TUBESYNC_DJANGO_TASKS_WORKERS:-${cores}}" +create_workers "${WORKERS}" + +if [[ -d "${service_directory}/instances/1/supervise" ]]; then + exec /command/s6-svwait -D -a "${service_directory}"/instance/* +fi + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name '*' --exclude-queues limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal new file mode 100644 index 000000000..d751378e1 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish new file mode 100755 index 000000000..fa7b6e379 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/finish @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py prune_db_task_results \ + --queue-name '*' diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run new file mode 100755 index 000000000..128259fea --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/run @@ -0,0 +1,6 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py db_worker -v 2 \ + --no-reload --interval 20 --max-tasks 100 \ + --queue-name '*' --exclude-queues limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type new file mode 100644 index 000000000..5883cff0c --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-default/worker@/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run index 5b33f8c6d..71cf3a8f4 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run +++ b/config/root/etc/s6-overlay/s6-rc.d/django-tasks-net-limited/run @@ -1,9 +1,5 @@ #!/command/with-contenv bash -nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ - /usr/bin/python3 /app/manage.py prune_db_task_results \ - --queue-name limited - exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py db_worker -v 2 \ --no-reload --interval 20 --max-tasks 100 \ diff --git a/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-default b/config/root/etc/s6-overlay/user-bundles.d/django-tasks-consumers/contents.d/django-tasks-default new file mode 100644 index 000000000..e69de29bb From b3df114b5090738093bce0c00edeee81ca85aeb1 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 21 Oct 2026 10:10:07 -0400 Subject: [PATCH 11/11] chore(tasks): add make_source_directory --- tubesync/sync/tasks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4cc15b3ad..d05e66df0 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -4,6 +4,7 @@ ''' +import contextlib import os import random import requests @@ -727,6 +728,15 @@ def index_source(source_id): return True +@django_task(priority=100, queue_name=Val(TaskQueue.FS)) +def make_source_directory(source_id): + with contextlib.suppress(Source.DoesNotExist): + source = Source.objects.get(pk=source_id) + if not source.directory_exists(): + log.info(f'Creating directory: {source.directory_path}') + source.make_directory() + + @db_task(priority=100, retries=15, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.FS)) def check_source_directory_exists(source_id): ''' @@ -742,8 +752,8 @@ def check_source_directory_exists(source_id): # Check the source output directory exists if not source.directory_exists(): # Try to create it - log.info(f'Creating directory: {source.directory_path}') source.make_directory() + log.info(f'Created directory: {source.directory_path}') @db_task(delay=10, priority=90, retries=15, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.LIMIT))