+ - - - + @@ -47,51 +45,47 @@ {% for queue in queues %} - + - - - - + + + diff --git a/scheduler/templates/admin/scheduler/worker_details.html b/scheduler/templates/admin/scheduler/worker_details.html index 12ddfb6..989a1c6 100644 --- a/scheduler/templates/admin/scheduler/worker_details.html +++ b/scheduler/templates/admin/scheduler/worker_details.html @@ -12,80 +12,84 @@ {% block content %} {% endblock %} diff --git a/scheduler/templates/admin/scheduler/workers-list.partial.html b/scheduler/templates/admin/scheduler/workers-list.partial.html index aff14a2..124a9b8 100644 --- a/scheduler/templates/admin/scheduler/workers-list.partial.html +++ b/scheduler/templates/admin/scheduler/workers-list.partial.html @@ -7,6 +7,9 @@ + @@ -41,14 +44,20 @@ {{ worker.name }} - - + + + - + - + {% endfor %} diff --git a/scheduler/templates/admin/scheduler/workers.html b/scheduler/templates/admin/scheduler/workers_list.html similarity index 91% rename from scheduler/templates/admin/scheduler/workers.html rename to scheduler/templates/admin/scheduler/workers_list.html index 09da96b..3eb3e15 100644 --- a/scheduler/templates/admin/scheduler/workers.html +++ b/scheduler/templates/admin/scheduler/workers_list.html @@ -17,7 +17,7 @@ {% endblock %} -{% block content_title %}

Django Workers

{% endblock %} +{% block content_title %}

{{ workers|length }} Tasks Workers

{% endblock %} {% block content %} diff --git a/scheduler/templatetags/scheduler_tags.py b/scheduler/templatetags/scheduler_tags.py index faa0835..fa26f7c 100644 --- a/scheduler/templatetags/scheduler_tags.py +++ b/scheduler/templatetags/scheduler_tags.py @@ -3,18 +3,21 @@ from django import template from django.utils.safestring import mark_safe -from scheduler.rq_classes import JobExecution, DjangoQueue, DjangoWorker -from scheduler.tools import get_scheduled_task +from scheduler.helpers.queues import Queue +from scheduler.models import Task, get_scheduled_task +from scheduler.models.task import run_task +from scheduler.redis_models import Result, JobModel +from scheduler.views.helpers import get_queue register = template.Library() @register.filter -def show_func_name(rq_job: JobExecution) -> str: +def show_func_name(job: JobModel) -> str: try: - res = rq_job.func_name - if res == "scheduler.tools.run_task": - task = get_scheduled_task(*rq_job.args) + res = job.func_name + if job.func == run_task: + task = get_scheduled_task(*job.args) res = task.function_string() return mark_safe(res) except Exception as e: @@ -27,31 +30,32 @@ def get_item(dictionary: Dict, key): @register.filter -def scheduled_job(job: JobExecution): - django_scheduled_job = get_scheduled_task(*job.args) - return django_scheduled_job.get_absolute_url() +def scheduled_task(job: JobModel) -> Task: + django_scheduled_task = get_scheduled_task(*job.args) + return django_scheduled_task.get_absolute_url() @register.filter -def worker_scheduler_pid(worker: Optional[DjangoWorker]) -> str: - scheduler_pid = worker.scheduler_pid() if worker is not None else None - return str(scheduler_pid) if scheduler_pid is not None else "-" +def job_result(job: JobModel) -> Optional[str]: + queue = get_queue(job.queue_name) + result = Result.fetch_latest(queue.connection, job.name) + return result.type.name.capitalize() if result is not None else None @register.filter -def job_result(job: JobExecution): - result = job.latest_result() - return result.type.name.capitalize() if result else None +def job_scheduled_task(job: JobModel) -> Optional[str]: + task = Task.objects.filter(id=job.scheduled_task_id).first() + return task.name if task is not None else None @register.filter -def job_status(job: JobExecution): - result = job.get_status() +def job_status(job: JobModel): + result = job.status return result.capitalize() @register.filter -def job_runtime(job: JobExecution): +def job_runtime(job: JobModel): ended_at = job.ended_at if ended_at: runtime = job.ended_at - job.started_at @@ -63,8 +67,5 @@ def job_runtime(job: JobExecution): @register.filter -def job_scheduled_time(job: JobExecution, queue: DjangoQueue): - try: - return queue.scheduled_job_registry.get_scheduled_time(job.id) - except Exception: - return None +def job_scheduled_time(job: JobModel, queue: Queue): + return queue.scheduled_job_registry.get_scheduled_time(job.name) diff --git a/scheduler/tests/conf.py b/scheduler/tests/conf.py new file mode 100644 index 0000000..8dd2e79 --- /dev/null +++ b/scheduler/tests/conf.py @@ -0,0 +1,94 @@ +# Settings for tests +import os + +from django.conf import settings + +from scheduler.settings import conf_settings + +settings.SCHEDULER_QUEUES = { + "default": {"HOST": "localhost", "PORT": 6379, "DB": 0}, + "test": {"HOST": "localhost", "PORT": 1, "DB": 1}, + "sentinel": { + "SENTINELS": [("localhost", 26736), ("localhost", 26737)], + "MASTER_NAME": "testmaster", + "DB": 1, + "USERNAME": "redis-user", + "PASSWORD": "secret", + "SENTINEL_KWARGS": {}, + }, + "test1": { + "HOST": "localhost", + "PORT": 1, + "DB": 1, + }, + "test2": { + "HOST": "localhost", + "PORT": 1, + "DB": 1, + }, + "test3": { + "HOST": "localhost", + "PORT": 6379, + "DB": 1, + }, + "async": { + "HOST": "localhost", + "PORT": 6379, + "DB": 1, + "ASYNC": False, + }, + "url": { + "URL": "redis://username:password@host:1234/", + "DB": 4, + }, + "url_with_db": { + "URL": "redis://username:password@host:1234/5", + }, + "url_default_db": { + "URL": "redis://username:password@host:1234", + }, + "django_tasks_scheduler_test": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + }, + "scheduler_scheduler_active_test": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + "ASYNC": False, + }, + "scheduler_scheduler_inactive_test": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + "ASYNC": False, + }, + "worker_scheduler_active_test": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + "ASYNC": False, + }, + "worker_scheduler_inactive_test": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + "ASYNC": False, + }, + "django_tasks_scheduler_test2": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + }, + "test_scheduler": { + "HOST": "localhost", + "PORT": 6379, + "DB": 0, + }, +} +if os.getenv("FAKEREDIS", "False") == "True": # pragma: no cover + for name, queue_settings in settings.SCHEDULER_QUEUES: # pragma: no cover + queue_settings["BROKER"] = "fakeredis" # pragma: no cover + +conf_settings() diff --git a/scheduler/tests/jobs.py b/scheduler/tests/jobs.py index a6b0871..0342e63 100644 --- a/scheduler/tests/jobs.py +++ b/scheduler/tests/jobs.py @@ -1,6 +1,7 @@ +import logging from time import sleep -from scheduler.queues import get_queue +from scheduler.helpers.queues import get_queue _counter = 0 @@ -18,8 +19,14 @@ def test_args_kwargs(*args, **kwargs): return func.format(", ".join(args_list + kwargs_list)) +def two_seconds_job(): + sleep(2) + logging.info(f"Job {_counter}") + + def long_job(): - sleep(10) + sleep(1000) + logging.info(f"Job {_counter}") test_non_callable = "I am a teapot" @@ -36,4 +43,4 @@ def test_job(): def enqueue_jobs(): queue = get_queue() for i in range(20): - queue.enqueue(test_job, job_id=f"job_{i}", args=()) + queue.create_and_enqueue_job(test_job, name=f"job_{i:03}", args=()) diff --git a/scheduler/tests/test_internals.py b/scheduler/tests/test_internals.py index d888d27..2d0ca38 100644 --- a/scheduler/tests/test_internals.py +++ b/scheduler/tests/test_internals.py @@ -1,15 +1,17 @@ from datetime import timedelta +from django.core.exceptions import ImproperlyConfigured +from django.test import override_settings from django.utils import timezone -from scheduler.models.task import TaskType, Task +from scheduler.helpers.callback import Callback, CallbackSetupError +from scheduler.models import TaskType, get_scheduled_task from scheduler.tests.testtools import SchedulerBaseCase, task_factory -from scheduler.tools import get_scheduled_task class TestInternals(SchedulerBaseCase): def test_get_scheduled_job(self): - task = task_factory(TaskType.ONCE, scheduled_time=timezone.now() - timedelta(hours=1)) + task = task_factory(TaskType.ONCE, scheduled_time=timezone.now() + timedelta(hours=1)) self.assertEqual(task, get_scheduled_task(TaskType.ONCE, task.id)) with self.assertRaises(ValueError): get_scheduled_task(task.task_type, task.id + 1) @@ -19,4 +21,62 @@ def test_get_scheduled_job(self): def test_task_update(self): task = task_factory(TaskType.ONCE) task.name = "new_name" - task.save(update_fields=["name"]) \ No newline at end of file + task.save(update_fields=["name"]) + + def test_callback_bad_arguments(self): + with self.assertRaises(CallbackSetupError) as cm: + Callback("scheduler.tests.jobs.test_job", "1m") + self.assertEqual(str(cm.exception), "Callback `timeout` must be a positive int, but received 1m") + with self.assertRaises(CallbackSetupError) as cm: + Callback("scheduler.tests.jobs.non_existing_method") + self.assertEqual(str(cm.exception), "Callback `func` is not callable: scheduler.tests.jobs.non_existing_method") + with self.assertRaises(CallbackSetupError) as cm: + Callback("scheduler.tests.non_existing_module.non_existing_method") + self.assertEqual( + str(cm.exception), + "Callback `func` is not callable: scheduler.tests.non_existing_module.non_existing_method", + ) + with self.assertRaises(CallbackSetupError) as cm: + Callback("non_existing_method") + self.assertEqual(str(cm.exception), "Callback `func` is not callable: non_existing_method") + with self.assertRaises(CallbackSetupError) as cm: + Callback(1) + self.assertEqual(str(cm.exception), "Callback `func` must be a string or function, received 1") + + +class TestConfSettings(SchedulerBaseCase): + @override_settings(SCHEDULER_CONFIG=[]) + def test_conf_settings__bad_scheduler_config(self): + from scheduler import settings + + with self.assertRaises(ImproperlyConfigured) as cm: + settings.conf_settings() + + self.assertEqual(str(cm.exception), "SCHEDULER_CONFIG should be a SchedulerConfiguration or dict") + + @override_settings(SCHEDULER_QUEUES=[]) + def test_conf_settings__bad_scheduler_queues_config(self): + from scheduler import settings + + with self.assertRaises(ImproperlyConfigured) as cm: + settings.conf_settings() + + self.assertEqual(str(cm.exception), "You have to define SCHEDULER_QUEUES in settings.py as dict") + + @override_settings(SCHEDULER_QUEUES={"default": []}) + def test_conf_settings__bad_queue_config(self): + from scheduler import settings + + with self.assertRaises(ImproperlyConfigured) as cm: + settings.conf_settings() + + self.assertEqual(str(cm.exception), "Queue default configuration should be a QueueConfiguration or dict") + + @override_settings(SCHEDULER_CONFIG={"UNKNOWN_SETTING": 10}) + def test_conf_settings__unknown_setting(self): + from scheduler import settings + + with self.assertRaises(ImproperlyConfigured) as cm: + settings.conf_settings() + + self.assertEqual(str(cm.exception), "Unknown setting UNKNOWN_SETTING in SCHEDULER_CONFIG") diff --git a/scheduler/tests/test_job_arg_models.py b/scheduler/tests/test_job_arg_models.py index 28e815d..453778a 100644 --- a/scheduler/tests/test_job_arg_models.py +++ b/scheduler/tests/test_job_arg_models.py @@ -2,7 +2,7 @@ from django.test import TestCase from django.utils import timezone -from scheduler.models.args import TaskArg, TaskKwarg +from scheduler.models import TaskArg, TaskKwarg from .jobs import arg_callable from .testtools import taskarg_factory diff --git a/scheduler/tests/test_job_decorator.py b/scheduler/tests/test_job_decorator.py index 820c014..bd437f5 100644 --- a/scheduler/tests/test_job_decorator.py +++ b/scheduler/tests/test_job_decorator.py @@ -2,13 +2,16 @@ from django.test import TestCase -from scheduler import job, settings -from . import test_settings # noqa -from ..decorators import JOB_METHODS_LIST -from ..queues import get_queue, QueueNotFoundError +from scheduler import settings +from scheduler.helpers.queues import get_queue +from . import conf # noqa +from ..decorators import JOB_METHODS_LIST, job +from ..redis_models import JobStatus +from ..redis_models.job import JobModel +from ..worker import create_worker -@job +@job() def test_job(): time.sleep(1) return 1 + 1 @@ -31,52 +34,93 @@ def test_job_result_ttl(): return 1 + 1 +class MyClass: + def run(self): + print("Hello") + + def __eq__(self, other): + if not isinstance(other, MyClass): + return False + return True + + +@job() +def long_running_func(x): + x.run() + + class JobDecoratorTest(TestCase): def setUp(self) -> None: get_queue("default").connection.flushall() def test_all_job_methods_registered(self): - self.assertEqual(1, len(JOB_METHODS_LIST)) + self.assertEqual(5, len(JOB_METHODS_LIST)) def test_job_decorator_no_params(self): test_job.delay() - config = settings.SCHEDULER_CONFIG - self._assert_job_with_func_and_props("default", test_job, config.DEFAULT_RESULT_TTL, config.DEFAULT_TIMEOUT) + self._assert_job_with_func_and_props( + "default", + test_job, + settings.SCHEDULER_CONFIG.DEFAULT_SUCCESS_TTL, + settings.SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT, + ) def test_job_decorator_timeout(self): test_job_timeout.delay() - config = settings.SCHEDULER_CONFIG - self._assert_job_with_func_and_props("default", test_job_timeout, config.DEFAULT_RESULT_TTL, 1) + self._assert_job_with_func_and_props( + "default", + test_job_timeout, + settings.SCHEDULER_CONFIG.DEFAULT_SUCCESS_TTL, + 1, + ) def test_job_decorator_result_ttl(self): test_job_result_ttl.delay() - config = settings.SCHEDULER_CONFIG - self._assert_job_with_func_and_props("default", test_job_result_ttl, 1, config.DEFAULT_TIMEOUT) + self._assert_job_with_func_and_props( + "default", + test_job_result_ttl, + 1, + settings.SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT, + ) def test_job_decorator_different_queue(self): test_job_diff_queue.delay() - config = settings.SCHEDULER_CONFIG self._assert_job_with_func_and_props( "django_tasks_scheduler_test", test_job_diff_queue, - config.DEFAULT_RESULT_TTL, - config.DEFAULT_TIMEOUT, + settings.SCHEDULER_CONFIG.DEFAULT_SUCCESS_TTL, + settings.SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT, ) def _assert_job_with_func_and_props(self, queue_name, expected_func, expected_result_ttl, expected_timeout): queue = get_queue(queue_name) - jobs = queue.get_jobs() + jobs = JobModel.get_many(queue.queued_job_registry.all(), queue.connection) self.assertEqual(1, len(jobs)) j = jobs[0] self.assertEqual(j.func, expected_func) - self.assertEqual(j.result_ttl, expected_result_ttl) + self.assertEqual(j.success_ttl, expected_result_ttl) self.assertEqual(j.timeout, expected_timeout) def test_job_decorator_bad_queue(self): - with self.assertRaises(QueueNotFoundError): + with self.assertRaises(settings.QueueNotFoundError): @job("bad-queue") def test_job_bad_queue(): time.sleep(1) return 1 + 1 + + def test_job_decorator_delay_with_param(self): + queue_name = "default" + long_running_func.delay(MyClass()) + + worker = create_worker(queue_name, burst=True) + worker.work() + + jobs_list = worker.queues[0].get_all_jobs() + self.assertEqual(1, len(jobs_list)) + job = jobs_list[0] + self.assertEqual(job.func, long_running_func) + self.assertEqual(job.kwargs, {}) + self.assertEqual(job.status, JobStatus.FINISHED) + self.assertEqual(job.args, (MyClass(),)) diff --git a/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py b/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py index cff48d8..20e90d4 100644 --- a/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py +++ b/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py @@ -1,19 +1,20 @@ from django.core.management import call_command -from scheduler.queues import get_queue +from scheduler.helpers.queues import get_queue +from scheduler.tests import conf # noqa from scheduler.tests.jobs import failing_job -from scheduler.tests.test_views import BaseTestCase -from scheduler.tools import create_worker -from scheduler.tests import test_settings # noqa +from scheduler.worker import create_worker +from scheduler.tests.test_views.base import BaseTestCase class DeleteFailedExecutionsTest(BaseTestCase): def test_delete_failed_executions__delete_jobs(self): queue = get_queue("default") call_command("delete_failed_executions", queue="default") - queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) + queue.create_and_enqueue_job(failing_job) + self.assertEqual(1, len(queue.queued_job_registry)) + worker = create_worker("default", burst=True) + worker.work() self.assertEqual(1, len(queue.failed_job_registry)) call_command("delete_failed_executions", queue="default") self.assertEqual(0, len(queue.failed_job_registry)) diff --git a/scheduler/tests/test_mgmt_commands/test_export.py b/scheduler/tests/test_mgmt_commands/test_export.py index 6ad1878..8b3d8f8 100644 --- a/scheduler/tests/test_mgmt_commands/test_export.py +++ b/scheduler/tests/test_mgmt_commands/test_export.py @@ -7,9 +7,9 @@ from django.core.management import call_command from django.test import TestCase -from scheduler.tests import test_settings # noqa +from scheduler.tests import conf # noqa from scheduler.tests.testtools import task_factory -from scheduler.tools import TaskType +from scheduler.models import TaskType class ExportTest(TestCase): @@ -22,34 +22,34 @@ def tearDown(self) -> None: os.remove(self.tmpfile.name) def test_export__should_export_job(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True)) # act call_command("export", filename=self.tmpfile.name) # assert result = json.load(self.tmpfile) - self.assertEqual(len(jobs), len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) - self.assertEqual(result[1], jobs[1].to_dict()) + self.assertEqual(len(tasks), len(result)) + self.assertEqual(result[0], tasks[0].to_dict()) + self.assertEqual(result[1], tasks[1].to_dict()) def test_export__should_export_enabled_jobs_only(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=False)) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=False)) # act call_command("export", filename=self.tmpfile.name, enabled=True) # assert result = json.load(self.tmpfile) - self.assertEqual(len(jobs) - 1, len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) + self.assertEqual(len(tasks) - 1, len(result)) + self.assertEqual(result[0], tasks[0].to_dict()) def test_export__should_export_job_yaml_without_yaml_lib(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True)) # act with mock.patch.dict("sys.modules", {"yaml": None}): @@ -58,16 +58,16 @@ def test_export__should_export_job_yaml_without_yaml_lib(self): self.assertEqual(cm.exception.code, 1) def test_export__should_export_job_yaml_green(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) - jobs.append(task_factory(TaskType.CRON, enabled=True)) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True)) + tasks.append(task_factory(TaskType.CRON, enabled=True)) # act call_command("export", filename=self.tmpfile.name, format="yaml") # assert result = yaml.load(self.tmpfile, yaml.SafeLoader) - self.assertEqual(len(jobs), len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) - self.assertEqual(result[1], jobs[1].to_dict()) - self.assertEqual(result[2], jobs[2].to_dict()) + self.assertEqual(len(tasks), len(result)) + self.assertEqual(result[0], tasks[0].to_dict()) + self.assertEqual(result[1], tasks[1].to_dict()) + self.assertEqual(result[2], tasks[2].to_dict()) diff --git a/scheduler/tests/test_mgmt_commands/test_import.py b/scheduler/tests/test_mgmt_commands/test_import.py index db95e66..b37a407 100644 --- a/scheduler/tests/test_mgmt_commands/test_import.py +++ b/scheduler/tests/test_mgmt_commands/test_import.py @@ -7,10 +7,10 @@ from django.core.management import call_command from django.test import TestCase -from scheduler.models.task import Task +from scheduler.models import Task +from scheduler.tests import conf # noqa from scheduler.tests.testtools import task_factory -from scheduler.tools import TaskType -from scheduler.tests import test_settings # noqa +from scheduler.models import TaskType class ImportTest(TestCase): @@ -21,10 +21,10 @@ def tearDown(self) -> None: os.remove(self.tmpfile.name) def test_import__should_schedule_job(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = json.dumps([j.to_dict() for j in jobs]) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() # act @@ -32,10 +32,10 @@ def test_import__should_schedule_job(self): # assert self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + db_task = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(db_task, attr)) def test_import__should_schedule_job_yaml(self): tasks = list() @@ -49,16 +49,16 @@ def test_import__should_schedule_job_yaml(self): # assert self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + task = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(task, attr)) def test_import__should_schedule_job_yaml_without_yaml_lib(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = yaml.dump([j.to_dict() for j in jobs], default_flow_style=False) + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = yaml.dump([j.to_dict() for j in tasks], default_flow_style=False) self.tmpfile.write(res) self.tmpfile.flush() # act @@ -68,31 +68,27 @@ def test_import__should_schedule_job_yaml_without_yaml_lib(self): self.assertEqual(cm.exception.code, 1) def test_import__should_schedule_job_reset(self): - jobs = list() + tasks = list() task_factory(TaskType.ONCE, enabled=True) task_factory(TaskType.ONCE, enabled=True) - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = json.dumps([j.to_dict() for j in jobs]) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() # act - call_command( - "import", - filename=self.tmpfile.name, - reset=True, - ) + call_command("import", filename=self.tmpfile.name, reset=True) # assert self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + task = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(task, attr)) self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.REPEATABLE).first() + task = Task.objects.filter(task_type=TaskType.REPEATABLE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[1], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[1], attr), getattr(task, attr)) def test_import__should_schedule_job_update_existing(self): tasks = list() @@ -102,17 +98,13 @@ def test_import__should_schedule_job_update_existing(self): self.tmpfile.write(res) self.tmpfile.flush() # act - call_command( - "import", - filename=self.tmpfile.name, - update=True, - ) + call_command("import", filename=self.tmpfile.name, update=True) # assert self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=tasks[0].name) + task = Task.objects.filter(task_type=TaskType.ONCE).get(name=tasks[0].name) attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(task, attr)) def test_import__should_schedule_job_without_update_existing(self): tasks = list() @@ -122,13 +114,10 @@ def test_import__should_schedule_job_without_update_existing(self): self.tmpfile.write(res) self.tmpfile.flush() # act - call_command( - "import", - filename=self.tmpfile.name, - ) + call_command("import", filename=self.tmpfile.name) # assert self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.get(name=tasks[0].name) + task = Task.objects.get(name=tasks[0].name) attrs = ["id", "name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(task, attr)) diff --git a/scheduler/tests/test_mgmt_commands/test_rq_stats.py b/scheduler/tests/test_mgmt_commands/test_rq_stats.py deleted file mode 100644 index dc43a49..0000000 --- a/scheduler/tests/test_mgmt_commands/test_rq_stats.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.core.management import call_command -from django.test import TestCase - -from scheduler.tests import test_settings # noqa - - -class RqstatsTest(TestCase): - def test_rqstats__does_not_fail(self): - call_command("rqstats", "-j") - call_command("rqstats", "-y") - call_command("rqstats") diff --git a/scheduler/tests/test_mgmt_commands/test_rq_worker.py b/scheduler/tests/test_mgmt_commands/test_rq_worker.py deleted file mode 100644 index b2c4967..0000000 --- a/scheduler/tests/test_mgmt_commands/test_rq_worker.py +++ /dev/null @@ -1,114 +0,0 @@ -from django.core.management import call_command -from django.test import TestCase - -from scheduler.queues import get_queue -from scheduler.tests.jobs import failing_job -from scheduler.tests import test_settings # noqa - - -class RqworkerTestCase(TestCase): - def test_rqworker__no_queues_params(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__job_class_param__green(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command( - "rqworker", "--job-class", "scheduler.rq_classes.JobExecution", fork_job_execution=False, burst=True - ) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__bad_job_class__fail(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - with self.assertRaises(ImportError): - call_command("rqworker", "--job-class", "rq.badclass", fork_job_execution=False, burst=True) - - def test_rqworker__run_jobs(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", "default", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__worker_with_two_queues(self): - queue = get_queue("default") - queue2 = get_queue("django_tasks_scheduler_test") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - job = queue2.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", "default", "django_tasks_scheduler_test", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__worker_with_one_queue__does_not_perform_other_queue_job(self): - queue = get_queue("default") - queue2 = get_queue("django_tasks_scheduler_test") - - job = queue.enqueue(failing_job) - other_job = queue2.enqueue(failing_job) - - # Create a worker to execute these jobs - call_command("rqworker", "default", fork_job_execution=False, burst=True) - # assert - self.assertTrue(job.is_failed) - self.assertTrue(other_job.is_queued) diff --git a/scheduler/tests/test_mgmt_commands/test_run_job.py b/scheduler/tests/test_mgmt_commands/test_run_job.py index 7429c63..50c3a84 100644 --- a/scheduler/tests/test_mgmt_commands/test_run_job.py +++ b/scheduler/tests/test_mgmt_commands/test_run_job.py @@ -1,19 +1,20 @@ from django.core.management import call_command from django.test import TestCase -from scheduler.queues import get_queue +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobModel +from scheduler.tests import conf # noqa from scheduler.tests.jobs import test_job -from scheduler.tests import test_settings # noqa class RunJobTest(TestCase): def test_run_job__should_schedule_job(self): queue = get_queue("default") - queue.empty() + queue.queued_job_registry.empty() func_name = f"{test_job.__module__}.{test_job.__name__}" # act call_command("run_job", func_name, queue="default") # assert - job_list = queue.get_jobs() + job_list = JobModel.get_many(queue.queued_job_registry.all(), queue.connection) self.assertEqual(1, len(job_list)) self.assertEqual(func_name + "()", job_list[0].get_call_string()) diff --git a/scheduler/tests/test_mgmt_commands/test_scheduler_stats.py b/scheduler/tests/test_mgmt_commands/test_scheduler_stats.py new file mode 100644 index 0000000..1a04648 --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_scheduler_stats.py @@ -0,0 +1,94 @@ +import json +import sys +from io import StringIO + +import yaml +from django.core.management import call_command +from django.test import TestCase, override_settings + +from scheduler import settings +from scheduler.helpers.queues import get_queue + + +@override_settings(SCHEDULER_QUEUES=dict(default={"HOST": "localhost", "PORT": 6379, "DB": 0})) +class SchedulerStatsTest(TestCase): + EXPECTED_OUTPUT = { + "queues": [ + { + "canceled_jobs": 0, + "failed_jobs": 0, + "finished_jobs": 0, + "name": "default", + "oldest_job_timestamp": None, + "queued_jobs": 0, + "scheduled_jobs": 0, + "scheduler_pid": None, + "started_jobs": 0, + "workers": 0, + } + ] + } + OLD_QUEUES = None + + def setUp(self): + super(SchedulerStatsTest, self).setUp() + SchedulerStatsTest.OLD_QUEUES = settings._QUEUES + settings._QUEUES = dict() + settings.conf_settings() + get_queue("default").connection.flushall() + + def tearDown(self): + super(SchedulerStatsTest, self).tearDown() + settings._QUEUES = SchedulerStatsTest.OLD_QUEUES + + def test_scheduler_stats__json_output(self): + test_stdout = StringIO() + sys.stdout = test_stdout + # act + call_command("scheduler_stats", "-j") + # assert + res = test_stdout.getvalue() + self.assertEqual(json.loads(res), SchedulerStatsTest.EXPECTED_OUTPUT) + + def test_scheduler_stats__yaml_output(self): + # arrange + test_stdout = StringIO() + sys.stdout = test_stdout + # act + call_command("scheduler_stats", "-y") + # assert + res = test_stdout.getvalue() + self.assertEqual(yaml.load(res, yaml.SafeLoader), SchedulerStatsTest.EXPECTED_OUTPUT) + + def test_scheduler_stats__plain_text_output(self): + test_stdout = StringIO() + sys.stdout = test_stdout + # act + call_command("scheduler_stats", "--no-color") + # assert + res = test_stdout.getvalue() + self.assertEqual( + res, + """ +Django-Scheduler CLI Dashboard + +-------------------------------------------------------------------------------- +| Name | Queued | Active | Finished | Canceled | Workers | +-------------------------------------------------------------------------------- +| default | 0 | 0 | 0 | 0 | 0 | +-------------------------------------------------------------------------------- +""", + ) + + def test_scheduler_stats__bad_args(self): + # arrange + sys.stderr = StringIO() + sys.stdout = StringIO() + # act + with self.assertRaises(SystemExit): + call_command("scheduler_stats", "-y", "-j") + # assert + res = sys.stdout.getvalue() + self.assertEqual(res, """""") + err = sys.stderr.getvalue() + self.assertEqual(err, """Aborting. Cannot output as both json and yaml\n""") diff --git a/scheduler/tests/test_mgmt_commands/test_scheduler_worker.py b/scheduler/tests/test_mgmt_commands/test_scheduler_worker.py new file mode 100644 index 0000000..5627ca7 --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_scheduler_worker.py @@ -0,0 +1,111 @@ +from unittest import mock + +from django.core.management import call_command +from django.test import TestCase + +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobModel +from scheduler.settings import SCHEDULER_CONFIG +from scheduler.tests import conf # noqa +from scheduler.tests.jobs import failing_job + + +class SchedulerWorkerTestCase(TestCase): + def test_scheduler_worker__no_queues_params(self): + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 + queue = get_queue("default") + + # enqueue some jobs that will fail + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(failing_job) + job_names.append(job.name) + + # Create a worker to execute these jobs + call_command("scheduler_worker", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job_name in job_names: + job = JobModel.get(name=job_name, connection=queue.connection) + self.assertTrue(job.is_failed) + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 10 + + @mock.patch("scheduler.management.commands.scheduler_worker.create_worker") + def test_scheduler_worker__without_scheduler(self, mock_create_worker): + # Create a worker to execute these jobs + call_command("scheduler_worker", "default", "--burst", "--without-scheduler") + mock_create_worker.assert_called_once_with( + "default", + name=None, + fork_job_execution=True, + burst=True, + with_scheduler=False, + ) + + @mock.patch("scheduler.management.commands.scheduler_worker.create_worker") + def test_scheduler_worker__with_scheduler(self, mock_create_worker): + # Create a worker to execute these jobs + call_command("scheduler_worker", "default", "--burst") + mock_create_worker.assert_called_once_with( + "default", name=None, fork_job_execution=True, burst=True, with_scheduler=True + ) + + def test_scheduler_worker__run_jobs(self): + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 + queue = get_queue("default") + + # enqueue some jobs that will fail + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(failing_job) + job_names.append(job.name) + + # Create a worker to execute these jobs + call_command("scheduler_worker", "default", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job_name in job_names: + job = JobModel.get(name=job_name, connection=queue.connection) + self.assertTrue(job.is_failed) + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 10 + + def test_scheduler_worker__worker_with_two_queues(self): + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 + queue = get_queue("default") + queue2 = get_queue("django_tasks_scheduler_test") + + # enqueue some jobs that will fail + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(failing_job) + job_names.append(job.name) + job = queue2.create_and_enqueue_job(failing_job) + job_names.append(job.name) + + # Create a worker to execute these jobs + call_command("scheduler_worker", "default", "django_tasks_scheduler_test", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job_name in job_names: + job = JobModel.get(name=job_name, connection=queue.connection) + self.assertTrue(job.is_failed) + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 10 + + def test_scheduler_worker__worker_with_one_queue__does_not_perform_other_queue_job(self): + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 + queue = get_queue("default") + queue2 = get_queue("django_tasks_scheduler_test") + + job = queue.create_and_enqueue_job(failing_job) + other_job = queue2.create_and_enqueue_job(failing_job) + + # Create a worker to execute these jobs + call_command("scheduler_worker", "default", fork_job_execution=False, burst=True) + + # assert + job = JobModel.get(job.name, connection=queue.connection) + self.assertTrue(job.is_failed) + other_job = JobModel.get(other_job.name, connection=queue.connection) + + self.assertTrue(other_job.is_queued, f"Expected other job to be queued but status={other_job.status}") + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 10 diff --git a/scheduler/tests/test_multiprocess/__init__.py b/scheduler/tests/test_multiprocess/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_multiprocess/test_integrity.py b/scheduler/tests/test_multiprocess/test_integrity.py new file mode 100644 index 0000000..145f5fa --- /dev/null +++ b/scheduler/tests/test_multiprocess/test_integrity.py @@ -0,0 +1,40 @@ +from time import sleep + +from django.test import tag +from django.urls import reverse + +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobStatus, JobModel, WorkerModel +from scheduler.tests.jobs import long_job +from .. import testtools +from ..test_views.base import BaseTestCase + + +@tag("multiprocess") +class MultiProcessTest(BaseTestCase): + def test_cancel_job_after_it_started(self): + # arrange + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(long_job) + self.assertTrue(job.is_queued) + process, worker_name = testtools.run_worker_in_process("django_tasks_scheduler_test") + sleep(0.2) + job = JobModel.get(job.name, connection=queue.connection) + self.assertEqual(JobStatus.STARTED, job.status) + # act + res = self.client.post(reverse("job_detail_action", args=[job.name, "cancel"]), {"post": "yes"}, follow=True) + + # assert + self.assertEqual(200, res.status_code) + job = JobModel.get(job.name, connection=queue.connection) + self.assertEqual(JobStatus.STOPPED, job.status) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + sleep(0.2) + process.terminate() + process.join(2) + process.kill() + worker_model = WorkerModel.get(worker_name, connection=queue.connection) + self.assertEqual(0, worker_model.completed_jobs) + self.assertEqual(0, worker_model.failed_job_count) + self.assertEqual(0, worker_model.successful_job_count) + self.assertIsNotNone(worker_model.shutdown_requested_date) diff --git a/scheduler/tests/test_settings.py b/scheduler/tests/test_settings.py index 821bd14..4d0e957 100644 --- a/scheduler/tests/test_settings.py +++ b/scheduler/tests/test_settings.py @@ -1,96 +1,88 @@ -import os +import dataclasses from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from scheduler.settings import conf_settings +from scheduler.tests.testtools import SchedulerBaseCase +from scheduler.types import Broker, SchedulerConfiguration -settings.SCHEDULER_QUEUES = { - "default": {"HOST": "localhost", "PORT": 6379, "DB": 0, "DEFAULT_TIMEOUT": 500}, - "test": {"HOST": "localhost", "PORT": 1, "DB": 1}, - "sentinel": { - "SENTINELS": [("localhost", 26736), ("localhost", 26737)], - "MASTER_NAME": "testmaster", - "DB": 1, - "USERNAME": "redis-user", - "PASSWORD": "secret", - "SOCKET_TIMEOUT": 10, - "SENTINEL_KWARGS": {}, - }, - "test1": { - "HOST": "localhost", - "PORT": 1, - "DB": 1, - "DEFAULT_TIMEOUT": 400, - }, - "test2": { - "HOST": "localhost", - "PORT": 1, - "DB": 1, - }, - "test3": { - "HOST": "localhost", - "PORT": 6379, - "DB": 1, - }, - "async": { - "HOST": "localhost", - "PORT": 6379, - "DB": 1, - "ASYNC": False, - }, - "url": { - "URL": "redis://username:password@host:1234/", - "DB": 4, - }, - "url_with_db": { - "URL": "redis://username:password@host:1234/5", - }, - "url_default_db": { - "URL": "redis://username:password@host:1234", - }, - "django_tasks_scheduler_test": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - }, - "scheduler_scheduler_active_test": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - "ASYNC": False, - }, - "scheduler_scheduler_inactive_test": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - "ASYNC": False, - }, - "worker_scheduler_active_test": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - "ASYNC": False, - }, - "worker_scheduler_inactive_test": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - "ASYNC": False, - }, - "django_tasks_scheduler_test2": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - }, - "test_scheduler": { - "HOST": "localhost", - "PORT": 6379, - "DB": 0, - "DEFAULT_TIMEOUT": 400, - }, -} -if os.getenv("FAKEREDIS", "False") == "True": - for name, queue_settings in settings.SCHEDULER_QUEUES: - queue_settings["BROKER"] = "fakeredis" -conf_settings() +class TestWorkerAdmin(SchedulerBaseCase): + def setUp(self): + from scheduler.settings import SCHEDULER_CONFIG + + self.old_settings = SCHEDULER_CONFIG + + def tearDown(self): + from scheduler import settings as scheduler_settings + + scheduler_settings.SCHEDULER_CONFIG = self.old_settings + + def test_scheduler_config_as_dict(self): + from scheduler.settings import SCHEDULER_CONFIG + + settings.SCHEDULER_CONFIG = dict( + EXECUTIONS_IN_PAGE=SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE + 1, + SCHEDULER_INTERVAL=SCHEDULER_CONFIG.SCHEDULER_INTERVAL + 1, + BROKER=Broker.REDIS, + CALLBACK_TIMEOUT=SCHEDULER_CONFIG.SCHEDULER_INTERVAL + 1, + DEFAULT_SUCCESS_TTL=SCHEDULER_CONFIG.DEFAULT_SUCCESS_TTL + 1, + DEFAULT_FAILURE_TTL=SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL + 1, + DEFAULT_JOB_TTL=SCHEDULER_CONFIG.DEFAULT_JOB_TTL + 1, + DEFAULT_JOB_TIMEOUT=SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT + 1, + # General configuration values + DEFAULT_WORKER_TTL=SCHEDULER_CONFIG.DEFAULT_WORKER_TTL + 1, + DEFAULT_MAINTENANCE_TASK_INTERVAL=SCHEDULER_CONFIG.DEFAULT_MAINTENANCE_TASK_INTERVAL + 1, + DEFAULT_JOB_MONITORING_INTERVAL=SCHEDULER_CONFIG.DEFAULT_JOB_MONITORING_INTERVAL + 1, + SCHEDULER_FALLBACK_PERIOD_SECS=SCHEDULER_CONFIG.SCHEDULER_FALLBACK_PERIOD_SECS + 1, + ) + conf_settings() + from scheduler.settings import SCHEDULER_CONFIG + + for key, value in settings.SCHEDULER_CONFIG.items(): + self.assertEqual(getattr(SCHEDULER_CONFIG, key), value) + + def test_scheduler_config_as_data_class(self): + from scheduler.settings import SCHEDULER_CONFIG + + self.assertEqual(SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE, 20) + settings.SCHEDULER_CONFIG = SchedulerConfiguration( + EXECUTIONS_IN_PAGE=1, + SCHEDULER_INTERVAL=60, + BROKER=Broker.REDIS, + CALLBACK_TIMEOUT=1111, + DEFAULT_SUCCESS_TTL=1111, + DEFAULT_FAILURE_TTL=111111, + DEFAULT_JOB_TTL=1111, + DEFAULT_JOB_TIMEOUT=11111, + # General configuration values + DEFAULT_WORKER_TTL=11111, + DEFAULT_MAINTENANCE_TASK_INTERVAL=111, + DEFAULT_JOB_MONITORING_INTERVAL=1111, + SCHEDULER_FALLBACK_PERIOD_SECS=1111, + ) + conf_settings() + from scheduler.settings import SCHEDULER_CONFIG + + for key, value in dataclasses.asdict(settings.SCHEDULER_CONFIG).items(): + self.assertEqual(getattr(SCHEDULER_CONFIG, key), value) + + def test_scheduler_config_as_dict_bad_param(self): + settings.SCHEDULER_CONFIG = dict( + EXECUTIONS_IN_PAGE=1, + SCHEDULER_INTERVAL=60, + BROKER=Broker.REDIS, + CALLBACK_TIMEOUT=1111, + DEFAULT_SUCCESS_TTL=1111, + DEFAULT_FAILURE_TTL=111111, + DEFAULT_JOB_TTL=1111, + DEFAULT_JOB_TIMEOUT=11111, + # General configuration values + DEFAULT_WORKER_TTL=11111, + DEFAULT_MAINTENANCE_TASK_INTERVAL=111, + DEFAULT_JOB_MONITORING_INTERVAL=1111, + SCHEDULER_FALLBACK_PERIOD_SECS=1111, + BAD_PARAM="bad_value", # This should raise an error + ) + self.assertRaises(ImproperlyConfigured, conf_settings) diff --git a/scheduler/tests/test_task_types/test_cron_task.py b/scheduler/tests/test_task_types/test_cron_task.py index a7d2a7a..d56e17a 100644 --- a/scheduler/tests/test_task_types/test_cron_task.py +++ b/scheduler/tests/test_task_types/test_cron_task.py @@ -1,26 +1,32 @@ from django.core.exceptions import ValidationError from scheduler import settings -from scheduler.queues import get_queue +from scheduler.helpers.queues import get_queue +from scheduler.models import TaskType +from scheduler.redis_models import JobModel from scheduler.tests.test_task_types.test_task_model import BaseTestCases from scheduler.tests.testtools import task_factory -from scheduler.tools import create_worker, TaskType +from scheduler.worker import create_worker class TestCronTask(BaseTestCases.TestBaseTask): task_type = TaskType.CRON + def setUp(self) -> None: + super().setUp() + self.queue_name = settings.get_queue_names()[0] + def test_clean(self): task = task_factory(self.task_type) task.cron_string = "* * * * *" - task.queue = list(settings.QUEUES)[0] + task.queue = self.queue_name task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean()) def test_clean_cron_string_invalid(self): task = task_factory(self.task_type) task.cron_string = "not-a-cron-string" - task.queue = list(settings.QUEUES)[0] + task.queue = self.queue_name task.callable = "scheduler.tests.jobs.test_job" with self.assertRaises(ValidationError): task.clean_cron_string() @@ -28,8 +34,9 @@ def test_clean_cron_string_invalid(self): def test_check_rescheduled_after_execution(self): task = task_factory(self.task_type) queue = task.rqueue - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) + self.assertIsNotNone(entry) queue.run_sync(entry) task.refresh_from_db() self.assertEqual(task.failed_runs, 0) @@ -37,16 +44,13 @@ def test_check_rescheduled_after_execution(self): self.assertEqual(task.successful_runs, 1) self.assertIsNotNone(task.last_successful_run) self.assertTrue(task.is_scheduled()) - self.assertNotEqual(task.job_id, first_run_id) + self.assertNotEqual(task.job_name, first_run_id) def test_check_rescheduled_after_failed_execution(self): - task = task_factory( - self.task_type, - callable_name="scheduler.tests.jobs.scheduler.tests.jobs.test_job", - ) + task = task_factory(self.task_type, callable_name="scheduler.tests.jobs.failing_job") queue = task.rqueue - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) queue.run_sync(entry) task.refresh_from_db() self.assertEqual(task.failed_runs, 1) @@ -54,25 +58,23 @@ def test_check_rescheduled_after_failed_execution(self): self.assertEqual(task.successful_runs, 0) self.assertIsNone(task.last_successful_run) self.assertTrue(task.is_scheduled()) - self.assertNotEqual(task.job_id, first_run_id) + self.assertNotEqual(task.job_name, first_run_id) def test_cron_task_enqueuing_jobs(self): queue = get_queue() - prev_queued = len(queue.scheduled_job_registry) - prev_finished = len(queue.finished_job_registry) + prev_queued = queue.scheduled_job_registry.count(connection=queue.connection) + prev_finished = queue.finished_job_registry.count(connection=queue.connection) + task = task_factory(self.task_type, callable_name="scheduler.tests.jobs.enqueue_jobs") - self.assertEqual(prev_queued + 1, len(queue.scheduled_job_registry)) - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + self.assertEqual(prev_queued + 1, queue.scheduled_job_registry.count(connection=queue.connection)) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) queue.run_sync(entry) - self.assertEqual(20, len(queue)) - self.assertEqual(prev_finished + 1, len(queue.finished_job_registry)) - worker = create_worker( - "default", - fork_job_execution=False, - ) - worker.work(burst=True) - self.assertEqual(prev_finished + 21, len(queue.finished_job_registry)) - worker.refresh() - self.assertEqual(20, worker.successful_job_count) - self.assertEqual(0, worker.failed_job_count) + self.assertEqual(20, len(queue.queued_job_registry)) + self.assertEqual(prev_finished + 1, queue.finished_job_registry.count(connection=queue.connection)) + worker = create_worker("default", fork_job_execution=False, burst=True) + worker.work() + self.assertEqual(prev_finished + 21, queue.finished_job_registry.count(connection=queue.connection)) + worker.refresh(update_queues=True) + self.assertEqual(20, worker._model.successful_job_count) + self.assertEqual(0, worker._model.failed_job_count) diff --git a/scheduler/tests/test_task_types/test_once_task.py b/scheduler/tests/test_task_types/test_once_task.py index f9b686c..6c4eb00 100644 --- a/scheduler/tests/test_task_types/test_once_task.py +++ b/scheduler/tests/test_task_types/test_once_task.py @@ -1,22 +1,53 @@ -from datetime import timedelta +from datetime import timedelta, datetime +import time_machine +from django.core.exceptions import ValidationError +from django.urls import reverse from django.utils import timezone from scheduler import settings -from scheduler.models.task import TaskType +from scheduler.models import TaskType from scheduler.tests.test_task_types.test_task_model import BaseTestCases from scheduler.tests.testtools import task_factory -class TestScheduledTask(BaseTestCases.TestSchedulableTask): +class TestScheduledOnceTask(BaseTestCases.TestSchedulableTask): task_type = TaskType.ONCE + queue_name = settings.get_queue_names()[0] def test_clean(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - self.assertIsNone(job.clean()) + task = task_factory(self.task_type) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean()) + + @time_machine.travel(datetime(2016, 12, 25)) + def test_admin_changelist_view__has_timezone_data(self): + # arrange + self.client.login(username="admin", password="admin") + task_factory(self.task_type) + url = reverse("admin:scheduler_task_changelist") + # act + res = self.client.get(url) + # assert + self.assertContains(res, "Run once: Dec. 26, 2016, midnight", count=1, status_code=200) + + def test_create_without_date__fail(self): + task = task_factory(self.task_type, scheduled_time=None, instance_only=True) + self.assertIsNone(task.scheduled_time) + with self.assertRaises(Exception) as cm: + task.clean() + self.assertTrue(isinstance(cm.exception, ValidationError)) + self.assertEqual(str(cm.exception), "{'scheduled_time': ['Scheduled time is required']}") + + def test_create_with_date_in_the_past__fail(self): + task = task_factory(self.task_type, scheduled_time=datetime.now() - timedelta(days=1), instance_only=True) + with self.assertRaises(Exception) as cm: + task.clean() + self.assertTrue(isinstance(cm.exception, ValidationError)) + self.assertEqual(str(cm.exception), "{'scheduled_time': ['Scheduled time must be in the future']}") def test_unschedulable_old_job(self): - job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1)) - self.assertFalse(job.is_scheduled()) + task = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1), instance_only=True) + task.save(clean=False) + self.assertFalse(task.is_scheduled()) diff --git a/scheduler/tests/test_task_types/test_repeatable_task.py b/scheduler/tests/test_task_types/test_repeatable_task.py index fce0d32..6d85c70 100644 --- a/scheduler/tests/test_task_types/test_repeatable_task.py +++ b/scheduler/tests/test_task_types/test_repeatable_task.py @@ -5,157 +5,181 @@ from django.utils import timezone from scheduler import settings +from scheduler.models import TaskType, Task +from scheduler.redis_models import JobModel from scheduler.tests.test_task_types.test_task_model import BaseTestCases -from scheduler.tests.testtools import task_factory, _get_task_job_execution_from_registry -from scheduler.tools import TaskType +from scheduler.tests.testtools import task_factory, _get_task_scheduled_job_from_registry +from scheduler.types import SchedulerConfiguration class TestRepeatableTask(BaseTestCases.TestSchedulableTask): task_type = TaskType.REPEATABLE + queue_name = settings.get_queue_names()[0] + + def test_create_task_error(self): + scheduled_time = timezone.now() + task = Task.objects.create( + name="konichiva_every_2s", + callable="scheduler.tests.jobs.test_args_kwargs", + task_type=TaskType.REPEATABLE, + interval=333, + interval_unit="seconds", + queue="default", + enabled=True, + scheduled_time=scheduled_time, + ) + self.assertEqual(task.name, "konichiva_every_2s") + self.assertEqual(task.callable, "scheduler.tests.jobs.test_args_kwargs") + self.assertEqual(task.task_type, TaskType.REPEATABLE) + self.assertEqual(task.interval, 333) + self.assertEqual(task.interval_unit, "seconds") + + def test_create_task_without_scheduled_time(self): + settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 10 + task = Task.objects.create( + name="konichiva_every_2s", + callable="scheduler.tests.jobs.test_args_kwargs", + task_type=TaskType.REPEATABLE, + interval=33, + interval_unit="seconds", + queue="default", + enabled=True, + ) + self.assertAlmostEqual(task.scheduled_time.timestamp(), timezone.now().timestamp(), delta=2) + self.assertEqual(task.name, "konichiva_every_2s") + self.assertEqual(task.callable, "scheduler.tests.jobs.test_args_kwargs") + self.assertEqual(task.task_type, TaskType.REPEATABLE) + self.assertEqual(task.interval, 33) + self.assertEqual(task.interval_unit, "seconds") def test_unschedulable_old_job(self): - job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) - self.assertFalse(job.is_scheduled()) + task = task_factory(TaskType.REPEATABLE, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) + self.assertFalse(task.is_scheduled()) def test_schedulable_old_job_repeat_none(self): # If repeat is None, the job should be scheduled - job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1), repeat=None) - self.assertTrue(job.is_scheduled()) + task = task_factory(TaskType.REPEATABLE, scheduled_time=timezone.now() - timedelta(hours=1), repeat=None) + self.assertTrue(task.is_scheduled()) def test_clean(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 1 - job.result_ttl = -1 - self.assertIsNone(job.clean()) + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 1 + task.success_ttl = -1 + self.assertIsNone(task.clean()) def test_clean_seconds(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 60 - job.result_ttl = -1 - job.interval_unit = "seconds" - self.assertIsNone(job.clean()) - - @override_settings( - SCHEDULER_CONFIG={ - "SCHEDULER_INTERVAL": 10, - } - ) + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 60 + task.success_ttl = -1 + task.interval_unit = "seconds" + task.clean() + + @override_settings(SCHEDULER_CONFIG=SchedulerConfiguration(SCHEDULER_INTERVAL=10)) def test_clean_too_frequent(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 2 # Smaller than 10 - job.result_ttl = -1 - job.interval_unit = "seconds" - with self.assertRaises(ValidationError): - job.clean_interval_unit() - - def test_clean_not_multiple(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 121 - job.interval_unit = "seconds" + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 2 # Smaller than 10 + task.success_ttl = -1 + task.interval_unit = "seconds" with self.assertRaises(ValidationError): - job.clean_interval_unit() + task.clean_interval_unit() def test_clean_short_result_ttl(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 1 - job.repeat = 1 - job.result_ttl = 3599 - job.interval_unit = "hours" - job.repeat = 42 + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 1 + task.repeat = 1 + task.result_ttl = 3599 + task.interval_unit = "hours" + task.repeat = 42 with self.assertRaises(ValidationError): - job.clean_result_ttl() + task.clean_result_ttl() def test_clean_indefinite_result_ttl(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 1 - job.result_ttl = -1 - job.interval_unit = "hours" - job.clean_result_ttl() + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 1 + task.result_ttl = -1 + task.interval_unit = "hours" + task.clean_result_ttl() def test_clean_undefined_result_ttl(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - job.interval = 1 - job.interval_unit = "hours" - job.clean_result_ttl() + task = task_factory(TaskType.REPEATABLE) + task.queue = self.queue_name + task.callable = "scheduler.tests.jobs.test_job" + task.interval = 1 + task.interval_unit = "hours" + task.clean_result_ttl() def test_interval_seconds_weeks(self): - job = task_factory(self.task_type, interval=2, interval_unit="weeks") - self.assertEqual(1209600.0, job.interval_seconds()) + task = task_factory(TaskType.REPEATABLE, interval=2, interval_unit="weeks") + self.assertEqual(1209600.0, task.interval_seconds()) def test_interval_seconds_days(self): - job = task_factory(self.task_type, interval=2, interval_unit="days") - self.assertEqual(172800.0, job.interval_seconds()) + task = task_factory(TaskType.REPEATABLE, interval=2, interval_unit="days") + self.assertEqual(172800.0, task.interval_seconds()) def test_interval_seconds_hours(self): - job = task_factory(self.task_type, interval=2, interval_unit="hours") - self.assertEqual(7200.0, job.interval_seconds()) + task = task_factory(TaskType.REPEATABLE, interval=2, interval_unit="hours") + self.assertEqual(7200.0, task.interval_seconds()) def test_interval_seconds_minutes(self): - job = task_factory(self.task_type, interval=15, interval_unit="minutes") - self.assertEqual(900.0, job.interval_seconds()) + task = task_factory(TaskType.REPEATABLE, interval=15, interval_unit="minutes") + self.assertEqual(900.0, task.interval_seconds()) def test_interval_seconds_seconds(self): - job = task_factory(self.task_type, interval=15, interval_unit="seconds") - self.assertEqual(15.0, job.interval_seconds()) + task = task_factory(TaskType.REPEATABLE, interval=15, interval_unit="seconds") + self.assertEqual(15.0, task.interval_seconds()) def test_result_interval(self): - job = task_factory( - self.task_type, - ) - entry = _get_task_job_execution_from_registry(job) + task = task_factory(TaskType.REPEATABLE) + entry = _get_task_scheduled_job_from_registry(task) self.assertEqual(entry.meta["interval"], 3600) def test_repeat(self): - job = task_factory(self.task_type, repeat=10) - entry = _get_task_job_execution_from_registry(job) + task = task_factory(TaskType.REPEATABLE, repeat=10) + entry = _get_task_scheduled_job_from_registry(task) self.assertEqual(entry.meta["repeat"], 10) def test_repeat_old_job_exhausted(self): base_time = timezone.now() - job = task_factory(self.task_type, scheduled_time=base_time - timedelta(hours=10), repeat=10) - self.assertEqual(job.is_scheduled(), False) + task = task_factory(TaskType.REPEATABLE, scheduled_time=base_time - timedelta(hours=10), repeat=10) + self.assertEqual(task.is_scheduled(), False) def test_repeat_old_job_last_iter(self): base_time = timezone.now() - job = task_factory(self.task_type, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) - self.assertEqual(job.repeat, 0) - self.assertEqual(job.is_scheduled(), True) + task = task_factory(TaskType.REPEATABLE, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) + self.assertEqual(task.repeat, 0) + self.assertEqual(task.is_scheduled(), True) def test_repeat_old_job_remaining(self): base_time = timezone.now() - job = task_factory(self.task_type, scheduled_time=base_time - timedelta(minutes=30), repeat=5) - self.assertEqual(job.repeat, 4) - self.assertEqual(job.scheduled_time, base_time + timedelta(minutes=30)) - self.assertEqual(job.is_scheduled(), True) + task = task_factory(TaskType.REPEATABLE, scheduled_time=base_time - timedelta(minutes=30), repeat=5) + self.assertEqual(task.repeat, 4) + self.assertEqual(task.scheduled_time, base_time + timedelta(minutes=30)) + self.assertEqual(task.is_scheduled(), True) def test_repeat_none_interval_2_min(self): base_time = timezone.now() - task = task_factory(self.task_type, scheduled_time=base_time - timedelta(minutes=29), repeat=None) + task = task_factory(TaskType.REPEATABLE, scheduled_time=base_time - timedelta(minutes=29), repeat=None) task.interval = 120 task.interval_unit = "seconds" - task._schedule() + task.save() self.assertTrue(task.scheduled_time > base_time) self.assertTrue(task.is_scheduled()) def test_check_rescheduled_after_execution(self): - task = task_factory(self.task_type, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10) + task = task_factory(TaskType.REPEATABLE, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10) queue = task.rqueue - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) queue.run_sync(entry) task.refresh_from_db() self.assertEqual(task.failed_runs, 0) @@ -163,18 +187,18 @@ def test_check_rescheduled_after_execution(self): self.assertEqual(task.successful_runs, 1) self.assertIsNotNone(task.last_successful_run) self.assertTrue(task.is_scheduled()) - self.assertNotEqual(task.job_id, first_run_id) + self.assertNotEqual(task.job_name, first_run_id) def test_check_rescheduled_after_execution_failed_job(self): task = task_factory( - self.task_type, + TaskType.REPEATABLE, callable_name="scheduler.tests.jobs.failing_job", scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10, ) queue = task.rqueue - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) queue.run_sync(entry) task.refresh_from_db() self.assertEqual(task.failed_runs, 1) @@ -182,21 +206,21 @@ def test_check_rescheduled_after_execution_failed_job(self): self.assertEqual(task.successful_runs, 0) self.assertIsNone(task.last_successful_run) self.assertTrue(task.is_scheduled()) - self.assertNotEqual(task.job_id, first_run_id) + self.assertNotEqual(task.job_name, first_run_id) def test_check_not_rescheduled_after_last_repeat(self): task = task_factory( - self.task_type, + TaskType.REPEATABLE, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=1, ) queue = task.rqueue - first_run_id = task.job_id - entry = queue.fetch_job(first_run_id) + first_run_id = task.job_name + entry = JobModel.get(first_run_id, connection=queue.connection) queue.run_sync(entry) task.refresh_from_db() self.assertEqual(task.failed_runs, 0) self.assertIsNone(task.last_failed_run) self.assertEqual(task.successful_runs, 1) self.assertIsNotNone(task.last_successful_run) - self.assertNotEqual(task.job_id, first_run_id) + self.assertNotEqual(task.job_name, first_run_id) diff --git a/scheduler/tests/test_task_types/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py index a6f6c3b..6688839 100644 --- a/scheduler/tests/test_task_types/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -1,25 +1,27 @@ import zoneinfo from datetime import datetime, timedelta +import time_machine from django.contrib.messages import get_messages from django.core.exceptions import ValidationError from django.test import override_settings from django.urls import reverse from django.utils import timezone -from freezegun import freeze_time from scheduler import settings -from scheduler.models.task import TaskType, Task, TaskArg, TaskKwarg -from scheduler.queues import get_queue -from scheduler.tests import jobs +from scheduler.helpers.queues import get_queue +from scheduler.helpers.queues import perform_job +from scheduler.models import TaskType, Task, TaskArg, TaskKwarg, run_task +from scheduler.redis_models import JobStatus, JobModel +from scheduler.tests import jobs, conf # noqa from scheduler.tests.testtools import ( task_factory, taskarg_factory, - _get_task_job_execution_from_registry, + _get_task_scheduled_job_from_registry, SchedulerBaseCase, _get_executions, ) -from scheduler.tools import run_task, create_worker +from scheduler.worker import create_worker def assert_response_has_msg(response, message): @@ -29,7 +31,7 @@ def assert_response_has_msg(response, message): def assert_has_execution_with_status(task, status): job_list = _get_executions(task) - job_list = [(j.id, j.get_status()) for j in job_list] + job_list = [(j.name, j.get_status(connection=task.rqueue.connection)) for j in job_list] for job in job_list: if job[1] == status: return @@ -39,6 +41,7 @@ def assert_has_execution_with_status(task, status): class BaseTestCases: class TestBaseTask(SchedulerBaseCase): task_type = None + queue_name = settings.get_queue_names()[0] def test_callable_func(self): task = task_factory(self.task_type) @@ -64,7 +67,7 @@ def test_clean_callable_invalid(self): task.clean_callable() def test_clean_queue(self): - for queue in settings.QUEUES.keys(): + for queue in settings.get_queue_names(): task = task_factory(self.task_type) task.queue = queue self.assertIsNone(task.clean_queue()) @@ -79,13 +82,13 @@ def test_clean_queue_invalid(self): # next 2 check the above are included in job.clean() function def test_clean_base(self): task = task_factory(self.task_type) - task.queue = list(settings.QUEUES)[0] + task.queue = self.queue_name task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean()) def test_clean_invalid_callable(self): task = task_factory(self.task_type) - task.queue = list(settings.QUEUES)[0] + task.queue = self.queue_name task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(ValidationError): task.clean() @@ -99,7 +102,7 @@ def test_clean_invalid_queue(self): def test_is_schedulable_already_scheduled(self): task = task_factory(self.task_type) - task._schedule() + task.save() self.assertTrue(task.is_scheduled()) def test_is_schedulable_disabled(self): @@ -108,51 +111,49 @@ def test_is_schedulable_disabled(self): self.assertFalse(task.enabled) def test_schedule(self): - task = task_factory( - self.task_type, - ) + task = task_factory(self.task_type) self.assertTrue(task.is_scheduled()) - self.assertIsNotNone(task.job_id) + self.assertIsNotNone(task.job_name) def test_unschedulable(self): task = task_factory(self.task_type, enabled=False) self.assertFalse(task.is_scheduled()) - self.assertIsNone(task.job_id) + self.assertIsNone(task.job_name) def test_unschedule(self): task = task_factory(self.task_type) self.assertTrue(task.unschedule()) - self.assertIsNone(task.job_id) + self.assertIsNone(task.job_name) def test_unschedule_not_scheduled(self): task = task_factory(self.task_type, enabled=False) self.assertTrue(task.unschedule()) - self.assertIsNone(task.job_id) + self.assertIsNone(task.job_name) def test_save_enabled(self): task = task_factory(self.task_type) - self.assertIsNotNone(task.job_id) + self.assertIsNotNone(task.job_name) def test_save_disabled(self): task = task_factory(self.task_type, enabled=False) task.save() - self.assertIsNone(task.job_id) + self.assertIsNone(task.job_name) def test_save_and_schedule(self): task = task_factory(self.task_type) - self.assertIsNotNone(task.job_id) + self.assertIsNotNone(task.job_name) self.assertTrue(task.is_scheduled()) def test_schedule2(self): task = task_factory(self.task_type) - task.queue = list(settings.QUEUES)[0] + task.queue = self.queue_name task.enabled = False task.scheduled_time = timezone.now() + timedelta(minutes=1) self.assertFalse(task._schedule()) def test_delete_and_unschedule(self): task = task_factory(self.task_type) - self.assertIsNotNone(task.job_id) + self.assertIsNotNone(task.job_name) self.assertTrue(task.is_scheduled()) task.delete() self.assertFalse(task.is_scheduled()) @@ -169,32 +170,34 @@ def test_str(self): def test_callable_passthrough(self): task = task_factory(self.task_type) - entry = _get_task_job_execution_from_registry(task) + entry = _get_task_scheduled_job_from_registry(task) self.assertEqual(entry.func, run_task) - job_model, job_id = entry.args + job_model, task_id = entry.args self.assertEqual(job_model, self.task_type.value) - self.assertEqual(job_id, task.id) + self.assertEqual(task_id, task.id) def test_timeout_passthrough(self): task = task_factory(self.task_type, timeout=500) - entry = _get_task_job_execution_from_registry(task) + entry = _get_task_scheduled_job_from_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): task = task_factory(self.task_type, at_front=True) queue = task.rqueue - jobs_to_schedule = queue.scheduled_job_registry.get_job_ids() - self.assertIn(task.job_id, jobs_to_schedule) + jobs_to_schedule = queue.scheduled_job_registry.all() + self.assertIn(task.job_name, jobs_to_schedule) def test_callable_result(self): task = task_factory(self.task_type) - entry = _get_task_job_execution_from_registry(task) - self.assertEqual(entry.perform(), 2) + entry = _get_task_scheduled_job_from_registry(task) + queue = get_queue("default") + self.assertEqual(perform_job(entry, connection=queue.connection), 2) def test_callable_empty_args_and_kwargs(self): task = task_factory(self.task_type, callable="scheduler.tests.jobs.test_args_kwargs") - entry = _get_task_job_execution_from_registry(task) - self.assertEqual(entry.perform(), "test_args_kwargs()") + entry = _get_task_scheduled_job_from_registry(task) + queue = get_queue("default") + self.assertEqual(perform_job(entry, connection=queue.connection), "test_args_kwargs()") def test_delete_args(self): task = task_factory(self.task_type) @@ -221,13 +224,13 @@ def test_parse_args(self): self.assertEqual(task.parse_args(), ["one", 2, True, False, date]) def test_parse_kwargs(self): - job = task_factory(self.task_type) + task = task_factory(self.task_type) date = timezone.now() - taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=job) - taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=job) - taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=True, content_object=job) - taskarg_factory(TaskKwarg, key="key4", arg_type="datetime", val=date, content_object=job) - kwargs = job.parse_kwargs() + taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) + taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=task) + taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=True, content_object=task) + taskarg_factory(TaskKwarg, key="key4", arg_type="datetime", val=date, content_object=task) + kwargs = task.parse_kwargs() self.assertEqual(kwargs, dict(key1="one", key2=2, key3=True, key4=date)) def test_callable_args_and_kwargs(self): @@ -238,8 +241,12 @@ def test_callable_args_and_kwargs(self): taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) task.save() - entry = _get_task_job_execution_from_registry(task) - self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) + entry = _get_task_scheduled_job_from_registry(task) + queue = get_queue("default") + self.assertEqual( + perform_job(entry, connection=queue.connection), + "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date), + ) def test_function_string(self): task = task_factory(self.task_type) @@ -258,12 +265,11 @@ def test_function_string(self): f"key1='one', key2=2, key3={repr(date)}, key4=False)", ) - def test_admin_list_view(self): + def test_admin_changelist_view(self): # arrange self.client.login(username="admin", password="admin") - job = task_factory(self.task_type) - model = job._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + task_factory(self.task_type) + url = reverse("admin:scheduler_task_changelist") # act res = self.client.get(url) # assert @@ -272,21 +278,10 @@ def test_admin_list_view(self): def test_admin_list_view_delete_model(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( - self.task_type, - ) - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + task = task_factory(self.task_type) + url = reverse("admin:scheduler_task_changelist") # act - res = self.client.post( - url, - data={ - "action": "delete_model", - "_selected_action": [ - task.pk, - ], - }, - ) + res = self.client.post(url, data={"action": "delete_model", "_selected_action": [task.pk]}) # assert self.assertEqual(302, res.status_code) @@ -294,8 +289,7 @@ def test_admin_run_job_now_enqueues_job_at(self): # arrange self.client.login(username="admin", password="admin") task = task_factory(self.task_type) - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + url = reverse("admin:scheduler_task_changelist") # act res = self.client.post( url, @@ -310,21 +304,14 @@ def test_admin_run_job_now_enqueues_job_at(self): self.assertEqual(302, res.status_code) task.refresh_from_db() queue = get_queue(task.queue) - self.assertIn(task.job_id, queue.get_job_ids()) + assert_has_execution_with_status(task, JobStatus.QUEUED) + self.assertIn(task.job_name, queue.scheduled_job_registry.all()) def test_admin_change_view(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( - self.task_type, - ) - model = task._meta.model.__name__.lower() - url = reverse( - f"admin:scheduler_{model}_change", - args=[ - task.pk, - ], - ) + task = task_factory(self.task_type) + url = reverse("admin:scheduler_task_change", args=[task.pk]) # act res = self.client.get(url) # assert @@ -335,13 +322,7 @@ def test_admin_change_view__bad_redis_connection(self): self.client.login(username="admin", password="admin") task = task_factory(self.task_type, queue="test2", instance_only=True) task.save(schedule_job=False) - model = task._meta.model.__name__.lower() - url = reverse( - f"admin:scheduler_{model}_change", - args=[ - task.pk, - ], - ) + url = reverse("admin:scheduler_task_change", args=[task.pk]) # act res = self.client.get(url) # assert @@ -351,7 +332,9 @@ def test_admin_enqueue_job_now(self): # arrange self.client.login(username="admin", password="admin") task = task_factory(self.task_type) - self.assertIsNotNone(task.job_id) + self.assertIsNotNone(task.job_name) + job = JobModel.get(task.job_name, connection=task.rqueue.connection) + self.assertEqual(job.status, JobStatus.SCHEDULED) self.assertTrue(task.is_scheduled()) data = { "action": "enqueue_job_now", @@ -359,30 +342,26 @@ def test_admin_enqueue_job_now(self): task.id, ], } - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + url = reverse("admin:scheduler_task_changelist") # act res = self.client.post(url, data=data, follow=True) # assert part 1 self.assertEqual(200, res.status_code) - entry = _get_task_job_execution_from_registry(task) - task_model, scheduled_task_id = entry.args - self.assertEqual(task_model, task.task_type) + assert_has_execution_with_status(task, JobStatus.QUEUED) + entry = _get_task_scheduled_job_from_registry(task) + task_type, scheduled_task_id = entry.args + self.assertEqual(task_type, task.task_type) self.assertEqual(scheduled_task_id, task.id) - self.assertEqual("scheduled", entry.get_status()) - assert_has_execution_with_status(task, "queued") + self.assertEqual(JobStatus.SCHEDULED, entry.get_status(connection=task.rqueue.connection)) # act 2 - worker = create_worker( - "default", - fork_job_execution=False, - ) - worker.work(burst=True) + worker = create_worker("default", fork_job_execution=False, burst=True) + worker.work() # assert 2 - entry = _get_task_job_execution_from_registry(task) - self.assertEqual(task_model, task.task_type) + entry = _get_task_scheduled_job_from_registry(task) + self.assertEqual(task_type, task.task_type) self.assertEqual(scheduled_task_id, task.id) assert_has_execution_with_status(task, "finished") @@ -390,7 +369,7 @@ def test_admin_enable_job(self): # arrange self.client.login(username="admin", password="admin") task = task_factory(self.task_type, enabled=False) - self.assertIsNone(task.job_id) + self.assertIsNone(task.job_name) self.assertFalse(task.is_scheduled()) data = { "action": "enable_selected", @@ -398,8 +377,7 @@ def test_admin_enable_job(self): task.id, ], } - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + url = reverse("admin:scheduler_task_changelist") # act res = self.client.post(url, data=data, follow=True) # assert @@ -420,8 +398,7 @@ def test_admin_disable_job(self): task.id, ], } - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + url = reverse("admin:scheduler_task_changelist") self.assertTrue(task.is_scheduled()) # act res = self.client.post(url, data=data, follow=True) @@ -436,28 +413,18 @@ def test_admin_single_delete(self): # arrange self.client.login(username="admin", password="admin") prev_count = Task.objects.filter(task_type=self.task_type).count() - task = task_factory( - self.task_type, - ) - self.assertIsNotNone(task.job_id) + task = task_factory(self.task_type) + self.assertIsNotNone(task.job_name) self.assertTrue(task.is_scheduled()) - prev = len(_get_executions(task)) - model = task._meta.model.__name__.lower() - url = reverse( - f"admin:scheduler_{model}_delete", - args=[ - task.pk, - ], - ) - data = { - "post": "yes", - } + prev_executions_count = len(_get_executions(task)) + url = reverse("admin:scheduler_task_delete", args=[task.pk]) + data = dict(post="yes") # act res = self.client.post(url, data=data, follow=True) # assert self.assertEqual(200, res.status_code) self.assertEqual(prev_count, Task.objects.filter(task_type=self.task_type).count()) - self.assertEqual(prev - 1, len(_get_executions(task))) + self.assertEqual(prev_executions_count - 1, len(_get_executions(task))) def test_admin_delete_selected(self): # arrange @@ -465,9 +432,9 @@ def test_admin_delete_selected(self): task = task_factory(self.task_type, enabled=True) task.save() queue = get_queue(task.queue) - scheduled_jobs = queue.scheduled_job_registry.get_job_ids() - job_id = task.job_id - self.assertIn(job_id, scheduled_jobs) + scheduled_jobs = queue.scheduled_job_registry.all() + job_name = task.job_name + self.assertIn(job_name, scheduled_jobs) data = { "action": "delete_selected", "_selected_action": [ @@ -475,29 +442,28 @@ def test_admin_delete_selected(self): ], "post": "yes", } - model = task._meta.model.__name__.lower() - url = reverse(f"admin:scheduler_{model}_changelist") + url = reverse("admin:scheduler_task_changelist") # act res = self.client.post(url, data=data, follow=True) # assert self.assertEqual(200, res.status_code) assert_response_has_msg(res, "Successfully deleted 1 task.") self.assertIsNone(Task.objects.filter(task_type=self.task_type).filter(id=task.id).first()) - scheduled_jobs = queue.scheduled_job_registry.get_job_ids() - self.assertNotIn(job_id, scheduled_jobs) + scheduled_jobs = queue.scheduled_job_registry.all() + self.assertNotIn(job_name, scheduled_jobs) class TestSchedulableTask(TestBaseTask): # Currently ScheduledJob and RepeatableJob task_type = TaskType.ONCE - @freeze_time("2016-12-25") + @time_machine.travel(datetime(2016, 12, 25)) @override_settings(USE_TZ=False) def test_schedule_time_no_tz(self): task = task_factory(self.task_type) task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=None) self.assertEqual("2016-12-25T08:00:00", task._schedule_time().isoformat()) - @freeze_time("2016-12-25") + @time_machine.travel(datetime(2016, 12, 25)) @override_settings(USE_TZ=True) def test_schedule_time_with_tz(self): task = task_factory(self.task_type) @@ -506,6 +472,6 @@ def test_schedule_time_with_tz(self): self.assertEqual("2016-12-25T13:00:00+00:00", task._schedule_time().isoformat()) def test_result_ttl_passthrough(self): - job = task_factory(self.task_type, result_ttl=500) - entry = _get_task_job_execution_from_registry(job) - self.assertEqual(entry.result_ttl, 500) + task = task_factory(self.task_type, result_ttl=500) + entry = _get_task_scheduled_job_from_registry(task) + self.assertEqual(entry.success_ttl, 500) diff --git a/scheduler/tests/test_views.py b/scheduler/tests/test_views.py deleted file mode 100644 index 1c76334..0000000 --- a/scheduler/tests/test_views.py +++ /dev/null @@ -1,537 +0,0 @@ -import uuid -from datetime import datetime -from unittest.mock import patch, PropertyMock - -from django.contrib.auth.models import User -from django.test import TestCase -from django.test.client import Client -from django.urls import reverse - -from scheduler.queues import get_queue -from scheduler.rq_classes import JobExecution, ExecutionStatus -from scheduler.tests import test_settings # noqa -from scheduler.tests.jobs import failing_job, long_job, test_job -from scheduler.tests.testtools import assert_message_in_response, task_factory, _get_task_job_execution_from_registry -from scheduler.tools import create_worker, TaskType - - -class BaseTestCase(TestCase): - def setUp(self): - self.user = User.objects.create_superuser("user", password="pass") - self.client = Client() - self.client.login(username=self.user.username, password="pass") - get_queue("django_tasks_scheduler_test").connection.flushall() - - -class SingleJobActionViewsTest(BaseTestCase): - def test_single_job_action_unknown_job(self): - res = self.client.get(reverse("queue_job_action", args=["unknown", "cancel"]), follow=True) - self.assertEqual(400, res.status_code) - - def test_single_job_action_unknown_action(self): - queue = get_queue("default") - job = queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) - job.refresh() - self.assertTrue(job.is_failed) - res = self.client.get(reverse("queue_job_action", args=[job.id, "unknown"]), follow=True) - self.assertEqual(404, res.status_code) - - def test_single_job_action_requeue_job(self): - queue = get_queue("default") - job = queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) - job.refresh() - self.assertTrue(job.is_failed) - res = self.client.get(reverse("queue_job_action", args=[job.id, "requeue"]), follow=True) - self.assertEqual(200, res.status_code) - self.client.post(reverse("queue_job_action", args=[job.id, "requeue"]), {"requeue": "Requeue"}, follow=True) - self.assertIn(job, queue.jobs) - job.delete() - - def test_single_job_action_delete_job(self): - queue = get_queue("django_tasks_scheduler_test") - job = queue.enqueue(test_job) - res = self.client.get(reverse("queue_job_action", args=[job.id, "delete"]), follow=True) - self.assertEqual(200, res.status_code) - self.client.post(reverse("queue_job_action", args=[job.id, "delete"]), {"post": "yes"}, follow=True) - self.assertFalse(JobExecution.exists(job.id, connection=queue.connection)) - self.assertNotIn(job.id, queue.get_job_ids()) - - def test_single_job_action_cancel_job(self): - queue = get_queue("django_tasks_scheduler_test") - job = queue.enqueue(long_job) - res = self.client.get(reverse("queue_job_action", args=[job.id, "cancel"]), follow=True) - self.assertEqual(200, res.status_code) - res = self.client.post(reverse("queue_job_action", args=[job.id, "cancel"]), {"post": "yes"}, follow=True) - self.assertEqual(200, res.status_code) - tmp = JobExecution.fetch(job.id, connection=queue.connection) - self.assertTrue(tmp.is_canceled) - self.assertNotIn(job.id, queue.get_job_ids()) - - def test_single_job_action_cancel_job_that_is_already_cancelled(self): - queue = get_queue("django_tasks_scheduler_test") - job = queue.enqueue(long_job) - res = self.client.post(reverse("queue_job_action", args=[job.id, "cancel"]), {"post": "yes"}, follow=True) - self.assertEqual(200, res.status_code) - tmp = JobExecution.fetch(job.id, connection=queue.connection) - self.assertTrue(tmp.is_canceled) - self.assertNotIn(job.id, queue.get_job_ids()) - res = self.client.post(reverse("queue_job_action", args=[job.id, "cancel"]), {"post": "yes"}, follow=True) - self.assertEqual(200, res.status_code) - assert_message_in_response(res, f"Could not perform action: Cannot cancel already canceled job: {job.id}") - - def test_single_job_action_enqueue_job(self): - queue = get_queue("django_tasks_scheduler_test") - job_list = [] - # enqueue some jobs that depends on other - previous_job = None - for _ in range(0, 3): - job = queue.enqueue(test_job, depends_on=previous_job) - job_list.append(job) - previous_job = job - - # This job is deferred - - self.assertEqual(job_list[-1].get_status(), ExecutionStatus.DEFERRED) - self.assertIsNone(job_list[-1].enqueued_at) - - # Try to force enqueue last job should do nothing - res = self.client.get(reverse("queue_job_action", args=[job_list[-1].id, "enqueue"]), follow=True) - self.assertEqual(200, res.status_code) - res = self.client.post(reverse("queue_job_action", args=[job_list[-1].id, "enqueue"]), follow=True) - - # Check that job is still deferred because it has dependencies (rq 1.14 change) - self.assertEqual(200, res.status_code) - tmp = queue.fetch_job(job_list[-1].id) - self.assertEqual(tmp.get_status(), ExecutionStatus.QUEUED) - self.assertIsNotNone(tmp.enqueued_at) - - -class JobListActionViewsTest(BaseTestCase): - def test_job_list_action_delete_jobs__with_bad_next_url(self): - queue = get_queue("django_tasks_scheduler_test") - - # enqueue some jobs - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(test_job) - job_ids.append(job.id) - - # remove those jobs using view - res = self.client.post( - reverse( - "queue_actions", - args=[ - queue.name, - ], - ), - { - "action": "delete", - "job_ids": job_ids, - "next_url": "bad_url", - }, - follow=True, - ) - assert_message_in_response(res, "Bad followup URL") - # check if jobs are removed - self.assertEqual(200, res.status_code) - for job_id in job_ids: - self.assertFalse(JobExecution.exists(job_id, connection=queue.connection)) - self.assertNotIn(job_id, queue.job_ids) - - def test_job_list_action_delete_jobs(self): - queue = get_queue("django_tasks_scheduler_test") - - # enqueue some jobs - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(test_job) - job_ids.append(job.id) - - # remove those jobs using view - res = self.client.post( - reverse( - "queue_actions", - args=[ - queue.name, - ], - ), - {"action": "delete", "job_ids": job_ids}, - follow=True, - ) - - # check if jobs are removed - self.assertEqual(200, res.status_code) - for job_id in job_ids: - self.assertFalse(JobExecution.exists(job_id, connection=queue.connection)) - self.assertNotIn(job_id, queue.job_ids) - - def test_job_list_action_requeue_jobs(self): - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # do those jobs = fail them - worker = create_worker("django_tasks_scheduler_test") - worker.work(burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - # re-nqueue failed jobs from failed queue - self.client.post(reverse("queue_actions", args=[queue_name]), {"action": "requeue", "job_ids": job_ids}) - - # check if we requeue all failed jobs - for job in jobs: - self.assertFalse(job.is_failed) - - def test_job_list_action_stop_jobs(self): - queue_name = "django_tasks_scheduler_test" - queue = get_queue(queue_name) - - # Enqueue some jobs - job_ids = [] - worker = create_worker("django_tasks_scheduler_test") - for _ in range(3): - job = queue.enqueue(test_job) - job_ids.append(job.id) - worker.prepare_job_execution(job) - - # Check if the jobs are started - for job_id in job_ids: - job = JobExecution.fetch(job_id, connection=queue.connection) - self.assertEqual(job.get_status(), ExecutionStatus.STARTED) - - # Stop those jobs using the view - started_job_registry = queue.started_job_registry - self.assertEqual(len(started_job_registry), len(job_ids)) - self.client.post(reverse("queue_actions", args=[queue_name]), {"action": "stop", "job_ids": job_ids}) - self.assertEqual(len(started_job_registry), 0) - - canceled_job_registry = queue.canceled_job_registry - self.assertEqual(len(canceled_job_registry), len(job_ids)) - - for job_id in job_ids: - self.assertIn(job_id, canceled_job_registry) - - -class QueueRegistryJobsViewTest(BaseTestCase): - def test_queue_jobs_unknown_registry(self): - queue_name = "default" - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "unknown"]), follow=True) - self.assertEqual(404, res.status_code) - - def test_queue_jobs_unknown_queue(self): - res = self.client.get(reverse("queue_registry_jobs", args=["UNKNOWN", "queued"])) - self.assertEqual(404, res.status_code) - - def test_queued_jobs(self): - """Jobs in queue are displayed properly""" - queue = get_queue("default") - job = queue.enqueue(test_job) - queue_name = "default" - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "queued"])) - self.assertEqual(res.context["jobs"], [job]) - - def test_finished_jobs(self): - """Ensure that finished jobs page works properly.""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - job = queue.enqueue(test_job) - registry = queue.finished_job_registry - registry.add(job, 2) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "finished"])) - self.assertEqual(res.context["jobs"], [job]) - - def test_failed_jobs(self): - """Ensure that failed jobs page works properly.""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - # Test that page doesn't fail when FailedJobRegistry is empty - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "failed"])) - self.assertEqual(res.status_code, 200) - - job = queue.enqueue(test_job) - registry = queue.failed_job_registry - registry.add(job, 2) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "failed"])) - self.assertEqual(res.context["jobs"], [job]) - - def test_scheduled_jobs(self): - """Ensure that scheduled jobs page works properly.""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - # Test that page doesn't fail when ScheduledJobRegistry is empty - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) - self.assertEqual(res.status_code, 200) - - job = queue.enqueue_at(datetime.now(), test_job) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) - self.assertEqual(res.context["jobs"], [job]) - - def test_scheduled_jobs_registry_removal(self): - """Ensure that non-existing job is being deleted from registry by view""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - registry = queue.scheduled_job_registry - job = queue.enqueue_at(datetime.now(), test_job) - self.assertEqual(len(registry), 1) - - queue.connection.delete(job.key) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) - self.assertEqual(res.context["jobs"], []) - - self.assertEqual(len(registry), 0) - - def test_started_jobs(self): - """Ensure that active jobs page works properly.""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - job = queue.enqueue(test_job) - registry = queue.started_job_registry - registry.add(job, 2) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "started"])) - self.assertEqual(res.context["jobs"], [job]) - - def test_deferred_jobs(self): - """Ensure that active jobs page works properly.""" - queue = get_queue("django_tasks_scheduler_test") - queue_name = "django_tasks_scheduler_test" - - job = queue.enqueue(test_job) - registry = queue.deferred_job_registry - registry.add(job, 2) - res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "deferred"])) - self.assertEqual(res.context["jobs"], [job]) - - -class ViewTest(BaseTestCase): - def test_job_details(self): - """Job data is displayed properly""" - queue = get_queue("default") - job = queue.enqueue(test_job) - - url = reverse( - "job_details", - args=[ - job.id, - ], - ) - res = self.client.get(url) - self.assertIn("job", res.context) - self.assertEqual(res.context["job"], job) - - # This page shouldn't fail when job.data is corrupt - queue.connection.hset(job.key, "data", "non-pickleable data") - res = self.client.get(url) - self.assertEqual(res.status_code, 200) - self.assertIn("DeserializationError", res.content.decode()) - - # Bad job-id should return 404 - url = reverse( - "job_details", - args=[ - "bad_job_id", - ], - ) - res = self.client.get(url) - self.assertEqual(400, res.status_code) - - def test_scheduled_job_details(self): - """Job data is displayed properly""" - scheduled_job = task_factory(TaskType.ONCE, enabled=True) - job = _get_task_job_execution_from_registry(scheduled_job) - - url = reverse( - "job_details", - args=[ - job.id, - ], - ) - res = self.client.get(url, follow=True) - self.assertIn("job", res.context) - self.assertEqual(res.context["job"], job) - - def test_job_details_on_deleted_dependency(self): - """Page doesn't crash even if job.dependency has been deleted""" - queue = get_queue("default") - - job = queue.enqueue(test_job) - second_job = queue.enqueue(test_job, depends_on=job) - job.delete() - url = reverse("job_details", args=[second_job.id]) - res = self.client.get(url) - self.assertEqual(res.status_code, 200) - self.assertIn(second_job._dependency_id, res.content.decode()) - - def test_requeue_all(self): - """ - Ensure that re-queuing all failed job work properly - """ - queue = get_queue("default") - queue_name = "default" - queue.enqueue(failing_job) - queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) - - res = self.client.get(reverse("queue_requeue_all", args=[queue_name, "failed"])) - self.assertEqual(res.context["total_jobs"], 2) - # After requeue_all is called, jobs are enqueued - res = self.client.post(reverse("queue_requeue_all", args=[queue_name, "failed"])) - self.assertEqual(len(queue), 2) - - def test_requeue_all_if_deleted_job(self): - """ - Ensure that re-queuing all failed job work properly - """ - queue = get_queue("default") - queue_name = "default" - job = queue.enqueue(failing_job) - queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) - - res = self.client.get(reverse("queue_requeue_all", args=[queue_name, "failed"])) - self.assertEqual(res.context["total_jobs"], 2) - job.delete() - - # After requeue_all is called, jobs are enqueued - res = self.client.post(reverse("queue_requeue_all", args=[queue_name, "failed"])) - self.assertEqual(len(queue), 1) - - def test_clear_queue_unknown_registry(self): - queue_name = "django_tasks_scheduler_test" - res = self.client.post(reverse("queue_clear", args=[queue_name, "unknown"]), {"post": "yes"}) - self.assertEqual(404, res.status_code) - - def test_clear_queue_enqueued(self): - queue = get_queue("django_tasks_scheduler_test") - job = queue.enqueue(test_job) - self.client.post(reverse("queue_clear", args=[queue.name, "queued"]), {"post": "yes"}) - self.assertFalse(JobExecution.exists(job.id, connection=queue.connection)) - self.assertNotIn(job.id, queue.job_ids) - - def test_clear_queue_scheduled(self): - queue = get_queue("django_tasks_scheduler_test") - job = queue.enqueue_at(datetime.now(), test_job) - - res = self.client.get(reverse("queue_clear", args=[queue.name, "scheduled"]), follow=True) - self.assertEqual(200, res.status_code) - self.assertEqual( - res.context["jobs"], - [ - job, - ], - ) - - res = self.client.post(reverse("queue_clear", args=[queue.name, "scheduled"]), {"post": "yes"}, follow=True) - assert_message_in_response(res, f"You have successfully cleared the scheduled jobs in queue {queue.name}") - self.assertEqual(200, res.status_code) - self.assertFalse(JobExecution.exists(job.id, connection=queue.connection)) - self.assertNotIn(job.id, queue.job_ids) - - def test_workers_home(self): - res = self.client.get(reverse("workers_home")) - prev_workers = res.context["workers"] - worker1 = create_worker("django_tasks_scheduler_test") - worker1.register_birth() - worker2 = create_worker("test3") - worker2.register_birth() - - res = self.client.get(reverse("workers_home")) - self.assertEqual(res.context["workers"], prev_workers + [worker1, worker2]) - - def test_queue_workers(self): - """Worker index page should show workers for a specific queue""" - queue_name = "django_tasks_scheduler_test" - - worker1 = create_worker("django_tasks_scheduler_test") - worker1.register_birth() - worker2 = create_worker("test3") - worker2.register_birth() - - res = self.client.get(reverse("queue_workers", args=[queue_name])) - self.assertEqual(res.context["workers"], [worker1]) - - def test_worker_details(self): - """Worker index page should show workers for a specific queue""" - - worker = create_worker("django_tasks_scheduler_test", name=uuid.uuid4().hex) - worker.register_birth() - - url = reverse( - "worker_details", - args=[ - worker.name, - ], - ) - res = self.client.get(url) - self.assertEqual(res.context["worker"], worker) - - def test_worker_details__non_existing_worker(self): - """Worker index page should show workers for a specific queue""" - - worker = create_worker("django_tasks_scheduler_test", name="WORKER") - worker.register_birth() - - res = self.client.get(reverse("worker_details", args=["bad-worker-name"])) - self.assertEqual(404, res.status_code) - - def test_statistics_json_view(self): - # Override testing SCHEDULER_QUEUES - queues = { - "default": { - "DB": 0, - "HOST": "localhost", - "PORT": 6379, - } - } - with patch("scheduler.settings.QUEUES", new_callable=PropertyMock(return_value=queues)): - res = self.client.get(reverse("queues_home")) - self.assertEqual(res.status_code, 200) - - res = self.client.get(reverse("queues_home_json")) - self.assertEqual(res.status_code, 200) - - # Not staff => return 404 - self.user.is_staff = False - self.user.save() - - res = self.client.get(reverse("queues_home")) - self.assertEqual(res.status_code, 302) - - # 404 code for stats - res = self.client.get(reverse("queues_home_json")) - self.assertEqual(res.status_code, 404) - - @staticmethod - def token_validation(token: str) -> bool: - return token == "valid" - - # @patch('scheduler.views.SCHEDULER_CONFIG') - # def test_statistics_json_view_token(self, configuration): - # configuration.get.return_value = ViewTest.token_validation - # self.user.is_staff = False - # self.user.save() - # res = self.client.get(reverse('queues_home_json'), headers={'Authorization': 'valid'}) - # self.assertEqual(res.status_code, 200) - # - # res = self.client.get(reverse('queues_home_json'), headers={'Authorization': 'invalid'}) - # self.assertEqual(res.status_code, 404) diff --git a/scheduler/tests/test_views/__init__.py b/scheduler/tests/test_views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_views/base.py b/scheduler/tests/test_views/base.py new file mode 100644 index 0000000..f951e9c --- /dev/null +++ b/scheduler/tests/test_views/base.py @@ -0,0 +1,14 @@ +from django.contrib.auth.models import User +from django.test import TestCase +from django.test.client import Client + +from scheduler.helpers.queues import get_queue +from scheduler.tests import conf # noqa + + +class BaseTestCase(TestCase): + def setUp(self): + self.user = User.objects.create_superuser("user", password="pass") + self.client = Client() + self.client.login(username=self.user.username, password="pass") + get_queue("django_tasks_scheduler_test").connection.flushall() diff --git a/scheduler/tests/test_views/test_job_detail_action.py b/scheduler/tests/test_views/test_job_detail_action.py new file mode 100644 index 0000000..58da999 --- /dev/null +++ b/scheduler/tests/test_views/test_job_detail_action.py @@ -0,0 +1,103 @@ +from django.urls import reverse + +from scheduler.helpers.queues import get_queue +from scheduler.worker import create_worker +from scheduler.redis_models import JobStatus, JobModel +from scheduler.tests.jobs import failing_job, long_job, test_job +from scheduler.tests.testtools import assert_message_in_response +from .base import BaseTestCase +from ..test_task_types.test_task_model import assert_response_has_msg + + +class SingleJobActionViewsTest(BaseTestCase): + def test_single_job_action_unknown_job(self): + res = self.client.get(reverse("job_detail_action", args=["unknown", "cancel"]), follow=True) + self.assertEqual(200, res.status_code) + assert_response_has_msg(res, "Job unknown does not exist, maybe its TTL has passed") + + def test_single_job_action_unknown_action(self): + queue = get_queue("default") + job = queue.create_and_enqueue_job(failing_job) + worker = create_worker("default", burst=True) + worker.work() + job = JobModel.get(job.name, connection=queue.connection) + self.assertTrue(job.is_failed) + res = self.client.get(reverse("job_detail_action", args=[job.name, "unknown"]), follow=True) + self.assertEqual(400, res.status_code) + + def test_single_job_action_delete_job(self): + queue = get_queue("default") + job = queue.create_and_enqueue_job(test_job, job_info_ttl=0) + res = self.client.get(reverse("job_detail_action", args=[job.name, "delete"]), follow=True) + self.assertEqual(200, res.status_code) + self.client.post(reverse("job_detail_action", args=[job.name, "delete"]), {"post": "yes"}, follow=True) + self.assertFalse(JobModel.exists(job.name, connection=queue.connection)) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + + def test_single_job_action_cancel_job(self): + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(long_job) + self.assertTrue(job.is_queued) + job = JobModel.get(job.name, connection=queue.connection) + res = self.client.get(reverse("job_detail_action", args=[job.name, "cancel"]), follow=True) + self.assertEqual(200, res.status_code) + res = self.client.post(reverse("job_detail_action", args=[job.name, "cancel"]), {"post": "yes"}, follow=True) + self.assertEqual(200, res.status_code) + job = JobModel.get(job.name, connection=queue.connection) + self.assertTrue(job.is_canceled) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + + def test_single_job_action_cancel_job_that_is_already_cancelled(self): + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(long_job) + res = self.client.post(reverse("job_detail_action", args=[job.name, "cancel"]), {"post": "yes"}, follow=True) + self.assertEqual(200, res.status_code) + tmp = JobModel.get(job.name, connection=queue.connection) + self.assertTrue(tmp.is_canceled) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + res = self.client.post(reverse("job_detail_action", args=[job.name, "cancel"]), {"post": "yes"}, follow=True) + self.assertEqual(200, res.status_code) + assert_message_in_response(res, f"Could not perform action: Cannot cancel already canceled job: {job.name}") + + def test_single_job_action_enqueue_job(self): + queue = get_queue("django_tasks_scheduler_test") + job_list = [] + # enqueue some jobs that depends on other + for _ in range(0, 3): + job = queue.create_and_enqueue_job(test_job) + job_list.append(job) + + self.assertEqual(job_list[-1].get_status(connection=queue.connection), JobStatus.QUEUED) + self.assertIsNotNone(job_list[-1].enqueued_at) + + # Try to force enqueue last job should do nothing + res = self.client.get(reverse("job_detail_action", args=[job_list[-1].name, "enqueue"]), follow=True) + self.assertEqual(200, res.status_code) + res = self.client.post(reverse("job_detail_action", args=[job_list[-1].name, "enqueue"]), follow=True) + + self.assertEqual(200, res.status_code) + tmp = JobModel.get(job_list[-1].name, connection=queue.connection) + self.assertEqual(tmp.get_status(connection=queue.connection), JobStatus.QUEUED) + self.assertIsNotNone(tmp.enqueued_at) + + def test_single_job_action_enqueue_job_sync_queue(self): + queue = get_queue("scheduler_scheduler_active_test") + job_list = [] + # enqueue some jobs that depends on other + for _ in range(0, 3): + job = queue.create_and_enqueue_job(test_job) + job_list.append(job) + + self.assertEqual(job_list[-1].status, JobStatus.FINISHED) + self.assertIsNotNone(job_list[-1].enqueued_at) + + # Try to force enqueue last job should do nothing + res = self.client.get(reverse("job_detail_action", args=[job_list[-1].name, "enqueue"]), follow=True) + self.assertEqual(200, res.status_code) + res = self.client.post(reverse("job_detail_action", args=[job_list[-1].name, "enqueue"]), follow=True) + + self.assertEqual(200, res.status_code) + tmp = JobModel.get(job_list[-1].name, connection=queue.connection) + self.assertEqual(tmp.get_status(connection=queue.connection), JobStatus.FINISHED) + self.assertIsNotNone(tmp.enqueued_at) + self.assertGreater(tmp.enqueued_at, job_list[-1].enqueued_at) diff --git a/scheduler/tests/test_views/test_job_details.py b/scheduler/tests/test_views/test_job_details.py new file mode 100644 index 0000000..d70654f --- /dev/null +++ b/scheduler/tests/test_views/test_job_details.py @@ -0,0 +1,191 @@ +import uuid +from datetime import datetime +from unittest.mock import patch, PropertyMock + +from django.urls import reverse + +from scheduler.helpers.queues import get_queue +from scheduler.models import TaskType +from scheduler.redis_models import JobModel, WorkerModel +from scheduler.tests import conf # noqa +from scheduler.tests.jobs import failing_job, test_job +from scheduler.tests.test_task_types.test_task_model import assert_response_has_msg +from scheduler.tests.test_views.base import BaseTestCase +from scheduler.tests.testtools import assert_message_in_response, task_factory, _get_task_scheduled_job_from_registry +from scheduler.types import QueueConfiguration +from scheduler.worker import create_worker + + +class TestViewJobDetails(BaseTestCase): + def test_job_details(self): + """Job data is displayed properly""" + queue = get_queue("default") + job = queue.create_and_enqueue_job(test_job) + + url = reverse("job_details", args=[job.name]) + res = self.client.get(url) + self.assertEqual(200, res.status_code) + self.assertIn("job", res.context) + self.assertEqual(res.context["job"], job) + + # Bad job-id should return 404 + url = reverse("job_details", args=["bad_job_name"]) + res = self.client.get(url, follow=True) + self.assertEqual(200, res.status_code) + assert_response_has_msg(res, "Job bad_job_name does not exist, maybe its TTL has passed") + + def test_scheduled_job_details(self): + """Job data is displayed properly""" + task = task_factory(TaskType.ONCE, enabled=True) + job = _get_task_scheduled_job_from_registry(task) + + url = reverse( + "job_details", + args=[job.name], + ) + res = self.client.get(url, follow=True) + self.assertIn("job", res.context) + self.assertEqual(res.context["job"], job) + + def test_job_details_on_deleted_dependency(self): + """Page doesn't crash even if job.dependency has been deleted""" + queue = get_queue("default") + + job = queue.create_and_enqueue_job(test_job) + second_job = queue.create_and_enqueue_job(test_job) + queue.delete_job(job.name) + url = reverse("job_details", args=[second_job.name]) + res = self.client.get(url) + self.assertEqual(res.status_code, 200) + + def test_requeue_all(self): + """Ensure that re-queuing all failed job work properly""" + queue = get_queue("default") + queue_name = "default" + queue.create_and_enqueue_job(failing_job) + queue.create_and_enqueue_job(failing_job) + worker = create_worker("default", burst=True) + worker.work() + + res = self.client.get(reverse("queue_registry_action", args=[queue_name, "failed", "requeue"])) + self.assertEqual(res.context["total_jobs"], 2) + # After requeue_all is called, jobs are enqueued + res = self.client.post(reverse("queue_registry_action", args=[queue_name, "failed", "requeue"])) + self.assertEqual(len(queue), 4) + + def test_requeue_all_if_deleted_job(self): + """ + Ensure that re-queuing all failed job work properly + """ + queue = get_queue("default") + queue_name = "default" + job = queue.create_and_enqueue_job(failing_job) + queue.create_and_enqueue_job(failing_job) + worker = create_worker("default", burst=True) + worker.work() + + res = self.client.get(reverse("queue_registry_action", args=[queue_name, "failed", "requeue"])) + self.assertEqual(res.context["total_jobs"], 2) + queue.delete_job(job.name) + + # After requeue_all is called, jobs are enqueued + res = self.client.post(reverse("queue_registry_action", args=[queue_name, "failed", "requeue"])) + self.assertEqual(len(queue.queued_job_registry), 1) + + def test_clear_queue_unknown_registry(self): + queue_name = "django_tasks_scheduler_test" + res = self.client.post(reverse("queue_registry_action", args=[queue_name, "unknown", "empty"]), {"post": "yes"}) + self.assertEqual(404, res.status_code) + + def test_clear_queue_enqueued(self): + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(test_job) + self.client.post(reverse("queue_registry_action", args=[queue.name, "queued", "empty"]), {"post": "yes"}) + self.assertFalse(JobModel.exists(job.name, connection=queue.connection), f"job {job.name} exists") + self.assertNotIn(job.name, queue.queued_job_registry.all()) + + def test_clear_queue_scheduled(self): + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(test_job, when=datetime.now()) + + res = self.client.get(reverse("queue_registry_action", args=[queue.name, "scheduled", "empty"]), follow=True) + self.assertEqual(200, res.status_code) + self.assertEqual(res.context["jobs"], [job]) + + res = self.client.post( + reverse("queue_registry_action", args=[queue.name, "scheduled", "empty"]), {"post": "yes"}, follow=True + ) + assert_message_in_response(res, f"You have successfully cleared the scheduled jobs in queue {queue.name}") + self.assertEqual(200, res.status_code) + self.assertFalse(JobModel.exists(job.name, connection=queue.connection)) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + + def test_queue_workers(self): + """Worker index page should show workers for a specific queue""" + queue_name = "django_tasks_scheduler_test" + + worker1 = create_worker(queue_name) + worker1.worker_start() + worker2 = create_worker("test3") + worker2.worker_start() + + res = self.client.get(reverse("queue_workers", args=[queue_name])) + worker1_model = WorkerModel.get(worker1.name, connection=worker1.connection) + self.assertEqual(res.context["workers"], [worker1_model]) + + def test_worker_details(self): + """Worker index page should show workers for a specific queue""" + + worker = create_worker("django_tasks_scheduler_test", name=uuid.uuid4().hex) + worker.worker_start() + + url = reverse("worker_details", args=[worker.name]) + res = self.client.get(url) + self.assertEqual(res.context["worker"], worker._model) + + def test_worker_details__non_existing_worker(self): + """Worker index page should show workers for a specific queue""" + + worker = create_worker("django_tasks_scheduler_test", name="WORKER") + worker.worker_start() + + res = self.client.get(reverse("worker_details", args=["bad-worker-name"])) + self.assertEqual(404, res.status_code) + + def test_statistics_json_view(self): + # Override testing SCHEDULER_QUEUES + queues = { + "default": QueueConfiguration(DB=0, HOST="localhost", PORT=6379), + } + with patch("scheduler.settings._QUEUES", new_callable=PropertyMock(return_value=queues)): + res = self.client.get(reverse("queues_home")) + self.assertEqual(res.status_code, 200) + + res = self.client.get(reverse("queues_home_json")) + self.assertEqual(res.status_code, 200) + + # Not staff => return 404 + self.user.is_staff = False + self.user.save() + + res = self.client.get(reverse("queues_home")) + self.assertEqual(res.status_code, 302) + + # 404 code for stats + res = self.client.get(reverse("queues_home_json")) + self.assertEqual(res.status_code, 404) + + @staticmethod + def token_validation(token: str) -> bool: + return token == "valid" + + # @patch('scheduler.views.SCHEDULER_CONFIG') + # def test_statistics_json_view_token(self, configuration): + # configuration.get.return_value = ViewTest.token_validation + # self.user.is_staff = False + # self.user.save() + # res = self.client.get(reverse('queues_home_json'), headers={'Authorization': 'valid'}) + # self.assertEqual(res.status_code, 200) + # + # res = self.client.get(reverse('queues_home_json'), headers={'Authorization': 'invalid'}) + # self.assertEqual(res.status_code, 404) diff --git a/scheduler/tests/test_views/test_queue_actions.py b/scheduler/tests/test_views/test_queue_actions.py new file mode 100644 index 0000000..2697dee --- /dev/null +++ b/scheduler/tests/test_views/test_queue_actions.py @@ -0,0 +1,109 @@ +from django.urls import reverse + +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobStatus, JobModel +from scheduler.tests.jobs import failing_job, test_job +from scheduler.tests.test_views.base import BaseTestCase +from scheduler.tests.testtools import assert_message_in_response +from scheduler.worker import create_worker + + +class QueueActionsViewsTest(BaseTestCase): + def test_job_list_action_delete_jobs__with_bad_next_url(self): + queue = get_queue("django_tasks_scheduler_test") + + # enqueue some jobs + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(test_job, job_info_ttl=0) + job_names.append(job.name) + + # remove those jobs using view + res = self.client.post( + reverse("queue_job_actions", args=[queue.name]), + {"action": "delete", "job_names": job_names, "next_url": "bad_url"}, + follow=True, + ) + assert_message_in_response(res, "Bad followup URL") + # check if jobs are removed + self.assertEqual(200, res.status_code) + for job_name in job_names: + self.assertFalse(JobModel.exists(job_name, connection=queue.connection), f"job {job_name} exists") + self.assertNotIn(job_name, queue.queued_job_registry.all()) + + def test_job_list_action_delete_jobs(self): + queue = get_queue("django_tasks_scheduler_test") + + # enqueue some jobs + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(test_job, job_info_ttl=0) + job_names.append(job.name) + + # remove those jobs using view + res = self.client.post( + reverse("queue_job_actions", args=[queue.name]), {"action": "delete", "job_names": job_names}, follow=True + ) + + # check if jobs are removed + self.assertEqual(200, res.status_code) + for job_name in job_names: + self.assertFalse(JobModel.exists(job_name, connection=queue.connection), f"job {job_name} exists") + self.assertNotIn(job_name, queue.queued_job_registry.all()) + + def test_job_list_action_requeue_jobs(self): + queue_name = "django_tasks_scheduler_test" + queue = get_queue(queue_name) + + # enqueue some jobs that will fail + job_names = [] + for _ in range(0, 3): + job = queue.create_and_enqueue_job(failing_job) + job_names.append(job.name) + + # do those jobs = fail them + worker = create_worker(queue_name, burst=True) + worker.work() + + # check if all jobs are really failed + for job_name in job_names: + job = JobModel.get(job_name, connection=queue.connection) + self.assertTrue(job.is_failed) + + # re-nqueue failed jobs from failed queue + self.client.post(reverse("queue_job_actions", args=[queue_name]), {"action": "requeue", "job_names": job_names}) + + # check if we requeue all failed jobs + for job_name in job_names: + job = JobModel.get(job_name, connection=queue.connection) + self.assertFalse(job.is_failed) + + def test_job_list_action_stop_jobs__move_to_finished_registry(self): + queue_name = "django_tasks_scheduler_test" + queue = get_queue(queue_name) + + # Enqueue some jobs + job_names = [] + worker = create_worker(queue_name) + worker.bootstrap() + for _ in range(3): + job = queue.create_and_enqueue_job(test_job) + job_names.append(job.name) + worker.worker_before_execution(job, connection=queue.connection) + job.prepare_for_execution(worker.name, queue.active_job_registry, connection=queue.connection) + + # Check if the jobs are started + for job_name in job_names: + job = JobModel.get(job_name, connection=queue.connection) + self.assertEqual(job.status, JobStatus.STARTED) + + # Stop those jobs using the view + self.assertEqual(len(queue.active_job_registry), len(job_names)) + self.client.post(reverse("queue_job_actions", args=[queue_name]), {"action": "stop", "job_names": job_names}) + self.assertEqual(0, len(queue.active_job_registry)) + + self.assertEqual(0, len(queue.canceled_job_registry)) + self.assertEqual(len(job_names), len(queue.finished_job_registry)) + + for job_name in job_names: + self.assertIn(job_name, queue.finished_job_registry) diff --git a/scheduler/tests/test_views/test_queue_registry_jobs.py b/scheduler/tests/test_views/test_queue_registry_jobs.py new file mode 100644 index 0000000..4b7bfe9 --- /dev/null +++ b/scheduler/tests/test_views/test_queue_registry_jobs.py @@ -0,0 +1,92 @@ +import time +from datetime import datetime + +from django.urls import reverse + +from scheduler.helpers.queues import get_queue +from scheduler.tests.jobs import test_job +from scheduler.tests.test_views.base import BaseTestCase + + +class QueueRegistryJobsViewTest(BaseTestCase): + def test_queue_jobs_unknown_registry(self): + queue_name = "default" + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "unknown"]), follow=True) + self.assertEqual(404, res.status_code) + + def test_queue_jobs_unknown_queue(self): + res = self.client.get(reverse("queue_registry_jobs", args=["UNKNOWN", "queued"])) + self.assertEqual(404, res.status_code) + + def test_queued_jobs(self): + """Jobs in queue are displayed properly""" + queue = get_queue("default") + job = queue.create_and_enqueue_job(test_job) + queue_name = "default" + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "queued"])) + self.assertEqual(res.context["jobs"], [job]) + + def test_finished_jobs(self): + """Ensure that finished jobs page works properly.""" + queue = get_queue("django_tasks_scheduler_test") + queue_name = "django_tasks_scheduler_test" + + job = queue.create_and_enqueue_job(test_job) + registry = queue.finished_job_registry + registry.add(queue.connection, job.name, time.time() + 2) + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "finished"])) + self.assertEqual(res.context["jobs"], [job]) + + def test_failed_jobs(self): + """Ensure that failed jobs page works properly.""" + queue = get_queue("django_tasks_scheduler_test") + queue_name = "django_tasks_scheduler_test" + + # Test that page doesn't fail when FailedJobRegistry is empty + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "failed"])) + self.assertEqual(res.status_code, 200) + + job = queue.create_and_enqueue_job(test_job) + registry = queue.failed_job_registry + registry.add(queue.connection, job.name, time.time() + 20) + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "failed"])) + self.assertEqual(res.context["jobs"], [job]) + + def test_scheduled_jobs(self): + """Ensure that scheduled jobs page works properly.""" + queue = get_queue("django_tasks_scheduler_test") + queue_name = "django_tasks_scheduler_test" + + # Test that page doesn't fail when ScheduledJobRegistry is empty + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) + self.assertEqual(res.status_code, 200) + + job = queue.create_and_enqueue_job(test_job, when=datetime.now()) + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) + self.assertEqual(res.context["jobs"], [job]) + + def test_scheduled_jobs_registry_removal(self): + """Ensure that non-existing job is being deleted from registry by view""" + queue = get_queue("django_tasks_scheduler_test") + queue_name = "django_tasks_scheduler_test" + + registry = queue.scheduled_job_registry + job = queue.create_and_enqueue_job(test_job, when=datetime.now()) + self.assertEqual(len(registry), 1) + + queue.delete_job(job.name) + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "scheduled"])) + self.assertEqual(res.context["jobs"], []) + + self.assertEqual(len(registry), 0) + + def test_started_jobs(self): + """Ensure that active jobs page works properly.""" + queue = get_queue("django_tasks_scheduler_test") + queue_name = "django_tasks_scheduler_test" + + job = queue.create_and_enqueue_job(test_job) + registry = queue.active_job_registry + registry.add(queue.connection, job.name, time.time() + 20) + res = self.client.get(reverse("queue_registry_jobs", args=[queue_name, "active"])) + self.assertEqual(res.context["jobs"], [job]) diff --git a/scheduler/tests/test_views/test_workers_view.py b/scheduler/tests/test_views/test_workers_view.py new file mode 100644 index 0000000..f17f61d --- /dev/null +++ b/scheduler/tests/test_views/test_workers_view.py @@ -0,0 +1,18 @@ +from django.urls import reverse + +from scheduler.worker import create_worker +from scheduler.tests import conf # noqa +from scheduler.tests.test_views.base import BaseTestCase + + +class TestViewWorkers(BaseTestCase): + def test_workers_home(self): + res = self.client.get(reverse("workers_home")) + prev_workers = res.context["workers"] + worker1 = create_worker("django_tasks_scheduler_test") + worker1.worker_start() + worker2 = create_worker("test3") + worker2.worker_start() + + res = self.client.get(reverse("workers_home")) + self.assertEqual(res.context["workers"], prev_workers + [worker1._model, worker2._model]) diff --git a/scheduler/tests/test_worker/__init__.py b/scheduler/tests/test_worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_worker/test_scheduler.py b/scheduler/tests/test_worker/test_scheduler.py new file mode 100644 index 0000000..2033d93 --- /dev/null +++ b/scheduler/tests/test_worker/test_scheduler.py @@ -0,0 +1,40 @@ +from datetime import timedelta + +import time_machine +from django.utils import timezone + +from scheduler.settings import SCHEDULER_CONFIG +from scheduler.worker import create_worker +from scheduler.models import TaskType +from scheduler.tests.testtools import SchedulerBaseCase, task_factory +from scheduler.worker import WorkerScheduler + + +class TestWorkerScheduler(SchedulerBaseCase): + def test_create_worker_with_scheduler__scheduler_started(self): + SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 + worker = create_worker("default", name="test", burst=True, with_scheduler=True) + worker.bootstrap() + self.assertIsNotNone(worker.scheduler) + worker.stop_scheduler() + self.assertIsNone(worker.scheduler) + + def test_scheduler_schedules_tasks(self): + with time_machine.travel(0.0, tick=False) as traveller: + # arrange + task = task_factory(TaskType.ONCE, scheduled_time=timezone.now() + timedelta(seconds=50)) + self.assertIsNotNone(task.job_name) + self.assertNotIn(task.job_name, task.rqueue.queued_job_registry) + self.assertIn(task.job_name, task.rqueue.scheduled_job_registry) + + scheduler = WorkerScheduler([task.rqueue], worker_name="fake-worker", connection=task.rqueue.connection) + + # act + traveller.move_to(50) + scheduler._acquire_locks() + scheduler.enqueue_scheduled_jobs() + + # assert + self.assertIsNotNone(task.job_name) + self.assertIn(task.job_name, task.rqueue.queued_job_registry) + self.assertNotIn(task.job_name, task.rqueue.scheduled_job_registry) diff --git a/scheduler/tests/test_worker/test_worker_commands.py b/scheduler/tests/test_worker/test_worker_commands.py new file mode 100644 index 0000000..b915c91 --- /dev/null +++ b/scheduler/tests/test_worker/test_worker_commands.py @@ -0,0 +1,42 @@ +from scheduler.helpers.queues import get_queue +from scheduler.tests.jobs import test_job +from ..test_views.base import BaseTestCase +from ...redis_models import JobModel +from ...worker import create_worker +from ...worker.commands import send_command +from ...worker.commands.suspend_worker import SuspendWorkCommand + + +class WorkerCommandsTest(BaseTestCase): + def test_stop_worker_command__green(self): + # Arrange + worker_name = "test" + queue = get_queue("default") + job = queue.create_and_enqueue_job(test_job) + self.assertTrue(job.is_queued) + worker = create_worker("default", name=worker_name, burst=True, with_scheduler=False) + worker.worker_start() + # Act + send_command(queue.connection, SuspendWorkCommand(worker_name=worker_name)) + worker.work() + + # Assert + self.assertTrue(job.is_queued) + self.assertTrue(worker._model.is_suspended) + + def test_stop_worker_command__bad_worker_name(self): + # Arrange + worker_name = "test" + queue = get_queue("default") + job = queue.create_and_enqueue_job(test_job) + self.assertTrue(job.is_queued) + worker = create_worker("default", name=worker_name, burst=True, with_scheduler=False) + worker.bootstrap() + # Act + send_command(queue.connection, SuspendWorkCommand(worker_name=worker_name + "1")) + worker.work() + + # Assert + self.assertFalse(worker._model.is_suspended) + job = JobModel.get(job.name, connection=queue.connection) + self.assertFalse(job.is_queued) diff --git a/scheduler/tests/test_worker/test_worker_commands_multiprocess.py b/scheduler/tests/test_worker/test_worker_commands_multiprocess.py new file mode 100644 index 0000000..5287b61 --- /dev/null +++ b/scheduler/tests/test_worker/test_worker_commands_multiprocess.py @@ -0,0 +1,84 @@ +from time import sleep + +from django.test import tag + +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobStatus, JobModel, WorkerModel +from scheduler.tests.jobs import long_job, two_seconds_job +from .. import testtools +from ..test_views.base import BaseTestCase +from ...worker.commands import KillWorkerCommand, send_command, StopJobCommand + + +@tag("multiprocess") +class WorkerCommandsTest(BaseTestCase): + def test_kill_job_command__current_job(self): + # Arrange + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(long_job) + self.assertTrue(job.is_queued) + process, worker_name = testtools.run_worker_in_process("django_tasks_scheduler_test") + sleep(0.1) + job = JobModel.get(job.name, connection=queue.connection) + self.assertEqual(JobStatus.STARTED, job.status) + + # Act + send_command(queue.connection, StopJobCommand(worker_name=worker_name, job_name=job.name)) + + # Assert + + process.terminate() + process.join(2) + process.kill() + + job = JobModel.get(job.name, connection=queue.connection) + worker_model = WorkerModel.get(worker_name, connection=queue.connection) + self.assertEqual(job.name, worker_model.stopped_job_name) + self.assertEqual(job.name, worker_model.current_job_name) + self.assertEqual(0, worker_model.completed_jobs) + self.assertEqual(0, worker_model.failed_job_count) + self.assertEqual(0, worker_model.successful_job_count) + self.assertEqual(JobStatus.STOPPED, job.status) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + + def test_kill_job_command__different_job(self): + # Arrange + queue = get_queue("django_tasks_scheduler_test") + job = queue.create_and_enqueue_job(two_seconds_job) + self.assertTrue(job.is_queued) + process, worker_name = testtools.run_worker_in_process("django_tasks_scheduler_test") + sleep(0.2) + job = JobModel.get(job.name, connection=queue.connection) + self.assertEqual(JobStatus.STARTED, job.status) + + # Act + send_command(queue.connection, StopJobCommand(worker_name=worker_name, job_name=job.name + "1")) + sleep(0.1) + process.kill() + process.join() + # Assert + job = JobModel.get(job.name, connection=queue.connection) + self.assertEqual(JobStatus.STARTED, job.status) + self.assertNotIn(job.name, queue.queued_job_registry.all()) + worker_model = WorkerModel.get(worker_name, connection=queue.connection) + self.assertEqual(0, worker_model.completed_jobs) + self.assertEqual(0, worker_model.failed_job_count) + self.assertEqual(0, worker_model.successful_job_count) + self.assertIsNone(worker_model.stopped_job_name) + self.assertEqual(job.name, worker_model.current_job_name) + + def test_kill_worker_command(self): + queue = get_queue("django_tasks_scheduler_test") + process, worker_name = testtools.run_worker_in_process("django_tasks_scheduler_test") + sleep(0.1) + # act + send_command(queue.connection, KillWorkerCommand(worker_name=worker_name)) + # assert + sleep(0.2) + process.kill() + process.join() + worker_model = WorkerModel.get(worker_name, connection=queue.connection) + self.assertEqual(0, worker_model.completed_jobs) + self.assertEqual(0, worker_model.failed_job_count) + self.assertEqual(0, worker_model.successful_job_count) + self.assertIsNotNone(worker_model.shutdown_requested_date) diff --git a/scheduler/tests/test_worker.py b/scheduler/tests/test_worker/test_worker_creation.py similarity index 54% rename from scheduler/tests/test_worker.py rename to scheduler/tests/test_worker/test_worker_creation.py index 4b40bfb..578fcc1 100644 --- a/scheduler/tests/test_worker.py +++ b/scheduler/tests/test_worker/test_worker_creation.py @@ -1,26 +1,26 @@ import os import uuid -from rq.job import Job -from scheduler.rq_classes import JobExecution +from scheduler import settings +from scheduler.redis_models import WorkerModel +from scheduler.worker import create_worker +from scheduler.tests import conf # noqa from scheduler.tests.testtools import SchedulerBaseCase -from scheduler.tools import create_worker -from . import test_settings # noqa -from .. import settings +from scheduler.worker.worker import QueueConnectionDiscrepancyError class TestWorker(SchedulerBaseCase): def test_create_worker__two_workers_same_queue(self): worker1 = create_worker("default", "django_tasks_scheduler_test") - worker1.register_birth() + worker1.worker_start() worker2 = create_worker("default") - worker2.register_birth() + worker2.worker_start() hostname = os.uname()[1] self.assertEqual(f"{hostname}-worker.1", worker1.name) self.assertEqual(f"{hostname}-worker.2", worker2.name) def test_create_worker__worker_with_queues_different_connection(self): - with self.assertRaises(ValueError): + with self.assertRaises(QueueConnectionDiscrepancyError): create_worker("default", "test1") def test_create_worker__with_name(self): @@ -36,17 +36,21 @@ def test_create_worker__with_name_containing_slash(self): def test_create_worker__scheduler_interval(self): prev = settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL = 1 - worker = create_worker("default") - worker.work(burst=True) + worker = create_worker("default", name="test", burst=True, with_scheduler=True) + worker.bootstrap() + self.assertEqual(worker.name, "test") self.assertEqual(worker.scheduler.interval, 1) settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL = prev - - def test_get_worker_with_custom_job_class(self): - # Test with string representation of job_class - worker = create_worker("default", job_class="scheduler.rq_classes.JobExecution") - self.assertTrue(issubclass(worker.job_class, Job)) - self.assertTrue(issubclass(worker.job_class, JobExecution)) - - def test_get_worker_without_custom_job_class(self): - worker = create_worker("default") - self.assertTrue(issubclass(worker.job_class, JobExecution)) + worker.teardown() + + def test_create_worker__cleanup(self): + worker = create_worker("default", name="test", burst=True, with_scheduler=False) + worker.bootstrap() + worker.connection.delete(WorkerModel.key_for(worker.name)) + all_names = WorkerModel.all_names(worker.connection) + self.assertIn(worker.name, all_names) + # act + WorkerModel.cleanup(worker.connection, "default") + # assert + all_names = WorkerModel.all_names(worker.connection) + self.assertNotIn(worker.name, all_names) diff --git a/scheduler/tests/testtools.py b/scheduler/tests/testtools.py index 2c987b5..0be7d49 100644 --- a/scheduler/tests/testtools.py +++ b/scheduler/tests/testtools.py @@ -1,4 +1,6 @@ +import multiprocessing from datetime import timedelta +from typing import List, Tuple from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -7,10 +9,26 @@ from django.utils import timezone from scheduler import settings -from scheduler.models.args import TaskKwarg -from scheduler.models.task import Task -from scheduler.queues import get_queue -from scheduler.tools import TaskType +from scheduler.admin.task_admin import job_execution_of +from scheduler.helpers.queues import get_queue +from scheduler.models import Task, TaskType +from scheduler.models import TaskKwarg +from scheduler.redis_models import JobModel +from scheduler.worker import Worker +from scheduler.worker import create_worker + +multiprocessing.set_start_method("fork") + + +def _run_worker_process(worker: Worker, **kwargs): + worker.work(**kwargs) + + +def run_worker_in_process(*args, name="test-worker") -> Tuple[multiprocessing.Process, str]: + worker = create_worker(*args, name=name, fork_job_execution=False) + process = multiprocessing.Process(target=_run_worker_process, args=(worker,), kwargs=dict(with_scheduler=False)) + process.start() + return process, name def assert_message_in_response(response, message): @@ -33,8 +51,7 @@ def task_factory( ): values = dict( name="Scheduled Job %d" % next(seq), - job_id=None, - queue=list(settings.QUEUES.keys())[0], + queue=list(settings._QUEUES.keys())[0], callable=callable_name, enabled=True, timeout=None, @@ -57,11 +74,7 @@ def task_factory( ) ) elif task_type == TaskType.CRON: - values.update( - dict( - cron_string="0 0 * * *", - ) - ) + values.update(dict(cron_string="0 0 * * *")) values.update(kwargs) if instance_only: instance = Task(task_type=task_type, **values) @@ -88,17 +101,16 @@ def taskarg_factory(cls, **kwargs): return instance -def _get_task_job_execution_from_registry(django_task: Task): - jobs_to_schedule = django_task.rqueue.scheduled_job_registry.get_job_ids() - entry = next(i for i in jobs_to_schedule if i == django_task.job_id) - return django_task.rqueue.fetch_job(entry) +def _get_task_scheduled_job_from_registry(django_task: Task) -> JobModel: + jobs_to_schedule = django_task.rqueue.scheduled_job_registry.all() + entry = next(i for i in jobs_to_schedule if i == django_task.job_name) + return JobModel.get(entry, connection=django_task.rqueue.connection) -def _get_executions(django_job: Task): - job_ids = django_job.rqueue.get_all_job_ids() - return list( - filter(lambda j: j.is_execution_of(django_job), map(lambda jid: django_job.rqueue.fetch_job(jid), job_ids)) - ) +def _get_executions(task: Task): + job_names = task.rqueue.get_all_job_names() + job_list: List[JobModel] = JobModel.get_many(job_names, connection=task.rqueue.connection) + return list(filter(lambda j: job_execution_of(j, task), job_list)) class SchedulerBaseCase(TestCase): @@ -114,15 +126,7 @@ def setUpTestData(cls) -> None: def setUp(self) -> None: super(SchedulerBaseCase, self).setUp() queue = get_queue("default") - queue.empty() + queue.connection.flushall() def tearDown(self) -> None: super(SchedulerBaseCase, self).tearDown() - queue = get_queue("default") - queue.empty() - - @classmethod - def setUpClass(cls): - super(SchedulerBaseCase, cls).setUpClass() - queue = get_queue("default") - queue.connection.flushall() diff --git a/scheduler/timeouts.py b/scheduler/timeouts.py new file mode 100644 index 0000000..3919ba1 --- /dev/null +++ b/scheduler/timeouts.py @@ -0,0 +1,114 @@ +import ctypes +import signal +import threading + + +class BaseTimeoutException(Exception): + """Base exception for timeouts.""" + + pass + + +class JobTimeoutException(BaseTimeoutException): + """Raised when a job takes longer to complete than the allowed maximum timeout value.""" + + pass + + +class JobExecutionMonitorTimeoutException(BaseTimeoutException): + """Raised when waiting for a job-execution-process exiting takes longer than the maximum timeout value.""" + + pass + + +class BaseDeathPenalty: + """Base class to setup job timeouts.""" + + def __init__(self, timeout, exception=BaseTimeoutException, **kwargs): + self._timeout = timeout + self._exception = exception + + def __enter__(self): + self.setup_death_penalty() + + def __exit__(self, type, value, traceback): + # Always cancel immediately, since we're done + try: + self.cancel_death_penalty() + except BaseTimeoutException: + # Weird case: we're done with the with body, but now the alarm is fired. We may safely ignore this + # situation and consider the body done. + pass + + # __exit__ may return True to supress further exception handling. We don't want to suppress any exceptions + # here, since all errors should just pass through, BaseTimeoutException being handled normally to the invoking + # context. + return False + + def setup_death_penalty(self): + raise NotImplementedError() + + def cancel_death_penalty(self): + raise NotImplementedError() + + +class UnixSignalDeathPenalty(BaseDeathPenalty): + def handle_death_penalty(self, signum, frame) -> None: + raise self._exception("Task exceeded maximum timeout value ({0} seconds)".format(self._timeout)) + + def setup_death_penalty(self) -> None: + """Sets up an alarm signal and a signal handler that raises an exception after the timeout amount (expressed + in seconds).""" + signal.signal(signal.SIGALRM, self.handle_death_penalty) + signal.alarm(self._timeout) + + def cancel_death_penalty(self) -> None: + """Removes the death penalty alarm and puts back the system into default signal handling.""" + signal.alarm(0) + signal.signal(signal.SIGALRM, signal.SIG_DFL) + + +class TimerDeathPenalty(BaseDeathPenalty): + def __init__(self, timeout, exception=JobTimeoutException, **kwargs): + super().__init__(timeout, exception, **kwargs) + self._target_thread_id = threading.current_thread().ident + self._timer = None + + # Monkey-patch exception with the message ahead of time + # since PyThreadState_SetAsyncExc can only take a class + def init_with_message(self, *args, **kwargs): # noqa + super(exception, self).__init__("Task exceeded maximum timeout value ({0} seconds)".format(timeout)) + + self._exception.__init__ = init_with_message + + def new_timer(self): + """Returns a new timer since timers can only be used once.""" + return threading.Timer(self._timeout, self.handle_death_penalty) + + def handle_death_penalty(self): + """Raises an asynchronous exception in another thread. + + Reference http://docs.python.org/c-api/init.html#PyThreadState_SetAsyncExc for more info. + """ + ret = ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_long(self._target_thread_id), ctypes.py_object(self._exception) + ) + if ret == 0: + raise ValueError(f"Invalid thread ID {self._target_thread_id}") + elif ret > 1: + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self._target_thread_id), 0) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def setup_death_penalty(self): + """Starts the timer.""" + if self._timeout <= 0: + return + self._timer = self.new_timer() + self._timer.start() + + def cancel_death_penalty(self): + """Cancels the timer.""" + if self._timeout <= 0: + return + self._timer.cancel() + self._timer = None diff --git a/scheduler/tools.py b/scheduler/tools.py deleted file mode 100644 index c73aae8..0000000 --- a/scheduler/tools.py +++ /dev/null @@ -1,111 +0,0 @@ -import importlib -import os -from typing import List, Any, Callable, Optional - -import croniter -from django.apps import apps -from django.db import models -from django.utils import timezone -from django.utils.module_loading import import_string -from django.utils.translation import gettext_lazy as _ - -from scheduler.queues import get_queues, logger, get_queue -from scheduler.rq_classes import DjangoWorker, JobExecution, TASK_TYPES, MODEL_NAMES -from scheduler.settings import SCHEDULER_CONFIG, Broker - - -class TaskType(models.TextChoices): - CRON = "CronTaskType", _("Cron Task") - REPEATABLE = "RepeatableTaskType", _("Repeatable Task") - ONCE = "OnceTaskType", _("Run once") - - -def callable_func(callable_str: str) -> Callable: - path = callable_str.split(".") - module = importlib.import_module(".".join(path[:-1])) - func = getattr(module, path[-1]) - if callable(func) is False: - raise TypeError("'{}' is not callable".format(callable_str)) - return func - - -def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime]: - """Calculate the next scheduled time by creating a crontab object with a cron string""" - if cron_string is None: - return None - now = timezone.now() - itr = croniter.croniter(cron_string, now) - next_itr = itr.get_next(timezone.datetime) - return next_itr - - -def get_scheduled_task(task_type_str: str, task_id: int) -> "BaseTask": # noqa: F821 - # Try with new model names - model = apps.get_model(app_label="scheduler", model_name="Task") - if task_type_str in TASK_TYPES: - try: - task_type = TaskType(task_type_str) - task = model.objects.filter(task_type=task_type, id=task_id).first() - if task is None: - raise ValueError(f"Job {task_type}:{task_id} does not exit") - return task - except ValueError: - raise ValueError(f"Invalid task type {task_type_str}") - elif task_type_str in MODEL_NAMES: - model = apps.get_model(app_label="scheduler", model_name=task_type_str) - task = model.objects.filter(id=task_id).first() - if task is None: - raise ValueError(f"Job {task_type_str}:{task_id} does not exit") - return task - raise ValueError(f"Job Model {task_type_str} does not exist, choices are {TASK_TYPES}") - - -def run_task(task_model: str, task_id: int) -> Any: - """Run a scheduled job""" - scheduled_task = get_scheduled_task(task_model, task_id) - logger.debug(f"Running task {str(scheduled_task)}") - args = scheduled_task.parse_args() - kwargs = scheduled_task.parse_kwargs() - res = scheduled_task.callable_func()(*args, **kwargs) - return res - - -def _calc_worker_name(existing_worker_names) -> str: - hostname = os.uname()[1] - c = 1 - worker_name = f"{hostname}-worker.{c}" - while worker_name in existing_worker_names: - c += 1 - worker_name = f"{hostname}-worker.{c}" - return worker_name - - -def create_worker(*queue_names, **kwargs) -> DjangoWorker: - """Returns a Django worker for all queues or specified ones.""" - - queues = get_queues(*queue_names) - existing_workers = DjangoWorker.all(connection=queues[0].connection) - existing_worker_names = set(map(lambda w: w.name, existing_workers)) - kwargs.setdefault("fork_job_execution", SCHEDULER_CONFIG.BROKER != Broker.FAKEREDIS) - if kwargs.get("name", None) is None: - kwargs["name"] = _calc_worker_name(existing_worker_names) - - kwargs["name"] = kwargs["name"].replace("/", ".") - - # Handle job_class if provided - if "job_class" not in kwargs or kwargs["job_class"] is None: - kwargs["job_class"] = "scheduler.rq_classes.JobExecution" - try: - kwargs["job_class"] = import_string(kwargs["job_class"]) - except ImportError: - raise ImportError(f"Could not import job class {kwargs['job_class']}") - - worker = DjangoWorker(queues, connection=queues[0].connection, **kwargs) - return worker - - -def get_job_executions_for_task(queue_name, scheduled_task) -> List[JobExecution]: - queue = get_queue(queue_name) - job_list = queue.get_all_jobs() - res = list(filter(lambda j: j.is_execution_of(scheduled_task), job_list)) - return res diff --git a/scheduler/types/__init__.py b/scheduler/types/__init__.py new file mode 100644 index 0000000..167bc58 --- /dev/null +++ b/scheduler/types/__init__.py @@ -0,0 +1,30 @@ +__all__ = [ + "ConnectionErrorTypes", + "ResponseErrorTypes", + "TimeoutErrorTypes", + "WatchErrorTypes", + "ConnectionType", + "PipelineType", + "SentinelType", + "FunctionReferenceType", + "BrokerMetaData", + "TASK_TYPES", + "Broker", + "SchedulerConfiguration", + "QueueConfiguration", + "Self", +] + +from .broker_types import ( + ConnectionErrorTypes, + ResponseErrorTypes, + TimeoutErrorTypes, + WatchErrorTypes, + ConnectionType, + PipelineType, + SentinelType, + FunctionReferenceType, + BrokerMetaData, + TASK_TYPES, +) +from .settings_types import Broker, SchedulerConfiguration, QueueConfiguration, Self diff --git a/scheduler/broker_types.py b/scheduler/types/broker_types.py similarity index 67% rename from scheduler/broker_types.py rename to scheduler/types/broker_types.py index eb779fb..2a80aaf 100644 --- a/scheduler/broker_types.py +++ b/scheduler/types/broker_types.py @@ -1,6 +1,7 @@ # This is a helper module to obfuscate types used by different broker implementations. from collections import namedtuple -from typing import Union, Dict, Tuple +from typing import Any, Callable, TypeVar, Union +from typing import Dict, Tuple import redis @@ -11,20 +12,25 @@ valkey.Valkey = redis.Redis valkey.StrictValkey = redis.StrictRedis -from scheduler.settings import Broker +from .settings_types import Broker ConnectionErrorTypes = (redis.ConnectionError, valkey.ConnectionError) ResponseErrorTypes = (redis.ResponseError, valkey.ResponseError) +TimeoutErrorTypes = (redis.TimeoutError, valkey.TimeoutError) +WatchErrorTypes = (redis.WatchError, valkey.WatchError) ConnectionType = Union[redis.Redis, valkey.Valkey] PipelineType = Union[redis.client.Pipeline, valkey.client.Pipeline] SentinelType = Union[redis.sentinel.Sentinel, valkey.sentinel.Sentinel] +FunctionReferenceType = TypeVar("FunctionReferenceType", str, Callable[..., Any]) -BrokerMetaDataType = namedtuple("BrokerMetaDataType", ["connection_type", "sentinel_type", "ssl_prefix"]) +BrokerMetaDataType = namedtuple("BrokerMetaDataType", ["connection_type", "sentinel_type"]) BrokerMetaData: Dict[Tuple[Broker, bool], BrokerMetaDataType] = { - # Map of (Broker, Strict flag) => Connection Class, Sentinel Class, SSL Connection Prefix - (Broker.REDIS, False): BrokerMetaDataType(redis.Redis, redis.sentinel.Sentinel, "rediss"), - (Broker.VALKEY, False): BrokerMetaDataType(valkey.Valkey, valkey.sentinel.Sentinel, "valkeys"), - (Broker.REDIS, True): BrokerMetaDataType(redis.StrictRedis, redis.sentinel.Sentinel, "rediss"), - (Broker.VALKEY, True): BrokerMetaDataType(valkey.StrictValkey, valkey.sentinel.Sentinel, "valkeys"), + # Map of (Broker, Strict flag) => Connection Class, Sentinel Class + (Broker.REDIS, False): BrokerMetaDataType(redis.Redis, redis.sentinel.Sentinel), + (Broker.VALKEY, False): BrokerMetaDataType(valkey.Valkey, valkey.sentinel.Sentinel), + (Broker.REDIS, True): BrokerMetaDataType(redis.StrictRedis, redis.sentinel.Sentinel), + (Broker.VALKEY, True): BrokerMetaDataType(valkey.StrictValkey, valkey.sentinel.Sentinel), } + +TASK_TYPES = ["OnceTaskType", "RepeatableTaskType", "CronTaskType"] diff --git a/scheduler/types/settings_types.py b/scheduler/types/settings_types.py new file mode 100644 index 0000000..1deb903 --- /dev/null +++ b/scheduler/types/settings_types.py @@ -0,0 +1,87 @@ +import sys +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Dict, Optional, List, Tuple, Any, Type + +from scheduler.timeouts import BaseDeathPenalty, UnixSignalDeathPenalty + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class Broker(Enum): + REDIS = "redis" + FAKEREDIS = "fakeredis" + VALKEY = "valkey" + + +def _token_validation(token: str) -> bool: + return False + + +@dataclass(slots=True, kw_only=True) +class SchedulerConfiguration: + """Configuration for django-tasks-scheduler""" + + EXECUTIONS_IN_PAGE: int = 20 + SCHEDULER_INTERVAL: int = 10 + BROKER: Broker = Broker.REDIS + TOKEN_VALIDATION_METHOD: Callable[[str], bool] = _token_validation + CALLBACK_TIMEOUT: int = 60 # Callback timeout in seconds (success/failure/stopped) + # Default values, can be override per task + DEFAULT_SUCCESS_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep successful job results + DEFAULT_FAILURE_TTL: int = 365 * 24 * 60 * 60 # Time To Live (TTL) in seconds to keep job failure information + DEFAULT_JOB_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep job information + DEFAULT_JOB_TIMEOUT: int = 5 * 60 # timeout (seconds) for a job + # General configuration values + DEFAULT_WORKER_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep worker information after last heartbeat + DEFAULT_MAINTENANCE_TASK_INTERVAL: int = 10 * 60 # The interval to run maintenance tasks in seconds. 10 minutes. + DEFAULT_JOB_MONITORING_INTERVAL: int = 30 # The interval to monitor jobs in seconds. + SCHEDULER_FALLBACK_PERIOD_SECS: int = 120 # Period (secs) to wait before requiring to reacquire locks + DEATH_PENALTY_CLASS: Type[BaseDeathPenalty] = UnixSignalDeathPenalty + + +@dataclass(slots=True, frozen=True, kw_only=True) +class QueueConfiguration: + __CONNECTION_FIELDS__ = { + "URL", + "DB", + "UNIX_SOCKET_PATH", + "HOST", + "PORT", + "PASSWORD", + "SENTINELS", + "MASTER_NAME", + "CONNECTION_KWARGS", + } + DB: Optional[int] = None + # Redis connection parameters, either UNIX_SOCKET_PATH/URL/separate params (HOST, PORT, PASSWORD) should be provided + UNIX_SOCKET_PATH: Optional[str] = None + URL: Optional[str] = None + HOST: Optional[str] = None + PORT: Optional[int] = None + USERNAME: Optional[str] = None + PASSWORD: Optional[str] = None + + ASYNC: Optional[bool] = True + + SENTINELS: Optional[List[Tuple[str, int]]] = None + SENTINEL_KWARGS: Optional[Dict[str, str]] = None + MASTER_NAME: Optional[str] = None + CONNECTION_KWARGS: Optional[Dict[str, Any]] = None + + def __post_init__(self): + if not any((self.URL, self.UNIX_SOCKET_PATH, self.HOST, self.SENTINELS)): + raise ValueError(f"At least one of URL, UNIX_SOCKET_PATH, HOST must be provided: {self}") + if sum((self.URL is not None, self.UNIX_SOCKET_PATH is not None, self.HOST is not None)) > 1: + raise ValueError(f"Only one of URL, UNIX_SOCKET_PATH, HOST should be provided: {self}") + if self.HOST is not None and (self.PORT is None or self.DB is None): + raise ValueError(f"HOST requires PORT and DB: {self}") + + def same_connection_params(self, other: Self) -> bool: + for field in self.__CONNECTION_FIELDS__: + if getattr(self, field) != getattr(other, field): + return False + return True diff --git a/scheduler/urls.py b/scheduler/urls.py index 4dfa9b3..214badf 100644 --- a/scheduler/urls.py +++ b/scheduler/urls.py @@ -6,16 +6,19 @@ path("queues/", views.stats, name="queues_home"), path("queues/stats.json", views.stats_json, name="queues_home_json"), path("queues//workers/", views.queue_workers, name="queue_workers"), - path("queues///jobs", views.jobs_view, name="queue_registry_jobs"), - path("queues///empty/", views.clear_queue_registry, name="queue_clear"), - path("queues///requeue-all/", views.requeue_all, name="queue_requeue_all"), - path("queues//confirm-action/", views.confirm_action, name="queue_confirm_action"), - path("queues//actions/", views.actions, name="queue_actions"), + path("queues///jobs", views.list_registry_jobs, name="queue_registry_jobs"), + path( + "queues////", + views.queue_registry_actions, + name="queue_registry_action", + ), + path("queues//confirm-action/", views.queue_confirm_job_action, name="queue_confirm_job_action"), + path("queues//actions/", views.queue_job_actions, name="queue_job_actions"), ] urlpatterns += [ - path("workers/", views.workers, name="workers_home"), + path("workers/", views.workers_list, name="workers_home"), path("workers//", views.worker_details, name="worker_details"), - path("jobs//", views.job_detail, name="job_details"), - path("jobs///", views.job_action, name="queue_job_action"), + path("jobs//", views.job_detail, name="job_details"), + path("jobs///", views.job_action, name="job_detail_action"), ] diff --git a/scheduler/views.py b/scheduler/views.py deleted file mode 100644 index 3394764..0000000 --- a/scheduler/views.py +++ /dev/null @@ -1,471 +0,0 @@ -from html import escape -from math import ceil -from typing import Tuple, Optional - -from django.contrib import admin, messages -from django.contrib.admin.views.decorators import staff_member_required -from django.core.paginator import Paginator -from django.http import JsonResponse, HttpResponse, HttpRequest -from django.http.response import HttpResponseNotFound, Http404, HttpResponseBadRequest -from django.shortcuts import redirect -from django.shortcuts import render -from django.urls import reverse, resolve -from django.views.decorators.cache import never_cache - -from .broker_types import ConnectionErrorTypes, ResponseErrorTypes -from .queues import get_all_workers, get_connection, QueueNotFoundError -from .queues import get_queue as get_queue_base -from .rq_classes import JobExecution, DjangoWorker, DjangoQueue, InvalidJobOperation -from .settings import SCHEDULER_CONFIG, logger - - -def get_queue(queue_name: str) -> DjangoQueue: - try: - return get_queue_base(queue_name) - except QueueNotFoundError as e: - logger.error(e) - raise Http404(e) - - -def get_worker_executions(worker): - res = list() - for queue in worker.queues: - curr_jobs = queue.get_all_jobs() - curr_jobs = [j for j in curr_jobs if j.worker_name == worker.name] - res.extend(curr_jobs) - return res - - -# Create your views here. -@never_cache -@staff_member_required -def stats(request): - context_data = {**admin.site.each_context(request), **get_statistics(run_maintenance_tasks=True)} - return render(request, "admin/scheduler/stats.html", context_data) - - -def stats_json(request): - auth_token = request.headers.get("Authorization") - token_validation_func = SCHEDULER_CONFIG.TOKEN_VALIDATION_METHOD - if request.user.is_staff or (token_validation_func and auth_token and token_validation_func(auth_token)): - return JsonResponse(get_statistics()) - - return HttpResponseNotFound() - - -def get_statistics(run_maintenance_tasks=False): - from scheduler.settings import QUEUES - - queues = [] - if run_maintenance_tasks: - workers = get_all_workers() - for worker in workers: - worker.clean_registries() - for queue_name in QUEUES: - try: - queue = get_queue(queue_name) - connection = get_connection(QUEUES[queue_name]) - connection_kwargs = connection.connection_pool.connection_kwargs - - if run_maintenance_tasks: - queue.clean_registries() - - # Raw access to the first item from left of the broker list. - # This might not be accurate since new job can be added from the left - # with `at_front` parameters. - # Ideally rq should supports Queue.oldest_job - - last_job_id = queue.last_job_id() - last_job = queue.fetch_job(last_job_id.decode("utf-8")) if last_job_id else None - if last_job and last_job.enqueued_at: - oldest_job_timestamp = last_job.enqueued_at.strftime("%Y-%m-%d, %H:%M:%S") - else: - oldest_job_timestamp = "-" - - # parse_class and connection_pool are not needed and not JSON serializable - connection_kwargs.pop("parser_class", None) - connection_kwargs.pop("connection_pool", None) - - queue_data = dict( - name=queue.name, - jobs=queue.count, - oldest_job_timestamp=oldest_job_timestamp, - connection_kwargs=connection_kwargs, - scheduler_pid=queue.scheduler_pid, - workers=DjangoWorker.count(queue=queue), - finished_jobs=len(queue.finished_job_registry), - started_jobs=len(queue.started_job_registry), - deferred_jobs=len(queue.deferred_job_registry), - failed_jobs=len(queue.failed_job_registry), - scheduled_jobs=len(queue.scheduled_job_registry), - canceled_jobs=len(queue.canceled_job_registry), - ) - queues.append(queue_data) - except ConnectionErrorTypes as e: - logger.error(f"Could not connect for queue {queue_name}: {e}") - continue - - return {"queues": queues} - - -def _get_registry_job_list(queue, registry, page): - items_per_page = SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE - num_jobs = len(registry) - job_list = [] - - if num_jobs == 0: - return job_list, num_jobs, [] - - last_page = int(ceil(num_jobs / items_per_page)) - page_range = range(1, last_page + 1) - offset = items_per_page * (page - 1) - job_ids = registry.get_job_ids(offset, offset + items_per_page - 1) - job_list = JobExecution.fetch_many(job_ids, connection=queue.connection) - remove_job_ids = [job_id for i, job_id in enumerate(job_ids) if job_list[i] is None] - valid_jobs = [job for job in job_list if job is not None] - if registry is not queue: - for job_id in remove_job_ids: - registry.remove(job_id) - - return valid_jobs, num_jobs, page_range - - -@never_cache -@staff_member_required -def jobs_view(request, queue_name: str, registry_name: str): - queue = get_queue(queue_name) - registry = queue.get_registry(registry_name) - if registry is None: - return HttpResponseNotFound() - title = registry_name.capitalize() - page = int(request.GET.get("page", 1)) - job_list, num_jobs, page_range = _get_registry_job_list(queue, registry, page) - - context_data = { - **admin.site.each_context(request), - "queue": queue, - "registry_name": registry_name, - "registry": registry, - "jobs": job_list, - "num_jobs": num_jobs, - "page": page, - "page_range": page_range, - "job_status": title, - } - return render(request, "admin/scheduler/jobs.html", context_data) - - -@never_cache -@staff_member_required -def queue_workers(request: HttpRequest, queue_name: str) -> HttpResponse: - queue = get_queue(queue_name) - all_workers = DjangoWorker.all(queue.connection) - for w in all_workers: - w.clean_registries() - worker_list = [worker for worker in all_workers if queue.name in worker.queue_names()] - - context_data = { - **admin.site.each_context(request), - "queue": queue, - "workers": worker_list, - } - return render(request, "admin/scheduler/queue_workers.html", context_data) - - -@never_cache -@staff_member_required -def workers(request: HttpRequest) -> HttpResponse: - all_workers = get_all_workers() - worker_list = [worker for worker in all_workers] - - context_data = { - **admin.site.each_context(request), - "workers": worker_list, - } - return render(request, "admin/scheduler/workers.html", context_data) - - -@never_cache -@staff_member_required -def worker_details(request: HttpRequest, name: str) -> HttpResponse: - queue, worker = None, None - workers = get_all_workers() - worker = next((w for w in workers if w.name == name), None) - - if worker is None: - raise Http404(f"Couldn't find worker with this ID: {name}") - # Convert microseconds to milliseconds - worker.total_working_time = worker.total_working_time / 1000 - - execution_list = get_worker_executions(worker) - paginator = Paginator(execution_list, SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE) - page_number = request.GET.get("p", 1) - page_obj = paginator.get_page(page_number) - page_range = paginator.get_elided_page_range(page_obj.number) - context_data = { - **admin.site.each_context(request), - "queue": queue, - "worker": worker, - "queue_names": ", ".join(worker.queue_names()), - "job": worker.get_current_job(), - "total_working_time": worker.total_working_time * 1000, - "executions": page_obj, - "page_range": page_range, - "page_var": "p", - } - return render(request, "admin/scheduler/worker_details.html", context_data) - - -def _find_job(job_id: str) -> Tuple[Optional[DjangoQueue], Optional[JobExecution]]: - from scheduler.settings import QUEUES - - for queue_name in QUEUES: - try: - queue = get_queue(queue_name) - job = JobExecution.fetch(job_id, connection=queue.connection) - if job.origin == queue_name: - return queue, job - except Exception: - pass - return None, None - - -@never_cache -@staff_member_required -def job_detail(request: HttpRequest, job_id: str) -> HttpResponse: - queue, job = _find_job(job_id) - if job is None: - return HttpResponseBadRequest(f"Job {escape(job_id)} does not exist, maybe its TTL has passed") - try: - job.func_name - data_is_valid = True - except Exception: - data_is_valid = False - - try: - exc_info = job._exc_info - except AttributeError: - exc_info = None - - context_data = { - **admin.site.each_context(request), - "job": job, - "dependency_id": job._dependency_id, - "queue": queue, - "data_is_valid": data_is_valid, - "exc_info": exc_info, - } - return render(request, "admin/scheduler/job_detail.html", context_data) - - -@never_cache -@staff_member_required -def clear_queue_registry(request: HttpRequest, queue_name: str, registry_name: str) -> HttpResponse: - queue = get_queue(queue_name) - registry = queue.get_registry(registry_name) - if registry is None: - return HttpResponseNotFound() - - next_url = request.META.get("HTTP_REFERER") or reverse("queue_registry_jobs", args=[queue_name, registry_name]) - if request.method == "POST": - try: - if registry is queue: - queue.empty() - else: - job_ids = registry.get_job_ids() - for job_id in job_ids: - registry.remove(job_id, delete_job=True) - messages.info(request, f"You have successfully cleared the {registry_name} jobs in queue {queue.name}") - except ResponseErrorTypes as e: - messages.error( - request, - f"error: {e}", - ) - raise e - return redirect("queue_registry_jobs", queue_name, registry_name) - job_ids = registry.get_job_ids() - job_list = JobExecution.fetch_many(job_ids, connection=queue.connection) - context_data = { - **admin.site.each_context(request), - "queue": queue, - "total_jobs": len(registry), - "action": "empty", - "jobs": job_list, - "next_url": next_url, - "action_url": reverse( - "queue_clear", - args=[ - queue_name, - registry_name, - ], - ), - } - return render(request, "admin/scheduler/confirm_action.html", context_data) - - -@never_cache -@staff_member_required -def requeue_all(request: HttpRequest, queue_name: str, registry_name: str) -> HttpResponse: - queue = get_queue(queue_name) - registry = queue.get_registry(registry_name) - if registry is None: - return HttpResponseNotFound() - next_url = request.META.get("HTTP_REFERER") or reverse("queue_registry_jobs", args=[queue_name, registry_name]) - job_ids = registry.get_job_ids() - if request.method == "POST": - count = 0 - # Confirmation received - jobs = JobExecution.fetch_many(job_ids, connection=queue.connection) - for job in jobs: - if job is None: - continue - try: - job.requeue() - count += 1 - except Exception: - pass - - messages.info(request, f"You have successfully re-queued {count} jobs!") - return redirect("queue_registry_jobs", queue_name, registry_name) - - context_data = { - **admin.site.each_context(request), - "queue": queue, - "total_jobs": len(queue.failed_job_registry), - "action": "requeue", - "jobs": [queue.fetch_job(job_id) for job_id in job_ids], - "next_url": next_url, - "action_url": reverse("queue_requeue_all", args=[queue_name, registry_name]), - } - - return render(request, "admin/scheduler/confirm_action.html", context_data) - - -@never_cache -@staff_member_required -def confirm_action(request: HttpRequest, queue_name: str) -> HttpResponse: - queue = get_queue(queue_name) - next_url = request.META.get("HTTP_REFERER") or reverse("queue_registry_jobs", args=[queue_name, "queued"]) - try: - resolve(next_url) - except Exception: - messages.warning(request, "Bad followup URL") - next_url = reverse("queue_registry_jobs", args=[queue_name, "queued"]) - - if request.method == "POST" and request.POST.get("action", False): - # confirm action - if request.POST.get("_selected_action", False): - job_id_list = request.POST.getlist("_selected_action") - context_data = { - **admin.site.each_context(request), - "action": request.POST["action"], - "jobs": [queue.fetch_job(job_id) for job_id in job_id_list], - "total_jobs": len(job_id_list), - "queue": queue, - "next_url": next_url, - "action_url": reverse( - "queue_actions", - args=[ - queue_name, - ], - ), - } - return render(request, "admin/scheduler/confirm_action.html", context_data) - - return redirect(next_url) - - -@never_cache -@staff_member_required -def actions(request: HttpRequest, queue_name: str) -> HttpResponse: - queue = get_queue(queue_name) - next_url = request.POST.get("next_url") or reverse("queue_registry_jobs", args=[queue_name, "queued"]) - try: - resolve(next_url) - except Exception: - messages.warning(request, "Bad followup URL") - next_url = reverse("queue_registry_jobs", args=[queue_name, "queued"]) - - action = request.POST.get("action", False) - job_ids = request.POST.get("job_ids", False) - if request.method != "POST" or not action or not job_ids: - return redirect(next_url) - job_ids = request.POST.getlist("job_ids") - if action == "delete": - jobs = JobExecution.fetch_many(job_ids, connection=queue.connection) - for job in jobs: - if job is None: - continue - # Remove job id from queue and delete the actual job - queue.remove_job_id(job.id) - job.delete() - messages.info(request, f"You have successfully deleted {len(job_ids)} jobs!") - elif action == "requeue": - jobs = JobExecution.fetch_many(job_ids, connection=queue.connection) - for job in jobs: - if job is None: - continue - job.requeue() - messages.info(request, f"You have successfully re-queued {len(job_ids)} jobs!") - elif action == "stop": - cancelled_jobs = 0 - jobs = JobExecution.fetch_many(job_ids, connection=queue.connection) - for job in jobs: - if job is None: - continue - try: - job.stop_execution(queue.connection) - job.cancel() - cancelled_jobs += 1 - except Exception as e: - logger.warning(f"Could not stop job: {e}") - pass - messages.info(request, f"You have successfully stopped {cancelled_jobs} jobs!") - return redirect(next_url) - - -SUPPORTED_JOB_ACTIONS = {"requeue", "delete", "enqueue", "cancel"} - - -@never_cache -@staff_member_required -def job_action(request: HttpRequest, job_id: str, action: str) -> HttpResponse: - queue, job = _find_job(job_id) - if job is None: - return HttpResponseBadRequest(f"Job {escape(job_id)} does not exist, maybe its TTL has passed") - if action not in SUPPORTED_JOB_ACTIONS: - return HttpResponseNotFound() - - if request.method != "POST": - context_data = { - **admin.site.each_context(request), - "job": job, - "queue": queue, - "action": action, - } - return render(request, "admin/scheduler/single_job_action.html", context_data) - - try: - if action == "requeue": - job.requeue() - messages.info(request, f"You have successfully re-queued {job.id}") - return redirect("job_details", job_id) - elif action == "delete": - # Remove job id from queue and delete the actual job - queue.remove_job_id(job.id) - job.delete() - messages.info(request, "You have successfully deleted %s" % job.id) - return redirect("queue_registry_jobs", queue.name, "queued") - elif action == "enqueue": - job.delete(remove_from_queue=False) - queue._enqueue_job(job) - messages.info(request, "You have successfully enqueued %s" % job.id) - return redirect("job_details", job_id) - elif action == "cancel": - job.cancel() - messages.info(request, "You have successfully enqueued %s" % job.id) - return redirect("job_details", job_id) - except InvalidJobOperation as e: - logger.warning(f"Could not perform action: {e}") - messages.warning(request, f"Could not perform action: {e}") - return redirect("job_details", job_id) diff --git a/scheduler/views/__init__.py b/scheduler/views/__init__.py new file mode 100644 index 0000000..9399e46 --- /dev/null +++ b/scheduler/views/__init__.py @@ -0,0 +1,20 @@ +__all__ = [ + "job_detail", + "job_action", + "stats", + "stats_json", + "queue_registry_actions", + "queue_confirm_job_action", + "queue_workers", + "queue_job_actions", + "list_registry_jobs", + "workers_list", + "worker_details", + "get_statistics", +] + +from .job_views import job_detail, job_action +from .queue_job_actions import queue_job_actions, queue_confirm_job_action +from .queue_registry_actions import queue_registry_actions +from .queue_views import stats, stats_json, queue_workers, list_registry_jobs, get_statistics +from .worker_views import workers_list, worker_details diff --git a/scheduler/views/helpers.py b/scheduler/views/helpers.py new file mode 100644 index 0000000..713b20c --- /dev/null +++ b/scheduler/views/helpers.py @@ -0,0 +1,74 @@ +from typing import Optional, List +from typing import Tuple +from urllib.parse import urlparse + +from django.contrib import messages +from django.http import Http404 +from django.http import HttpRequest +from django.urls import resolve +from django.utils.http import url_has_allowed_host_and_scheme + +from scheduler.helpers.queues import Queue +from scheduler.helpers.queues import get_queue as get_queue_base +from scheduler.redis_models import JobModel +from scheduler.settings import QueueNotFoundError +from scheduler.settings import get_queue_names, logger + +_QUEUES_WITH_BAD_CONFIGURATION = set() + + +def get_queue(queue_name: str) -> Queue: + try: + return get_queue_base(queue_name) + except QueueNotFoundError as e: + logger.error(e) + raise Http404(e) + + +def _find_job(job_name: str) -> Tuple[Optional[Queue], Optional[JobModel]]: + queue_names = get_queue_names() + for queue_name in queue_names: + if queue_name in _QUEUES_WITH_BAD_CONFIGURATION: + continue + try: + queue = get_queue(queue_name) + job = JobModel.get(job_name, connection=queue.connection) + if job is not None and job.queue_name == queue_name: + return queue, job + except Exception as e: + _QUEUES_WITH_BAD_CONFIGURATION.add(queue_name) + logger.debug(f"Queue {queue_name} added to bad configuration - Got exception: {e}") + pass + return None, None + + +def _check_next_url(request: HttpRequest, default_next_url: str) -> str: + next_url = request.POST.get("next_url", default_next_url) + next_url = next_url.replace("\\", "") + if ( + not url_has_allowed_host_and_scheme(next_url, allowed_hosts=None) + or urlparse(next_url).netloc + or urlparse(next_url).scheme + ): + messages.warning(request, "Bad followup URL") + next_url = default_next_url + try: + resolve(next_url) + except Exception: + messages.warning(request, "Bad followup URL") + next_url = default_next_url + return next_url + + +def _enqueue_multiple_jobs(queue: Queue, job_names: List[str], at_front: bool = False) -> int: + jobs = JobModel.get_many(job_names, connection=queue.connection) + jobs_requeued = 0 + with queue.connection.pipeline() as pipe: + for job in jobs: + if job is None: + continue + job.save(connection=pipe) + queue.enqueue_job(job, connection=pipe, at_front=at_front) + jobs_requeued += 1 + pipe.execute() + return jobs_requeued diff --git a/scheduler/views/job_views.py b/scheduler/views/job_views.py new file mode 100644 index 0000000..c57468c --- /dev/null +++ b/scheduler/views/job_views.py @@ -0,0 +1,92 @@ +from enum import Enum +from html import escape + +from django.contrib import admin, messages +from django.contrib.admin.views.decorators import staff_member_required +from django.http import HttpResponse, HttpRequest +from django.http.response import HttpResponseBadRequest +from django.shortcuts import render, redirect +from django.views.decorators.cache import never_cache + +from scheduler.helpers.queues import InvalidJobOperation +from scheduler.redis_models import Result +from scheduler.settings import logger +from scheduler.views.helpers import _find_job +from scheduler.worker.commands import send_command, StopJobCommand + + +class JobDetailAction(str, Enum): + DELETE = "delete" + ENQUEUE = "enqueue" + CANCEL = "cancel" + + +@never_cache +@staff_member_required +def job_detail(request: HttpRequest, job_name: str) -> HttpResponse: + queue, job = _find_job(job_name) + if job is None: + messages.warning(request, f"Job {escape(job_name)} does not exist, maybe its TTL has passed") + return redirect("queues_home") + try: + job.func_name + data_is_valid = True + except Exception: + data_is_valid = False + + try: + last_result = Result.fetch_latest(queue.connection, job.name) + except AttributeError: + last_result = None + + context_data = { + **admin.site.each_context(request), + "job": job, + "last_result": last_result, + "results": Result.all(connection=queue.connection, parent=job.name), + "queue": queue, + "data_is_valid": data_is_valid, + } + return render(request, "admin/scheduler/job_detail.html", context_data) + + +@never_cache +@staff_member_required +def job_action(request: HttpRequest, job_name: str, action: str) -> HttpResponse: + queue, job = _find_job(job_name) + if job is None: + messages.warning(request, f"Job {escape(job_name)} does not exist, maybe its TTL has passed") + return redirect("queues_home") + if action not in [item.value for item in JobDetailAction]: + return HttpResponseBadRequest(f"Action {escape(action)} is not supported") + + if request.method != "POST": + context_data = { + **admin.site.each_context(request), + "job": job, + "queue": queue, + "action": action, + } + return render(request, "admin/scheduler/single_job_action.html", context_data) + + try: + if action == JobDetailAction.DELETE: + queue.delete_job(job.name) + messages.info(request, f"You have successfully deleted {job.name}") + return redirect("queue_registry_jobs", queue.name, "queued") + elif action == JobDetailAction.ENQUEUE: + queue.delete_job(job.name, expire_job_model=False) + queue.enqueue_job(job) + messages.info(request, f"You have successfully enqueued {job.name}") + return redirect("job_details", job_name) + elif action == JobDetailAction.CANCEL: + send_command( + connection=queue.connection, command=StopJobCommand(job_name=job.name, worker_name=job.worker_name) + ) + queue.cancel_job(job.name) + messages.info(request, f"You have successfully cancelled {job.name}") + return redirect("job_details", job_name) + except InvalidJobOperation as e: + logger.warning(f"Could not perform action: {e}") + messages.warning(request, f"Could not perform action: {e}") + return redirect("job_details", job_name) diff --git a/scheduler/views/queue_job_actions.py b/scheduler/views/queue_job_actions.py new file mode 100644 index 0000000..d7956cc --- /dev/null +++ b/scheduler/views/queue_job_actions.py @@ -0,0 +1,90 @@ +"""list_registry_jobs actions on multiple selected jobs""" + +from enum import Enum + +from django.contrib import admin, messages +from django.contrib.admin.views.decorators import staff_member_required +from django.http import HttpResponse, HttpRequest +from django.shortcuts import render, redirect +from django.urls import reverse +from django.views.decorators.cache import never_cache + +from scheduler.redis_models import JobModel +from scheduler.settings import logger +from scheduler.views.helpers import get_queue, _check_next_url, _enqueue_multiple_jobs +from scheduler.worker.commands import StopJobCommand, send_command + + +class QueueJobAction(Enum): + DELETE = "delete" + REQUEUE = "requeue" + STOP = "stop" + + def __contains__(self, item) -> bool: + return item in [a.value for a in self.__class__] + + +@never_cache +@staff_member_required +def queue_job_actions(request: HttpRequest, queue_name: str) -> HttpResponse: + queue = get_queue(queue_name) + next_url = _check_next_url(request, reverse("queue_registry_jobs", args=[queue_name, "queued"])) + action = request.POST.get("action", False) + job_names = request.POST.get("job_names", False) + if request.method != "POST" or not action or not job_names or action not in [item.value for item in QueueJobAction]: + return redirect(next_url) + job_names = request.POST.getlist("job_names") + if action == QueueJobAction.DELETE.value: + jobs = JobModel.get_many(job_names, connection=queue.connection) + for job in jobs: + if job is None: + continue + queue.delete_job(job.name) + messages.info(request, f"You have successfully deleted {len(job_names)} jobs!") + elif action == QueueJobAction.REQUEUE.value: + requeued_jobs_count = _enqueue_multiple_jobs(queue, job_names) + messages.info(request, f"You have successfully re-queued {requeued_jobs_count}/{len(job_names)} jobs!") + elif action == QueueJobAction.STOP.value: + cancelled_jobs = 0 + jobs = JobModel.get_many(job_names, connection=queue.connection) + for job in jobs: + if job is None: + continue + try: + command = StopJobCommand(job_name=job.name, worker_name=job.worker_name) + send_command(connection=queue.connection, command=command) + queue.cancel_job(job.name) + cancelled_jobs += 1 + except Exception as e: + logger.warning(f"Could not stop job: {e}") + pass + messages.info(request, f"You have successfully stopped {cancelled_jobs} jobs!") + return redirect(next_url) + + +@never_cache +@staff_member_required +def queue_confirm_job_action(request: HttpRequest, queue_name: str) -> HttpResponse: + queue = get_queue(queue_name) + next_url = _check_next_url(request, reverse("queue_registry_jobs", args=[queue_name, "queued"])) + action = request.POST.get("action", None) + job_names = request.POST.getlist("_selected_action", None) + if request.method != "POST" or action is None or job_names is None or action not in QueueJobAction: + return redirect(next_url) + + # confirm action + context_data = { + **admin.site.each_context(request), + "action": action, + "jobs": [JobModel.get(job_name, connection=queue.connection) for job_name in job_names], + "total_jobs": len(job_names), + "queue": queue, + "next_url": next_url, + "action_url": reverse( + "queue_job_actions", + args=[ + queue_name, + ], + ), + } + return render(request, "admin/scheduler/confirm_action.html", context_data) diff --git a/scheduler/views/queue_registry_actions.py b/scheduler/views/queue_registry_actions.py new file mode 100644 index 0000000..2c9db5b --- /dev/null +++ b/scheduler/views/queue_registry_actions.py @@ -0,0 +1,70 @@ +"""list_registry_jobs actions on all jobs in the registry""" + +from enum import Enum + +from django.contrib import admin, messages +from django.contrib.admin.views.decorators import staff_member_required +from django.http import HttpResponse, HttpRequest, HttpResponseNotFound +from django.shortcuts import render, redirect +from django.urls import reverse +from django.views.decorators.cache import never_cache + +from scheduler.helpers.queues import Queue +from scheduler.redis_models import JobModel, JobNamesRegistry +from scheduler.types import ResponseErrorTypes +from scheduler.views.helpers import get_queue, _check_next_url, _enqueue_multiple_jobs + + +class QueueRegistryActions(Enum): + EMPTY = "empty" + REQUEUE = "requeue" + + +def _clear_registry(request: HttpRequest, queue: Queue, registry_name: str, registry: JobNamesRegistry): + try: + job_names = registry.all() + for job_name in job_names: + registry.delete(registry.connection, job_name) + job_model = JobModel.get(job_name, connection=registry.connection) + job_model.delete(connection=registry.connection) + messages.info(request, f"You have successfully cleared the {registry_name} jobs in queue {queue.name}") + except ResponseErrorTypes as e: + messages.error(request, f"error: {e}") + raise e + + +def _requeue_job_names(request: HttpRequest, queue: Queue, registry_name: str): + registry = queue.get_registry(registry_name) + job_names = registry.all() + jobs_requeued_count = _enqueue_multiple_jobs(queue, job_names) + messages.info(request, f"You have successfully re-queued {jobs_requeued_count} jobs!") + + +@never_cache +@staff_member_required +def queue_registry_actions(request: HttpRequest, queue_name: str, registry_name: str, action: str) -> HttpResponse: + queue = get_queue(queue_name) + registry = queue.get_registry(registry_name) + if registry is None: + return HttpResponseNotFound() + next_url = _check_next_url(request, reverse("queue_registry_jobs", args=[queue_name, registry_name])) + if action not in [item.value for item in QueueRegistryActions]: + return redirect(next_url) + if request.method == "POST": + if action == QueueRegistryActions.EMPTY.value: + _clear_registry(request, queue, registry_name, registry) + elif action == QueueRegistryActions.REQUEUE.value: + _requeue_job_names(request, queue, registry_name) + return redirect("queue_registry_jobs", queue_name, registry_name) + job_names = registry.all() + job_list = JobModel.get_many(job_names, connection=queue.connection) + context_data = { + **admin.site.each_context(request), + "queue": queue, + "total_jobs": len(registry), + "action": action, + "jobs": job_list, + "next_url": next_url, + "action_url": reverse("queue_registry_action", args=[queue_name, registry_name, action]), + } + return render(request, "admin/scheduler/confirm_action.html", context_data) diff --git a/scheduler/views/queue_views.py b/scheduler/views/queue_views.py new file mode 100644 index 0000000..7800886 --- /dev/null +++ b/scheduler/views/queue_views.py @@ -0,0 +1,165 @@ +import dataclasses +from math import ceil +from typing import Tuple, List, Dict, Union, Any, Optional + +from django.contrib import admin +from django.contrib.admin.views.decorators import staff_member_required +from django.http import HttpResponse, HttpRequest, HttpResponseNotFound, JsonResponse +from django.shortcuts import render +from django.views.decorators.cache import never_cache + +from scheduler import settings +from scheduler.helpers.queues import Queue, get_all_workers +from scheduler.redis_models import JobModel, JobNamesRegistry, WorkerModel +from scheduler.settings import get_queue_names, logger +from scheduler.types import ConnectionErrorTypes +from scheduler.views.helpers import get_queue + + +def _get_registry_job_list(queue: Queue, registry: JobNamesRegistry, page: int) -> Tuple[List[JobModel], int, range]: + items_per_page = settings.SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE + num_jobs = registry.count(queue.connection) + job_list = list() + + if num_jobs == 0: + return job_list, num_jobs, range(1, 1) + + last_page = int(ceil(num_jobs / items_per_page)) + page_range = range(1, last_page + 1) + offset = items_per_page * (page - 1) + job_names = registry.all(offset, offset + items_per_page - 1) + job_list = JobModel.get_many(job_names, connection=queue.connection) + remove_job_names = [job_name for i, job_name in enumerate(job_names) if job_list[i] is None] + valid_jobs = [job for job in job_list if job is not None] + if registry is not queue: + for job_name in remove_job_names: + registry.delete(queue.connection, job_name) + + return valid_jobs, num_jobs, page_range + + +@never_cache +@staff_member_required +def list_registry_jobs(request: HttpRequest, queue_name: str, registry_name: str) -> HttpResponse: + queue = get_queue(queue_name) + registry = queue.get_registry(registry_name) + if registry is None: + return HttpResponseNotFound() + title = registry_name.capitalize() + page = int(request.GET.get("page", 1)) + job_list, num_jobs, page_range = _get_registry_job_list(queue, registry, page) + + context_data = { + **admin.site.each_context(request), + "queue": queue, + "registry_name": registry_name, + "registry": registry, + "jobs": job_list, + "num_jobs": num_jobs, + "page": page, + "page_range": page_range, + "job_status": title, + } + return render(request, "admin/scheduler/jobs.html", context_data) + + +@never_cache +@staff_member_required +def queue_workers(request: HttpRequest, queue_name: str) -> HttpResponse: + queue = get_queue(queue_name) + queue.clean_registries() + + all_workers = WorkerModel.all(queue.connection) + worker_list = [worker for worker in all_workers if queue.name in worker.queue_names] + + context_data = { + **admin.site.each_context(request), + "queue": queue, + "workers": worker_list, + } + return render(request, "admin/scheduler/queue_workers.html", context_data) + + +def stats_json(request: HttpRequest) -> Union[JsonResponse, HttpResponseNotFound]: + auth_token = request.headers.get("Authorization") + token_validation_func = settings.SCHEDULER_CONFIG.TOKEN_VALIDATION_METHOD + if request.user.is_staff or (token_validation_func and auth_token and token_validation_func(auth_token)): + return JsonResponse(get_statistics()) + + return HttpResponseNotFound() + + +@never_cache +@staff_member_required +def stats(request: HttpRequest) -> HttpResponse: + context_data = {**admin.site.each_context(request), **get_statistics(run_maintenance_tasks=True)} + return render(request, "admin/scheduler/stats.html", context_data) + + +@dataclasses.dataclass +class QueueData: + name: str + queued_jobs: int + oldest_job_timestamp: Optional[str] + scheduler_pid: Optional[int] + workers: int + finished_jobs: int + started_jobs: int + failed_jobs: int + scheduled_jobs: int + canceled_jobs: int + + +def get_statistics(run_maintenance_tasks: bool = False) -> Dict[str, List[Dict[str, Any]]]: + queue_names = get_queue_names() + queues: List[QueueData] = [] + queue_workers_count: Dict[str, int] = {queue_name: 0 for queue_name in queue_names} + workers = get_all_workers() + for worker in workers: + for queue_name in worker.queue_names: + if queue_name not in queue_workers_count: + logger.warning(f"Worker {worker.name} is working on queue {queue_name} which is not defined") + queue_workers_count[queue_name] = 0 + queue_workers_count[queue_name] += 1 + for queue_name in queue_names: + try: + queue = get_queue(queue_name) + connection_kwargs = queue.connection.connection_pool.connection_kwargs + + if run_maintenance_tasks: + queue.clean_registries() + + # Raw access to the first item from left of the broker list. + # This might not be accurate since new job can be added from the left + # with `at_front` parameters. + # Ideally rq should supports Queue.oldest_job + + last_job_name = queue.first_queued_job_name() + last_job = JobModel.get(last_job_name, connection=queue.connection) if last_job_name else None + if last_job and last_job.enqueued_at: + oldest_job_timestamp = last_job.enqueued_at.isoformat() + else: + oldest_job_timestamp = None + + # parse_class and connection_pool are not needed and not JSON serializable + connection_kwargs.pop("parser_class", None) + connection_kwargs.pop("connection_pool", None) + + queue_data = QueueData( + name=queue.name, + queued_jobs=len(queue.queued_job_registry), + oldest_job_timestamp=oldest_job_timestamp, + scheduler_pid=queue.scheduler_pid, + workers=queue_workers_count[queue.name], + finished_jobs=len(queue.finished_job_registry), + started_jobs=len(queue.active_job_registry), + failed_jobs=len(queue.failed_job_registry), + scheduled_jobs=len(queue.scheduled_job_registry), + canceled_jobs=len(queue.canceled_job_registry), + ) + queues.append(queue_data) + except ConnectionErrorTypes as e: + logger.error(f"Could not connect for queue {queue_name}: {e}") + continue + + return {"queues": [dataclasses.asdict(q) for q in queues]} diff --git a/scheduler/views/worker_views.py b/scheduler/views/worker_views.py new file mode 100644 index 0000000..3f133e4 --- /dev/null +++ b/scheduler/views/worker_views.py @@ -0,0 +1,66 @@ +from typing import List + +from django.contrib import admin +from django.contrib.admin.views.decorators import staff_member_required +from django.core.paginator import Paginator +from django.http import HttpResponse, HttpRequest, Http404 +from django.shortcuts import render +from django.views.decorators.cache import never_cache + +from scheduler.helpers.queues import get_all_workers +from scheduler.redis_models import WorkerModel, JobModel +from scheduler.settings import SCHEDULER_CONFIG +from scheduler.views.helpers import get_queue + + +def get_worker_executions(worker: WorkerModel) -> List[JobModel]: + res = list() + for queue_name in worker.queue_names: + queue = get_queue(queue_name) + curr_jobs = queue.get_all_jobs() + curr_jobs = [j for j in curr_jobs if j.worker_name == worker.name] + res.extend(curr_jobs) + return res + + +@never_cache +@staff_member_required +def worker_details(request: HttpRequest, name: str) -> HttpResponse: + workers = get_all_workers() + worker = next((w for w in workers if w.name == name), None) + + if worker is None: + raise Http404(f"Couldn't find worker with this ID: {name}") + + execution_list = get_worker_executions(worker) + paginator = Paginator(execution_list, SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE) + page_number = request.GET.get("p", 1) + page_obj = paginator.get_page(page_number) + page_range = paginator.get_elided_page_range(page_obj.number) + current_job = None + if worker.current_job_name is not None: + queue = get_queue(worker.queue_names[0]) + current_job = JobModel.get(worker.current_job_name, connection=queue.connection) + context_data = { + **admin.site.each_context(request), + "worker": worker, + "queue_names": ", ".join(worker.queue_names), + "current_job": current_job, + "executions": page_obj, + "page_range": page_range, + "page_var": "p", + } + return render(request, "admin/scheduler/worker_details.html", context_data) + + +@never_cache +@staff_member_required +def workers_list(request: HttpRequest) -> HttpResponse: + all_workers = get_all_workers() + worker_list = [worker for worker in all_workers] + + context_data = { + **admin.site.each_context(request), + "workers": worker_list, + } + return render(request, "admin/scheduler/workers_list.html", context_data) diff --git a/scheduler/worker/__init__.py b/scheduler/worker/__init__.py new file mode 100644 index 0000000..4998e74 --- /dev/null +++ b/scheduler/worker/__init__.py @@ -0,0 +1,8 @@ +__all__ = [ + "Worker", + "create_worker", + "WorkerScheduler", +] + +from .scheduler import WorkerScheduler +from .worker import Worker, create_worker diff --git a/scheduler/worker/commands/__init__.py b/scheduler/worker/commands/__init__.py new file mode 100644 index 0000000..2d259d9 --- /dev/null +++ b/scheduler/worker/commands/__init__.py @@ -0,0 +1,13 @@ +__all__ = [ + "WorkerCommandsChannelListener", + "StopJobCommand", + "ShutdownCommand", + "KillWorkerCommand", + "WorkerCommandError", + "send_command", +] + +from .kill_worker import KillWorkerCommand +from .shutdown import ShutdownCommand +from .stop_job import StopJobCommand +from .worker_commands import WorkerCommandsChannelListener, WorkerCommandError, send_command diff --git a/scheduler/worker/commands/kill_worker.py b/scheduler/worker/commands/kill_worker.py new file mode 100644 index 0000000..595271a --- /dev/null +++ b/scheduler/worker/commands/kill_worker.py @@ -0,0 +1,40 @@ +import errno +import os +import signal +from typing import Optional + +from scheduler.types import ConnectionType +from scheduler.redis_models import WorkerModel +from scheduler.settings import logger +from scheduler.worker.commands.worker_commands import WorkerCommand + + +class KillWorkerCommand(WorkerCommand): + """kill-worker command""" + + command_name = "kill-worker" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.worker_pid: Optional[int] = None + + def process_command(self, connection: ConnectionType) -> None: + from scheduler.worker import Worker + + logger.info("Received kill-worker command.") + worker_model = WorkerModel.get(self.worker_name, connection) + self.worker_pid = worker_model.pid + if self.worker_pid is None: + raise ValueError("Worker PID is not set") + logger.info(f"Killing worker main process {self.worker_pid}...") + try: + Worker.from_model(worker_model).request_stop(signal.SIGTERM, None) + os.killpg(os.getpgid(self.worker_pid), signal.SIGTERM) + logger.info(f"Killed worker main process pid {self.worker_pid}") + except OSError as e: + if e.errno == errno.ESRCH: + logger.debug( + f"Worker main process for {self.worker_name}:{self.worker_pid} already dead" + ) # "No such process" is fine with us + else: + raise diff --git a/scheduler/worker/commands/shutdown.py b/scheduler/worker/commands/shutdown.py new file mode 100644 index 0000000..1ed93e6 --- /dev/null +++ b/scheduler/worker/commands/shutdown.py @@ -0,0 +1,17 @@ +import os +import signal + +from scheduler.types import ConnectionType +from scheduler.settings import logger +from scheduler.worker.commands.worker_commands import WorkerCommand + + +class ShutdownCommand(WorkerCommand): + """shutdown command""" + + command_name = "shutdown" + + def process_command(self, connection: ConnectionType) -> None: + logger.info("Received shutdown command, sending SIGINT signal.") + pid = os.getpid() + os.kill(pid, signal.SIGINT) diff --git a/scheduler/worker/commands/stop_job.py b/scheduler/worker/commands/stop_job.py new file mode 100644 index 0000000..5f1dc41 --- /dev/null +++ b/scheduler/worker/commands/stop_job.py @@ -0,0 +1,58 @@ +import os +import signal +from typing import Dict, Any + +from scheduler.types import ConnectionType +from scheduler.redis_models import WorkerModel, JobModel +from scheduler.settings import logger +from scheduler.worker.commands.worker_commands import WorkerCommand, WorkerCommandError + + +class StopJobCommand(WorkerCommand): + """stop-job command""" + + command_name = "stop-job" + + def __init__(self, *args, job_name: str, worker_name: str, **kwargs) -> None: + super().__init__(*args, worker_name=worker_name, **kwargs) + self.job_name = job_name + if self.job_name is None: + raise WorkerCommandError("job_name for kill-job command is required") + + def command_payload(self) -> Dict[str, Any]: + return super().command_payload(job_name=self.job_name) + + def process_command(self, connection: ConnectionType) -> None: + logger.debug(f"Received command to stop job {self.job_name}") + worker_model = WorkerModel.get(self.worker_name, connection) + job_model = JobModel.get(self.job_name, connection) + if worker_model is None: + logger.error(f"Worker {self.worker_name} not found") + return + if job_model is None: + logger.error(f"Job {self.job_name} not found") + return + if worker_model.pid == worker_model.job_execution_process_pid: + logger.warning(f"Job execution process ID and worker process id {worker_model.pid} are equal, skipping") + return + if not worker_model.job_execution_process_pid: + logger.error(f"Worker {self.worker_name} has no job execution process") + return + if worker_model.current_job_name != self.job_name: + logger.info( + f"{self.worker_name} working on job {worker_model.current_job_name}, " + f"not on {self.job_name}, kill-job command ignored." + ) + return + worker_model.set_field("stopped_job_name", self.job_name, connection) + try: + pgid = os.getpgid(worker_model.job_execution_process_pid) + logger.debug( + f"worker_pid {worker_model.pid}, job_execution_process {worker_model.job_execution_process_pid}" + ) + if pgid == worker_model.pid: + logger.error("No separate process for job execution, skipping") + return + os.killpg(pgid, signal.SIGTERM) + except ProcessLookupError as e: + logger.error(f"Error killing job {self.job_name}: {e}") diff --git a/scheduler/worker/commands/suspend_worker.py b/scheduler/worker/commands/suspend_worker.py new file mode 100644 index 0000000..e56e444 --- /dev/null +++ b/scheduler/worker/commands/suspend_worker.py @@ -0,0 +1,38 @@ +from scheduler.types import ConnectionType +from scheduler.redis_models import WorkerModel +from scheduler.settings import logger +from scheduler.worker.commands.worker_commands import WorkerCommand + + +class SuspendWorkCommand(WorkerCommand): + """Suspend worker command""" + + command_name = "suspend" + + def process_command(self, connection: ConnectionType) -> None: + logger.debug(f"Received command to suspend worker {self.worker_name}") + worker_model = WorkerModel.get(self.worker_name, connection) + if worker_model is None: + logger.warning(f"Worker {self.worker_name} not found") + if worker_model.is_suspended: + logger.warning(f"Worker {self.worker_name} already suspended") + return + worker_model.set_field("is_suspended", True, connection=connection) + logger.info(f"Worker {self.worker_name} suspended") + + +class ResumeWorkCommand(WorkerCommand): + """Resume worker command""" + + command_name = "resume" + + def process_command(self, connection: ConnectionType) -> None: + logger.debug(f"Received command to resume worker {self.worker_name}") + worker_model = WorkerModel.get(self.worker_name, connection) + if worker_model is None: + logger.warning(f"Worker {self.worker_name} not found") + if not worker_model.is_suspended: + logger.warning(f"Worker {self.worker_name} not suspended and therefore can't be resumed") + return + worker_model.set_field("is_suspended", False, connection=connection) + logger.info(f"Worker {self.worker_name} resumed") diff --git a/scheduler/worker/commands/worker_commands.py b/scheduler/worker/commands/worker_commands.py new file mode 100644 index 0000000..b2d9f25 --- /dev/null +++ b/scheduler/worker/commands/worker_commands.py @@ -0,0 +1,95 @@ +import json +from abc import ABC +from datetime import datetime, timezone +from typing import Type, Dict, Any + +from scheduler.settings import logger +from scheduler.types import ConnectionType, Self + +_PUBSUB_CHANNEL_TEMPLATE: str = ":workers:pubsub:{}" + + +class WorkerCommandError(Exception): + pass + + +class WorkerCommand(ABC): + """Abstract class for commands to be sent to a worker and processed by worker""" + + _registry: Dict[str, Type[Self]] = dict() + command_name: str = "" + + def __init__(self, *args, worker_name: str, **kwargs) -> None: + self.worker_name = worker_name + + def command_payload(self, **kwargs) -> Dict[str, Any]: + commands_channel = WorkerCommandsChannelListener._commands_channel(self.worker_name) + payload = { + "command": self.command_name, + "worker_name": self.worker_name, + "channel_name": commands_channel, + "created_at": datetime.now(tz=timezone.utc).isoformat(), + } + if kwargs: + payload.update(kwargs) + return payload + + def __str__(self) -> str: + return f"{self.command_name}[{self.command_payload()}]" + + def process_command(self, connection: ConnectionType) -> None: + raise NotImplementedError + + @classmethod + def __init_subclass__(cls, *args, **kwargs): + if cls is WorkerCommand: + return + if not cls.command_name: + raise NotImplementedError(f"{cls.__name__} must have a name attribute") + WorkerCommand._registry[cls.command_name] = cls + + @classmethod + def from_payload(cls, payload: Dict[str, Any]) -> Type[Self]: + command_name = payload.get("command") + command_class = WorkerCommand._registry.get(command_name) + if command_class is None: + raise WorkerCommandError(f"Invalid command: {command_name}") + return command_class(**payload) + + +def send_command(connection: ConnectionType, command: WorkerCommand) -> None: + """Send a command to the worker""" + payload = command.command_payload() + connection.publish(payload["channel_name"], json.dumps(payload)) + + +class WorkerCommandsChannelListener(object): + def __init__(self, connection: ConnectionType, worker_name: str) -> None: + self.connection = connection + self.pubsub_channel_name = WorkerCommandsChannelListener._commands_channel(worker_name) + + @staticmethod + def _commands_channel(worker_name: str) -> str: + return _PUBSUB_CHANNEL_TEMPLATE.format(worker_name) + + def start(self): + """Subscribe to this worker's channel""" + logger.info(f"Subscribing to channel {self.pubsub_channel_name}") + self.pubsub = self.connection.pubsub() + self.pubsub.subscribe(**{self.pubsub_channel_name: self.handle_payload}) + self.pubsub_thread = self.pubsub.run_in_thread(sleep_time=0.2, daemon=True) + + def stop(self): + """Unsubscribe from pubsub channel""" + if self.pubsub_thread: + logger.info(f"Unsubscribing from channel {self.pubsub_channel_name}") + self.pubsub_thread.stop() + self.pubsub_thread.join() + self.pubsub.unsubscribe() + self.pubsub.close() + + def handle_payload(self, payload: str) -> None: + """Handle commands""" + command = WorkerCommand.from_payload(json.loads(payload["data"])) + logger.debug(f"Received command: {command}") + command.process_command(self.connection) diff --git a/scheduler/worker/scheduler.py b/scheduler/worker/scheduler.py new file mode 100644 index 0000000..af7c85b --- /dev/null +++ b/scheduler/worker/scheduler.py @@ -0,0 +1,172 @@ +import os +import time +import traceback +from datetime import datetime +from enum import Enum +from threading import Thread +from typing import List, Set, Optional, Sequence, Dict + +import django + +from scheduler.helpers.queues import Queue +from scheduler.helpers.queues import get_queue +from scheduler.helpers.utils import current_timestamp +from scheduler.models import Task +from scheduler.redis_models import SchedulerLock, JobModel, ScheduledJobRegistry +from scheduler.settings import SCHEDULER_CONFIG, logger +from scheduler.types import ConnectionType + + +class SchedulerStatus(str, Enum): + STARTED = "started" + WORKING = "working" + STOPPED = "stopped" + + +def _reschedule_tasks(): + enabled_jobs = list(Task.objects.filter(enabled=True)) + for item in enabled_jobs: + logger.debug(f"Rescheduling {str(item)}") + item.save() + + +class WorkerScheduler: + def __init__( + self, + queues: Sequence[Queue], + connection: ConnectionType, + worker_name: str, + interval: Optional[int] = None, + ) -> None: + interval = interval or SCHEDULER_CONFIG.SCHEDULER_INTERVAL + self._queues = queues + self._scheduled_job_registries: List[ScheduledJobRegistry] = [] + self.lock_acquisition_time = None + self._pool_class = connection.connection_pool.connection_class + self._pool_kwargs = connection.connection_pool.connection_kwargs.copy() + self._locks: Dict[str, SchedulerLock] = dict() + self.connection = connection + self.interval = interval + self._stop_requested = False + self.status = SchedulerStatus.STOPPED + self._thread = None + self._pid: Optional[int] = None + self.worker_name = worker_name + + @property + def pid(self) -> Optional[int]: + return self._pid + + def _should_reacquire_locks(self) -> bool: + """Returns True if lock_acquisition_time is longer than 10 minutes ago""" + if not self.lock_acquisition_time: + return True + seconds_since = (datetime.now() - self.lock_acquisition_time).total_seconds() + return seconds_since > SCHEDULER_CONFIG.SCHEDULER_FALLBACK_PERIOD_SECS + + def _acquire_locks(self) -> Set[str]: + """Returns names of queue it successfully acquires lock on""" + successful_locks = set() + if self.pid is None: + self._pid = os.getpid() + queue_names = [queue.name for queue in self._queues] + logger.debug( + f"""[Scheduler {self.worker_name}/{self.pid}] Trying to acquire locks for {", ".join(queue_names)}""" + ) + for queue in self._queues: + lock = SchedulerLock(queue.name) + if lock.acquire(self.pid, connection=queue.connection, expire=self.interval + 60): + self._locks[queue.name] = lock + successful_locks.add(queue.name) + + # Always reset _scheduled_job_registries when acquiring locks + self.lock_acquisition_time = datetime.now() + self._scheduled_job_registries = [] + for queue_name in self._locks: + queue = get_queue(queue_name) + self._scheduled_job_registries.append(queue.scheduled_job_registry) + logger.debug(f"[Scheduler {self.worker_name}/{self.pid}] Locks acquired for {', '.join(self._locks.keys())}") + return successful_locks + + def start(self) -> None: + locks = self._acquire_locks() + if len(locks) == 0: + return + self.status = SchedulerStatus.STARTED + self._thread = Thread(target=run_scheduler, args=(self,), name="scheduler-thread") + self._thread.start() + + def request_stop_and_wait(self): + """Toggle self._stop_requested that's checked on every loop""" + logger.debug(f"[Scheduler {self.worker_name}/{self.pid}] Stop Scheduler requested") + self._stop_requested = True + if self._thread is not None: + self._thread.join() + + def heartbeat(self): + """Updates the TTL on scheduler keys and the locks""" + lock_keys = ", ".join(self._locks.keys()) + logger.debug(f"[Scheduler {self.worker_name}/{self.pid}] Scheduler updating lock for queue {lock_keys}") + with self.connection.pipeline() as pipeline: + for lock in self._locks.values(): + lock.expire(self.connection, expire=self.interval + 60) + pipeline.execute() + + def stop(self): + logger.info( + f"[Scheduler {self.worker_name}/{self.pid}] Stopping scheduler, releasing locks for {', '.join(self._locks.keys())}..." + ) + self.release_locks() + self.status = SchedulerStatus.STOPPED + + def release_locks(self): + """Release acquired locks""" + with self.connection.pipeline() as pipeline: + for lock in self._locks.values(): + lock.release(self.connection) + pipeline.execute() + + def work(self) -> None: + queue_names = [queue.name for queue in self._queues] + logger.info(f"""[Scheduler {self.worker_name}/{self.pid}] Scheduler for {", ".join(queue_names)} started""") + django.setup() + + while True: + if self._stop_requested: + self.stop() + break + + if self._should_reacquire_locks(): + self._acquire_locks() + + self.enqueue_scheduled_jobs() + self.heartbeat() + time.sleep(self.interval) + + def enqueue_scheduled_jobs(self) -> None: + """Enqueue jobs whose timestamp is in the past""" + self.status = SchedulerStatus.WORKING + _reschedule_tasks() + + for registry in self._scheduled_job_registries: + timestamp = current_timestamp() + job_names = registry.get_jobs_to_schedule(timestamp) + if len(job_names) == 0: + continue + queue = get_queue(registry.name) + jobs = JobModel.get_many(job_names, connection=self.connection) + with self.connection.pipeline() as pipeline: + for job in jobs: + if job is not None: + queue.enqueue_job(job, connection=pipeline, at_front=job.at_front) + pipeline.execute() + self.status = SchedulerStatus.STARTED + + +def run_scheduler(scheduler: WorkerScheduler): + try: + scheduler.work() + except: # noqa + logger.error(f"Scheduler [PID {os.getpid()}] raised an exception.\n{traceback.format_exc()}") + raise + logger.info(f"Scheduler with PID {os.getpid()} has stopped") diff --git a/scheduler/worker/worker.py b/scheduler/worker/worker.py new file mode 100644 index 0000000..34662d0 --- /dev/null +++ b/scheduler/worker/worker.py @@ -0,0 +1,956 @@ +import contextlib +import errno +import math +import os +import random +import signal +import socket +import sys +import threading +import time +import traceback +import warnings +from datetime import timedelta +from enum import Enum +from random import shuffle +from types import FrameType +from typing import List, Optional, Tuple, Any, Iterable + +import scheduler +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import WorkerModel, JobModel, JobStatus, DequeueTimeout +from scheduler.settings import SCHEDULER_CONFIG, logger, get_queue_configuration +from scheduler.types import Broker, Self +from scheduler.types import ConnectionType, TimeoutErrorTypes, ConnectionErrorTypes, WatchErrorTypes, ResponseErrorTypes +from .commands import WorkerCommandsChannelListener +from .scheduler import WorkerScheduler, SchedulerStatus +from ..redis_models.lock import QueueLock +from ..redis_models.worker import WorkerStatus + +try: + from signal import SIGKILL +except ImportError: + from signal import SIGTERM as SIGKILL + +from contextlib import suppress + +from scheduler.helpers.queues import Queue, perform_job +from scheduler.timeouts import JobExecutionMonitorTimeoutException, JobTimeoutException +from scheduler.helpers.utils import utcnow, current_timestamp + +try: + from setproctitle import setproctitle as setprocname +except ImportError: + + def setprocname(*args, **kwargs): # noqa + pass + + +class StopRequested(Exception): + pass + + +class WorkerNotFound(Exception): + pass + + +class QueueConnectionDiscrepancyError(Exception): + pass + + +_signames = dict( + (getattr(signal, signame), signame) for signame in dir(signal) if signame.startswith("SIG") and "_" not in signame +) + + +def signal_name(signum): + try: + return signal.Signals(signum).name + except KeyError: + return "SIG_UNKNOWN" + except ValueError: + return "SIG_UNKNOWN" + + +class DequeueStrategy(str, Enum): + DEFAULT = "default" + ROUND_ROBIN = "round_robin" + RANDOM = "random" + + +class Worker: + queue_class = Queue + + # factor to increase connection_wait_time in case of continuous connection failures. + exponential_backoff_factor = 2.0 + # Max Wait time (in seconds) after which exponential_backoff_factor won't be applicable. + max_connection_wait_time = 60.0 + + @classmethod + def from_model(cls, model: WorkerModel) -> Self: + connection = get_queue(model.queue_names[0]).connection + res = cls( + queues=[get_queue(queue_name) for queue_name in model.queue_names], + name=model.name, + connection=connection, + with_scheduler=False, + model=model, + ) + return res + + def __init__( + self, + queues, + name: str, + connection: Optional[ConnectionType] = None, + maintenance_interval: int = SCHEDULER_CONFIG.DEFAULT_MAINTENANCE_TASK_INTERVAL, + job_monitoring_interval=SCHEDULER_CONFIG.DEFAULT_JOB_MONITORING_INTERVAL, + dequeue_strategy: DequeueStrategy = DequeueStrategy.DEFAULT, + disable_default_exception_handler: bool = False, + fork_job_execution: bool = True, + with_scheduler: bool = True, + burst: bool = False, + model: Optional[WorkerModel] = None, + ): # noqa + self.fork_job_execution = fork_job_execution + self.job_monitoring_interval = job_monitoring_interval + self.maintenance_interval = maintenance_interval + + connection = self._set_connection(connection) + self.connection = connection + + self.queues = [ + (Queue(name=q, connection=connection) if isinstance(q, str) else q) for q in _ensure_list(queues) + ] + self.name: str = name + self._validate_name_uniqueness() + self._ordered_queues = self.queues[:] + + self._is_job_execution_process: bool = False + + self.scheduler: Optional[WorkerScheduler] = None + self._command_listener = WorkerCommandsChannelListener(connection, self.name) + self._dequeue_strategy = dequeue_strategy + + self.disable_default_exception_handler = disable_default_exception_handler + self.with_scheduler = with_scheduler + self.burst = burst + self._model = ( + model + if model is not None + else WorkerModel( + name=self.name, + queue_names=[queue.name for queue in self.queues], + birth=None, + last_heartbeat=None, + pid=os.getpid(), + hostname=socket.gethostname(), + ip_address=_get_ip_address_from_connection(self.connection, self.name), + version=scheduler.__version__, + python_version=sys.version, + state=WorkerStatus.CREATED, + ) + ) + self._model.save(self.connection) + + @property + def _pid(self) -> int: + return self._model.pid + + def should_run_maintenance_tasks(self): + """Maintenance tasks should run on first startup or every 10 minutes.""" + if self._model.last_cleaned_at is None: + return True + if (utcnow() - self._model.last_cleaned_at) > timedelta(seconds=self.maintenance_interval): + return True + return False + + def _set_connection(self, connection: ConnectionType) -> ConnectionType: + """Configures the Broker connection to have a socket timeout. + This should timouet the connection in case any specific command hangs at any given time (eg. BLPOP). + If the connection provided already has a `socket_timeout` defined, skips. + + :param connection: Broker connection to configure. + """ + current_socket_timeout = connection.connection_pool.connection_kwargs.get("socket_timeout") + if current_socket_timeout is None: + timeout_config = {"socket_timeout": self.connection_timeout} + connection.connection_pool.connection_kwargs.update(timeout_config) + return connection + + def clean_registries(self): + """Runs maintenance jobs on each Queue's registries.""" + for queue in self.queues: + # If there are multiple workers running, we only want 1 worker + # to run clean_registries(). + queue_lock = QueueLock(self.name) + if queue_lock.acquire(1, expire=899, connection=self.connection): + logger.info(f"[Worker {self.name}/{self._pid}]: Cleaning registries for queue: {queue.name}") + queue.clean_registries() + WorkerModel.cleanup(queue.connection, queue.name) + queue_lock.release(self.connection) + self._model.last_cleaned_at = utcnow() + + def _install_signal_handlers(self) -> None: + """Installs signal handlers for handling SIGINT and SIGTERM gracefully.""" + if threading.current_thread() is not threading.main_thread(): + logger.warning( + f"[Worker {self.name}/{self._pid}]: Running in a thread, skipping signal handlers installation" + ) + return + signal.signal(signal.SIGINT, self.request_stop) + signal.signal(signal.SIGTERM, self.request_stop) + + def work( + self, + max_jobs: Optional[int] = None, + max_idle_time: Optional[int] = None, + ) -> bool: + """Starts the work loop. + + Pops and performs all jobs on the current list of queues. When all + queues are empty, block and wait for new jobs to arrive on any of the + queues, unless `burst` mode is enabled. + If `max_idle_time` is provided, worker will die when it's idle for more than the provided value. + + The return value indicates whether any jobs were processed. + + :param max_jobs: Max number of jobs. Defaults to None. + :param max_idle_time: Max seconds for a worker to be idle. Defaults to None. + :return: Whether any jobs were processed. + """ + self.bootstrap() + + self._install_signal_handlers() + try: + while True: + self.refresh() + self._check_for_suspension(self.burst) + + if self.should_run_maintenance_tasks(): + self.run_maintenance_tasks() + + if self._model.shutdown_requested_date: + logger.info(f"[Worker {self.name}/{self._pid}]: stopping on request") + break + + timeout = None if self.burst else (SCHEDULER_CONFIG.DEFAULT_WORKER_TTL - 15) + job, queue = self.dequeue_job_and_maintain_ttl(timeout, max_idle_time) + if job is None: + if self.burst: + logger.info(f"[Worker {self.name}/{self._pid}]: done, quitting") + break + elif max_idle_time is not None: + logger.info(f"[Worker {self.name}/{self._pid}]: idle for {max_idle_time} seconds, quitting") + break + continue + + self.execute_job(job, queue) + + self.refresh() + with self.connection.pipeline() as pipeline: + self._model.heartbeat(pipeline) + self._model.save(pipeline) + pipeline.execute() + if max_jobs is not None and self._model.completed_jobs >= max_jobs: + logger.info( + f"[Worker {self.name}/{self._pid}]: finished executing {self._model.completed_jobs} jobs, quitting" + ) + break + return self._model.completed_jobs > 0 + + except TimeoutErrorTypes: + logger.error(f"[Worker {self.name}/{self._pid}]: Redis connection timeout, quitting...") + except StopRequested: + logger.info(f"[Worker {self.name}/{self._pid}]: Worker was requested to stop, quitting") + pass + except SystemExit: # Cold shutdown detected + raise + except Exception: + logger.error(f"[Worker {self.name}/{self._pid}]: found an unhandled exception, quitting...", exc_info=True) + finally: + self.teardown() + + def handle_job_failure(self, job: JobModel, queue: Queue, exc_string=""): + """ + Handles the failure or an executing job by: + 1. Setting the job status to failed + 2. Removing the job from active_job_registry + 3. Setting the workers current job to None + 4. Add the job to FailedJobRegistry + `save_exc_to_job` should only be used for testing purposes + """ + logger.debug(f"[Worker {self.name}/{self._pid}]: Handling failed execution of job {job.name}") + # check whether a job was stopped intentionally and set the job status appropriately if it was this job. + + new_job_status = ( + JobStatus.STOPPED + if self._model.get_field("stopped_job_name", self.connection) == job.name + else JobStatus.FAILED + ) + self._model.current_job_name = None + with self.connection.pipeline() as pipeline: + if new_job_status == JobStatus.STOPPED: + logger.debug(f"[Worker {self.name}/{self._pid}]: Job was stopped, setting status to STOPPED") + else: + logger.debug(f"[Worker {self.name}/{self._pid}]: Job has failed, setting status to FAILED") + + queue.active_job_registry.delete(connection=pipeline, job_name=job.name) + + if not self.disable_default_exception_handler: + queue.job_handle_failure(new_job_status, job, exc_string, connection=pipeline) + with suppress(ConnectionErrorTypes): + pipeline.execute() + + self._model.current_job_working_time = 0 + if job.status == JobStatus.FAILED: + self._model.failed_job_count += 1 + self._model.completed_jobs += 1 + if job.started_at and job.ended_at: + self._model.total_working_time_ms += (job.ended_at - job.started_at).microseconds / 1000.0 + self._model.save(connection=self.connection) + + try: + pipeline.execute() + except Exception: + # Ensure that custom exception handlers are called even if the Broker is down + pass + + def bootstrap(self): + """Bootstraps the worker. + Runs the basic tasks that should run when the worker actually starts working. + Used so that new workers can focus on the work loop implementation rather + than the full bootstraping process. + """ + self.worker_start() + logger.info(f"[Worker {self.name}/{self._pid}]: Worker {self.name} started with PID {os.getpid()}") + self._command_listener.start() + if self.with_scheduler: + self.scheduler = WorkerScheduler(self.queues, worker_name=self.name, connection=self.connection) + self.scheduler.start() + self._model.has_scheduler = True + self._model.save(connection=self.connection) + if self.with_scheduler and self.burst: + self.scheduler.request_stop_and_wait() + self._model.has_scheduler = False + self._model.save(connection=self.connection) + qnames = [queue.name for queue in self.queues] + logger.info(f"""[Worker {self.name}/{self._pid}]: Listening to queues {", ".join(qnames)}...""") + + def _check_for_suspension(self, burst: bool) -> None: + """Check to see if workers have been suspended by `rq suspend`""" + before_state = None + notified = False + while self._model.is_suspended: + if burst: + logger.info( + f"[Worker {self.name}/{self._pid}]: Suspended in burst mode, exiting, " + f"Note: There could still be unfinished jobs on the queue" + ) + raise StopRequested() + + if not notified: + logger.info(f"[Worker {self.name}/{self._pid}]: Worker suspended, trigger ResumeCommand") + before_state = self._model.state + self._model.set_field("state", WorkerStatus.SUSPENDED, connection=self.connection) + notified = True + time.sleep(1) + + if before_state: + self._model.set_field("state", before_state, connection=self.connection) + + def run_maintenance_tasks(self): + """Runs periodic maintenance tasks, these include: + 1. Check if scheduler should be started. + 2. Cleaning registries + """ + self.clean_registries() + if not self.with_scheduler: + return + if self.scheduler is None: + self.scheduler = WorkerScheduler(self.queues, worker_name=self.name, connection=self.connection) + if self.scheduler.status == SchedulerStatus.STOPPED: + self.scheduler.start() + self._model.has_scheduler = True + self._model.save(connection=self.connection) + if self.burst: + self.scheduler.request_stop_and_wait() + self._model.has_scheduler = False + self._model.save(connection=self.connection) + + def dequeue_job_and_maintain_ttl( + self, timeout: Optional[int], max_idle_time: Optional[int] = None + ) -> Tuple[JobModel, Queue]: + """Dequeues a job while maintaining the TTL. + :param timeout: The timeout for the dequeue operation. + :param max_idle_time: The maximum idle time for the worker. + :returns: A tuple with the job and the queue. + """ + qnames = ",".join([queue.name for queue in self.queues]) + + self._model.set_field("state", WorkerStatus.IDLE, connection=self.connection) + self.procline(f"Listening on {qnames}") + logger.debug(f"[Worker {self.name}/{self._pid}]: listening on {qnames}...") + connection_wait_time = 1.0 + idle_since = utcnow() + idle_time_left = max_idle_time + job, queue = None, None + while True: + try: + self._model.heartbeat(self.connection) + + if self.should_run_maintenance_tasks(): + self.run_maintenance_tasks() + + if timeout is not None and idle_time_left is not None: + timeout = min(timeout, idle_time_left) + + logger.debug( + f"[Worker {self.name}/{self._pid}]: Fetching jobs on queues {qnames} and timeout {timeout}" + ) + job, queue = Queue.dequeue_any(self._ordered_queues, timeout, connection=self.connection) + if job is not None: + self.reorder_queues(reference_queue=queue) + logger.info(f"[Worker {self.name}/{self._pid}]: Popped job `{job.name}` from `{queue.name}`") + break + except DequeueTimeout: + if max_idle_time is not None: + idle_for = (utcnow() - idle_since).total_seconds() + idle_time_left = math.ceil(max_idle_time - idle_for) + if idle_time_left <= 0: + break + except ConnectionErrorTypes as conn_err: + logger.error( + f"[Worker {self.name}/{self._pid}]: Could not connect to Broker: {conn_err} Retrying in {connection_wait_time} seconds..." + ) + time.sleep(connection_wait_time) + connection_wait_time *= self.exponential_backoff_factor + connection_wait_time = min(connection_wait_time, self.max_connection_wait_time) + + self._model.heartbeat(self.connection) + return job, queue + + @property + def connection_timeout(self) -> int: + return SCHEDULER_CONFIG.DEFAULT_WORKER_TTL - 5 + + def procline(self, message): + """Changes the current procname for the process. + + This can be used to make `ps -ef` output more readable. + """ + setprocname(f"{self._model._key}: {message}") + + def _validate_name_uniqueness(self): + """Validates that the worker name is unique.""" + worker_model = WorkerModel.get(self.name, connection=self.connection) + if worker_model is not None and worker_model.death is None: + raise ValueError(f"There exists an active worker named {self.name!r} already") + + def worker_start(self): + """Registers its own birth.""" + logger.debug(f"[Worker {self.name}/{self._pid}]: Registering birth") + now = utcnow() + self._model.birth = now + self._model.last_heartbeat = now + self._model.state = WorkerStatus.STARTED + self._model.save(self.connection) + + def kill_job_execution_process(self, sig: signal.Signals = SIGKILL): + """Kill the job execution process but catch "No such process" error has the job execution process could already + be dead. + + :param sig: Optional, Defaults to SIGKILL. + """ + try: + os.killpg(os.getpgid(self._model.job_execution_process_pid), sig) + logger.info( + f"[Worker {self.name}/{self._pid}]: Killed job execution process pid {self._model.job_execution_process_pid}" + ) + except OSError as e: + if e.errno == errno.ESRCH: + # "No such process" is fine with us + logger.debug("[Worker {self.name}/{self._pid}]: Job execution process already dead") + else: + raise + + def wait_for_job_execution_process(self) -> Tuple[Optional[int], Optional[int]]: + """Waits for the job execution process to complete. + Uses `0` as argument as to include "any child in the process group of the current process". + """ + pid = stat = None + with contextlib.suppress(ChildProcessError): # ChildProcessError: [Errno 10] No child processes + pid, stat = os.waitpid(self._model.job_execution_process_pid, 0) + return pid, stat + + def request_force_stop(self, signum: int, frame: Optional[FrameType]): + """Terminates the application (cold shutdown). + + :param signum: Signal number + :param frame: Frame + :raises SystemExit: SystemExit + """ + # When a worker is run through a worker pool, it may receive duplicate signals. + # One is sent by the pool when it calls `pool.stop_worker()` and another is sent by the OS + # when a user hits Ctrl+C. In this case, if we receive the second signal within 1 second, we ignore it. + shutdown_date = self._model.shutdown_requested_date + if shutdown_date is not None and (utcnow() - shutdown_date) < timedelta(seconds=1): + logger.debug( + f"[Worker {self.name}/{self._pid}]: Shutdown signal ignored, received twice in less than 1 second" + ) + return + + logger.warning(f"[Worker {self.name}/{self._pid}]: Cold shut down") + + # Take down the job execution process with the worker + if self._model.job_execution_process_pid: + logger.debug( + f"[Worker {self.name}/{self._pid}]: Taking down job execution process {self._model.job_execution_process_pid} with me" + ) + self.kill_job_execution_process() + self.wait_for_job_execution_process() + raise SystemExit() + + def request_stop(self, signum: int, frame: Optional[FrameType]) -> None: + """Stops the current worker loop but waits for child processes to end gracefully (warm shutdown). + :param signum: Signal number + :param frame: Frame + """ + logger.debug(f"[Worker {self.name}/{self._pid}]: Got signal {signal_name(signum)}") + self._model.set_field("shutdown_requested_date", utcnow(), self.connection) + + signal.signal(signal.SIGINT, self.request_force_stop) + signal.signal(signal.SIGTERM, self.request_force_stop) + + logger.info(f"[Worker {self.name}/{self._pid}]: warm shut down requested") + + self.stop_scheduler() + # If shutdown is requested in the middle of a job, wait until finish before shutting down and save the request. + if self._model.state == WorkerStatus.BUSY: + self._model.set_field("shutdown_requested_date", utcnow(), connection=self.connection) + + logger.debug( + f"[Worker {self.name}/{self._pid}]: Stopping after current job execution process is finished. " + f"Press Ctrl+C again for a cold shutdown." + ) + else: + raise StopRequested() + + def reorder_queues(self, reference_queue: Queue): + """Reorder the queues according to the strategy. + As this can be defined both in the `Worker` initialization or in the `work` method, + it doesn't take the strategy directly, but rather uses the private `_dequeue_strategy` attribute. + + :param reference_queue: The queues to reorder + """ + if self._dequeue_strategy is None: + self._dequeue_strategy = DequeueStrategy.DEFAULT + + if self._dequeue_strategy not in [e.value for e in DequeueStrategy]: + raise ValueError( + f"""[Worker {self.name}/{self._pid}]: Dequeue strategy should be one of {", ".join([e.value for e in DequeueStrategy])}""" + ) + if self._dequeue_strategy == DequeueStrategy.DEFAULT: + return + if self._dequeue_strategy == DequeueStrategy.ROUND_ROBIN: + pos = self._ordered_queues.index(reference_queue) + self._ordered_queues = self._ordered_queues[pos + 1 :] + self._ordered_queues[: pos + 1] + return + if self._dequeue_strategy == DequeueStrategy.RANDOM: + shuffle(self._ordered_queues) + return + + def teardown(self) -> None: + if self._is_job_execution_process: + return + self.stop_scheduler() + self._command_listener.stop() + self._model.delete(self.connection) + + def stop_scheduler(self): + """Stop the scheduler thread. + Will send the kill signal to the scheduler process, + if there's an OSError, just passes and `join()`'s the scheduler process, waiting for the process to finish. + """ + if self.scheduler is None: + return + logger.info(f"[Worker {self.name}/{self._pid}]: Stopping scheduler thread {self.scheduler.pid}") + self.scheduler.request_stop_and_wait() + logger.debug(f"[Worker {self.name}/{self._pid}]: Scheduler thread stopped") + self.scheduler = None + + def refresh(self, update_queues: bool = False): + """Refreshes the worker data. + It will get the data from the datastore and update the Worker's attributes + """ + self._model = WorkerModel.get(self.name, connection=self.connection) + if self._model is None: + msg = f"[Worker {self.name}/{self._pid}]: Worker broker record for {self.name} not found, quitting..." + logger.error(msg) + raise WorkerNotFound(msg) + if update_queues: + self.queues = [Queue(name=queue_name, connection=self.connection) for queue_name in self._model.queue_names] + + def fork_job_execution_process(self, job: JobModel, queue: Queue) -> None: + """Spawns a job execution process to perform the actual work and passes it a job. + This is where the `fork()` actually happens. + + :param job: The job to be executed + :param queue: The queue from which the job was dequeued + """ + child_pid = os.fork() + os.environ["SCHEDULER_WORKER_NAME"] = self.name + os.environ["SCHEDULER_JOB_NAME"] = job.name + if child_pid == 0: # Child process/Job executor process to run the job + os.setsid() + self._model.job_execution_process_pid = os.getpid() + self._model.save(connection=self.connection) + self.execute_in_separate_process(job, queue) + os._exit(0) # just in case + else: # Parent worker process + logger.debug( + f"[Worker {self.name}/{self._pid}]: Forking job execution process, job_execution_process_pid={child_pid}" + ) + self._model.job_execution_process_pid = child_pid + self._model.save(connection=self.connection) + self.procline(f"Forked {child_pid} at {time.time()}") + + def get_heartbeat_ttl(self, job: JobModel) -> int: + """Get's the TTL for the next heartbeat. + :param job: The Job + :return: The heartbeat TTL + """ + if job.timeout and job.timeout > 0: + remaining_execution_time = int(job.timeout - self._model.current_job_working_time) + return min(remaining_execution_time, self.job_monitoring_interval) + 60 + else: + return self.job_monitoring_interval + 60 + + def monitor_job_execution_process(self, job: JobModel, queue: Queue) -> None: + """The worker will monitor the job execution process and make sure that it either executes successfully or the + status of the job is set to failed + + :param job: The Job + :param queue: The Queue + """ + retpid = ret_val = None + job.started_at = utcnow() + while True: + try: + with SCHEDULER_CONFIG.DEATH_PENALTY_CLASS( + self.job_monitoring_interval, JobExecutionMonitorTimeoutException + ): + retpid, ret_val = self.wait_for_job_execution_process() + break + except JobExecutionMonitorTimeoutException: + # job execution process has not exited yet and is still running. Send a heartbeat to keep the worker alive. + self._model.set_current_job_working_time((utcnow() - job.started_at).total_seconds(), self.connection) + + # Kill the job from this side if something is really wrong (interpreter lock/etc). + if job.timeout != -1 and self._model.current_job_working_time > (job.timeout + 60): + self._model.heartbeat(self.connection, self.job_monitoring_interval + 60) + self.kill_job_execution_process() + self.wait_for_job_execution_process() + break + + self.maintain_heartbeats(job, queue) + + except OSError as e: + # In case we encountered an OSError due to EINTR (which is + # caused by a SIGINT or SIGTERM signal during + # os.waitpid()), we simply ignore it and enter the next + # iteration of the loop, waiting for the child to end. In + # any other case, this is some other unexpected OS error, + # which we don't want to catch, so we re-raise those ones. + if e.errno != errno.EINTR: + raise + # Send a heartbeat to keep the worker alive. + self._model.heartbeat(self.connection) + + self._model = WorkerModel.get(self.name, connection=self.connection) + self._model.current_job_working_time = 0 + self._model.save(connection=self.connection) + if ret_val == os.EX_OK: # The process exited normally. + return + + job_status = job.get_status(self.connection) + stopped_job_name = self._model.get_field("stopped_job_name", self.connection) + + if job_status is None: # Job completed and its ttl has expired + logger.debug(f"[Worker {self.name}/{self._pid}]: Job status is None, completed and expired?") + return + elif stopped_job_name == job.name: # job execution process killed deliberately + logger.warning(f"[Worker {self.name}/{self._pid}]: Job stopped by user, moving job to failed-jobs-registry") + if job.stopped_callback: + job.stopped_callback() + self.handle_job_failure( + job, queue=queue, exc_string="Job stopped by user, job execution process terminated." + ) + elif job_status not in [JobStatus.FINISHED, JobStatus.FAILED]: + if not job.ended_at: + job.ended_at = utcnow() + + # Unhandled failure: move the job to the failed queue + signal_msg = f" (signal {os.WTERMSIG(ret_val)})" if ret_val and os.WIFSIGNALED(ret_val) else "" + exc_string = f"job-execution-process terminated unexpectedly; waitpid returned {ret_val}{signal_msg}; " + logger.warning( + f"[Worker {self.name}/{self._pid}]: Moving job to {queue.name}/failed-job-registry ({exc_string})" + ) + + self.handle_job_failure(job, queue=queue, exc_string=exc_string) + + def execute_job(self, job: JobModel, queue: Queue): + """Spawns a job execution process to perform the actual work and passes it a job. + The worker will wait for the job execution process and make sure it executes within the given timeout bounds, or + will end the job execution process with SIGALRM. + """ + if self.fork_job_execution: + self._model.set_field("state", WorkerStatus.BUSY, connection=self.connection) + self.fork_job_execution_process(job, queue) + self.monitor_job_execution_process(job, queue) + self._model.set_field("state", WorkerStatus.IDLE, connection=self.connection) + else: + self._model.set_field("state", WorkerStatus.BUSY, connection=self.connection) + self.perform_job(job, queue) + self._model.set_field("state", WorkerStatus.IDLE, connection=self.connection) + + def maintain_heartbeats(self, job: JobModel, queue: Queue): + """Updates worker and job's last heartbeat field.""" + with self.connection.pipeline() as pipeline: + self._model.heartbeat(pipeline, self.job_monitoring_interval + 60) + ttl = self.get_heartbeat_ttl(job) + + queue.active_job_registry.add(pipeline, self.name, current_timestamp() + ttl, update_existing_only=False) + results = pipeline.execute() + if results[2] == 1: + job.delete(self.connection) + + def execute_in_separate_process(self, job: JobModel, queue: Queue): + """This is the entry point of the newly spawned job execution process. + After fork()'ing, assure we are generating random sequences that are different from the worker. + + os._exit() is the way to exit from child processes after a fork(), in contrast to the regular sys.exit() + """ + random.seed() + self.setup_job_execution_process_signals() + self._is_job_execution_process = True + try: + self.perform_job(job, queue) + except: # noqa + os._exit(1) + os._exit(0) + + def setup_job_execution_process_signals(self): + """Setup signal handing for the newly spawned job execution process + + Always ignore Ctrl+C in the job execution process, as it might abort the currently running job. + + The main worker catches the Ctrl+C and requests graceful shutdown after the current work is done. + When cold shutdown is requested, it kills the current job anyway. + """ + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + def worker_before_execution(self, job: JobModel, connection: ConnectionType) -> None: + logger.debug(f"[Worker {self.name}/{self._pid}]: Preparing for execution of job: `{job.name}`") + current_pid = os.getpid() + self._model.current_job_name = job.name + self._model.current_job_working_time = 0 + self._model.job_execution_process_pid = current_pid + heartbeat_ttl = self.get_heartbeat_ttl(job) + self._model.heartbeat(self.connection, heartbeat_ttl) + self.procline( + f"[Worker {self.name}/{self._pid}]: Processing {job.func_name} from {job.queue_name} since {time.time()}" + ) + self._model.save(connection=connection) + + def handle_job_success(self, job: JobModel, return_value: Any, queue: Queue): + """Handles the successful execution of certain job. + It will remove the job from the `active_job_registry`, adding it to the `SuccessfulJobRegistry`, + and run a few maintenance tasks including: + - Resting the current job name + - Enqueue dependents + - Incrementing the job count and working time + - Handling of the job successful execution + + Runs within a loop with the `watch` method so that protects interactions with dependents keys. + + :param job: The job that was successful. + :param queue: The queue + """ + logger.debug(f"[Worker {self.name}/{self._pid}]: Handling successful execution of job {job.name}") + with self.connection.pipeline() as pipeline: + while True: + try: + queue.job_handle_success( + job, + result=return_value, + job_info_ttl=job.job_info_ttl, + result_ttl=job.success_ttl, + connection=pipeline, + ) + self._model.current_job_name = None + self._model.successful_job_count += 1 + self._model.completed_jobs += 1 + self._model.total_working_time_ms += (job.ended_at - job.started_at).microseconds / 1000.0 + self._model.save(connection=self.connection) + + job.expire(job.success_ttl, connection=pipeline) + logger.debug(f"[Worker {self.name}/{self._pid}]: Removing job {job.name} from active_job_registry") + pipeline.execute() + logger.debug( + f"[Worker {self.name}/{self._pid}]: Finished handling successful execution of job {job.name}" + ) + break + except WatchErrorTypes: + continue + + def perform_job(self, job: JobModel, queue: Queue) -> bool: + """Performs the actual work of a job. + Called from the process executing the job (forked job execution process). + + :param job: The job to perform + :param queue: The queue the job was dequeued from + :returns: True after finished. + """ + logger.debug(f"[Worker {self.name}/{self._pid}]: Performing {job.name} code.") + + try: + with self.connection.pipeline() as pipeline: + self.worker_before_execution(job, connection=pipeline) + job.prepare_for_execution(self.name, queue.active_job_registry, connection=pipeline) + pipeline.execute() + timeout = job.timeout or SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT + with SCHEDULER_CONFIG.DEATH_PENALTY_CLASS(timeout, JobTimeoutException, job_name=job.name): + logger.debug(f"[Worker {self.name}/{self._pid}]: Performing job `{job.name}`...") + rv = perform_job(job, self.connection) + logger.debug(f"[Worker {self.name}/{self._pid}]: Finished performing job `{job.name}`") + + self.handle_job_success(job=job, return_value=rv, queue=queue) + except: # NOQA + logger.debug(f"[Worker {self.name}/{self._pid}]: Job {job.name} raised an exception.") + exc_info = sys.exc_info() + exc_string = "".join(traceback.format_exception(*exc_info)) + + self.handle_job_failure(job=job, exc_string=exc_string, queue=queue) + self.handle_exception(job, *exc_info) + return False + + logger.info(f"[Worker {self.name}/{self._pid}]: queue:{queue.name}/job:{job.name} performed.") + logger.debug(f"[Worker {self.name}/{self._pid}]: job:{job.name} result: {str(rv)}") + + return True + + def handle_exception(self, job: JobModel, *exc_info): + """Walks the exception handler stack to delegate exception handling. + If the job cannot be deserialized, it will raise when func_name or + the other properties are accessed, which will stop exceptions from + being properly logged, so we guard against it here. + """ + logger.debug(f"Handling exception caused while performing job:{job.name}.") + exc_string = "".join(traceback.format_exception(*exc_info)) + + extra = { + "func": job.func_name, + "arguments": job.args, + "kwargs": job.kwargs, + Queue: job.queue_name, + "job_name": job.name, + } + func_name = job.func_name + + # func_name + logger.error( + f"[Worker {self.name}/{self._pid}]: exception raised while executing ({func_name})\n{exc_string}", + extra=extra, + ) + + +class SimpleWorker(Worker): + def execute_job(self, job: JobModel, queue: Queue): + """Execute job in same thread/process, do not fork()""" + self._model.set_field("state", WorkerStatus.BUSY, connection=self.connection) + self.perform_job(job, queue) + self._model.set_field("state", WorkerStatus.IDLE, connection=self.connection) + + +class RoundRobinWorker(Worker): + """Modified version of Worker that dequeues jobs from the queues using a round-robin strategy.""" + + def reorder_queues(self, reference_queue): + pos = self._ordered_queues.index(reference_queue) + self._ordered_queues = self._ordered_queues[pos + 1 :] + self._ordered_queues[: pos + 1] + + +class RandomWorker(Worker): + """Modified version of Worker that dequeues jobs from the queues using a random strategy.""" + + def reorder_queues(self, reference_queue): + shuffle(self._ordered_queues) + + +def _get_ip_address_from_connection(connection: ConnectionType, client_name: str) -> str: + try: + connection.client_setname(client_name) + except ResponseErrorTypes: + warnings.warn("CLIENT SETNAME command not supported, setting ip_address to unknown", Warning) + return "unknown" + client_adresses = [client["addr"] for client in connection.client_list() if client["name"] == client_name] + if len(client_adresses) > 0: + return client_adresses[0] + else: + warnings.warn("CLIENT LIST command not supported, setting ip_address to unknown", Warning) + return "unknown" + + +def _ensure_list(obj: Any) -> List: + """When passed an iterable of objects, does nothing, otherwise, it returns a list with just that object in it. + + :param obj: The object to ensure is a list + :return: + """ + is_nonstring_iterable = isinstance(obj, Iterable) and not isinstance(obj, str) + return obj if is_nonstring_iterable else [obj] + + +def _calc_worker_name(existing_worker_names) -> str: + hostname = os.uname()[1] + c = 1 + worker_name = f"{hostname}-worker.{c}" + while worker_name in existing_worker_names: + c += 1 + worker_name = f"{hostname}-worker.{c}" + return worker_name + + +def get_queues(*queue_names: str) -> List[Queue]: + """Return queue instances from specified queue names. All instances must use the same connection configuration.""" + + queue_config = get_queue_configuration(queue_names[0]) + queues = [get_queue(queue_names[0])] + # perform consistency checks while building return list + for queue_name in queue_names[1:]: + curr_queue_config = get_queue_configuration(queue_name) + if not queue_config.same_connection_params(curr_queue_config): + raise QueueConnectionDiscrepancyError( + f'Queues must have the same broker connection. "{queue_name}" and "{queue_names[0]}" have different connection settings' + ) + queue = get_queue(queue_name) + queues.append(queue) + + return queues + + +def create_worker(*queue_names: str, **kwargs) -> Worker: + """Returns a Django worker for all queues or specified ones.""" + queues = get_queues(*queue_names) + existing_worker_names = WorkerModel.all_names(connection=queues[0].connection) + kwargs.setdefault("fork_job_execution", SCHEDULER_CONFIG.BROKER != Broker.FAKEREDIS) + if kwargs.get("name", None) is None: + kwargs["name"] = _calc_worker_name(existing_worker_names) + if kwargs["name"] in existing_worker_names: + raise ValueError(f"Worker {kwargs['name']} already exists") + kwargs["name"] = kwargs["name"].replace("/", ".") + kwargs.setdefault("with_scheduler", False) + worker = Worker(queues, connection=queues[0].connection, **kwargs) + return worker diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index e076068..e01d1ff 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -1,7 +1,15 @@ import os -import django -from fakeredis import FakeConnection +from scheduler.types import QueueConfiguration + +BROKER_PORT = os.getenv("BROKER_PORT", "6379") +SCHEDULER_QUEUES = { + "default": QueueConfiguration(URL=f"redis://localhost:{BROKER_PORT}/0"), + "low": QueueConfiguration(URL=f"redis://localhost:{BROKER_PORT}/0"), + "high": QueueConfiguration(URL=f"redis://localhost:{BROKER_PORT}/1"), + "medium": QueueConfiguration(URL=f"redis://localhost:{BROKER_PORT}/1"), + "another": QueueConfiguration(URL=f"redis://localhost:{BROKER_PORT}/1"), +} BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -14,7 +22,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ["*"] # Application definition @@ -25,7 +33,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "scheduler", + "scheduler.apps.SchedulerConfig", ] MIDDLEWARE = [ @@ -111,25 +119,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.9/howto/static-files/ -BROKER_PORT = os.getenv("BROKER_PORT", "6379") STATIC_URL = "/static/" -SCHEDULER_QUEUES = { - "default": { - "URL": f"redis://localhost:{BROKER_PORT}/0", - }, - "low": { - "URL": f"redis://localhost:{BROKER_PORT}/0", - }, - "high": { - "URL": f"redis://localhost:{BROKER_PORT}/1", - }, - "medium": { - "URL": f"redis://localhost:{BROKER_PORT}/1", - }, - "another": { - "URL": f"redis://localhost:{BROKER_PORT}/1", - }, -} DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" LOGGING = { @@ -145,16 +135,16 @@ }, }, "handlers": { - "console": {"level": "INFO", "class": "logging.StreamHandler", "formatter": "simple"}, + "console": {"level": "DEBUG", "class": "logging.StreamHandler", "formatter": "simple"}, }, "root": { "handlers": ["console"], - "level": "INFO", + "level": "DEBUG", }, "loggers": { "scheduler": { "handlers": ["console"], - "level": "INFO", + "level": "DEBUG", }, }, } diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..95cc2e4 --- /dev/null +++ b/uv.lock @@ -0,0 +1,445 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "asgiref" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/38/b3395cc9ad1b56d2ddac9970bc8f4141312dbaec28bc7c218b0dfafd0f42/asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590", size = 35186, upload-time = "2024-03-22T14:39:36.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828, upload-time = "2024-03-22T14:39:34.521Z" }, +] + +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + +[[package]] +name = "certifi" +version = "2025.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/f7/f14b46d4bcd21092d7d3ccef689615220d8a08fb25e564b65d20738e672e/certifi-2025.6.15.tar.gz", hash = "sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b", size = 158753, upload-time = "2025-06-15T02:45:51.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/ae/320161bd181fc06471eed047ecce67b693fd7515b16d495d8932db763426/certifi-2025.6.15-py3-none-any.whl", hash = "sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057", size = 157650, upload-time = "2025-06-15T02:45:49.977Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/e0/98670a80884f64578f0c22cd70c5e81a6e07b08167721c7487b4d70a7ca0/coverage-7.9.1.tar.gz", hash = "sha256:6cf43c78c4282708a28e466316935ec7489a9c487518a77fa68f716c67909cec", size = 813650, upload-time = "2025-06-13T13:02:28.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/78/1c1c5ec58f16817c09cbacb39783c3655d54a221b6552f47ff5ac9297603/coverage-7.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cc94d7c5e8423920787c33d811c0be67b7be83c705f001f7180c7b186dcf10ca", size = 212028, upload-time = "2025-06-13T13:00:29.293Z" }, + { url = "https://files.pythonhosted.org/packages/98/db/e91b9076f3a888e3b4ad7972ea3842297a52cc52e73fd1e529856e473510/coverage-7.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16aa0830d0c08a2c40c264cef801db8bc4fc0e1892782e45bcacbd5889270509", size = 212420, upload-time = "2025-06-13T13:00:34.027Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d0/2b3733412954576b0aea0a16c3b6b8fbe95eb975d8bfa10b07359ead4252/coverage-7.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf95981b126f23db63e9dbe4cf65bd71f9a6305696fa5e2262693bc4e2183f5b", size = 241529, upload-time = "2025-06-13T13:00:35.786Z" }, + { url = "https://files.pythonhosted.org/packages/b3/00/5e2e5ae2e750a872226a68e984d4d3f3563cb01d1afb449a17aa819bc2c4/coverage-7.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f05031cf21699785cd47cb7485f67df619e7bcdae38e0fde40d23d3d0210d3c3", size = 239403, upload-time = "2025-06-13T13:00:37.399Z" }, + { url = "https://files.pythonhosted.org/packages/37/3b/a2c27736035156b0a7c20683afe7df498480c0dfdf503b8c878a21b6d7fb/coverage-7.9.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4fbcab8764dc072cb651a4bcda4d11fb5658a1d8d68842a862a6610bd8cfa3", size = 240548, upload-time = "2025-06-13T13:00:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/98/f5/13d5fc074c3c0e0dc80422d9535814abf190f1254d7c3451590dc4f8b18c/coverage-7.9.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0f16649a7330ec307942ed27d06ee7e7a38417144620bb3d6e9a18ded8a2d3e5", size = 240459, upload-time = "2025-06-13T13:00:40.934Z" }, + { url = "https://files.pythonhosted.org/packages/36/24/24b9676ea06102df824c4a56ffd13dc9da7904478db519efa877d16527d5/coverage-7.9.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:cea0a27a89e6432705fffc178064503508e3c0184b4f061700e771a09de58187", size = 239128, upload-time = "2025-06-13T13:00:42.343Z" }, + { url = "https://files.pythonhosted.org/packages/be/05/242b7a7d491b369ac5fee7908a6e5ba42b3030450f3ad62c645b40c23e0e/coverage-7.9.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e980b53a959fa53b6f05343afbd1e6f44a23ed6c23c4b4c56c6662bbb40c82ce", size = 239402, upload-time = "2025-06-13T13:00:43.634Z" }, + { url = "https://files.pythonhosted.org/packages/73/e0/4de7f87192fa65c9c8fbaeb75507e124f82396b71de1797da5602898be32/coverage-7.9.1-cp310-cp310-win32.whl", hash = "sha256:70760b4c5560be6ca70d11f8988ee6542b003f982b32f83d5ac0b72476607b70", size = 214518, upload-time = "2025-06-13T13:00:45.622Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ab/5e4e2fe458907d2a65fab62c773671cfc5ac704f1e7a9ddd91996f66e3c2/coverage-7.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:a66e8f628b71f78c0e0342003d53b53101ba4e00ea8dabb799d9dba0abbbcebe", size = 215436, upload-time = "2025-06-13T13:00:47.245Z" }, + { url = "https://files.pythonhosted.org/packages/60/34/fa69372a07d0903a78ac103422ad34db72281c9fc625eba94ac1185da66f/coverage-7.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:95c765060e65c692da2d2f51a9499c5e9f5cf5453aeaf1420e3fc847cc060582", size = 212146, upload-time = "2025-06-13T13:00:48.496Z" }, + { url = "https://files.pythonhosted.org/packages/27/f0/da1894915d2767f093f081c42afeba18e760f12fdd7a2f4acbe00564d767/coverage-7.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ba383dc6afd5ec5b7a0d0c23d38895db0e15bcba7fb0fa8901f245267ac30d86", size = 212536, upload-time = "2025-06-13T13:00:51.535Z" }, + { url = "https://files.pythonhosted.org/packages/10/d5/3fc33b06e41e390f88eef111226a24e4504d216ab8e5d1a7089aa5a3c87a/coverage-7.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37ae0383f13cbdcf1e5e7014489b0d71cc0106458878ccde52e8a12ced4298ed", size = 245092, upload-time = "2025-06-13T13:00:52.883Z" }, + { url = "https://files.pythonhosted.org/packages/0a/39/7aa901c14977aba637b78e95800edf77f29f5a380d29768c5b66f258305b/coverage-7.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:69aa417a030bf11ec46149636314c24c8d60fadb12fc0ee8f10fda0d918c879d", size = 242806, upload-time = "2025-06-13T13:00:54.571Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/30e5cfeaf560b1fc1989227adedc11019ce4bb7cce59d65db34fe0c2d963/coverage-7.9.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a4be2a28656afe279b34d4f91c3e26eccf2f85500d4a4ff0b1f8b54bf807338", size = 244610, upload-time = "2025-06-13T13:00:56.932Z" }, + { url = "https://files.pythonhosted.org/packages/bf/15/cca62b13f39650bc87b2b92bb03bce7f0e79dd0bf2c7529e9fc7393e4d60/coverage-7.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:382e7ddd5289f140259b610e5f5c58f713d025cb2f66d0eb17e68d0a94278875", size = 244257, upload-time = "2025-06-13T13:00:58.545Z" }, + { url = "https://files.pythonhosted.org/packages/cd/1a/c0f2abe92c29e1464dbd0ff9d56cb6c88ae2b9e21becdb38bea31fcb2f6c/coverage-7.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e5532482344186c543c37bfad0ee6069e8ae4fc38d073b8bc836fc8f03c9e250", size = 242309, upload-time = "2025-06-13T13:00:59.836Z" }, + { url = "https://files.pythonhosted.org/packages/57/8d/c6fd70848bd9bf88fa90df2af5636589a8126d2170f3aade21ed53f2b67a/coverage-7.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a39d18b3f50cc121d0ce3838d32d58bd1d15dab89c910358ebefc3665712256c", size = 242898, upload-time = "2025-06-13T13:01:02.506Z" }, + { url = "https://files.pythonhosted.org/packages/c2/9e/6ca46c7bff4675f09a66fe2797cd1ad6a24f14c9c7c3b3ebe0470a6e30b8/coverage-7.9.1-cp311-cp311-win32.whl", hash = "sha256:dd24bd8d77c98557880def750782df77ab2b6885a18483dc8588792247174b32", size = 214561, upload-time = "2025-06-13T13:01:04.012Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/166978c6302010742dabcdc425fa0f938fa5a800908e39aff37a7a876a13/coverage-7.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:6b55ad10a35a21b8015eabddc9ba31eb590f54adc9cd39bcf09ff5349fd52125", size = 215493, upload-time = "2025-06-13T13:01:05.702Z" }, + { url = "https://files.pythonhosted.org/packages/60/07/a6d2342cd80a5be9f0eeab115bc5ebb3917b4a64c2953534273cf9bc7ae6/coverage-7.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ad935f0016be24c0e97fc8c40c465f9c4b85cbbe6eac48934c0dc4d2568321e", size = 213869, upload-time = "2025-06-13T13:01:09.345Z" }, + { url = "https://files.pythonhosted.org/packages/68/d9/7f66eb0a8f2fce222de7bdc2046ec41cb31fe33fb55a330037833fb88afc/coverage-7.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8de12b4b87c20de895f10567639c0797b621b22897b0af3ce4b4e204a743626", size = 212336, upload-time = "2025-06-13T13:01:10.909Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/e07cb920ef3addf20f052ee3d54906e57407b6aeee3227a9c91eea38a665/coverage-7.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5add197315a054e92cee1b5f686a2bcba60c4c3e66ee3de77ace6c867bdee7cb", size = 212571, upload-time = "2025-06-13T13:01:12.518Z" }, + { url = "https://files.pythonhosted.org/packages/78/f8/96f155de7e9e248ca9c8ff1a40a521d944ba48bec65352da9be2463745bf/coverage-7.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:600a1d4106fe66f41e5d0136dfbc68fe7200a5cbe85610ddf094f8f22e1b0300", size = 246377, upload-time = "2025-06-13T13:01:14.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/cf/1d783bd05b7bca5c10ded5f946068909372e94615a4416afadfe3f63492d/coverage-7.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a876e4c3e5a2a1715a6608906aa5a2e0475b9c0f68343c2ada98110512ab1d8", size = 243394, upload-time = "2025-06-13T13:01:16.23Z" }, + { url = "https://files.pythonhosted.org/packages/02/dd/e7b20afd35b0a1abea09fb3998e1abc9f9bd953bee548f235aebd2b11401/coverage-7.9.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81f34346dd63010453922c8e628a52ea2d2ccd73cb2487f7700ac531b247c8a5", size = 245586, upload-time = "2025-06-13T13:01:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/4e/38/b30b0006fea9d617d1cb8e43b1bc9a96af11eff42b87eb8c716cf4d37469/coverage-7.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:888f8eee13f2377ce86d44f338968eedec3291876b0b8a7289247ba52cb984cd", size = 245396, upload-time = "2025-06-13T13:01:19.164Z" }, + { url = "https://files.pythonhosted.org/packages/31/e4/4d8ec1dc826e16791f3daf1b50943e8e7e1eb70e8efa7abb03936ff48418/coverage-7.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9969ef1e69b8c8e1e70d591f91bbc37fc9a3621e447525d1602801a24ceda898", size = 243577, upload-time = "2025-06-13T13:01:22.433Z" }, + { url = "https://files.pythonhosted.org/packages/25/f4/b0e96c5c38e6e40ef465c4bc7f138863e2909c00e54a331da335faf0d81a/coverage-7.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:60c458224331ee3f1a5b472773e4a085cc27a86a0b48205409d364272d67140d", size = 244809, upload-time = "2025-06-13T13:01:24.143Z" }, + { url = "https://files.pythonhosted.org/packages/8a/65/27e0a1fa5e2e5079bdca4521be2f5dabf516f94e29a0defed35ac2382eb2/coverage-7.9.1-cp312-cp312-win32.whl", hash = "sha256:5f646a99a8c2b3ff4c6a6e081f78fad0dde275cd59f8f49dc4eab2e394332e74", size = 214724, upload-time = "2025-06-13T13:01:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/9b/a8/d5b128633fd1a5e0401a4160d02fa15986209a9e47717174f99dc2f7166d/coverage-7.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:30f445f85c353090b83e552dcbbdad3ec84c7967e108c3ae54556ca69955563e", size = 215535, upload-time = "2025-06-13T13:01:27.861Z" }, + { url = "https://files.pythonhosted.org/packages/a3/37/84bba9d2afabc3611f3e4325ee2c6a47cd449b580d4a606b240ce5a6f9bf/coverage-7.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:af41da5dca398d3474129c58cb2b106a5d93bbb196be0d307ac82311ca234342", size = 213904, upload-time = "2025-06-13T13:01:29.202Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a7/a027970c991ca90f24e968999f7d509332daf6b8c3533d68633930aaebac/coverage-7.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:31324f18d5969feef7344a932c32428a2d1a3e50b15a6404e97cba1cc9b2c631", size = 212358, upload-time = "2025-06-13T13:01:30.909Z" }, + { url = "https://files.pythonhosted.org/packages/f2/48/6aaed3651ae83b231556750280682528fea8ac7f1232834573472d83e459/coverage-7.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c804506d624e8a20fb3108764c52e0eef664e29d21692afa375e0dd98dc384f", size = 212620, upload-time = "2025-06-13T13:01:32.256Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2a/f4b613f3b44d8b9f144847c89151992b2b6b79cbc506dee89ad0c35f209d/coverage-7.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef64c27bc40189f36fcc50c3fb8f16ccda73b6a0b80d9bd6e6ce4cffcd810bbd", size = 245788, upload-time = "2025-06-13T13:01:33.948Z" }, + { url = "https://files.pythonhosted.org/packages/04/d2/de4fdc03af5e4e035ef420ed26a703c6ad3d7a07aff2e959eb84e3b19ca8/coverage-7.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4fe2348cc6ec372e25adec0219ee2334a68d2f5222e0cba9c0d613394e12d86", size = 243001, upload-time = "2025-06-13T13:01:35.285Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e8/eed18aa5583b0423ab7f04e34659e51101135c41cd1dcb33ac1d7013a6d6/coverage-7.9.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34ed2186fe52fcc24d4561041979a0dec69adae7bce2ae8d1c49eace13e55c43", size = 244985, upload-time = "2025-06-13T13:01:36.712Z" }, + { url = "https://files.pythonhosted.org/packages/17/f8/ae9e5cce8885728c934eaa58ebfa8281d488ef2afa81c3dbc8ee9e6d80db/coverage-7.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:25308bd3d00d5eedd5ae7d4357161f4df743e3c0240fa773ee1b0f75e6c7c0f1", size = 245152, upload-time = "2025-06-13T13:01:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/5a/c8/272c01ae792bb3af9b30fac14d71d63371db227980682836ec388e2c57c0/coverage-7.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:73e9439310f65d55a5a1e0564b48e34f5369bee943d72c88378f2d576f5a5751", size = 243123, upload-time = "2025-06-13T13:01:40.727Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d0/2819a1e3086143c094ab446e3bdf07138527a7b88cb235c488e78150ba7a/coverage-7.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:37ab6be0859141b53aa89412a82454b482c81cf750de4f29223d52268a86de67", size = 244506, upload-time = "2025-06-13T13:01:42.184Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4e/9f6117b89152df7b6112f65c7a4ed1f2f5ec8e60c4be8f351d91e7acc848/coverage-7.9.1-cp313-cp313-win32.whl", hash = "sha256:64bdd969456e2d02a8b08aa047a92d269c7ac1f47e0c977675d550c9a0863643", size = 214766, upload-time = "2025-06-13T13:01:44.482Z" }, + { url = "https://files.pythonhosted.org/packages/27/0f/4b59f7c93b52c2c4ce7387c5a4e135e49891bb3b7408dcc98fe44033bbe0/coverage-7.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:be9e3f68ca9edb897c2184ad0eee815c635565dbe7a0e7e814dc1f7cbab92c0a", size = 215568, upload-time = "2025-06-13T13:01:45.772Z" }, + { url = "https://files.pythonhosted.org/packages/09/1e/9679826336f8c67b9c39a359352882b24a8a7aee48d4c9cad08d38d7510f/coverage-7.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:1c503289ffef1d5105d91bbb4d62cbe4b14bec4d13ca225f9c73cde9bb46207d", size = 213939, upload-time = "2025-06-13T13:01:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5b/5c6b4e7a407359a2e3b27bf9c8a7b658127975def62077d441b93a30dbe8/coverage-7.9.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0b3496922cb5f4215bf5caaef4cf12364a26b0be82e9ed6d050f3352cf2d7ef0", size = 213079, upload-time = "2025-06-13T13:01:48.554Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/1e2e07279fd2fd97ae26c01cc2186e2258850e9ec125ae87184225662e89/coverage-7.9.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9565c3ab1c93310569ec0d86b017f128f027cab0b622b7af288696d7ed43a16d", size = 213299, upload-time = "2025-06-13T13:01:49.997Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/4c5125a4b69d66b8c85986d3321520f628756cf524af810baab0790c7647/coverage-7.9.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2241ad5dbf79ae1d9c08fe52b36d03ca122fb9ac6bca0f34439e99f8327ac89f", size = 256535, upload-time = "2025-06-13T13:01:51.314Z" }, + { url = "https://files.pythonhosted.org/packages/81/8b/e36a04889dda9960be4263e95e777e7b46f1bb4fc32202612c130a20c4da/coverage-7.9.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bb5838701ca68b10ebc0937dbd0eb81974bac54447c55cd58dea5bca8451029", size = 252756, upload-time = "2025-06-13T13:01:54.403Z" }, + { url = "https://files.pythonhosted.org/packages/98/82/be04eff8083a09a4622ecd0e1f31a2c563dbea3ed848069e7b0445043a70/coverage-7.9.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a25f814591a8c0c5372c11ac8967f669b97444c47fd794926e175c4047ece", size = 254912, upload-time = "2025-06-13T13:01:56.769Z" }, + { url = "https://files.pythonhosted.org/packages/0f/25/c26610a2c7f018508a5ab958e5b3202d900422cf7cdca7670b6b8ca4e8df/coverage-7.9.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2d04b16a6062516df97969f1ae7efd0de9c31eb6ebdceaa0d213b21c0ca1a683", size = 256144, upload-time = "2025-06-13T13:01:58.19Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/fb9425c4684066c79e863f1e6e7ecebb49e3a64d9f7f7860ef1688c56f4a/coverage-7.9.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7931b9e249edefb07cd6ae10c702788546341d5fe44db5b6108a25da4dca513f", size = 254257, upload-time = "2025-06-13T13:01:59.645Z" }, + { url = "https://files.pythonhosted.org/packages/93/df/27b882f54157fc1131e0e215b0da3b8d608d9b8ef79a045280118a8f98fe/coverage-7.9.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52e92b01041151bf607ee858e5a56c62d4b70f4dac85b8c8cb7fb8a351ab2c10", size = 255094, upload-time = "2025-06-13T13:02:01.37Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/cad1c3dbed8b3ee9e16fa832afe365b4e3eeab1fb6edb65ebbf745eabc92/coverage-7.9.1-cp313-cp313t-win32.whl", hash = "sha256:684e2110ed84fd1ca5f40e89aa44adf1729dc85444004111aa01866507adf363", size = 215437, upload-time = "2025-06-13T13:02:02.905Z" }, + { url = "https://files.pythonhosted.org/packages/99/4d/fad293bf081c0e43331ca745ff63673badc20afea2104b431cdd8c278b4c/coverage-7.9.1-cp313-cp313t-win_amd64.whl", hash = "sha256:437c576979e4db840539674e68c84b3cda82bc824dd138d56bead1435f1cb5d7", size = 216605, upload-time = "2025-06-13T13:02:05.638Z" }, + { url = "https://files.pythonhosted.org/packages/1f/56/4ee027d5965fc7fc126d7ec1187529cc30cc7d740846e1ecb5e92d31b224/coverage-7.9.1-cp313-cp313t-win_arm64.whl", hash = "sha256:18a0912944d70aaf5f399e350445738a1a20b50fbea788f640751c2ed9208b6c", size = 214392, upload-time = "2025-06-13T13:02:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e5/c723545c3fd3204ebde3b4cc4b927dce709d3b6dc577754bb57f63ca4a4a/coverage-7.9.1-pp39.pp310.pp311-none-any.whl", hash = "sha256:db0f04118d1db74db6c9e1cb1898532c7dcc220f1d2718f058601f7c3f499514", size = 204009, upload-time = "2025-06-13T13:02:25.787Z" }, + { url = "https://files.pythonhosted.org/packages/08/b8/7ddd1e8ba9701dea08ce22029917140e6f66a859427406579fd8d0ca7274/coverage-7.9.1-py3-none-any.whl", hash = "sha256:66b974b145aa189516b6bf2d8423e888b742517d37872f6ee4c5be0073bd9a3c", size = 204000, upload-time = "2025-06-13T13:02:27.173Z" }, +] + +[[package]] +name = "croniter" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/2f/44d1ae153a0e27be56be43465e5cb39b9650c781e001e7864389deb25090/croniter-6.0.0.tar.gz", hash = "sha256:37c504b313956114a983ece2c2b07790b1f1094fe9d81cc94739214748255577", size = 64481, upload-time = "2024-12-17T17:17:47.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4b/290b4c3efd6417a8b0c284896de19b1d5855e6dbdb97d2a35e68fa42de85/croniter-6.0.0-py2.py3-none-any.whl", hash = "sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368", size = 25468, upload-time = "2024-12-17T17:17:45.359Z" }, +] + +[[package]] +name = "django" +version = "5.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "sqlparse" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/af/77b403926025dc6f7fd7b31256394d643469418965eb528eab45d0505358/django-5.2.3.tar.gz", hash = "sha256:335213277666ab2c5cac44a792a6d2f3d58eb79a80c14b6b160cd4afc3b75684", size = 10850303, upload-time = "2025-06-10T10:14:05.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/11/7aff961db37e1ea501a2bb663d27a8ce97f3683b9e5b83d3bfead8b86fa4/django-5.2.3-py3-none-any.whl", hash = "sha256:c517a6334e0fd940066aa9467b29401b93c37cec2e61365d663b80922542069d", size = 8301935, upload-time = "2025-06-10T10:13:58.993Z" }, +] + +[[package]] +name = "django-tasks-scheduler" +version = "4.0.5" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "croniter" }, + { name = "django" }, +] + +[package.optional-dependencies] +sentry = [ + { name = "sentry-sdk" }, +] +valkey = [ + { name = "valkey" }, +] +yaml = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "fakeredis" }, + { name = "pyyaml" }, + { name = "ruff" }, + { name = "time-machine" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = "~=8.2" }, + { name = "croniter", specifier = ">=2.0" }, + { name = "django", specifier = ">=5" }, + { name = "pyyaml", marker = "extra == 'yaml'", specifier = "~=6.0" }, + { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = "~=2.19" }, + { name = "valkey", marker = "extra == 'valkey'", specifier = ">=6.0.2,<7" }, +] +provides-extras = ["yaml", "valkey", "sentry"] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", specifier = "~=7.6" }, + { name = "fakeredis", specifier = "~=2.28" }, + { name = "pyyaml", specifier = ">=6,<7" }, + { name = "ruff", specifier = ">=0.11" }, + { name = "time-machine", specifier = ">=2.16.0,<3" }, +] + +[[package]] +name = "fakeredis" +version = "2.30.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "redis" }, + { name = "sortedcontainers" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/3b/eb1d4d0fdc138df1d8e625dfa6b500189030e6c1c265b8dd22b783b2f9ec/fakeredis-2.30.1.tar.gz", hash = "sha256:6489f2926e39815c9bf0fce80751635e0898e333c43a767825adf101180dbc45", size = 167724, upload-time = "2025-06-19T17:55:45.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ee/acc3de71b8c66029ea4567d83e9c736d79836b2d97aa2cacf1b83f96c678/fakeredis-2.30.1-py3-none-any.whl", hash = "sha256:b594a9c20aef8b94c4d923f489210ef443e4001e62ad3cd73b9a01298dcef743", size = 116215, upload-time = "2025-06-19T17:55:43.893Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + +[[package]] +name = "redis" +version = "6.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/9a/0551e01ba52b944f97480721656578c8a7c46b51b99d66814f85fe3a4f3e/redis-6.2.0.tar.gz", hash = "sha256:e821f129b75dde6cb99dd35e5c76e8c49512a5a0d8dfdc560b2fbd44b85ca977", size = 4639129, upload-time = "2025-05-28T05:01:18.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/67/e60968d3b0e077495a8fee89cf3f2373db98e528288a48f1ee44967f6e8c/redis-6.2.0-py3-none-any.whl", hash = "sha256:c8ddf316ee0aab65f04a11229e94a64b2618451dab7a67cb2f77eb799d872d5e", size = 278659, upload-time = "2025-05-28T05:01:16.955Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/90/5255432602c0b196a0da6720f6f76b93eb50baef46d3c9b0025e2f9acbf3/ruff-0.12.0.tar.gz", hash = "sha256:4d047db3662418d4a848a3fdbfaf17488b34b62f527ed6f10cb8afd78135bc5c", size = 4376101, upload-time = "2025-06-17T15:19:26.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/fd/b46bb20e14b11ff49dbc74c61de352e0dc07fb650189513631f6fb5fc69f/ruff-0.12.0-py3-none-linux_armv6l.whl", hash = "sha256:5652a9ecdb308a1754d96a68827755f28d5dfb416b06f60fd9e13f26191a8848", size = 10311554, upload-time = "2025-06-17T15:18:45.792Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/021dde5a988fa3e25d2468d1dadeea0ae89dc4bc67d0140c6e68818a12a1/ruff-0.12.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:05ed0c914fabc602fc1f3b42c53aa219e5736cb030cdd85640c32dbc73da74a6", size = 11118435, upload-time = "2025-06-17T15:18:49.064Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/01a5acf495265c667686ec418f19fd5c32bcc326d4c79ac28824aecd6a32/ruff-0.12.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:07a7aa9b69ac3fcfda3c507916d5d1bca10821fe3797d46bad10f2c6de1edda0", size = 10466010, upload-time = "2025-06-17T15:18:51.341Z" }, + { url = "https://files.pythonhosted.org/packages/4c/57/7caf31dd947d72e7aa06c60ecb19c135cad871a0a8a251723088132ce801/ruff-0.12.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7731c3eec50af71597243bace7ec6104616ca56dda2b99c89935fe926bdcd48", size = 10661366, upload-time = "2025-06-17T15:18:53.29Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ba/aa393b972a782b4bc9ea121e0e358a18981980856190d7d2b6187f63e03a/ruff-0.12.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:952d0630eae628250ab1c70a7fffb641b03e6b4a2d3f3ec6c1d19b4ab6c6c807", size = 10173492, upload-time = "2025-06-17T15:18:55.262Z" }, + { url = "https://files.pythonhosted.org/packages/d7/50/9349ee777614bc3062fc6b038503a59b2034d09dd259daf8192f56c06720/ruff-0.12.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c021f04ea06966b02614d442e94071781c424ab8e02ec7af2f037b4c1e01cc82", size = 11761739, upload-time = "2025-06-17T15:18:58.906Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/ad459de67c70ec112e2ba7206841c8f4eb340a03ee6a5cabc159fe558b8e/ruff-0.12.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:7d235618283718ee2fe14db07f954f9b2423700919dc688eacf3f8797a11315c", size = 12537098, upload-time = "2025-06-17T15:19:01.316Z" }, + { url = "https://files.pythonhosted.org/packages/ed/50/15ad9c80ebd3c4819f5bd8883e57329f538704ed57bac680d95cb6627527/ruff-0.12.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0758038f81beec8cc52ca22de9685b8ae7f7cc18c013ec2050012862cc9165", size = 12154122, upload-time = "2025-06-17T15:19:03.727Z" }, + { url = "https://files.pythonhosted.org/packages/76/e6/79b91e41bc8cc3e78ee95c87093c6cacfa275c786e53c9b11b9358026b3d/ruff-0.12.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:139b3d28027987b78fc8d6cfb61165447bdf3740e650b7c480744873688808c2", size = 11363374, upload-time = "2025-06-17T15:19:05.875Z" }, + { url = "https://files.pythonhosted.org/packages/db/c3/82b292ff8a561850934549aa9dc39e2c4e783ab3c21debe55a495ddf7827/ruff-0.12.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68853e8517b17bba004152aebd9dd77d5213e503a5f2789395b25f26acac0da4", size = 11587647, upload-time = "2025-06-17T15:19:08.246Z" }, + { url = "https://files.pythonhosted.org/packages/2b/42/d5760d742669f285909de1bbf50289baccb647b53e99b8a3b4f7ce1b2001/ruff-0.12.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3a9512af224b9ac4757f7010843771da6b2b0935a9e5e76bb407caa901a1a514", size = 10527284, upload-time = "2025-06-17T15:19:10.37Z" }, + { url = "https://files.pythonhosted.org/packages/19/f6/fcee9935f25a8a8bba4adbae62495c39ef281256693962c2159e8b284c5f/ruff-0.12.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b08df3d96db798e5beb488d4df03011874aff919a97dcc2dd8539bb2be5d6a88", size = 10158609, upload-time = "2025-06-17T15:19:12.286Z" }, + { url = "https://files.pythonhosted.org/packages/37/fb/057febf0eea07b9384787bfe197e8b3384aa05faa0d6bd844b94ceb29945/ruff-0.12.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6a315992297a7435a66259073681bb0d8647a826b7a6de45c6934b2ca3a9ed51", size = 11141462, upload-time = "2025-06-17T15:19:15.195Z" }, + { url = "https://files.pythonhosted.org/packages/10/7c/1be8571011585914b9d23c95b15d07eec2d2303e94a03df58294bc9274d4/ruff-0.12.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e55e44e770e061f55a7dbc6e9aed47feea07731d809a3710feda2262d2d4d8a", size = 11641616, upload-time = "2025-06-17T15:19:17.6Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/b960ab4818f90ff59e571d03c3f992828d4683561095e80f9ef31f3d58b7/ruff-0.12.0-py3-none-win32.whl", hash = "sha256:7162a4c816f8d1555eb195c46ae0bd819834d2a3f18f98cc63819a7b46f474fb", size = 10525289, upload-time = "2025-06-17T15:19:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/34/93/8b16034d493ef958a500f17cda3496c63a537ce9d5a6479feec9558f1695/ruff-0.12.0-py3-none-win_amd64.whl", hash = "sha256:d00b7a157b8fb6d3827b49d3324da34a1e3f93492c1f97b08e222ad7e9b291e0", size = 11598311, upload-time = "2025-06-17T15:19:21.785Z" }, + { url = "https://files.pythonhosted.org/packages/d0/33/4d3e79e4a84533d6cd526bfb42c020a23256ae5e4265d858bd1287831f7d/ruff-0.12.0-py3-none-win_arm64.whl", hash = "sha256:8cd24580405ad8c1cc64d61725bca091d6b6da7eb3d36f72cc605467069d7e8b", size = 10724946, upload-time = "2025-06-17T15:19:23.952Z" }, +] + +[[package]] +name = "sentry-sdk" +version = "2.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/4c/af31e0201b48469786ddeb1bf6fd3dfa3a291cc613a0fe6a60163a7535f9/sentry_sdk-2.30.0.tar.gz", hash = "sha256:436369b02afef7430efb10300a344fb61a11fe6db41c2b11f41ee037d2dd7f45", size = 326767, upload-time = "2025-06-12T10:34:34.733Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/99/31ac6faaae33ea698086692638f58d14f121162a8db0039e68e94135e7f1/sentry_sdk-2.30.0-py2.py3-none-any.whl", hash = "sha256:59391db1550662f746ea09b483806a631c3ae38d6340804a1a4c0605044f6877", size = 343149, upload-time = "2025-06-12T10:34:32.896Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "sqlparse" +version = "0.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/40/edede8dd6977b0d3da179a342c198ed100dd2aba4be081861ee5911e4da4/sqlparse-0.5.3.tar.gz", hash = "sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272", size = 84999, upload-time = "2024-12-10T12:05:30.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/5c/bfd6bd0bf979426d405cc6e71eceb8701b148b16c21d2dc3c261efc61c7b/sqlparse-0.5.3-py3-none-any.whl", hash = "sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca", size = 44415, upload-time = "2024-12-10T12:05:27.824Z" }, +] + +[[package]] +name = "time-machine" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/dd/5022939b9cadefe3af04f4012186c29b8afbe858b1ec2cfa38baeec94dab/time_machine-2.16.0.tar.gz", hash = "sha256:4a99acc273d2f98add23a89b94d4dd9e14969c01214c8514bfa78e4e9364c7e2", size = 24626, upload-time = "2024-10-08T14:21:59.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/47/32fdb8e70122edbc8be9db1f032d22b38e3d9ef0bf52c64470d0815cdb62/time_machine-2.16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:09531af59fdfb39bfd24d28bd1e837eff5a5d98318509a31b6cfd57d27801e52", size = 20493, upload-time = "2024-10-08T14:20:51.241Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e6/f3bc391d5642e69299f2d1f0a46e7f98d1669e82b1e16c8cf3c6e4615059/time_machine-2.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:92d0b0f3c49f34dd76eb462f0afdc61ed1cb318c06c46d03e99b44ebb489bdad", size = 16757, upload-time = "2024-10-08T14:20:52.614Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7f/3a78d50fec64edd9964bf42b66a2e659a9846669ac8f705acc363ee79d3a/time_machine-2.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c29616e18e2349a8766d5b6817920fc74e39c00fa375d202231e9d525a1b882", size = 34527, upload-time = "2024-10-08T14:20:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/61/00/7cf1324d8f8db8f5dab71c44ed1e9c11c4f1cecca9d4363abf44154aa13b/time_machine-2.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1ceb6035a64cb00650e3ab203cf3faffac18576a3f3125c24df468b784077c7", size = 32537, upload-time = "2024-10-08T14:20:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/8e/c2/edf5ccb2fa529251eb7f1cfb34098c0ef236dbb88f0a6564d06f6f8378f5/time_machine-2.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64c205ea37b8c4ba232645335fc3b75bc2d03ce30f0a34649e36cae85652ee96", size = 34353, upload-time = "2024-10-08T14:20:56.414Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1e/178b9e3d0054300a4dd0485747c89359e5f719f090ae5165c88618793700/time_machine-2.16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dfe92412bd11104c4f0fb2da68653e6c45b41f7217319a83a8b66ed4f20148b3", size = 34045, upload-time = "2024-10-08T14:20:58.117Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4d/068ad9660f00f88a54f3ff7e9d423ed5c08a5f8147518f6c66fd0393dde7/time_machine-2.16.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d5fe7a6284e3dce87ae13a25029c53542dd27a28d151f3ef362ec4dd9c3e45fd", size = 32356, upload-time = "2024-10-08T14:20:59.8Z" }, + { url = "https://files.pythonhosted.org/packages/a5/25/c0f26294808946ec5b665f17a0072049a3f9e2468abc18aa8fe22580b4cf/time_machine-2.16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0fca3025266d88d1b48be162a43b7c2d91c81cc5b3bee9f01194678ffb9969a", size = 33737, upload-time = "2024-10-08T14:21:00.868Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/ae909a269828eaa7672e1201403976e794ea679ae7ba04fe0c0c0c65c2b6/time_machine-2.16.0-cp310-cp310-win32.whl", hash = "sha256:4149e17018af07a5756a1df84aea71e6e178598c358c860c6bfec42170fa7970", size = 19133, upload-time = "2024-10-08T14:21:01.975Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e7/5946d62d49e79b97c6772fe2918eccbd069d74effa8d50bdca4056502aeb/time_machine-2.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:01bc257e9418980a4922de94775be42a966e1a082fb01a1635917f9afc7b84ca", size = 19995, upload-time = "2024-10-08T14:21:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/54/cb/6507c6594f086bc955ff200cc4fd415d2ab229371ca3ba8fc3d27429a9cc/time_machine-2.16.0-cp310-cp310-win_arm64.whl", hash = "sha256:6895e3e84119594ab12847c928f619d40ae9cedd0755515dc154a5b5dc6edd9f", size = 18109, upload-time = "2024-10-08T14:21:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/38/7b/34aad93f75f86503dd1fa53bc120d8129fe4de83aef58ffa78c62b044ef9/time_machine-2.16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8f936566ef9f09136a3d5db305961ef6d897b76b240c9ff4199144aed6dd4fe5", size = 20169, upload-time = "2024-10-08T14:21:06.288Z" }, + { url = "https://files.pythonhosted.org/packages/68/cb/7d020d5c05d0460a4a96232b0777882ef989c1e6144d11ba984c4b0b4d1a/time_machine-2.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5886e23ede3478ca2a3e0a641f5d09dd784dfa9e48c96e8e5e31fc4fe77b6dc0", size = 16614, upload-time = "2024-10-08T14:21:07.253Z" }, + { url = "https://files.pythonhosted.org/packages/0d/24/ce1ff76c9a4f3be88c2b947f2411a5a8019390734597d3106a151f8a9416/time_machine-2.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76caf539fa4941e1817b7c482c87c65c52a1903fea761e84525955c6106fafb", size = 32507, upload-time = "2024-10-08T14:21:08.289Z" }, + { url = "https://files.pythonhosted.org/packages/08/d7/ba1135587bd2ed105e59ed7e05969c913277d110fecc0ed871006ea3f763/time_machine-2.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:298aa423e07c8b21b991782f01d7749c871c792319c2af3e9755f9ab49033212", size = 30627, upload-time = "2024-10-08T14:21:09.246Z" }, + { url = "https://files.pythonhosted.org/packages/da/c6/f490aaddc80c54238f4b8fe97870bbfe0d2c70fe4a57269badc94f5f38a6/time_machine-2.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391ae9c484736850bb44ef125cbad52fe2d1b69e42c95dc88c43af8ead2cc7", size = 32362, upload-time = "2024-10-08T14:21:10.178Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f7/2522ae1c1995a39d6d8b7ee7efed47ec8bd7ff3240fdb2662a8b7e11b84a/time_machine-2.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:503e7ff507c2089699d91885fc5b9c8ff16774a7b6aff48b4dcee0c0a0685b61", size = 32188, upload-time = "2024-10-08T14:21:11.743Z" }, + { url = "https://files.pythonhosted.org/packages/e9/53/b1ccb55f39e7e62660f852d7aedef438d2872ea9c73f64be46d0d3b3f3d7/time_machine-2.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eee7b0fc4fbab2c6585ea17606c6548be83919c70deea0865409fe9fc2d8cdce", size = 30600, upload-time = "2024-10-08T14:21:12.728Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/37a5a9333a2da35b0fc43e8ac693b82dd5492892131bc3cc0c8f5835af94/time_machine-2.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9db5e5b3ccdadaafa5730c2f9db44c38b013234c9ad01f87738907e19bdba268", size = 31896, upload-time = "2024-10-08T14:21:14.451Z" }, + { url = "https://files.pythonhosted.org/packages/fc/97/e1a8bd64e5432adf47859cb63847b4472efc644b508602141c60ccf52112/time_machine-2.16.0-cp311-cp311-win32.whl", hash = "sha256:2552f0767bc10c9d668f108fef9b487809cdeb772439ce932e74136365c69baf", size = 19030, upload-time = "2024-10-08T14:21:15.662Z" }, + { url = "https://files.pythonhosted.org/packages/34/c9/f4764e447aa9da4031c89da60fa69f4f73fd45571415788c298cbd4620e9/time_machine-2.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:12474fcdbc475aa6fe5275fe7224e685c5b9777f5939647f35980e9614ae7558", size = 19924, upload-time = "2024-10-08T14:21:17.324Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c0/788500d33656a044e3289b814106c2277209ac73316c00b9668012ce6027/time_machine-2.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:ac2df0fa564356384515ed62cb6679f33f1f529435b16b0ec0f88414635dbe39", size = 17993, upload-time = "2024-10-08T14:21:18.346Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/603a84e7ae6427a53953db9f61b689dc6adf233e03c5f5ca907a901452fd/time_machine-2.16.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:84788f4d62a8b1bf5e499bb9b0e23ceceea21c415ad6030be6267ce3d639842f", size = 20155, upload-time = "2024-10-08T14:21:20.055Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/dbe69aecb4b84be52d34814e63176c5ca61f38ee9e6ecda11104653405b5/time_machine-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:15ec236b6571730236a193d9d6c11d472432fc6ab54e85eac1c16d98ddcd71bf", size = 16640, upload-time = "2024-10-08T14:21:22.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/13/27f11be25d7bd298e033b9da93217e5b68309bf724b6e494cdadb471d00d/time_machine-2.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cedc989717c8b44a3881ac3d68ab5a95820448796c550de6a2149ed1525157f0", size = 33721, upload-time = "2024-10-08T14:21:23.059Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9d/70e4640fed1fd8122204ae825c688d0ef8c04f515ec6bf3c5f3086d6510e/time_machine-2.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d26d79de1c63a8c6586c75967e09b0ff306aa7e944a1eaddb74595c9b1839ca", size = 31646, upload-time = "2024-10-08T14:21:24.037Z" }, + { url = "https://files.pythonhosted.org/packages/a1/cb/93bc0e51bea4e171a85151dbba3c3b3f612b50b953cd3076f5b4f0db9e14/time_machine-2.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:317b68b56a9c3731e0cf8886e0f94230727159e375988b36c60edce0ddbcb44a", size = 33403, upload-time = "2024-10-08T14:21:24.975Z" }, + { url = "https://files.pythonhosted.org/packages/89/71/2c6a63ad4fbce3d62d46bbd9ac4433f30bade7f25978ce00815b905bcfcf/time_machine-2.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:43e1e18279759897be3293a255d53e6b1cb0364b69d9591d0b80c51e461c94b0", size = 33327, upload-time = "2024-10-08T14:21:25.958Z" }, + { url = "https://files.pythonhosted.org/packages/68/4e/205c2b26763b8817cd6b8868242843800a1fbf275f2af35f5ba35ff2b01a/time_machine-2.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e43adb22def972a29d2b147999b56897116085777a0fea182fd93ee45730611e", size = 31454, upload-time = "2024-10-08T14:21:27.367Z" }, + { url = "https://files.pythonhosted.org/packages/d7/95/44c1aa3994919f93534244c40cfd2fb9416d7686dc0c8b9b262c751b5118/time_machine-2.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0c766bea27a0600e36806d628ebc4b47178b12fcdfb6c24dc0a566a9c06bfe7f", size = 32972, upload-time = "2024-10-08T14:21:28.351Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ee/75243df9c7cf30f108758e887141a58e6544baaa46e2e647b9ccc56db819/time_machine-2.16.0-cp312-cp312-win32.whl", hash = "sha256:6dae82ab647d107817e013db82223e20a9853fa88543fec853ae326382d03c2e", size = 19078, upload-time = "2024-10-08T14:21:29.425Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7c/d4e67cc031f9653c92167ccf87d241e3208653d191c96ac79281c273ab92/time_machine-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:265462c77dc9576267c3c7f20707780a171a9fdbac93ac22e608c309efd68c33", size = 19923, upload-time = "2024-10-08T14:21:30.759Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b6/7047226fcb9afefe47fc80f605530535bf71ad99b6797f057abbfa4cd9a5/time_machine-2.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:ef768e14768eebe3bb1196c0dece8e14c1c6991605721214a0c3c68cf77eb216", size = 18003, upload-time = "2024-10-08T14:21:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/a6/18/3087d0eb185cedbc82385f46bf16032ec7102a0e070205a2c88c4ecf9952/time_machine-2.16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7751bf745d54e9e8b358c0afa332815da9b8a6194b26d0fd62876ab6c4d5c9c0", size = 20209, upload-time = "2024-10-08T14:21:34.222Z" }, + { url = "https://files.pythonhosted.org/packages/03/a3/fcc3eaf69390402ecf491d718e533b6d0e06d944d77fc8d87be3a2839102/time_machine-2.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1784edf173ca840ba154de6eed000b5727f65ab92972c2f88cec5c4d6349c5f2", size = 16681, upload-time = "2024-10-08T14:21:35.14Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/8b76d264014bf9dc21873218de50d67223c71736f87fe6c65e582f7c29ac/time_machine-2.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f5876a5682ce1f517e55d7ace2383432627889f6f7e338b961f99d684fd9e8d", size = 33768, upload-time = "2024-10-08T14:21:36.942Z" }, + { url = "https://files.pythonhosted.org/packages/5c/13/59ae8259be02b6c657ef6e3b6952bf274b43849f6f35cc61a576c68ce301/time_machine-2.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:806672529a2e255cd901f244c9033767dc1fa53466d0d3e3e49565a1572a64fe", size = 31685, upload-time = "2024-10-08T14:21:37.881Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c1/9f142beb4d373a2a01ebb58d5117289315baa5131d880ec804db49e94bf7/time_machine-2.16.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:667b150fedb54acdca2a4bea5bf6da837b43e6dd12857301b48191f8803ba93f", size = 33447, upload-time = "2024-10-08T14:21:38.809Z" }, + { url = "https://files.pythonhosted.org/packages/95/f7/ed9ecd93c2d38dca77d0a28e070020f3ce0fb23e0d4a6edb14bcfffa5526/time_machine-2.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:da3ae1028af240c0c46c79adf9c1acffecc6ed1701f2863b8132f5ceae6ae4b5", size = 33408, upload-time = "2024-10-08T14:21:39.785Z" }, + { url = "https://files.pythonhosted.org/packages/91/40/d0d274d70fa2c4cad531745deb8c81346365beb0a2736be05a3acde8b94a/time_machine-2.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:520a814ea1b2706c89ab260a54023033d3015abef25c77873b83e3d7c1fafbb2", size = 31526, upload-time = "2024-10-08T14:21:40.769Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ba/a27cdbb324d9a6d779cde0d514d47b696b5a6a653705d4b511fd65ef1514/time_machine-2.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8243664438bb468408b29c6865958662d75e51f79c91842d2794fa22629eb697", size = 33042, upload-time = "2024-10-08T14:21:41.722Z" }, + { url = "https://files.pythonhosted.org/packages/72/63/64e9156c9e38c18720d0cc41378168635241de44013ffe3dd5b099447eb0/time_machine-2.16.0-cp313-cp313-win32.whl", hash = "sha256:32d445ce20d25c60ab92153c073942b0bac9815bfbfd152ce3dcc225d15ce988", size = 19108, upload-time = "2024-10-08T14:21:43.596Z" }, + { url = "https://files.pythonhosted.org/packages/3d/40/27f5738fbd50b78dcc0682c14417eac5a49ccf430525dd0c5a058be125a2/time_machine-2.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:f6927dda86425f97ffda36131f297b1a601c64a6ee6838bfa0e6d3149c2f0d9f", size = 19935, upload-time = "2024-10-08T14:21:45.277Z" }, + { url = "https://files.pythonhosted.org/packages/35/75/c4d8b2f0fe7dac22854d88a9c509d428e78ac4bf284bc54cfe83f75cc13b/time_machine-2.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:4d3843143c46dddca6491a954bbd0abfd435681512ac343169560e9bab504129", size = 18047, upload-time = "2024-10-08T14:21:46.261Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] + +[[package]] +name = "valkey" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/d4/04b6a234e584e21ccb63b895ca9dfb4e759e4c139c1ab3f9484982ee6491/valkey-6.1.0.tar.gz", hash = "sha256:a652df15ed89c41935ffae6dfd09c56f4a9ab80b592e5ed9204d538e2ddad6d3", size = 4600944, upload-time = "2025-02-11T17:20:46.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/b0/c4d47032bbda89cff7af99c0b096db9b9453b9f0c1e24cf027aa616be389/valkey-6.1.0-py3-none-any.whl", hash = "sha256:cfe769edae894f74ac946eff1e93f7d7f466032c3030ba7e9d089a742459ac9c", size = 259302, upload-time = "2025-02-11T17:20:42.96Z" }, +]

Name	Scheduled Jobs	Queued Jobs	Oldest Queued Job	Active Jobs	Deferred Jobs	Finished Jobs	Failed Jobs	Scheduled Jobs	Canceled Jobs	Oldest Queued Job	Workers	Host	Port
+	{{ queue.name }} - +	+ + {{ queue.scheduled_jobs }} + +	- {{ queue.jobs }} + {{ queue.queued_jobs }}	{{ queue.oldest_job_timestamp }}	- +	+ {{ queue.started_jobs }} - -	- - {{ queue.deferred_jobs }} - -	+ +	{{ queue.finished_jobs }} - -	+ +	{{ queue.failed_jobs }} - -	- - {{ queue.scheduled_jobs }} - -	+ +	{{ queue.canceled_jobs }} - -	- {{ queue.workers }} - -	{{ queue.oldest_job_timestamp }}	+ + {{ queue.workers }} + +	{{ queue.connection_kwargs.host }}	{{ queue.connection_kwargs.port }}	{{ queue.connection_kwargs.db }}	Name	+ Queues +	State	{{ worker.get_state }}	{{ worker.birth_date | date:"Y-m-d, H:i:s" }}	+ {% for queue_name in worker.queue_names %} + {{ queue_name }} + {% if not forloop.last %},{% endif %} + {% endfor %} +	{{ worker.state.value | capfirst }}	{{ worker.birth | date:"Y-m-d, H:i:s" }}	{{ worker.hostname }}	{{ worker.pid | unlocalize }}	{{ worker.total_working_time | default:0 | floatformat }} secs	{{ worker.total_working_time_ms | default:0 | floatformat }} secs	{{ worker.successful_job_count | default:0 }}	{{ worker.failed_job_count | default:0 }}	{{ worker | worker_scheduler_pid }}	{{ worker.has_scheduler }}