Add new cleanup task

gso/migrations/versions/2024-08-13_844aa61c09ce_add_new_cleanup_task.py

+"""Add new cleanup task.
+
+Revision ID: 844aa61c09ce
+Revises: 88dd5a44150d
+Create Date: 2024-08-13 12:23:11.043293
+
+"""
+
+from uuid import uuid4
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '844aa61c09ce'
+down_revision = '88dd5a44150d'
+branch_labels = None
+depends_on = None
+
+workflows = [
+    {
+        "name": "task_clean_old_tasks",
+        "target": "SYSTEM",
+        "description": "Remove old cleanup tasks",
+        "workflow_id": uuid4(),
+    }
+]
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    for workflow in workflows:
+        conn.execute(
+            sa.text(
+                "INSERT INTO workflows VALUES (:workflow_id, :name, :target, :description, now())"
+            ),
+            workflow,
+        )
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    for workflow in workflows:
+        conn.execute(sa.text("DELETE FROM workflows WHERE name = :name"), {"name": workflow["name"]})

gso/schedules/clean_old_tasks.py

+"""Metatask that cleans up old cleanup tasks."""
+
+from orchestrator.services.processes import start_process
+
+from gso.schedules.scheduling import CronScheduleConfig, scheduler
+from gso.worker import celery
+
+
+@celery.task
+@scheduler(CronScheduleConfig(name="Clean up tasks", hour="3"))
+def clean_old_tasks() -> None:
+    """Run all cleanup tasks every 3 AM UTC."""
+    start_process("task_clean_old_tasks")

gso/translations/en-GB.json

@@ -72,6 +72,7 @@
         "task_create_partners": "Create partner task",
         "task_modify_partners": "Modify partner task",
         "task_delete_partners": "Delete partner task",
+        "task_clean_old_tasks": "Remove old cleanup tasks",
         "promote_p_to_pe": "Promote P to PE"
     }
 }

gso/worker.py

@@ -25,6 +25,7 @@ celery = OrchestratorCelery(
         "gso.schedules.validate_products",
         "gso.schedules.validate_subscriptions",
         "gso.schedules.send_email_notifications",
+        "gso.schedules.clean_old_tasks",
     ],
 )
 

gso/workflows/__init__.py

@@ -74,3 +74,4 @@ LazyWorkflowInstance("gso.workflows.tasks.validate_geant_products", "task_valida
 LazyWorkflowInstance("gso.workflows.tasks.create_partners", "task_create_partners")
 LazyWorkflowInstance("gso.workflows.tasks.modify_partners", "task_modify_partners")
 LazyWorkflowInstance("gso.workflows.tasks.delete_partners", "task_delete_partners")
+LazyWorkflowInstance("gso.workflows.tasks.clean_old_tasks", "task_clean_old_tasks")

gso/workflows/tasks/clean_old_tasks.py

+"""A cleanup task for removing past runs."""
+
+from orchestrator.db import ProcessTable, WorkflowTable, db
+from orchestrator.targets import Target
+from orchestrator.workflow import ProcessStatus, StepList, begin, done, step, workflow
+from sqlalchemy import or_, select
+
+
+@step("Clean up all cleanup tasks")
+def remove_meta_tasks() -> None:
+    """Find and remove runs of old cleanup tasks."""
+    cleanup_tasks = WorkflowTable.query.filter(
+        or_(WorkflowTable.name == "task_clean_up_tasks", WorkflowTable.name == "task_clean_old_tasks")
+    ).all()
+
+    for cleanup_task in cleanup_tasks:
+        tasks = db.session.scalars(
+            select(ProcessTable)
+            .filter(ProcessTable.is_task.is_(True))
+            .filter(ProcessTable.workflow_id == cleanup_task.workflow_id)
+            .filter(
+                or_(
+                    ProcessTable.last_status == ProcessStatus.COMPLETED.value,
+                    ProcessTable.last_status == ProcessStatus.CREATED.value,
+                )
+            )
+        )
+
+        for task in tasks:
+            db.session.delete(task)
+
+
+@workflow("Remove old cleanup tasks", target=Target.SYSTEM)
+def task_clean_old_tasks() -> StepList:
+    """Remove all runs of old cleanup tasks.
+
+    This will look for all past executions of the ``orchestrator-core`` built-in cleanup task, and this current cleanup
+    task. Once they have run to completion, or are stuck in a "created" state, they serve no purpose in the database and
+    can therefore be removed.
+    """
+    return begin >> remove_meta_tasks >> done

test-docs.sh

 #!/bin/bash
 
-if [ ! -d ./docs/vale/styles/proselint ] || [ ! -d ./docs/vale/styles/Microsoft ]; then
-  docker run -it --rm -v "$(pwd)"/docs:/docs jdkato/vale:latest --config="/docs/vale/.vale.ini" sync
-fi
-
-docker run -it --rm -v "$(pwd)":/gso jdkato/vale:latest --glob='!*/migrations/*' \
---config="/gso/docs/vale/.vale.ini" /gso/docs/source /gso/gso
+vale --config=docs/vale/.vale.ini sync
+vale --glob='!*/migrations/*' --config=docs/vale/.vale.ini docs/source gso