From 25ca6784003df71f1ffcd62ee6fd19d7d3ab833c Mon Sep 17 00:00:00 2001 From: Karel van Klink <karel.vanklink@geant.org> Date: Fri, 20 Jun 2025 11:20:58 +0200 Subject: [PATCH 1/2] Add modify_note workflow to all existing products --- ...5_add_modify_note_workflow_to_existing_.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py diff --git a/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py b/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py new file mode 100644 index 000000000..7414647ef --- /dev/null +++ b/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py @@ -0,0 +1,70 @@ +"""Add modify note workflow to existing products. + +Revision ID: 24858fd1d805 +Revises: 550e3aebc1c5 +Create Date: 2025-06-20 10:51:57.321841 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '24858fd1d805' +down_revision = '550e3aebc1c5' +branch_labels = None +depends_on = None + + +from orchestrator.migrations.helpers import create_workflow, delete_workflow, add_products_to_workflow_by_product_tag, \ + remove_products_from_workflow_by_product_tag + +product_tags = [ + "ER", + "COP", + "EP", + "G_IP", + "G_PLUS", + "IAS", + "IMP_ER", + "IMP_COP", + "IMP_EP", + "IMP_G_IP", + "IMP_G_PLUS", + "IMP_IAS", + "IMP_IP_TRUNK", + "IMP_LSI", + "IMP_LHC", + "IMP_OFFICE_RTR", + "IMPORTED_OPENGEAR", + "IMP_RE_LHCONE", + "IMP_RE_PEER", + "IMP_RTR", + "IMP_SITE", + "IMP_SPOP_SWITCH", + "IMP_SWITCH", + "IPTRUNK", + "LSI", + "LHC", + "OFFICE_ROUTER", + "OPENGEAR", + "POP_VLAN", + "RE_LHCONE", + "RE_PEER", + "RTR", + "SITE", + "Super_POP_SWITCH", + "SWITCH", + "VRF", +] + + +def upgrade() -> None: + conn = op.get_bind() + for product in product_tags: + add_products_to_workflow_by_product_tag(conn, "modify_note", product) + + +def downgrade() -> None: + conn = op.get_bind() + for product in product_tags: + remove_products_from_workflow_by_product_tag(conn, "modify_note", product) -- GitLab From e8e0da07a2163d886fdbbce61c23288e71e7468f Mon Sep 17 00:00:00 2001 From: Karel van Klink <karel.vanklink@geant.org> Date: Fri, 20 Jun 2025 11:43:43 +0200 Subject: [PATCH 2/2] Solve a FIXME and revert to the vanilla implementation of task_validate_products over our custom one --- ...4cd282a_remove_obsolete_validation_task.py | 35 ++++++++++++++ gso/schedules/validate_products.py | 4 +- gso/services/processes.py | 13 ------ gso/translations/en-GB.json | 1 - gso/workflows/__init__.py | 1 - .../tasks/validate_geant_products.py | 46 ------------------- ...y => test_massive_redeploy_base_config.py} | 0 .../tasks/test_task_validate_products.py | 7 +-- 8 files changed, 40 insertions(+), 67 deletions(-) create mode 100644 gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py delete mode 100644 gso/workflows/tasks/validate_geant_products.py rename test/tasks/{test_masssive_redeploy_base_config.py => test_massive_redeploy_base_config.py} (100%) diff --git a/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py b/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py new file mode 100644 index 000000000..9efde60de --- /dev/null +++ b/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py @@ -0,0 +1,35 @@ +"""Remove obsolete validation task. + +Revision ID: 7c3094cd282a +Revises: 24858fd1d805 +Create Date: 2025-06-20 11:34:08.439370 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '7c3094cd282a' +down_revision = '24858fd1d805' +branch_labels = None +depends_on = None + + +from orchestrator.migrations.helpers import create_task, delete_workflow + +old_task = { + "name": "task_validate_geant_products", + "description": "Validate GEANT products" +} + +def upgrade() -> None: + conn = op.get_bind() + conn.execute(sa.text(f""" +DELETE FROM processes WHERE workflow_id = (SELECT workflow_id FROM workflows WHERE name = '{old_task["name"]}') + """)) + delete_workflow(conn, old_task["name"]) + + +def downgrade() -> None: + conn = op.get_bind() + create_task(conn, old_task) diff --git a/gso/schedules/validate_products.py b/gso/schedules/validate_products.py index efc441ad2..1e693fe3a 100644 --- a/gso/schedules/validate_products.py +++ b/gso/schedules/validate_products.py @@ -4,12 +4,10 @@ from celery import shared_task from orchestrator.services.processes import start_process from gso.schedules.scheduling import CronScheduleConfig, scheduler -from gso.services.processes import count_incomplete_validate_products @shared_task @scheduler(CronScheduleConfig(name="Validate Products and inactive subscriptions", minute="30", hour="2")) def validate_products() -> None: """Validate all products.""" - if count_incomplete_validate_products() == 0: - start_process("task_validate_geant_products") + start_process("task_validate_products") diff --git a/gso/services/processes.py b/gso/services/processes.py index 30caa96fd..22bfe4e0d 100644 --- a/gso/services/processes.py +++ b/gso/services/processes.py @@ -18,19 +18,6 @@ def get_processes_by_workflow_name(workflow_name: str) -> Query: return ProcessTable.query.join(WorkflowTable).filter(WorkflowTable.name == workflow_name) -def count_incomplete_validate_products() -> int: - """Count the number of incomplete validate_geant_products processes. - - Returns: - The count of incomplete 'validate_geant_products' processes. - """ - return ( - get_processes_by_workflow_name("validate_geant_products") - .filter(ProcessTable.last_status != ProcessStatus.COMPLETED) - .count() - ) - - def get_failed_tasks() -> list[ProcessTable]: """Get all tasks that have failed.""" return ProcessTable.query.filter( diff --git a/gso/translations/en-GB.json b/gso/translations/en-GB.json index e3ea7bc6c..bbb37da1b 100644 --- a/gso/translations/en-GB.json +++ b/gso/translations/en-GB.json @@ -163,7 +163,6 @@ "task_modify_partners": "Modify partner task", "task_redeploy_base_config": "Redeploy base config on multiple routers", "task_send_email_notifications": "Send email notifications for failed tasks", - "task_validate_geant_products": "Validation task for GEANT products", "terminate_edge_port": "Terminate Edge Port", "terminate_iptrunk": "Terminate IP Trunk", "terminate_geant_ip": "Terminate GÉANT IP", diff --git a/gso/workflows/__init__.py b/gso/workflows/__init__.py index 3456115a4..827d2a07c 100644 --- a/gso/workflows/__init__.py +++ b/gso/workflows/__init__.py @@ -104,7 +104,6 @@ LazyWorkflowInstance("gso.workflows.opengear.import_opengear", "import_opengear" # Tasks LazyWorkflowInstance("gso.workflows.tasks.send_email_notifications", "task_send_email_notifications") -LazyWorkflowInstance("gso.workflows.tasks.validate_geant_products", "task_validate_geant_products") LazyWorkflowInstance("gso.workflows.tasks.create_partners", "task_create_partners") LazyWorkflowInstance("gso.workflows.tasks.modify_partners", "task_modify_partners") LazyWorkflowInstance("gso.workflows.tasks.delete_partners", "task_delete_partners") diff --git a/gso/workflows/tasks/validate_geant_products.py b/gso/workflows/tasks/validate_geant_products.py deleted file mode 100644 index 8fe39f61b..000000000 --- a/gso/workflows/tasks/validate_geant_products.py +++ /dev/null @@ -1,46 +0,0 @@ -"""A task that checks for all products in the database to be well-kept.""" - -# <!-- vale off --> -# Copyright 2019-2020 SURF. -# Copyright 2024 GÉANT Vereniging. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# <!-- vale on --> - -from orchestrator.targets import Target -from orchestrator.workflow import StepList, done, init, workflow -from orchestrator.workflows.tasks.validate_products import ( - check_all_workflows_are_in_db, - check_db_fixed_input_config, - check_subscription_models, - check_that_products_have_create_modify_and_terminate_workflows, - check_workflows_for_matching_targets_and_descriptions, -) - - -@workflow("Validate GEANT products", target=Target.SYSTEM) -def task_validate_geant_products() -> StepList: - """Validate products in the database. - - This task is based on the ``task_validate_products`` present in ``orchestrator-core`` but it does not check for the - existence of the ``modify_note`` workflow on all products, since this workflow is currently not used in GEANT. - """ - return ( - init - >> check_all_workflows_are_in_db - >> check_workflows_for_matching_targets_and_descriptions - # >> check_that_products_have_at_least_one_workflow FIXME: Uncomment as soon as this would pass again - >> check_db_fixed_input_config - >> check_that_products_have_create_modify_and_terminate_workflows - >> check_subscription_models - >> done - ) diff --git a/test/tasks/test_masssive_redeploy_base_config.py b/test/tasks/test_massive_redeploy_base_config.py similarity index 100% rename from test/tasks/test_masssive_redeploy_base_config.py rename to test/tasks/test_massive_redeploy_base_config.py diff --git a/test/workflows/tasks/test_task_validate_products.py b/test/workflows/tasks/test_task_validate_products.py index 66853d257..6c7398b92 100644 --- a/test/workflows/tasks/test_task_validate_products.py +++ b/test/workflows/tasks/test_task_validate_products.py @@ -4,14 +4,15 @@ from test.workflows import assert_complete, extract_state, run_workflow @pytest.mark.workflow() -def test_task_validate_geant_products(): - result, _, _ = run_workflow("task_validate_geant_products", [{}]) +def test_task_validate_products(): + result, _, _ = run_workflow("task_validate_products", [{}]) assert_complete(result) state = extract_state(result) assert state["check_all_workflows_are_in_db"] assert state["check_workflows_for_matching_targets_and_descriptions"] - # assert state["check_that_products_have_at_least_one_workflow"] FIXME: Uncomment when the task is reverted again + assert state["check_that_products_have_at_least_one_workflow"] + assert state["check_that_active_products_have_a_modify_note"] assert state["check_db_fixed_input_config"] assert state["check_that_products_have_create_modify_and_terminate_workflows"] assert state["check_subscription_models"] -- GitLab