From 25ca6784003df71f1ffcd62ee6fd19d7d3ab833c Mon Sep 17 00:00:00 2001
From: Karel van Klink <karel.vanklink@geant.org>
Date: Fri, 20 Jun 2025 11:20:58 +0200
Subject: [PATCH 1/2] Add modify_note workflow to all existing products
---
...5_add_modify_note_workflow_to_existing_.py | 70 +++++++++++++++++++
1 file changed, 70 insertions(+)
create mode 100644 gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py
diff --git a/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py b/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py
new file mode 100644
index 000000000..7414647ef
--- /dev/null
+++ b/gso/migrations/versions/2025-06-20_24858fd1d805_add_modify_note_workflow_to_existing_.py
@@ -0,0 +1,70 @@
+"""Add modify note workflow to existing products.
+
+Revision ID: 24858fd1d805
+Revises: 550e3aebc1c5
+Create Date: 2025-06-20 10:51:57.321841
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '24858fd1d805'
+down_revision = '550e3aebc1c5'
+branch_labels = None
+depends_on = None
+
+
+from orchestrator.migrations.helpers import create_workflow, delete_workflow, add_products_to_workflow_by_product_tag, \
+ remove_products_from_workflow_by_product_tag
+
+product_tags = [
+ "ER",
+ "COP",
+ "EP",
+ "G_IP",
+ "G_PLUS",
+ "IAS",
+ "IMP_ER",
+ "IMP_COP",
+ "IMP_EP",
+ "IMP_G_IP",
+ "IMP_G_PLUS",
+ "IMP_IAS",
+ "IMP_IP_TRUNK",
+ "IMP_LSI",
+ "IMP_LHC",
+ "IMP_OFFICE_RTR",
+ "IMPORTED_OPENGEAR",
+ "IMP_RE_LHCONE",
+ "IMP_RE_PEER",
+ "IMP_RTR",
+ "IMP_SITE",
+ "IMP_SPOP_SWITCH",
+ "IMP_SWITCH",
+ "IPTRUNK",
+ "LSI",
+ "LHC",
+ "OFFICE_ROUTER",
+ "OPENGEAR",
+ "POP_VLAN",
+ "RE_LHCONE",
+ "RE_PEER",
+ "RTR",
+ "SITE",
+ "Super_POP_SWITCH",
+ "SWITCH",
+ "VRF",
+]
+
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ for product in product_tags:
+ add_products_to_workflow_by_product_tag(conn, "modify_note", product)
+
+
+def downgrade() -> None:
+ conn = op.get_bind()
+ for product in product_tags:
+ remove_products_from_workflow_by_product_tag(conn, "modify_note", product)
--
GitLab
From e8e0da07a2163d886fdbbce61c23288e71e7468f Mon Sep 17 00:00:00 2001
From: Karel van Klink <karel.vanklink@geant.org>
Date: Fri, 20 Jun 2025 11:43:43 +0200
Subject: [PATCH 2/2] Solve a FIXME and revert to the vanilla implementation of
task_validate_products over our custom one
---
...4cd282a_remove_obsolete_validation_task.py | 35 ++++++++++++++
gso/schedules/validate_products.py | 4 +-
gso/services/processes.py | 13 ------
gso/translations/en-GB.json | 1 -
gso/workflows/__init__.py | 1 -
.../tasks/validate_geant_products.py | 46 -------------------
...y => test_massive_redeploy_base_config.py} | 0
.../tasks/test_task_validate_products.py | 7 +--
8 files changed, 40 insertions(+), 67 deletions(-)
create mode 100644 gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py
delete mode 100644 gso/workflows/tasks/validate_geant_products.py
rename test/tasks/{test_masssive_redeploy_base_config.py => test_massive_redeploy_base_config.py} (100%)
diff --git a/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py b/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py
new file mode 100644
index 000000000..9efde60de
--- /dev/null
+++ b/gso/migrations/versions/2025-06-20_7c3094cd282a_remove_obsolete_validation_task.py
@@ -0,0 +1,35 @@
+"""Remove obsolete validation task.
+
+Revision ID: 7c3094cd282a
+Revises: 24858fd1d805
+Create Date: 2025-06-20 11:34:08.439370
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '7c3094cd282a'
+down_revision = '24858fd1d805'
+branch_labels = None
+depends_on = None
+
+
+from orchestrator.migrations.helpers import create_task, delete_workflow
+
+old_task = {
+ "name": "task_validate_geant_products",
+ "description": "Validate GEANT products"
+}
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ conn.execute(sa.text(f"""
+DELETE FROM processes WHERE workflow_id = (SELECT workflow_id FROM workflows WHERE name = '{old_task["name"]}')
+ """))
+ delete_workflow(conn, old_task["name"])
+
+
+def downgrade() -> None:
+ conn = op.get_bind()
+ create_task(conn, old_task)
diff --git a/gso/schedules/validate_products.py b/gso/schedules/validate_products.py
index efc441ad2..1e693fe3a 100644
--- a/gso/schedules/validate_products.py
+++ b/gso/schedules/validate_products.py
@@ -4,12 +4,10 @@ from celery import shared_task
from orchestrator.services.processes import start_process
from gso.schedules.scheduling import CronScheduleConfig, scheduler
-from gso.services.processes import count_incomplete_validate_products
@shared_task
@scheduler(CronScheduleConfig(name="Validate Products and inactive subscriptions", minute="30", hour="2"))
def validate_products() -> None:
"""Validate all products."""
- if count_incomplete_validate_products() == 0:
- start_process("task_validate_geant_products")
+ start_process("task_validate_products")
diff --git a/gso/services/processes.py b/gso/services/processes.py
index 30caa96fd..22bfe4e0d 100644
--- a/gso/services/processes.py
+++ b/gso/services/processes.py
@@ -18,19 +18,6 @@ def get_processes_by_workflow_name(workflow_name: str) -> Query:
return ProcessTable.query.join(WorkflowTable).filter(WorkflowTable.name == workflow_name)
-def count_incomplete_validate_products() -> int:
- """Count the number of incomplete validate_geant_products processes.
-
- Returns:
- The count of incomplete 'validate_geant_products' processes.
- """
- return (
- get_processes_by_workflow_name("validate_geant_products")
- .filter(ProcessTable.last_status != ProcessStatus.COMPLETED)
- .count()
- )
-
-
def get_failed_tasks() -> list[ProcessTable]:
"""Get all tasks that have failed."""
return ProcessTable.query.filter(
diff --git a/gso/translations/en-GB.json b/gso/translations/en-GB.json
index e3ea7bc6c..bbb37da1b 100644
--- a/gso/translations/en-GB.json
+++ b/gso/translations/en-GB.json
@@ -163,7 +163,6 @@
"task_modify_partners": "Modify partner task",
"task_redeploy_base_config": "Redeploy base config on multiple routers",
"task_send_email_notifications": "Send email notifications for failed tasks",
- "task_validate_geant_products": "Validation task for GEANT products",
"terminate_edge_port": "Terminate Edge Port",
"terminate_iptrunk": "Terminate IP Trunk",
"terminate_geant_ip": "Terminate GÉANT IP",
diff --git a/gso/workflows/__init__.py b/gso/workflows/__init__.py
index 3456115a4..827d2a07c 100644
--- a/gso/workflows/__init__.py
+++ b/gso/workflows/__init__.py
@@ -104,7 +104,6 @@ LazyWorkflowInstance("gso.workflows.opengear.import_opengear", "import_opengear"
# Tasks
LazyWorkflowInstance("gso.workflows.tasks.send_email_notifications", "task_send_email_notifications")
-LazyWorkflowInstance("gso.workflows.tasks.validate_geant_products", "task_validate_geant_products")
LazyWorkflowInstance("gso.workflows.tasks.create_partners", "task_create_partners")
LazyWorkflowInstance("gso.workflows.tasks.modify_partners", "task_modify_partners")
LazyWorkflowInstance("gso.workflows.tasks.delete_partners", "task_delete_partners")
diff --git a/gso/workflows/tasks/validate_geant_products.py b/gso/workflows/tasks/validate_geant_products.py
deleted file mode 100644
index 8fe39f61b..000000000
--- a/gso/workflows/tasks/validate_geant_products.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""A task that checks for all products in the database to be well-kept."""
-
-# <!-- vale off -->
-# Copyright 2019-2020 SURF.
-# Copyright 2024 GÉANT Vereniging.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# <!-- vale on -->
-
-from orchestrator.targets import Target
-from orchestrator.workflow import StepList, done, init, workflow
-from orchestrator.workflows.tasks.validate_products import (
- check_all_workflows_are_in_db,
- check_db_fixed_input_config,
- check_subscription_models,
- check_that_products_have_create_modify_and_terminate_workflows,
- check_workflows_for_matching_targets_and_descriptions,
-)
-
-
-@workflow("Validate GEANT products", target=Target.SYSTEM)
-def task_validate_geant_products() -> StepList:
- """Validate products in the database.
-
- This task is based on the ``task_validate_products`` present in ``orchestrator-core`` but it does not check for the
- existence of the ``modify_note`` workflow on all products, since this workflow is currently not used in GEANT.
- """
- return (
- init
- >> check_all_workflows_are_in_db
- >> check_workflows_for_matching_targets_and_descriptions
- # >> check_that_products_have_at_least_one_workflow FIXME: Uncomment as soon as this would pass again
- >> check_db_fixed_input_config
- >> check_that_products_have_create_modify_and_terminate_workflows
- >> check_subscription_models
- >> done
- )
diff --git a/test/tasks/test_masssive_redeploy_base_config.py b/test/tasks/test_massive_redeploy_base_config.py
similarity index 100%
rename from test/tasks/test_masssive_redeploy_base_config.py
rename to test/tasks/test_massive_redeploy_base_config.py
diff --git a/test/workflows/tasks/test_task_validate_products.py b/test/workflows/tasks/test_task_validate_products.py
index 66853d257..6c7398b92 100644
--- a/test/workflows/tasks/test_task_validate_products.py
+++ b/test/workflows/tasks/test_task_validate_products.py
@@ -4,14 +4,15 @@ from test.workflows import assert_complete, extract_state, run_workflow
@pytest.mark.workflow()
-def test_task_validate_geant_products():
- result, _, _ = run_workflow("task_validate_geant_products", [{}])
+def test_task_validate_products():
+ result, _, _ = run_workflow("task_validate_products", [{}])
assert_complete(result)
state = extract_state(result)
assert state["check_all_workflows_are_in_db"]
assert state["check_workflows_for_matching_targets_and_descriptions"]
- # assert state["check_that_products_have_at_least_one_workflow"] FIXME: Uncomment when the task is reverted again
+ assert state["check_that_products_have_at_least_one_workflow"]
+ assert state["check_that_active_products_have_a_modify_note"]
assert state["check_db_fixed_input_config"]
assert state["check_that_products_have_create_modify_and_terminate_workflows"]
assert state["check_subscription_models"]
--
GitLab