first half of remaining legacy db publisher

compendium_v2/db/presentation_models.py

@@ -206,7 +206,7 @@ class Standards(db.Model):
     crisis_management_procedure: Mapped[Optional[bool]]
 
 
-class CrisisExcercises(db.Model):
+class CrisisExercises(db.Model):
     __tablename__ = 'crisis_excercises'
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')

compendium_v2/publishers/survey_publisher.py

@@ -18,7 +18,7 @@ from compendium_v2.db import db
 from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, \
     InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection, \
     FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings, \
-    Standards, CrisisExcercises, SecurityControls, ConnectedProportion, ConnectivityLevel, \
+    Standards, CrisisExercises, SecurityControls, ConnectedProportion, ConnectivityLevel, \
     ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity, \
     CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
     NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
@@ -59,7 +59,7 @@ def _map_2023(nren, answers) -> None:
     for table_class in [BudgetEntry, ChargingStructure, ECProject, ExternalConnections,
                         InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume,
                         FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings,
-                        Standards, CrisisExcercises, SecurityControls, ConnectedProportion, ConnectivityLevel,
+                        Standards, CrisisExercises, SecurityControls, ConnectedProportion, ConnectivityLevel,
                         ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity,
                         CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
                         NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
@@ -210,7 +210,7 @@ def _map_2023(nren, answers) -> None:
 
     crisis_exercises = answers.get("crisis_exercises")
     if crisis_exercises:
-        db.session.add(CrisisExcercises(
+        db.session.add(CrisisExercises(
             nren_id=nren.id, nren=nren, year=year,
             exercise_descriptions=crisis_exercises
         ))

compendium_v2/publishers/survey_publisher_old_db_2022.py

@@ -6,6 +6,7 @@ This module loads the survey data from 2022 from the old survey database into pr
 Registered as click cli command when installing compendium-v2.
 
 """
+from decimal import Decimal
 import logging
 import click
 import enum
@@ -17,7 +18,8 @@ from sqlalchemy import delete, text
 from collections import defaultdict
 
 import compendium_v2
-from compendium_v2.db.presentation_model_enums import FeeType
+from compendium_v2.conversion.mapping import CHARGING_LEVELS, CONNECTION, SERVICE_USER_TYPE_TO_CODE
+from compendium_v2.db.presentation_model_enums import CommercialCharges, CommercialConnectivityCoverage, FeeType, ServiceCategory, UserCategory
 from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
 from compendium_v2.publishers.helpers import extract_urls
@@ -60,14 +62,14 @@ WHERE
 ORDER BY n.id, a.question_id, a.updated_at DESC
 """
 
-INSTITUTIONS_URLS_QUERY_UNTIL_2022 = """
+RECURSIVE_QUERY = """
     WITH RECURSIVE parent_questions AS (
         -- Base case
         SELECT q.id, q.equivalent_question_id, c.year, q.title
         FROM questions q
                  JOIN sections s ON q.section_id = s.id
                  JOIN compendia c ON s.compendium_id = c.id
-        WHERE q.id = 16507
+        WHERE q.id = {}
         UNION ALL
         -- Recursive case
         SELECT q.id, q.equivalent_question_id, c.year, q.title
@@ -147,9 +149,10 @@ def query_budget():
     return db.session.execute(text(BUDGET_QUERY), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
 
 
-def query_institutions_urls():
-    return db.session.execute(text(INSTITUTIONS_URLS_QUERY_UNTIL_2022),
-                              bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
+def recursive_query(question_id_2022):
+    assert question_id_2022
+    query = RECURSIVE_QUERY.format(question_id_2022)
+    return db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
 
 
 def query_funding_sources():
@@ -210,7 +213,7 @@ def transfer_institutions_urls(nren_dict):
             logger.info(f'JSON decode error for institution urls for {nren_name}.')
             return []
 
-    rows = query_institutions_urls()
+    rows = recursive_query(16507)
 
     for row in rows:
         answer_id, nren_name, year, answer = row
@@ -556,18 +559,477 @@ def transfer_policies(nren_dict):
     db.session.commit()
 
 
-def _cli(config, app):
+def transfer_central_procurement(nren_dict):
+    rows = recursive_query(16482)
+    amounts = recursive_query(16483)
+    amounts = {(nren_name, year): Decimal(answer.strip('"')) for answer_id, nren_name, year, answer in amounts}
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.CentralProcurement(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            central_procurement=answer == '"Yes"',
+            amount=amounts.get((nren_name, year))
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_service_management(nren_dict):
+    framework = recursive_query(16484)
+    framework = {(nren_name, year): answer == '"Yes"' for answer_id, nren_name, year, answer in framework}
+    targets = recursive_query(16485)
+    targets = {(nren_name, year): answer == '"Yes"' for answer_id, nren_name, year, answer in targets}
+
+    for nren_name, year in framework.keys() | targets.keys():
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.ServiceManagement(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            service_management_framework=framework.get((nren_name, year)),
+            service_level_targets=targets.get((nren_name, year))
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_service_user_types(nren_dict):
+    categories = [
+        (ServiceCategory.identity, 16488),
+        (ServiceCategory.network_services, 16489),
+        (ServiceCategory.collaboration, 16490),
+        (ServiceCategory.security, 16491),
+        (ServiceCategory.isp_support, 16492),
+        (ServiceCategory.storage_and_hosting, 16493),
+        (ServiceCategory.multimedia, 16494),
+        (ServiceCategory.professional_services, 16495)
+    ]
+    for service_category, question_id in categories:
+        rows = recursive_query(question_id)
+        for answer_id, nren_name, year, answer in rows:
+            if nren_name not in nren_dict:
+                logger.warning(f'{nren_name} unknown. Skipping.')
+                continue
+            for user_cat_db in json.loads(answer):
+                user_cat = UserCategory[SERVICE_USER_TYPE_TO_CODE[user_cat_db]]
+                new_entry = presentation_models.ServiceUserTypes(
+                    nren=nren_dict[nren_name],
+                    nren_id=nren_dict[nren_name].id,
+                    year=year,
+                    user_category=user_cat,
+                    service_category=service_category
+                )
+                db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_standards(nren_dict):
+    audits = recursive_query(16499)
+    audits = {(nren_name, year): answer == '"Yes"' for answer_id, nren_name, year, answer in audits}
+    audit_specifics = recursive_query(16500)
+    audit_specifics = {(nren_name, year): answer.strip('"') for answer_id, nren_name, year, answer in audit_specifics}
+    bcp = recursive_query(16501)
+    bcp = {(nren_name, year): answer == '"Yes"' for answer_id, nren_name, year, answer in bcp}
+    bcp_specifics = recursive_query(16502)
+    bcp_specifics = {(nren_name, year): answer.strip('"') for answer_id, nren_name, year, answer in bcp_specifics}
+    cmp = recursive_query(16762)
+    cmp = {(nren_name, year): answer == '"Yes"' for answer_id, nren_name, year, answer in cmp}
+
+    for nren_name, year in audits.keys() | audit_specifics.keys() | bcp.keys() | bcp_specifics.keys() | cmp.keys():
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.Standards(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            audits=audits.get((nren_name, year)),
+            audit_specifics=audit_specifics.get((nren_name, year), ""),
+            business_continuity_plans=bcp.get((nren_name, year)),
+            business_continuity_plans_specifics=bcp_specifics.get((nren_name, year), ""),
+            crisis_management_procedure=cmp.get((nren_name, year))
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_crisis_exercises(nren_dict):
+    rows = recursive_query(16763)
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.CrisisExercises(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            exercise_descriptions=json.loads(answer)
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_security_controls(nren_dict):
+    sc = recursive_query(16503)
+    sc = {(nren_name, year): json.loads(answer) for answer_id, nren_name, year, answer in sc}
+    sc_other = recursive_query(16504)
+    sc_other = {(nren_name, year): json.loads(answer) for answer_id, nren_name, year, answer in sc_other}
+    for key, value in sc_other.items():
+        if not isinstance(value, list):
+            sc_other[key] = [value]
+
+    for nren_name, year in sc.keys() | sc_other.keys():
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        full_list = sc.get((nren_name, year), [])
+        other_entries = [e for e in sc_other.get((nren_name, year), []) if e and e.lower() not in ["n/a", "-"]]
+        other_entry = ", ".join(other_entries)
+        if other_entry:
+            full_list.append(other_entry)
+        if full_list:
+            new_entry = presentation_models.SecurityControls(
+                nren=nren_dict[nren_name],
+                nren_id=nren_dict[nren_name].id,
+                year=year,
+                security_control_descriptions=full_list
+            )
+            db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_eosc_listings(nren_dict):
+    rows = recursive_query(16497)
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.EOSCListings(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            service_names=[x for x in json.loads(answer) if x]
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_commercial_connectivity(nren_dict):
+
+    simple_connection = {
+        key.replace(" ", "").replace("-", "").replace("/", "").lower(): value for key, value in CONNECTION.items()
+    }
+
+    def get_coverage(db_string):
+        cleaned_str = db_string.strip('"').replace(" ", "").replace("-", "").replace("/", "").lower()
+        key = simple_connection[cleaned_str]
+        return CommercialConnectivityCoverage[key]
+
+    sp = recursive_query(16646)
+    sp = {(nren_name, year): get_coverage(answer) for answer_id, nren_name, year, answer in sp}
+    collab = recursive_query(16647)
+    collab = {(nren_name, year): get_coverage(answer) for answer_id, nren_name, year, answer in collab}
+    r_e = recursive_query(16648)
+    r_e = {(nren_name, year): get_coverage(answer) for answer_id, nren_name, year, answer in r_e}
+    general = recursive_query(16649)
+    general = {(nren_name, year): get_coverage(answer) for answer_id, nren_name, year, answer in general}
+    spin_off = recursive_query(16650)
+    spin_off = {(nren_name, year): get_coverage(answer) for answer_id, nren_name, year, answer in spin_off}
+
+    for nren_name, year in sp.keys() | collab.keys() | r_e.keys() | general.keys() | spin_off.keys():
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.CommercialConnectivity(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            commercial_r_and_e=r_e.get((nren_name, year)),
+            commercial_general=general.get((nren_name, year)),
+            commercial_collaboration=collab.get((nren_name, year)),
+            commercial_service_provider=sp.get((nren_name, year)),
+            university_spin_off=spin_off.get((nren_name, year))
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_commercial_charging_level(nren_dict):
+    simple_charging = {
+        key.replace(" ", "").replace("-", "").replace("/", "").lower(): value for key, value in CHARGING_LEVELS.items()
+    }
+    simple_charging["nochargesapplied"] = "no_charges_if_r_e_requested"
+    simple_charging['nochargesappliedifrequestedbyr+eusers\\"needed?'] = "no_charges_if_r_e_requested"
+
+    def get_charging(db_string):
+        if db_string[0] == '[':
+            db_string = json.loads(db_string)[0]
+        cleaned_str = db_string.strip('"').replace(" ", "").replace("-", "").replace("/", "").lower()
+        key = simple_charging[cleaned_str]
+        return CommercialCharges[key]
+
+    collab = recursive_query(16652)
+    collab = {(nren_name, year): get_charging(answer) for answer_id, nren_name, year, answer in collab}
+    services = recursive_query(16653)
+    services = {(nren_name, year): get_charging(answer) for answer_id, nren_name, year, answer in services}
+    peering = recursive_query(16654)
+    peering = {(nren_name, year): get_charging(answer) for answer_id, nren_name, year, answer in peering}
+
+    for nren_name, year in collab.keys() | services.keys() | peering.keys():
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.CommercialChargingLevel(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+            collaboration=collab.get((nren_name, year)),
+            service_supplier=services.get((nren_name, year)),
+            direct_peering=peering.get((nren_name, year))
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_fibre_light(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.FibreLight(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_network_map_urls(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.NetworkMapUrls(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_traffic_statistics(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.TrafficStatistics(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_siem_vendors(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.SiemVendors(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_certificate_providers(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.CertificateProviders(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_weather_map(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.WeatherMap(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_pert_team(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.PertTeam(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_alien_wave(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.AlienWave(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_external_connections(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.ExternalConnections(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_traffic_ratio(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.TrafficRatio(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def transfer_network_automation(nren_dict):
+    rows = recursive_query()
+
+    for answer_id, nren_name, year, answer in rows:
+        if nren_name not in nren_dict:
+            logger.warning(f'{nren_name} unknown. Skipping.')
+            continue
+
+        new_entry = presentation_models.NetworkAutomation(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            year=year,
+        )
+        db.session.merge(new_entry)
+    db.session.commit()
+
+
+def _cli(app):
     with app.app_context():
         nren_dict = helpers.get_uppercase_nren_dict()
-        transfer_budget(nren_dict)
-        transfer_funding_sources(nren_dict)
-        transfer_staff_data(nren_dict)
-        transfer_nren_parent_org(nren_dict)
-        transfer_nren_sub_org(nren_dict)
-        transfer_charging_structure(nren_dict)
-        transfer_ec_projects(nren_dict)
-        transfer_policies(nren_dict)
-        transfer_institutions_urls(nren_dict)
+        # transfer_budget(nren_dict)
+        # transfer_funding_sources(nren_dict)
+        # transfer_staff_data(nren_dict)
+        # transfer_nren_parent_org(nren_dict)
+        # transfer_nren_sub_org(nren_dict)
+        # transfer_charging_structure(nren_dict)
+        # transfer_ec_projects(nren_dict)
+        # transfer_policies(nren_dict)
+        # transfer_institutions_urls(nren_dict)
+
+        transfer_central_procurement(nren_dict)
+        transfer_service_management(nren_dict)
+        transfer_service_user_types(nren_dict)
+        transfer_standards(nren_dict)
+        transfer_crisis_exercises(nren_dict)
+        transfer_security_controls(nren_dict)
+        transfer_eosc_listings(nren_dict)
+
+        transfer_commercial_connectivity(nren_dict)
+        transfer_commercial_charging_level(nren_dict)
+
+        transfer_fibre_light(nren_dict)
+        transfer_network_map_urls(nren_dict)
+        transfer_traffic_statistics(nren_dict)
+        transfer_siem_vendors(nren_dict)
+        transfer_certificate_providers(nren_dict)
+        transfer_weather_map(nren_dict)
+        transfer_pert_team(nren_dict)
+        transfer_alien_wave(nren_dict)
+        transfer_external_connections(nren_dict)
+        transfer_traffic_ratio(nren_dict)
+        transfer_network_automation(nren_dict)
 
 
 @click.command()
@@ -578,7 +1040,7 @@ def cli(config):
     app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
 
     app = compendium_v2._create_app_with_db(app_config)
-    _cli(app_config, app)
+    _cli(app)
 
 
 if __name__ == "__main__":

test/test_survey_publisher.py

@@ -128,7 +128,7 @@ def test_v2_publisher_full(app):
         assert standards.business_continuity_plans_specifics == "no"
         assert not standards.crisis_management_procedure
 
-        crisis_excercises = db.session.scalar(select(presentation_models.CrisisExcercises))
+        crisis_excercises = db.session.scalar(select(presentation_models.CrisisExercises))
         assert crisis_excercises.exercise_descriptions == [
             "geant_workshops", "national_excercises", "tabletop_exercises", "other_excercises",
             "none", "simulation_excercises", "real_crisis"

test/test_survey_publisher_old_db_2022.py

@@ -110,7 +110,7 @@ org_dataKTU,"NOC, administrative authority"
         ]
 
 
-def test_publisher(app_with_survey_db, mocker, dummy_config):
+def test_publisher(app_with_survey_db, mocker):
     global org_data
 
     def get_rows_as_tuples(*args, **kwargs):
@@ -194,18 +194,20 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
                 ('nren3', 'n.a. online'),
             ]
 
-    def institutions_urls_data():
+    def institutions_urls_data(question_id):
+        if question_id == 16507:
             return [
                 (87483, 'ANA', 2013, "http://www.rash.al/index.php/network/points-of-presence-pop"),
                 (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"),
             ]
+        else:
+            return []
 
     mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_budget', get_rows_as_tuples)
     mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_funding_sources', funding_source_data)
     mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question', question_data)
     mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question_id', question_id_data)
-    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_institutions_urls',
-                 institutions_urls_data)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.recursive_query', institutions_urls_data)
 
     nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
                   'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
@@ -213,7 +215,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
         db.session.commit()
 
-    _cli(dummy_config, app_with_survey_db)
+    _cli(app_with_survey_db)
 
     with app_with_survey_db.app_context():
         budgets = db.session.scalars(