From eff36daec9bc492542f79f3ec192f5d2876d8db1 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Fri, 22 Sep 2023 17:27:07 +0200
Subject: [PATCH 1/3] last db publish model as the data in the excel turned out
to be insufficient
---
.../survey_publisher_old_db_2022.py | 35 ++++++++++++++++++-
1 file changed, 34 insertions(+), 1 deletion(-)
diff --git a/compendium_v2/publishers/survey_publisher_old_db_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
index 5c72d1c5..b6ceea90 100644
--- a/compendium_v2/publishers/survey_publisher_old_db_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -1167,6 +1167,38 @@ def transfer_network_automation(nren_dict):
db.session.commit()
+def transfer_network_function_virtualisation(nren_dict):
+ rows = recursive_query(16754)
+ types = recursive_query(16755)
+ types = {(nren_name, year): json.loads(answer) for answer_id, nren_name, year, answer in types}
+ types_comment = recursive_query(16756)
+ types_comment = {(nren_name, year): answer.strip('" ') for answer_id, nren_name, year, answer in types_comment}
+
+ for answer_id, nren_name, year, answer in rows:
+ if nren_name not in nren_dict:
+ logger.warning(f'{nren_name} unknown. Skipping.')
+ continue
+
+ nfv = YesNoPlanned[answer.strip('"').lower()]
+ specifics = types.get((nren_name, year), [])
+ specifics = [s for s in specifics if s]
+ comment = types_comment.get((nren_name, year), "").replace("-", "")
+ if comment:
+ specifics.append(comment)
+ if "Other" in specifics:
+ specifics.remove("Other")
+
+ new_entry = presentation_models.NetworkFunctionVirtualisation(
+ nren=nren_dict[nren_name],
+ nren_id=nren_dict[nren_name].id,
+ year=year,
+ nfv=nfv,
+ nfv_specifics=specifics
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
def _cli(app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
@@ -1200,7 +1232,8 @@ def _cli(app):
transfer_pert_team(nren_dict)
transfer_alien_wave(nren_dict)
transfer_external_connections(nren_dict)
- # traffic ratio was freeform text
+ # traffic ratio was freeform text so we don't transfer it
+ transfer_network_function_virtualisation(nren_dict)
transfer_network_automation(nren_dict)
--
GitLab
From cac86203220caef9cc156225b38e0a0c21768875 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Fri, 22 Sep 2023 20:25:19 +0200
Subject: [PATCH 2/3] and one more old db publisher due to incomplete excel
data
---
.../survey_publisher_old_db_2022.py | 34 +++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/compendium_v2/publishers/survey_publisher_old_db_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
index b6ceea90..2719fbf5 100644
--- a/compendium_v2/publishers/survey_publisher_old_db_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -1199,6 +1199,39 @@ def transfer_network_function_virtualisation(nren_dict):
db.session.commit()
+def transfer_monitoring_tools(nren_dict):
+ tools = recursive_query(16672)
+ tools = {(nren_name, year): json.loads(answer) for answer_id, nren_name, year, answer in tools}
+ tools_comment = recursive_query(16673)
+ tools_comment = {(nren_name, year): answer.strip('" ') for answer_id, nren_name, year, answer in tools_comment}
+ netflow = recursive_query(16674)
+ netflow = {(nren_name, year): answer.strip('" ') for answer_id, nren_name, year, answer in netflow}
+
+ for nren_name, year in tools.keys() | tools_comment.keys() | netflow.keys():
+ if nren_name not in nren_dict:
+ logger.warning(f'{nren_name} unknown. Skipping.')
+ continue
+
+ tool_descriptions = tools.get((nren_name, year), [])
+ comment = tools_comment.get((nren_name, year), "").replace("-", "")
+ if comment:
+ tool_descriptions.append(comment)
+ if "Other" in tool_descriptions:
+ tool_descriptions.remove("Other")
+ if "Other " in tool_descriptions:
+ tool_descriptions.remove("Other ")
+
+ new_entry = presentation_models.MonitoringTools(
+ nren=nren_dict[nren_name],
+ nren_id=nren_dict[nren_name].id,
+ year=year,
+ tool_descriptions=tool_descriptions,
+ netflow_processing_description=netflow.get((nren_name, year), "")
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
def _cli(app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
@@ -1225,6 +1258,7 @@ def _cli(app):
transfer_fibre_light(nren_dict)
transfer_network_map_urls(nren_dict)
+ transfer_monitoring_tools(nren_dict)
transfer_traffic_statistics(nren_dict)
transfer_siem_vendors(nren_dict)
transfer_certificate_providers(nren_dict)
--
GitLab
From e4936163dce0ccadd17ede113acf6a03da4385b1 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Fri, 22 Sep 2023 22:04:38 +0200
Subject: [PATCH 3/3] publishing the network excel file
---
compendium_v2/publishers/excel_parser.py | 239 +++++++++++++++++-
compendium_v2/publishers/helpers.py | 1 +
.../survey_publisher_legacy_excel.py | 130 +++++++++-
test/test_survey_publisher_legacy_excel.py | 3 +-
test/test_survey_publisher_old_db_2022.py | 2 +-
5 files changed, 370 insertions(+), 5 deletions(-)
diff --git a/compendium_v2/publishers/excel_parser.py b/compendium_v2/publishers/excel_parser.py
index b2d5a3c8..a08ea4bd 100644
--- a/compendium_v2/publishers/excel_parser.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -3,7 +3,8 @@ import logging
import openpyxl
from compendium_v2.conversion import mapping
-from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectivityCoverage, UserCategory, FeeType
+from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectivityCoverage, MonitoringMethod, \
+ UserCategory, FeeType, YesNoPlanned
from compendium_v2.environment import setup_logging
from compendium_v2.resources import get_resource_file_path
@@ -827,3 +828,239 @@ def fetch_remote_campuses_excel_data():
yield from create_points_for_year(2020, 16)
yield from create_points_for_year(2021, 10)
yield from create_points_for_year(2022, 4)
+
+
+def fetch_dark_fibre_iru_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Dark Fibre"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+
+ def parse_int(excel_value):
+ if excel_value is None or excel_value == "":
+ return None
+ return int(str(excel_value).replace("-", "").replace(" ", "").replace(".", ""))
+
+ def create_points_for_year(year, start_column):
+ for i in range(10, 53):
+ nren_name = rows[i][start_column].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ s = start_column
+ iru = ""
+ if year > 2019:
+ s += 1
+ iru = rows[i][s].value
+ elif parse_int(rows[i][s + 1].value) is not None:
+ iru = "Yes" if parse_int(rows[i][s + 1].value) else "No"
+
+ if iru:
+ length_in_country = parse_int(rows[i][s + 1].value)
+ length_out_country = parse_int(rows[i][s + 3].value)
+ iru = iru == "Yes"
+ yield nren_name, year, iru, length_in_country, length_out_country
+
+ yield from create_points_for_year(2016, 42)
+ yield from create_points_for_year(2017, 36)
+ yield from create_points_for_year(2018, 30)
+ yield from create_points_for_year(2019, 24)
+ yield from create_points_for_year(2020, 17)
+ yield from create_points_for_year(2021, 10)
+ yield from create_points_for_year(2022, 3)
+
+
+def fetch_dark_fibre_installed_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Dark Fibre"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+
+ def parse_int(excel_value):
+ if excel_value is None or excel_value == "":
+ return None
+ return int(str(excel_value).replace("-", "").replace(" ", "").replace(".", ""))
+
+ def create_points_for_year(year, start_column):
+ for i in range(10, 53):
+ nren_name = rows[i][start_column].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ s = start_column
+ if year > 2019:
+ s += 1
+ installed_length = parse_int(rows[i][s + 2].value)
+ if installed_length is not None:
+ installed = bool(installed_length)
+ yield nren_name, year, installed, installed_length
+
+ yield from create_points_for_year(2016, 42)
+ yield from create_points_for_year(2017, 36)
+ yield from create_points_for_year(2018, 30)
+ yield from create_points_for_year(2019, 24)
+ yield from create_points_for_year(2020, 17)
+ yield from create_points_for_year(2021, 10)
+ yield from create_points_for_year(2022, 3)
+
+
+def fetch_iru_duration_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "IRU duration"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+ result = {}
+
+ def create_points_for_year(year, start_column):
+ for i in range(3, 46):
+ nren_name = rows[i][start_column].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ years = rows[i][start_column + 1].value
+ if not years:
+ continue
+ years = str(years).split(" ")[0].split("+")[0].split("-")[0]
+ if not years:
+ continue
+ try:
+ years = int(years)
+ except ValueError:
+ logger.warning(f'Invalid iru duration Value :{nren_name} ({year}) with value ({years})')
+ continue
+ result[(nren_name, year)] = years
+
+ create_points_for_year(2019, 10)
+ create_points_for_year(2020, 7)
+ create_points_for_year(2021, 4)
+ create_points_for_year(2022, 1)
+ return result
+
+
+def fetch_passive_monitoring_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Traffic monitoring"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+
+ def create_points_for_year(year, start_column):
+ for i in range(6, 48):
+ nren_name = rows[i][1].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ monitoring = rows[i][start_column].value
+ method = rows[i][start_column + 1].value
+ if monitoring:
+ monitoring = monitoring == "Yes"
+ method = {
+ "SPAN ports": MonitoringMethod.span_ports,
+ "Passive optical TAPS": MonitoringMethod.taps,
+ "Both": MonitoringMethod.both,
+ None: None
+ }[method]
+ yield nren_name, year, monitoring, method
+
+ yield from create_points_for_year(2021, 4)
+ yield from create_points_for_year(2022, 2)
+
+
+def fetch_largest_link_capacity_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Largest IP Trunk capacity"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+ result = {}
+
+ def create_points_for_year(year, start_column):
+ for i in range(5, 47):
+ nren_name = rows[i][5].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ largest_capacity = rows[i][start_column].value
+ if largest_capacity:
+ result[(nren_name, year)] = int(largest_capacity)
+
+ create_points_for_year(2016, 12)
+ create_points_for_year(2017, 11)
+ create_points_for_year(2018, 10)
+ create_points_for_year(2019, 9)
+ create_points_for_year(2020, 8)
+ create_points_for_year(2021, 7)
+ create_points_for_year(2022, 6)
+ return result
+
+
+def fetch_typical_backbone_capacity_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Typical IP Trunk capacity"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+ result = {}
+
+ def create_points_for_year(year, start_column):
+ for i in range(5, 47):
+ nren_name = rows[i][4].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ typical_capacity = rows[i][start_column].value
+ if typical_capacity:
+ result[(nren_name, year)] = int(typical_capacity)
+
+ create_points_for_year(2016, 11)
+ create_points_for_year(2017, 10)
+ create_points_for_year(2018, 9)
+ create_points_for_year(2019, 8)
+ create_points_for_year(2020, 7)
+ create_points_for_year(2021, 6)
+ create_points_for_year(2022, 5)
+ return result
+
+
+def fetch_non_r_e_peers_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Peering-Non R& Network"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+
+ def create_points_for_year(year, start_column):
+ for i in range(5, 48):
+ nren_name = rows[i][2].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ nr_peers = rows[i][start_column].value
+ if nr_peers:
+ yield nren_name, year, int(nr_peers)
+
+ yield from create_points_for_year(2016, 10)
+ yield from create_points_for_year(2017, 9)
+ yield from create_points_for_year(2018, 8)
+ yield from create_points_for_year(2019, 7)
+ yield from create_points_for_year(2020, 6)
+ yield from create_points_for_year(2021, 5)
+ yield from create_points_for_year(2022, 3)
+
+
+def fetch_ops_automation_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
+ sheet_name = "Automation"
+ ws = wb[sheet_name]
+ rows = list(ws.rows)
+
+ def create_points_for_year(year, start_column):
+ for i in range(5, 48):
+ nren_name = rows[i][1].value
+ if not nren_name:
+ continue
+ nren_name = nren_name.upper()
+ automation = rows[i][start_column].value
+ specifics = rows[i][start_column + 1].value or ""
+ if automation:
+ automation = YesNoPlanned[automation.lower()]
+ yield nren_name, year, automation, specifics
+
+ yield from create_points_for_year(2021, 5)
+ yield from create_points_for_year(2022, 3)
diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py
index 1668bb90..b9c2ce68 100644
--- a/compendium_v2/publishers/helpers.py
+++ b/compendium_v2/publishers/helpers.py
@@ -33,6 +33,7 @@ def get_uppercase_nren_dict():
nren_dict['AZSCIENCENET'] = nren_dict['ANAS']
nren_dict['GRNET S.A.'] = nren_dict['GRNET']
nren_dict['FUNET'] = nren_dict['CSC']
+ nren_dict['PIONEER'] = nren_dict['PIONIER']
return nren_dict
diff --git a/compendium_v2/publishers/survey_publisher_legacy_excel.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
index fadee87a..50cb9b87 100644
--- a/compendium_v2/publishers/survey_publisher_legacy_excel.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -418,15 +418,134 @@ def db_remote_campuses_migration(nren_dict):
connections.append({'country': country, 'local_r_and_e_connection': connected_to_r_e})
nren = nren_dict[abbrev]
- connection_carrier = presentation_models.RemoteCampuses(
+ new_entry = presentation_models.RemoteCampuses(
nren=nren,
nren_id=nren.id,
year=year,
remote_campus_connectivity=connectivity,
connections=connections
)
- db.session.merge(connection_carrier)
+ db.session.merge(new_entry)
+
+ db.session.commit()
+
+
+def db_dark_fibre_lease_migration(nren_dict):
+ data_rows = excel_parser.fetch_dark_fibre_iru_excel_data()
+ iru_duration = excel_parser.fetch_iru_duration_excel_data()
+
+ for (abbrev, year, iru, length_in_country, length_out_country) in data_rows:
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.DarkFibreLease(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ iru_or_lease=iru,
+ fibre_length_in_country=length_in_country,
+ fibre_length_outside_country=length_out_country,
+ iru_duration=iru_duration.get((abbrev, year))
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
+def db_dark_fibre_installed_migration(nren_dict):
+ data_rows = excel_parser.fetch_dark_fibre_installed_excel_data()
+ for (abbrev, year, installed, length) in data_rows:
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.DarkFibreInstalled(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ installed=installed,
+ fibre_length_in_country=length
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
+def db_passive_monitoring_migration(nren_dict):
+ data_rows = excel_parser.fetch_passive_monitoring_excel_data()
+ for (abbrev, year, monitoring, method) in data_rows:
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.PassiveMonitoring(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ monitoring=monitoring,
+ method=method
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
+def db_capacity_migration(nren_dict):
+ largest_data_rows = excel_parser.fetch_largest_link_capacity_excel_data()
+ typical_data_rows = excel_parser.fetch_typical_backbone_capacity_excel_data()
+
+ for abbrev, year in largest_data_rows.keys() | typical_data_rows.keys():
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.Capacity(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ largest_link_capacity=largest_data_rows.get((abbrev, year)),
+ typical_backbone_capacity=typical_data_rows.get((abbrev, year))
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
+def db_non_r_e_peers_migration(nren_dict):
+ data_rows = excel_parser.fetch_non_r_e_peers_excel_data()
+ for (abbrev, year, nr_of_non_r_and_e_peers) in data_rows:
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.NonREPeers(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ nr_of_non_r_and_e_peers=nr_of_non_r_and_e_peers
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
+def db_ops_automation_migration(nren_dict):
+ data_rows = excel_parser.fetch_ops_automation_excel_data()
+ for (abbrev, year, automation, specifics) in data_rows:
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ new_entry = presentation_models.OpsAutomation(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ ops_automation=automation,
+ ops_automation_specifics=specifics
+ )
+ db.session.merge(new_entry)
db.session.commit()
@@ -450,6 +569,13 @@ def _cli(app):
db_connectivity_load_migration(nren_dict)
db_remote_campuses_migration(nren_dict)
+ db_dark_fibre_lease_migration(nren_dict)
+ db_dark_fibre_installed_migration(nren_dict)
+ db_passive_monitoring_migration(nren_dict)
+ db_capacity_migration(nren_dict)
+ db_non_r_e_peers_migration(nren_dict)
+ db_ops_automation_migration(nren_dict)
+
@click.command()
@click.option('--config', type=click.STRING, default='config.json')
diff --git a/test/test_survey_publisher_legacy_excel.py b/test/test_survey_publisher_legacy_excel.py
index b066767e..9a14b393 100644
--- a/test/test_survey_publisher_legacy_excel.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -13,7 +13,8 @@ def test_excel_publisher(app_with_survey_db, mocker):
mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE)
with app_with_survey_db.app_context():
- nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
+ nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC',
+ 'PIONIER']
db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
diff --git a/test/test_survey_publisher_old_db_2022.py b/test/test_survey_publisher_old_db_2022.py
index 7b2983f9..39259ab6 100644
--- a/test/test_survey_publisher_old_db_2022.py
+++ b/test/test_survey_publisher_old_db_2022.py
@@ -210,7 +210,7 @@ def test_publisher(app_with_survey_db, mocker):
mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.recursive_query', institutions_urls_data)
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
- 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
+ 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC', 'PIONIER']
with app_with_survey_db.app_context():
db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
--
GitLab