diff --git a/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy_db.py b/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy_db.py
index 4d12a84e01cf8b4a5ed04d3b98becbd49dd1783b..8512f37ab8a57ba991c60fed4cbb0c7472587ece 100644
--- a/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy_db.py
+++ b/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy_db.py
@@ -6,13 +6,15 @@ This module loads the survey data from 2022 from the old survey database and ret
ready to be used to generate new-style surveys.
"""
-from decimal import Decimal
import logging
import enum
import json
import html
import itertools
+import os
+import csv
+from decimal import Decimal
from sqlalchemy import text
from collections import defaultdict
@@ -88,6 +90,8 @@ RECURSIVE_QUERY = """
ORDER BY n.id, answers.question_id, answers.updated_at DESC;
"""
+SECURITY_CONTROLS_CSV = os.path.join(os.path.dirname(__file__), "../../resources/security_controls_parsed.csv")
+
class FundingSource(enum.Enum):
CLIENT_INSTITUTIONS = 16405
@@ -690,11 +694,11 @@ def security_controls(nren_dict):
if year < 2021: # prior to 2022, the mapping is different, use a different data source
continue
- # TODO: import the pre-2022 data from a handmade CSV.
-
if nren_name not in nren_dict:
continue
+ nren = nren_dict[nren_name]
+
full_list = sc.get((nren_name, year), [])
other_entries = [e.strip() for e in sc_other.get((nren_name, year), [])
if e.strip() and e.lower() not in ["n/a", "-"]]
@@ -709,9 +713,35 @@ def security_controls(nren_dict):
if other_entry:
full_list.append('other')
- yield ('security_controls', nren_dict[nren_name], nren_dict[nren_name].id, year, full_list)
+ yield ('security_controls', nren, nren.id, year, full_list)
if other_entry:
- yield ('security_controls-Comment', nren_dict[nren_name], nren_dict[nren_name].id, year, other_entry)
+ yield ('security_controls-Comment', nren, nren.id, year, other_entry)
+
+ # pop on data manually fixed from free-text fields for <2021
+ with open(SECURITY_CONTROLS_CSV, 'r') as f:
+ reader = csv.DictReader(f)
+ for row in reader:
+ nren_name = row['nren'].upper()
+ year = int(row['year'])
+ if nren_name not in nren_dict:
+ continue
+
+ nren = nren_dict[nren_name]
+
+ controls = {**row}
+ del controls['nren']
+ del controls['year']
+
+ other = ''
+ if controls.get('other'):
+ other = controls['other'].strip()
+ del controls['other']
+
+ controls_for_year = [k for k, v in controls.items() if v]
+
+ yield ('security_controls', nren, nren.id, year, controls_for_year)
+ if other:
+ yield ('security_controls-Comment', nren, nren.id, year, other)
def institutions_urls(nren_dict):
diff --git a/compendium_v2/publishers/survey_publisher_legacy_excel.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
index c73ef1aff54c14256539e06161df9a0be90cd004..d9bb2a604b06089bdf69f0effa1e70acbe50ecc6 100644
--- a/compendium_v2/publishers/survey_publisher_legacy_excel.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -12,6 +12,8 @@ import itertools
import logging
import math
import click
+import os
+import csv
from sqlalchemy import select, delete
from collections import defaultdict
@@ -27,6 +29,8 @@ setup_logging()
logger = logging.getLogger('survey-publisher-legacy-excel')
+SECURITY_CONTROLS_CSV = os.path.join(os.path.dirname(__file__), "../resources/security_controls_parsed.csv")
+
def db_budget_migration(nren_dict):
# move data from Survey DB budget table
@@ -595,6 +599,42 @@ def db_ops_automation_migration(nren_dict):
db.session.commit()
+def security_controls_migration(nren_dict):
+ db.session.execute(delete(presentation_models.SecurityControls).where(
+ presentation_models.SecurityControls.year < 2021))
+
+ # pop on data manually fixed from free-text fields
+ with open(SECURITY_CONTROLS_CSV, 'r') as f:
+ reader = csv.DictReader(f)
+ for row in reader:
+ nren_name = row['nren'].upper()
+ year = int(row['year'])
+ if nren_name not in nren_dict:
+ continue
+
+ nren = nren_dict[nren_name]
+
+ controls = {**row}
+ del controls['nren']
+ del controls['year']
+
+ other = ''
+ if controls.get('other'):
+ other = controls['other'].strip()
+ del controls['other']
+
+ controls_for_year = [k for k, v in controls.items() if v]
+
+ new_entry = presentation_models.SecurityControls(
+ nren=nren,
+ nren_id=nren.id,
+ year=year,
+ security_control_descriptions=[*controls_for_year, other] if other else controls_for_year
+ )
+ db.session.merge(new_entry)
+ db.session.commit()
+
+
def _cli(app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
@@ -620,6 +660,7 @@ def _cli(app):
db_capacity_migration(nren_dict)
db_non_r_e_peers_migration(nren_dict)
db_ops_automation_migration(nren_dict)
+ security_controls_migration(nren_dict)
@click.command()
diff --git a/compendium_v2/resources/security_controls_parsed.csv b/compendium_v2/resources/security_controls_parsed.csv
new file mode 100644
index 0000000000000000000000000000000000000000..dfb080bf50377f6932c331bf8c3c9eb18f43044b
--- /dev/null
+++ b/compendium_v2/resources/security_controls_parsed.csv
@@ -0,0 +1,117 @@
+"nren","year","anti_virus","anti_spam","firewall","ddos_mitigation","monitoring","ips_ids","acl","segmentation","integrity_checking","other"
+"ACONET",2017,,,1,1,,,,,,
+"ACONET",2018,,,1,1,,,,,,
+"ACONET",2019,,,1,1,,,,,,
+"ACONET",2020,,,1,,,,,,,
+"BELNET",2018,1,1,1,,,,1,,1,"version management"
+"BELNET",2019,1,1,1,1,,,1,,1,"version management"
+"BELNET",2020,,,1,1,,,,,,"Anti-malware, vulnerability management"
+"CARNET",2017,1,,1,1,,,1,,,
+"CARNET",2018,1,,1,1,,,1,,,
+"CARNET",2019,1,,1,1,,,1,,,
+"CARNET",2020,1,,1,,,,1,,,
+"CYNET",2020,,,,,,1,,,,"Endpoint security"
+"CESNET",2016,,,,,,,,,,"Not a public information."
+"CESNET",2017,1,1,1,,,1,,,,"active monitoring, honeypots"
+"CESNET",2018,1,1,1,,,1,,,,"active monitoring, honeypots, Rate-limiting, RTBH filtering (BGP community)"
+"CESNET",2019,1,1,1,,,1,,,,"active monitoring, honeypots, Rate-limiting, RTBH filtering, Flowspec, RPKI - BGP zone signed, DDoS Protector (in-house research)"
+"CESNET",2020,,,,,,,,,,"Regular penetration testing of the infrastructure, Monitoring and application of FlowSpec, rate-limiting, RTBH communities etc."
+"DEIC",2016,,1,,,,,,,,
+"DEIC",2017,,1,,,,,,,,
+"DEIC",2018,1,1,1,,,,,,,
+"DEIC",2020,,,1,,,1,1,,,"log analysis"
+"EENET",2016,1,,,,,,,,,
+"EENET",2017,1,,,,,,,,,
+"EENET",2018,1,,1,,,,,,,
+"EENET",2019,1,,1,,,,,,,
+"FUNET",2017,,,,,,,,,,
+"FUNET",2018,,,,,,,,,,
+"FUNET",2019,,,,,,,,,,
+"FUNET",2020,,,,,,,,,,
+"RENATER",2016,1,,1,,,,,,1,
+"RENATER",2018,1,,1,,,,,,1,"anti-malware. All servers are behind at least one firewall, two firewalls if they are back-office servers"
+"RENATER",2019,1,,1,,,,,,1,"anti-malware. All servers are behind at least one firewall, two firewalls if they are back-office servers"
+"MARNET",2018,,,1,,,,,,,
+"MARNET",2019,,,1,,,,,,,
+"GRENA",2016,1,,1,,,,,,,
+"GRENA",2017,1,,1,,,,,,,
+"GRENA",2018,1,1,1,,,,,,,
+"GRENA",2019,1,1,1,1,,,,,,
+"GRENA",2020,1,1,1,1,,,,,,
+"DFN",2018,1,1,1,,,,,,1,
+"DFN",2019,1,1,1,,,,,,1,
+"DFN",2020,1,,1,,,,,,1,"VPN, Strong Password, server root-kit detection, 4-eyes-principle for changes"
+"GRNET S.A.",2020,1,,1,,,1,,,,"PeakFlow DDoS monitoring"
+"KIFU",2016,1,,,,,,,,,
+"KIFU",2017,1,,,,,,,,,
+"KIFU",2018,1,,,,,,,,,
+"KIFU",2019,1,,,,,,,,,
+"KIFU",2020,1,,1,,,,,,1,
+"HEANET",2017,1,,1,,,,1,1,,
+"HEANET",2018,1,,1,,,,1,1,,
+"HEANET",2019,1,,,,,,1,1,,"Flowspec"
+"HEANET",2020,,,1,,,,1,,,
+"IUCC",2018,,,,,,,,,,"WAF"
+"IUCC",2019,1,,,,,,,,,"WAF"
+"IUCC",2020,1,,1,,,,,,,
+"GARR",2017,1,,,,,,1,,,
+"GARR",2018,1,,,,,,1,,,
+"GARR",2019,1,,,,,,1,,,
+"GARR",2020,1,1,,,,,1,,,
+"LITNET",2018,1,1,,,,,,,,
+"LITNET",2019,1,1,1,,,,1,,,
+"LITNET",2020,,,1,,,,,,,"Next generation end point protection Forti-client, Integrated security, performance, and availability monitoring FortiSIEM, Windows Defender"
+"RESTENA",2016,,,,,,,,,,
+"RENAM",2018,,,,,,,,,,
+"RENAM",2019,,,,,,,,,,
+"SURF",2016,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2017,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2018,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2019,1,,1,,,,,,,"device encryption, mail encryption"
+"PIONIER",2016,1,,1,,,,,,,
+"PIONIER",2017,1,,1,,,,,,,
+"PIONIER",2018,1,,1,,,,,,,
+"PIONIER",2019,1,,1,,,,,,,
+"PIONIER",2020,,,,,,1,,,,"VPNs"
+"FCCN",2016,1,,,,,,,,," Network analyzer"
+"FCCN",2017,1,,,,,,,,,"Network analyzer"
+"FCCN",2018,1,,,,,,,,,"Network analyzer"
+"FCCN",2019,1,,1,,,1,,,,"Network analyzer"
+"FCCN",2020,1,,1,,,,,,,"VPN"
+"SANET",2017,1,,,,,,,,,
+"SANET",2018,1,,,,,,,,,
+"SANET",2019,1,,,,,,,,,
+"ARNES",2016,1,,,,,,,,,
+"ARNES",2017,1,,,,,,,,,
+"ARNES",2018,1,,,,,,,,,
+"ARNES",2019,1,,,,,,,,,
+"REDIRIS",2016,1,,1,,,,,,,
+"REDIRIS",2019,1,,1,,,,,,,
+"SWITCH",2016,1,,1,,,,,,,
+"SWITCH",2017,1,,1,,1,,,,,"Log Management"
+"SWITCH",2018,1,,1,,1,,,,,"Log Management"
+"SWITCH",2019,1,,1,,1,,,,,"Log Management"
+"ULAKBIM",2017,,,1,,,,,,,
+"ULAKBIM",2018,,,1,,,,,,,
+"ULAKBIM",2019,,,1,,,,,,,
+"URAN",2020,,,,,,,,,,"RIPE RPKI validation tool"
+"JISC",2018,1,,1,,,,,,,"DLP, MFA, SAML, DMARC, netflow, flowspec"
+"JISC",2019,1,,1,,,,,,,"DLP, MFA, SAML, DMARC, netflow, flowspec, RPZ, STIX/TAXI, MFA"
+"AMRES",2018,1,,1,,,,1,,,"web content filtering"
+"AMRES",2019,1,,1,,,,1,,,"web content filtering"
+"AMRES",2020,1,,1,1,,,,,,
+"AZSCIENCENET",2016,1,,,,,,,,,
+"AZSCIENCENET",2017,1,,,,,,,,,
+"AZSCIENCENET",2018,1,,1,,,,,,,
+"AZSCIENCENET",2019,,,1,,,,,,,"own authentication system, traffic log system, monitoring system"
+"LAT",2019,,,,1,,,,,,
+"ROEDUNET",2020,1,,1,,,1,,,,
+"BASNET",2016,1,,,,,,,,1,
+"BASNET",2017,1,,,,,,,,1,
+"BASNET",2018,1,,1,,,,,,1,
+"BASNET",2019,1,,1,,,,,,1,
+"MREN",2020,,,1,,,,,,,
+"ASNET-AM",2016,1,,,,,,,,,
+"ASNET-AM",2017,1,,,,,,,,,
+"ASNET-AM",2018,1,,,,,,,,,
+"ASNET-AM",2019,1,1,1,,,,,,,