Add security controls data

compendium_v2/publishers/legacy_publisher/survey_publisher_legacy_db.py

@@ -6,13 +6,15 @@ This module loads the survey data from 2022 from the old survey database and ret
 ready to be used to generate new-style surveys.
 
 """
-from decimal import Decimal
 import logging
 import enum
 import json
 import html
 import itertools
+import os
+import csv
 
+from decimal import Decimal
 from sqlalchemy import text
 from collections import defaultdict
 
@@ -88,6 +90,8 @@ RECURSIVE_QUERY = """
     ORDER BY n.id, answers.question_id, answers.updated_at DESC;
 """
 
+SECURITY_CONTROLS_CSV = os.path.join(os.path.dirname(__file__), "../../resources/security_controls_parsed.csv")
+
 
 class FundingSource(enum.Enum):
     CLIENT_INSTITUTIONS = 16405
@@ -690,11 +694,11 @@ def security_controls(nren_dict):
         if year < 2021:  # prior to 2022, the mapping is different, use a different data source
             continue
 
-        # TODO: import the pre-2022 data from a handmade CSV.
-
         if nren_name not in nren_dict:
             continue
 
+        nren = nren_dict[nren_name]
+
         full_list = sc.get((nren_name, year), [])
         other_entries = [e.strip() for e in sc_other.get((nren_name, year), [])
                          if e.strip() and e.lower() not in ["n/a", "-"]]
@@ -709,9 +713,35 @@ def security_controls(nren_dict):
         if other_entry:
             full_list.append('other')
 
-        yield ('security_controls', nren_dict[nren_name], nren_dict[nren_name].id, year, full_list)
+        yield ('security_controls', nren, nren.id, year, full_list)
         if other_entry:
-            yield ('security_controls-Comment', nren_dict[nren_name], nren_dict[nren_name].id, year, other_entry)
+            yield ('security_controls-Comment', nren, nren.id, year, other_entry)
+
+    # pop on data manually fixed from free-text fields for <2021
+    with open(SECURITY_CONTROLS_CSV, 'r') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            nren_name = row['nren'].upper()
+            year = int(row['year'])
+            if nren_name not in nren_dict:
+                continue
+
+            nren = nren_dict[nren_name]
+
+            controls = {**row}
+            del controls['nren']
+            del controls['year']
+
+            other = ''
+            if controls.get('other'):
+                other = controls['other'].strip()
+                del controls['other']
+
+            controls_for_year = [k for k, v in controls.items() if v]
+
+            yield ('security_controls', nren, nren.id, year, controls_for_year)
+            if other:
+                yield ('security_controls-Comment', nren, nren.id, year, other)
 
 
 def institutions_urls(nren_dict):

compendium_v2/publishers/survey_publisher_legacy_excel.py

@@ -12,6 +12,8 @@ import itertools
 import logging
 import math
 import click
+import os
+import csv
 
 from sqlalchemy import select, delete
 from collections import defaultdict
@@ -27,6 +29,8 @@ setup_logging()
 
 logger = logging.getLogger('survey-publisher-legacy-excel')
 
+SECURITY_CONTROLS_CSV = os.path.join(os.path.dirname(__file__), "../resources/security_controls_parsed.csv")
+
 
 def db_budget_migration(nren_dict):
     # move data from Survey DB budget table
@@ -595,6 +599,42 @@ def db_ops_automation_migration(nren_dict):
     db.session.commit()
 
 
+def security_controls_migration(nren_dict):
+    db.session.execute(delete(presentation_models.SecurityControls).where(
+        presentation_models.SecurityControls.year < 2021))
+
+    # pop on data manually fixed from free-text fields
+    with open(SECURITY_CONTROLS_CSV, 'r') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            nren_name = row['nren'].upper()
+            year = int(row['year'])
+            if nren_name not in nren_dict:
+                continue
+
+            nren = nren_dict[nren_name]
+
+            controls = {**row}
+            del controls['nren']
+            del controls['year']
+
+            other = ''
+            if controls.get('other'):
+                other = controls['other'].strip()
+                del controls['other']
+
+            controls_for_year = [k for k, v in controls.items() if v]
+
+            new_entry = presentation_models.SecurityControls(
+                nren=nren,
+                nren_id=nren.id,
+                year=year,
+                security_control_descriptions=[*controls_for_year, other] if other else controls_for_year
+            )
+            db.session.merge(new_entry)
+    db.session.commit()
+
+
 def _cli(app):
     with app.app_context():
         nren_dict = helpers.get_uppercase_nren_dict()
@@ -620,6 +660,7 @@ def _cli(app):
         db_capacity_migration(nren_dict)
         db_non_r_e_peers_migration(nren_dict)
         db_ops_automation_migration(nren_dict)
+        security_controls_migration(nren_dict)
 
 
 @click.command()

compendium_v2/resources/security_controls_parsed.csv

+"nren","year","anti_virus","anti_spam","firewall","ddos_mitigation","monitoring","ips_ids","acl","segmentation","integrity_checking","other"
+"ACONET",2017,,,1,1,,,,,,
+"ACONET",2018,,,1,1,,,,,,
+"ACONET",2019,,,1,1,,,,,,
+"ACONET",2020,,,1,,,,,,,
+"BELNET",2018,1,1,1,,,,1,,1,"version management"
+"BELNET",2019,1,1,1,1,,,1,,1,"version management"
+"BELNET",2020,,,1,1,,,,,,"Anti-malware, vulnerability management"
+"CARNET",2017,1,,1,1,,,1,,,
+"CARNET",2018,1,,1,1,,,1,,,
+"CARNET",2019,1,,1,1,,,1,,,
+"CARNET",2020,1,,1,,,,1,,,
+"CYNET",2020,,,,,,1,,,,"Endpoint security"
+"CESNET",2016,,,,,,,,,,"Not a public information."
+"CESNET",2017,1,1,1,,,1,,,,"active monitoring, honeypots"
+"CESNET",2018,1,1,1,,,1,,,,"active monitoring, honeypots, Rate-limiting, RTBH filtering (BGP community)"
+"CESNET",2019,1,1,1,,,1,,,,"active monitoring, honeypots, Rate-limiting, RTBH filtering, Flowspec, RPKI - BGP zone signed, DDoS Protector (in-house research)"
+"CESNET",2020,,,,,,,,,,"Regular penetration testing of the infrastructure, Monitoring and application of FlowSpec, rate-limiting, RTBH communities etc."
+"DEIC",2016,,1,,,,,,,,
+"DEIC",2017,,1,,,,,,,,
+"DEIC",2018,1,1,1,,,,,,,
+"DEIC",2020,,,1,,,1,1,,,"log analysis"
+"EENET",2016,1,,,,,,,,,
+"EENET",2017,1,,,,,,,,,
+"EENET",2018,1,,1,,,,,,,
+"EENET",2019,1,,1,,,,,,,
+"FUNET",2017,,,,,,,,,,
+"FUNET",2018,,,,,,,,,,
+"FUNET",2019,,,,,,,,,,
+"FUNET",2020,,,,,,,,,,
+"RENATER",2016,1,,1,,,,,,1,
+"RENATER",2018,1,,1,,,,,,1,"anti-malware. All servers are behind at least one firewall, two firewalls if they are back-office servers"
+"RENATER",2019,1,,1,,,,,,1,"anti-malware. All servers are behind at least one firewall, two firewalls if they are back-office servers"
+"MARNET",2018,,,1,,,,,,,
+"MARNET",2019,,,1,,,,,,,
+"GRENA",2016,1,,1,,,,,,,
+"GRENA",2017,1,,1,,,,,,,
+"GRENA",2018,1,1,1,,,,,,,
+"GRENA",2019,1,1,1,1,,,,,,
+"GRENA",2020,1,1,1,1,,,,,,
+"DFN",2018,1,1,1,,,,,,1,
+"DFN",2019,1,1,1,,,,,,1,
+"DFN",2020,1,,1,,,,,,1,"VPN, Strong Password, server root-kit detection, 4-eyes-principle for changes"
+"GRNET S.A.",2020,1,,1,,,1,,,,"PeakFlow DDoS monitoring"
+"KIFU",2016,1,,,,,,,,,
+"KIFU",2017,1,,,,,,,,,
+"KIFU",2018,1,,,,,,,,,
+"KIFU",2019,1,,,,,,,,,
+"KIFU",2020,1,,1,,,,,,1,
+"HEANET",2017,1,,1,,,,1,1,,
+"HEANET",2018,1,,1,,,,1,1,,
+"HEANET",2019,1,,,,,,1,1,,"Flowspec"
+"HEANET",2020,,,1,,,,1,,,
+"IUCC",2018,,,,,,,,,,"WAF"
+"IUCC",2019,1,,,,,,,,,"WAF"
+"IUCC",2020,1,,1,,,,,,,
+"GARR",2017,1,,,,,,1,,,
+"GARR",2018,1,,,,,,1,,,
+"GARR",2019,1,,,,,,1,,,
+"GARR",2020,1,1,,,,,1,,,
+"LITNET",2018,1,1,,,,,,,,
+"LITNET",2019,1,1,1,,,,1,,,
+"LITNET",2020,,,1,,,,,,,"Next generation end point protection Forti-client, Integrated security, performance, and availability monitoring FortiSIEM, Windows Defender"
+"RESTENA",2016,,,,,,,,,,
+"RENAM",2018,,,,,,,,,,
+"RENAM",2019,,,,,,,,,,
+"SURF",2016,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2017,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2018,1,,1,,,,,,,"device encryption, mail encryption"
+"SURF",2019,1,,1,,,,,,,"device encryption, mail encryption"
+"PIONIER",2016,1,,1,,,,,,,
+"PIONIER",2017,1,,1,,,,,,,
+"PIONIER",2018,1,,1,,,,,,,
+"PIONIER",2019,1,,1,,,,,,,
+"PIONIER",2020,,,,,,1,,,,"VPNs"
+"FCCN",2016,1,,,,,,,,," Network analyzer"
+"FCCN",2017,1,,,,,,,,,"Network analyzer"
+"FCCN",2018,1,,,,,,,,,"Network analyzer"
+"FCCN",2019,1,,1,,,1,,,,"Network analyzer"
+"FCCN",2020,1,,1,,,,,,,"VPN"
+"SANET",2017,1,,,,,,,,,
+"SANET",2018,1,,,,,,,,,
+"SANET",2019,1,,,,,,,,,
+"ARNES",2016,1,,,,,,,,,
+"ARNES",2017,1,,,,,,,,,
+"ARNES",2018,1,,,,,,,,,
+"ARNES",2019,1,,,,,,,,,
+"REDIRIS",2016,1,,1,,,,,,,
+"REDIRIS",2019,1,,1,,,,,,,
+"SWITCH",2016,1,,1,,,,,,,
+"SWITCH",2017,1,,1,,1,,,,,"Log Management"
+"SWITCH",2018,1,,1,,1,,,,,"Log Management"
+"SWITCH",2019,1,,1,,1,,,,,"Log Management"
+"ULAKBIM",2017,,,1,,,,,,,
+"ULAKBIM",2018,,,1,,,,,,,
+"ULAKBIM",2019,,,1,,,,,,,
+"URAN",2020,,,,,,,,,,"RIPE RPKI validation tool"
+"JISC",2018,1,,1,,,,,,,"DLP, MFA, SAML, DMARC, netflow, flowspec"
+"JISC",2019,1,,1,,,,,,,"DLP, MFA, SAML, DMARC, netflow, flowspec, RPZ, STIX/TAXI, MFA"
+"AMRES",2018,1,,1,,,,1,,,"web content filtering"
+"AMRES",2019,1,,1,,,,1,,,"web content filtering"
+"AMRES",2020,1,,1,1,,,,,,
+"AZSCIENCENET",2016,1,,,,,,,,,
+"AZSCIENCENET",2017,1,,,,,,,,,
+"AZSCIENCENET",2018,1,,1,,,,,,,
+"AZSCIENCENET",2019,,,1,,,,,,,"own authentication system, traffic log system, monitoring system"
+"LAT",2019,,,,1,,,,,,
+"ROEDUNET",2020,1,,1,,,1,,,,
+"BASNET",2016,1,,,,,,,,1,
+"BASNET",2017,1,,,,,,,,1,
+"BASNET",2018,1,,1,,,,,,1,
+"BASNET",2019,1,,1,,,,,,1,
+"MREN",2020,,,1,,,,,,,
+"ASNET-AM",2016,1,,,,,,,,,
+"ASNET-AM",2017,1,,,,,,,,,
+"ASNET-AM",2018,1,,,,,,,,,
+"ASNET-AM",2019,1,1,1,,,,,,,