add testcase and fix v2 publisher bugs

compendium_v2/db/model.py

@@ -124,6 +124,7 @@ class Policy(db.Model):
     acceptable_use: Mapped[str]
     privacy_notice: Mapped[str]
     data_protection: Mapped[str]
+    gender_equality: Mapped[str]
 
 
 class TrafficVolume(db.Model):

compendium_v2/migrations/versions/3cef2a22ebf1_add_gender_equality_policy.py

+"""empty message
+
+Revision ID: 3cef2a22ebf1
+Revises: 3730c7f1ea1b
+Create Date: 2023-09-07 14:29:11.149798
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3cef2a22ebf1'
+down_revision = '3730c7f1ea1b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('policy', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('gender_equality', sa.String(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('policy', schema=None) as batch_op:
+        batch_op.drop_column('gender_equality')
+
+    # ### end Alembic commands ###

compendium_v2/publishers/survey_publisher_2022.py

@@ -500,7 +500,8 @@ def transfer_policies(nren_dict):
         'connectiviy':     {2022: 16475, 2021: 16068, 2020: 15724, 2019: 15309, 2018: 14914},
         'acceptable_use':  {2022: 16477, 2021: 16070, 2020: 15726, 2019: 15311, 2018: 14916},
         'privacy':         {2022: 16479, 2021: 16072, 2020: 15728, 2019: 15575},
-        'data_protection': {2022: 16481, 2021: 16074, 2020: 15730, 2019: 15577}
+        'data_protection': {2022: 16481, 2021: 16074, 2020: 15730, 2019: 15577},
+        'gender':          {2022: 16761}  # TODO older years too??
     }
 
     data = {}

compendium_v2/publishers/survey_publisher_v2.py

@@ -8,7 +8,9 @@ from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, Fe
 from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
 
 
-def map_2023(year, nren, answers):
+def map_2023(nren, answers):
+    year = 2023
+
     for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
                         NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume]:
         db.session.execute(delete(table_class).where(table_class.year == 2023))
@@ -51,16 +53,22 @@ def map_2023(year, nren, answers):
     subs = answers.get("suborganization_details")
     if subs:
         for sub in subs:
+            role = sub.get("suborganization_role", "")
+            if role == "other":
+                role = sub.get("suborganization_role-Comment", "")
             db.session.add(SubOrganization(
                 nren_id=nren.id, nren=nren, year=year,
                 organization=sub.get("suborganization_name"),
-                role=sub.get("suborganization_role")  # TODO handle 'other' option properly
+                role=role
             ))
 
     ec_projects = answers.get("ec_project_names")
     if ec_projects:
         for ec_project in ec_projects:
-            db.session.add(ECProject(nren_id=nren.id, nren=nren, year=year, project=ec_project.get("ec_project_name")))
+            if ec_project:
+                db.session.add(
+                    ECProject(nren_id=nren.id, nren=nren, year=year, project=ec_project.get("ec_project_name"))
+                )
 
     strategy = answers.get("corporate_strategy_url", "")
     policies = answers.get("policies", {})
@@ -73,15 +81,14 @@ def map_2023(year, nren, answers):
             connectivity=policies.get("connectivity_policy", {}).get("url", ""),
             acceptable_use=policies.get("acceptable_use_policy", {}).get("url", ""),
             privacy_notice=policies.get("privacy_notice", {}).get("url", ""),
-            data_protection=policies.get("data_protection_contact", {}).get("url", "")
-            # TODO gender_equality_policy missing?
+            data_protection=policies.get("data_protection_contact", {}).get("url", ""),
+            gender_equality=policies.get("gender_equality_policy", {}).get("url", "")
         ))
 
     traffic_estimate = answers.get("traffic_estimate")
     if traffic_estimate:
         db.session.add(TrafficVolume(
             nren_id=nren.id, nren=nren, year=year,
-            strategic_plan=strategy,
             to_customers=Decimal(traffic_estimate.get("to_customers")),
             from_customers=Decimal(traffic_estimate.get("from_customers")),
             to_external=Decimal(traffic_estimate.get("to_external")),
@@ -90,10 +97,12 @@ def map_2023(year, nren, answers):
 
     institution_urls = answers.get("connected_sites_lists")
     if institution_urls:
-        db.session.add(InstitutionURLs(
-            nren_id=nren.id, nren=nren, year=year,
-            urls=institution_urls,
-        ))
+        urls=[i.get("connected_sites_url", "") for i in institution_urls if i.get("connected_sites_url", "") != ""]
+        if urls:
+            db.session.add(InstitutionURLs(
+                nren_id=nren.id, nren=nren, year=year,
+                urls=urls
+            ))
 
 
 def publish(year):
@@ -109,4 +118,4 @@ def publish(year):
     mapping_function = question_mapping[year]
 
     for response in responses:
-        mapping_function(year, response.nren, response.answers)
+        mapping_function(response.nren, response.answers)

test/test_survey_publisher_v2.py

+from decimal import Decimal
+import json
+import os
+
+from sqlalchemy import func, select
+
+from compendium_v2 import db
+from compendium_v2.db import model
+from compendium_v2.publishers.survey_publisher_v2 import map_2023
+
+
+JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
+
+
+def test_v2_publisher_empty(app):
+    data = {}
+
+    with app.app_context():
+        nren = model.NREN(name='name', country='country')
+        db.session.commit()
+
+    with app.app_context():
+        map_2023(nren, {"data": data})
+        db.session.commit()
+
+    with app.app_context():
+        budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+        assert budget_count == 0
+        # the main thing is actually that it doesnt crash
+
+
+def test_v2_publisher_full(app):
+    with open(JSON_FILE) as json_data:
+        data = json.load(json_data)
+
+    with app.app_context():
+        nren = model.NREN(name='name', country='country')
+        db.session.commit()
+
+    with app.app_context():
+        map_2023(nren, {"data": data})
+        db.session.commit()
+
+    with app.app_context():
+        budget = db.session.scalar(select(model.BudgetEntry.budget))
+        assert budget == Decimal("124.76")
+
+        funding_source = db.session.scalar(select(model.FundingSource))
+        assert funding_source.client_institutions == Decimal("0")
+        assert funding_source.european_funding == Decimal("20")
+        assert funding_source.gov_public_bodies == Decimal("70")
+        assert funding_source.commercial == Decimal("0")
+        assert funding_source.other == Decimal("10")
+
+        charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
+        assert charging_structure == model.FeeType.usage_based_fee
+
+        staff = db.session.scalar(select(model.NrenStaff))
+        assert staff.permanent_fte == Decimal("5.6")
+        assert staff.subcontracted_fte == Decimal("56")
+        assert staff.technical_fte == Decimal("2")
+        assert staff.non_technical_fte == Decimal("1")
+
+        parent = db.session.scalar(select(model.ParentOrganization.organization))
+        assert parent == "sdtfgd"
+
+        subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization))
+        subs = [s for s in subs]
+        assert subs[0].organization == "Aaerer"
+        assert subs[0].role == "Treer"
+        assert subs[1].organization == "gg"
+        assert subs[1].role == "hpc"
+        assert subs[2].organization == "werser"
+        assert subs[2].role == ""
+
+        projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project))
+        projects = [p for p in projects]
+        assert projects[0] == "dgdg"
+        assert projects[1] == "rrrrr"
+        assert projects[2] == "st"
+
+        policy = db.session.scalar(select(model.Policy))
+        assert policy.strategic_plan == "https://serere.com"
+        assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf"
+        assert policy.equal_opportunity == ""
+        assert policy.connectivity == "http://www.ucg.ac.me/skladiste/blog_1028/objava_10/fajlovi/Pravila%20o%20koriscenju%20Akademske%20mreze%20Univerziteta%20Crne%20Gore.pdf"
+        assert policy.acceptable_use == "http://www.ucg.ac.me/skladiste/blog_1028/objava_10/fajlovi/Pravila%20o%20koriscenju%20Akademske%20mreze%20Univerziteta%20Crne%20Gore.pdf"
+        assert policy.privacy_notice == ""
+        assert policy.data_protection == ""
+        assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore"
+
+        traffic = db.session.scalar(select(model.TrafficVolume))
+        assert traffic.to_customers == Decimal("3")
+        assert traffic.from_customers == Decimal("34")
+        assert traffic.to_external == Decimal("22")
+        assert traffic.from_external == Decimal("3")
+
+        client_urls = db.session.scalar(select(model.InstitutionURLs))
+        assert client_urls.urls == ["http://erse.com", "https://wwe.com"]