Add EC project model and publisher changes

compendium_v2/db/model.py

@@ -99,3 +99,11 @@ class SubOrganization(base_schema):
     year = sa.Column(sa.Integer, primary_key=True)
     organization = sa.Column(sa.String(128), primary_key=True)
     role = sa.Column(sa.String(128), nullable=False)
+
+
+class ECProject(base_schema):
+    __tablename__ = 'ec_project'
+    nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
+    nren = relationship(NREN, lazy='joined')
+    year = sa.Column(sa.Integer, primary_key=True)
+    project = sa.Column(sa.String(256), primary_key=True)

compendium_v2/migrations/versions/2edefbaa9db4_add_ec_project_model.py

+"""Add EC project model
+
+Revision ID: 2edefbaa9db4
+Revises: faf15719e2c6
+Create Date: 2023-04-25 13:58:48.504179
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2edefbaa9db4'
+down_revision = 'faf15719e2c6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'ec_project',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('project', sa.String(length=256), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_ec_project_nren_id_nren')),
+        sa.PrimaryKeyConstraint('nren_id', 'year', 'project', name=op.f('pk_ec_project'))
+    )
+
+
+def downgrade():
+    op.drop_table('ec_project')

compendium_v2/publishers/survey_publisher_2022.py

@@ -2,6 +2,8 @@ import logging
 import click
 import enum
 import math
+import json
+import html
 
 from sqlalchemy import text
 from collections import defaultdict
@@ -42,7 +44,7 @@ JOIN compendia c ON s.compendium_id = c.id
 WHERE
     a.question_id = {}
     AND c.year = 2022
-    AND a.value NOT IN ('""', '"NA"', '"N/A"')
+    AND a.value NOT IN ('""', '"NA"', '"N/A"', '[]', '[""]', '["-"]', '["/"]')
 ORDER BY n.id, a.question_id, a.updated_at DESC
 """
 
@@ -59,37 +61,41 @@ class StaffQuestion(enum.Enum):
     """
     Answers are numbers expressed in FTEs (full time equivalents)
     """
-    permanent_fte = 16414
-    subcontracted_fte = 16413
-    technical_fte = 16416
-    non_technical_fte = 16417
+    PERMANENT_FTE = 16414
+    SUBCONTRACTED_FTE = 16413
+    TECHNICAL_FTE = 16416
+    NON_TECHNICAL_FTE = 16417
 
 
 class OrgQuestion(enum.Enum):
     """
     Answers are strings
     """
-    parent_org_name = 16419
+    PARENT_ORG_NAME = 16419
 
-    sub_orgs_1_name = 16422
-    sub_orgs_1_choice = 16449
-    sub_orgs_1_role = 16426
+    SUB_ORGS_1_NAME = 16422
+    SUB_ORGS_1_CHOICE = 16449
+    SUB_ORGS_1_ROLE = 16426
 
-    sub_orgs_2_name = 16429
-    sub_orgs_2_choice = 16448
-    sub_orgs_2_role = 16434
+    SUB_ORGS_2_NAME = 16429
+    SUB_ORGS_2_CHOICE = 16448
+    SUB_ORGS_2_ROLE = 16434
 
-    sub_orgs_3_name = 16430
-    sub_orgs_3_choice = 16446
-    sub_orgs_3_role = 16435
+    SUB_ORGS_3_NAME = 16430
+    SUB_ORGS_3_CHOICE = 16446
+    SUB_ORGS_3_ROLE = 16435
 
-    sub_orgs_4_name = 16432
-    sub_orgs_4_choice = 16451
-    sub_orgs_4_role = 16438
+    SUB_ORGS_4_NAME = 16432
+    SUB_ORGS_4_CHOICE = 16451
+    SUB_ORGS_4_ROLE = 16438
 
-    sub_orgs_5_name = 16433
-    sub_orgs_5_choice = 16450
-    sub_orgs_5_role = 16439
+    SUB_ORGS_5_NAME = 16433
+    SUB_ORGS_5_CHOICE = 16450
+    SUB_ORGS_5_ROLE = 16439
+
+
+class ECQuestion(enum.Enum):
+    EC_PROJECT = 16453
 
 
 def query_budget():
@@ -225,8 +231,8 @@ def transfer_staff_data():
                 ).delete()
                 continue
 
-            employed = nren_info[StaffQuestion.permanent_fte] + nren_info[StaffQuestion.subcontracted_fte]
-            technical = nren_info[StaffQuestion.technical_fte] + nren_info[StaffQuestion.non_technical_fte]
+            employed = nren_info[StaffQuestion.PERMANENT_FTE] + nren_info[StaffQuestion.SUBCONTRACTED_FTE]
+            technical = nren_info[StaffQuestion.TECHNICAL_FTE] + nren_info[StaffQuestion.NON_TECHNICAL_FTE]
             if not math.isclose(
                 employed,
                 technical,
@@ -239,10 +245,10 @@ def transfer_staff_data():
             staff_data = model.NrenStaff(
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
-                permanent_fte=nren_info[StaffQuestion.permanent_fte],
-                subcontracted_fte=nren_info[StaffQuestion.subcontracted_fte],
-                technical_fte=nren_info[StaffQuestion.technical_fte],
-                non_technical_fte=nren_info[StaffQuestion.non_technical_fte],
+                permanent_fte=nren_info[StaffQuestion.PERMANENT_FTE],
+                subcontracted_fte=nren_info[StaffQuestion.SUBCONTRACTED_FTE],
+                technical_fte=nren_info[StaffQuestion.TECHNICAL_FTE],
+                non_technical_fte=nren_info[StaffQuestion.NON_TECHNICAL_FTE],
             )
             session.merge(staff_data)
         session.commit()
@@ -258,7 +264,7 @@ def transfer_nren_parent_org():
     with db.session_scope() as session:
         nren_dict = helpers.get_uppercase_nren_dict(session)
 
-        rows = query_question(OrgQuestion.parent_org_name)
+        rows = query_question(OrgQuestion.PARENT_ORG_NAME)
         for row in rows:
             nren_name = row[0].upper()
             value = str(row[1]).replace('"', '')
@@ -282,11 +288,11 @@ def transfer_nren_parent_org():
 def transfer_nren_sub_org():
 
     suborg_questions = [
-        (OrgQuestion.sub_orgs_1_name, OrgQuestion.sub_orgs_1_choice, OrgQuestion.sub_orgs_1_role),
-        (OrgQuestion.sub_orgs_2_name, OrgQuestion.sub_orgs_2_choice, OrgQuestion.sub_orgs_2_role),
-        (OrgQuestion.sub_orgs_3_name, OrgQuestion.sub_orgs_3_choice, OrgQuestion.sub_orgs_3_role),
-        (OrgQuestion.sub_orgs_4_name, OrgQuestion.sub_orgs_4_choice, OrgQuestion.sub_orgs_4_role),
-        (OrgQuestion.sub_orgs_5_name, OrgQuestion.sub_orgs_5_choice, OrgQuestion.sub_orgs_5_role)
+        (OrgQuestion.SUB_ORGS_1_NAME, OrgQuestion.SUB_ORGS_1_CHOICE, OrgQuestion.SUB_ORGS_1_ROLE),
+        (OrgQuestion.SUB_ORGS_2_NAME, OrgQuestion.SUB_ORGS_2_CHOICE, OrgQuestion.SUB_ORGS_2_ROLE),
+        (OrgQuestion.SUB_ORGS_3_NAME, OrgQuestion.SUB_ORGS_3_CHOICE, OrgQuestion.SUB_ORGS_3_ROLE),
+        (OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
+        (OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
     ]
 
     with db.session_scope() as session:
@@ -329,6 +335,49 @@ def transfer_nren_sub_org():
         session.commit()
 
 
+def transfer_ec_projects():
+    with db.session_scope() as session:
+        nren_dict = helpers.get_uppercase_nren_dict(session)
+
+        # delete all existing EC projects, in case something changed
+        session.query(model.ECProject).filter(
+            model.ECProject.year == 2022,
+        ).delete()
+
+        rows = query_question(ECQuestion.EC_PROJECT)
+        for row in rows:
+            nren_name = row[0].upper()
+
+            if nren_name not in nren_dict:
+                logger.info(f'{nren_name} unknown. Skipping.')
+                continue
+
+            try:
+                value = json.loads(row[1])
+            except json.decoder.JSONDecodeError:
+                logger.info(f'JSON decode error for EC project data for {nren_name}. Skipping.')
+                continue
+
+            for val in value:
+                if not val:
+                    logger.info(f'Invalid EC project value for {nren_name}: {val}.')
+                    continue
+
+                # strip html entities/NBSP from val
+                val = html.unescape(val).replace('\xa0', ' ')
+
+                # some answers include contract numbers, which we don't want here
+                val = val.split('(contract n')[0]
+
+                ec_project = model.ECProject(
+                    nren_id=nren_dict[nren_name].id,
+                    year=2022,
+                    project=str(val).strip()
+                )
+                session.add(ec_project)
+        session.commit()
+
+
 def _cli(config):
     helpers.init_db(config)
     transfer_budget()
@@ -336,6 +385,7 @@ def _cli(config):
     transfer_staff_data()
     transfer_nren_parent_org()
     transfer_nren_sub_org()
+    transfer_ec_projects()
 
 
 @click.command()