Finished release 0.7.

Changelog.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.8] - 2023-03-21
+- Add survey-publisher-v1
+- Add survey-publisher-2022
+- Add funding sources API
+- Add initial funding sources frontend work
+
 ## [0.7] - 2023-02-22
 - Fix API call URL http -> https redirect bug
 

compendium_v2/app.py

@@ -3,7 +3,6 @@ default app creation
 """
 import compendium_v2
 from compendium_v2 import environment
-
 environment.setup_logging()
 
 app = compendium_v2.create_app()

compendium_v2/background_task/csv/BudgetCsvFile.csv

+ACOnet,6.4,2021
+ACOnet,6.4,2020
+ACOnet,6.1,2019
+ACOnet,6.1,2018
+ACOnet,5.7,2017
+AMRES,3.1,2021
+AMRES,2.35,2020
+AMRES,2.47,2019
+AMRES,2.02,2018
+AMRES,1.77,2017
+ANA,0.8,2021
+ANA,0.8,2020
+ANA,0.8,2019
+ANA,0.8,2018
+ARNES,9.8,2021
+ARNES,9.0,2020
+ARNES,8.0,2019
+ARNES,7.5,2018
+ARNES,6.0,2017
+ASNET,0.5,2021
+ASNET,0.48,2020
+ASNET,0.48,2019
+ASNET,0.25,2018
+ASNET,0.15,2017
+AzScienceNet,1.2,2021
+AzScienceNet,1.2,2020
+AzScienceNet,1.2,2019
+AzScienceNet,2.0,2018
+AzScienceNet,0.0,2017
+BASNET,0.63,2021
+BASNET,0.94,2020
+BASNET,0.92,2019
+BASNET,0.72,2018
+BASNET,0.82,2017
+BELNET,17.3,2021
+BELNET,13.85,2020
+BELNET,14.48,2019
+BELNET,15.52,2018
+BREN,0.03,2021
+CARNet,86.08,2021
+CARNet,30.31,2020
+CARNet,36.45,2019
+CARNet,20.1,2018
+CARNet,34.73,2017
+CESNET,19.5,2021
+CESNET,18.86,2020
+CESNET,18.97,2019
+CESNET,17.88,2018
+CESNET,14.7,2017
+CYNET,0.97,2021
+CYNET,0.92,2020
+CYNET,0.92,2019
+CYNET,3.6,2018
+CYNET,0.77,2017
+DeIC,9.92,2021
+DeIC,7.95,2020
+DeIC,7.1,2019
+DeIC,7.41,2018
+DFN,51.78,2021
+DFN,50.37,2020
+DFN,42.34,2019
+DFN,42.0,2018
+DFN,45.0,2017
+EENet,1.29,2021
+EENet,1.29,2020
+EENet,4.77,2019
+EENet,2.34,2018
+EENet,2.34,2017
+FCCN,24.82,2021
+FCCN,20.11,2020
+FCCN,16.25,2019
+FCCN,14.25,2018
+FCCN,8.07,2017
+Funet,8.3,2021
+Funet,8.3,2020
+Funet,7.9,2019
+Funet,7.5,2018
+Funet,7.5,2017
+GARR,22.0,2021
+GARR,23.0,2020
+GARR,22.33,2019
+GARR,22.15,2018
+GARR,21.97,2017
+GRENA,0.4,2021
+GRENA,0.4,2020
+GRENA,0.4,2019
+GRENA,0.3,2018
+GRENA,0.3,2017
+GRNET S.A.,7.4,2021
+GRNET S.A.,7.0,2020
+GRNET S.A.,6.9,2019
+GRNET S.A.,6.9,2018
+GRNET S.A.,6.5,2017
+HEAnet,25.2,2021
+HEAnet,24.89,2020
+HEAnet,25.59,2019
+HEAnet,25.06,2018
+HEAnet,29.6,2017
+IUCC,4.28,2021
+IUCC,4.3,2020
+IUCC,3.48,2019
+IUCC,3.72,2018
+IUCC,3.9,2017
+Jisc,69.61,2021
+Jisc,63.41,2020
+Jisc,65.23,2019
+KIFU (NIIF),34.0,2021
+KIFU (NIIF),34.0,2020
+KIFU (NIIF),48.0,2019
+KIFU (NIIF),12.0,2018
+KIFU (NIIF),12.0,2017
+LAT,1.24,2019
+LAT,1.03,2018
+LAT,0.0,2017
+LITNET,2.65,2021
+LITNET,2.33,2020
+LITNET,2.28,2019
+LITNET,2.28,2018
+LITNET,2.06,2017
+MARNET,0.63,2021
+MARNET,1.14,2020
+MARNET,0.95,2019
+MARNET,0.33,2018
+MREN,0.07,2021
+MREN,0.07,2020
+MREN,0.07,2019
+MREN,0.07,2018
+PIONIER,0.0,2020
+RedIRIS,23.0,2021
+RedIRIS,8.0,2020
+RedIRIS,8.0,2019
+RedIRIS,8.0,2018
+RedIRIS,8.0,2017
+RENAM,0.45,2021
+RENAM,0.32,2020
+RENAM,0.38,2019
+RENAM,0.34,2018
+RENAM,0.38,2017
+RENATER,39.0,2021
+RENATER,31.1,2020
+RENATER,32.2,2019
+RENATER,33.9,2018
+RENATER,29.1,2017
+RESTENA,4.54,2021
+RESTENA,4.75,2020
+RESTENA,4.03,2019
+RESTENA,3.48,2018
+RESTENA,2.06,2017
+RhNET,0.0,2019
+RoEduNet,2.0,2020
+RoEduNet,2.0,2019
+RoEduNet,2.0,2018
+RoEduNet,2.0,2017
+SANET,1.98,2021
+SANET,2.18,2020
+SANET,1.98,2019
+SANET,1.98,2018
+SANET,1.98,2017
+SUNET,28.0,2021
+SUNET,26.0,2020
+SUNET,24.0,2019
+SUNET,19.0,2018
+SUNET,0.0,2017
+SURF,54.97,2020
+SURF,52.86,2019
+SURF,40.5,2018
+SURF,40.2,2017
+SWITCH,30.56,2021
+SWITCH,27.74,2020
+SWITCH,13.46,2019
+SWITCH,14.0,2018
+SWITCH,14.2,2017
+ULAKBIM,17.0,2021
+ULAKBIM,17.2,2020
+ULAKBIM,17.0,2019
+ULAKBIM,21.0,2018
+ULAKBIM,18.5,2017
+UNINETT,15.0,2019
+UNINETT,20.0,2018
+UNINETT,30.0,2017

compendium_v2/background_task/csv/FundingSourceCsvFile.csv

+ACOnet,2020,100.0,0.0,0.0,0.0,0.0
+AMRES,2020,0.0,0.0,100.0,0.0,0.0
+ANA,2020,13.0,1.0,1.0,1.0,0.0
+ARNES,2020,0.0,5.0,84.0,11.0,0.0
+ASNET-AM,2020,10.0,48.0,40.0,0.0,1.0
+AzScienceNet,2020,60.0,40.0,60.0,0.0,0.0
+BASNET,2020,26.0,64.0,10.0,0.0,0.0
+Belnet,2020,39.0,2.0,52.0,7.0,0.0
+BREN,2020,25.0,0.0,0.0,0.0,0.0
+CARNET,2020,0.0,59.32,37.64,3.04,0.0
+CESNET,2020,21.0,3.0,73.0,0.0,2.0
+CYNET,2020,63.0,25.0,12.0,0.0,0.0
+DeIC,2020,95.0,1.0,0.0,4.0,0.0
+DFN,2020,94.0,3.0,1.5,0.0,1.5
+EENet,2020,0.0,71.0,26.0,0.0,0.0
+FCCN,2020,3.07,0.21,95.85,0.0,0.87
+Funet,2020,60.0,0.0,40.0,0.0,0.0
+GARR,2020,88.92,3.69,7.34,0.0,0.04
+GRENA,2020,40.0,50.0,0.0,5.0,5.0
+GRNET S.A.,2020,0.0,20.0,75.0,0.0,5.0
+HEAnet,2020,16.0,1.0,74.0,9.0,0.0
+IUCC,2020,91.0,9.0,0.0,0.0,0.0
+Jisc,2020,20.069,1.562,48.554,29.814,0.0
+KIFU (NIIF),2020,4.0,2.0,94.0,0.0,0.0
+LITNET,2020,10.0,33.0,57.0,0.0,0.0
+MARnet,2020,0.0,20.0,56.0,24.0,0.0
+MREN,2020,0.0,20.0,70.0,0.0,5.0
+PIONIER,2020,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2020,0.0,7.0,92.0,0.0,1.0
+RENAM,2020,24.0,74.0,2.0,0.0,0.0
+RENATER,2020,11.0,4.0,85.0,0.0,0.0
+RESTENA,2020,6.0,3.0,34.0,37.0,20.0
+RhNET,2020,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2020,0.0,0.0,100.0,0.0,0.0
+SANET,2020,7.0,0.0,93.0,0.0,0.0
+LAT,2020,0.0,0.0,0.0,0.0,0.0
+SUNET,2020,70.0,0.0,20.0,0.0,10.0
+SURFnet,2020,60.0,2.0,38.0,0.0,0.0
+SWITCH,2020,55.5,0.9,1.1,42.5,0.0
+ULAKBIM,2020,0.0,0.3,99.5,0.0,0.2
+UNINETT,2020,0.0,0.0,0.0,0.0,0.0
+UoM,2020,0.0,0.0,0.0,0.0,0.0
+ACOnet,2019,100.0,0.0,0.0,0.0,0.0
+AMRES,2019,0.0,0.0,100.0,0.0,0.0
+ANA,2019,13.0,1.0,1.0,1.0,0.0
+ARNES,2019,0.0,15.0,72.0,13.0,0.0
+ASNET-AM,2019,0.0,0.0,0.0,0.0,0.0
+AzScienceNet,2019,58.0,40.0,60.0,0.0,0.0
+BASNET,2019,27.0,64.0,9.0,0.0,0.0
+Belnet,2019,38.46,2.4,52.16,6.98,0.0
+BREN,2019,25.0,0.0,0.0,0.0,0.0
+CARNET,2019,0.0,59.11,37.89,3.0,0.0
+CESNET,2019,23.0,4.0,72.0,1.0,0.0
+CYNET,2019,54.6,29.6,15.8,0.0,0.0
+DeIC,2019,0.0,0.0,0.0,0.0,0.0
+DFN,2019,98.0,2.0,0.0,0.0,0.0
+EENet,2019,3.0,71.0,26.0,0.0,0.0
+FCCN,2019,2.0,0.22,96.4,0.0,1.29
+Funet,2019,60.0,0.0,40.0,0.0,0.0
+GARR,2019,87.66,4.73,7.57,0.0,0.04
+GRENA,2019,40.0,50.0,0.0,5.0,5.0
+GRNET S.A.,2019,0.0,20.0,75.0,0.0,5.0
+HEAnet,2019,14.0,1.0,84.0,1.0,0.0
+IUCC,2019,86.0,14.0,0.0,0.0,0.0
+Jisc,2019,23.0,1.0,70.0,6.0,0.0
+KIFU (NIIF),2019,3.0,2.0,95.0,0.0,0.0
+LITNET,2019,0.0,0.0,0.0,0.0,0.0
+MARnet,2019,14.0,17.0,69.0,0.0,0.0
+MREN,2019,0.0,14.0,66.0,20.0,0.0
+PIONIER,2019,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2019,0.0,7.0,92.0,0.0,1.0
+RENAM,2019,39.0,60.0,1.0,0.0,0.0
+RENATER,2019,30.0,4.0,64.0,1.0,1.0
+RESTENA,2019,5.0,5.0,37.0,38.0,15.0
+RhNET,2019,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2019,0.0,0.0,100.0,0.0,0.0
+SANET,2019,7.0,0.0,93.0,0.0,0.0
+LAT,2019,0.0,0.0,0.0,0.0,0.0
+SUNET,2019,70.0,0.0,25.0,0.0,5.0
+SURFnet,2019,58.0,2.0,40.0,0.0,0.0
+SWITCH,2019,48.0,1.0,1.0,50.0,0.0
+ULAKBIM,2019,0.0,0.0,99.5,0.0,0.0
+UNINETT,2019,0.0,0.0,0.0,0.0,0.0
+UoM,2019,0.0,0.0,0.0,0.0,0.0
+ACOnet,2018,100.0,0.0,0.0,0.0,0.0
+AMRES,2018,0.0,0.0,100.0,0.0,0.0
+ANA,2018,13.0,1.0,1.0,1.0,0.0
+ARNES,2018,0.0,4.0,84.0,12.0,0.0
+ASNET-AM,2018,20.0,40.0,40.0,0.0,0.0
+AzScienceNet,2018,0.0,95.0,5.0,0.0,0.0
+BASNET,2018,34.0,58.0,8.0,0.0,0.0
+Belnet,2018,20.83,1.99,71.41,5.77,0.0
+BREN,2018,0.0,0.0,0.0,0.0,0.0
+CARNET,2018,0.0,34.6,62.21,3.19,0.0
+CESNET,2018,22.0,7.0,70.0,1.0,0.0
+CYNET,2018,13.0,20.0,63.3,0.0,0.0
+DeIC,2018,99.0,1.0,0.0,0.0,0.0
+DFN,2018,96.0,2.0,0.0,0.0,2.0
+EENet,2018,5.0,42.0,50.0,0.0,0.0
+FCCN,2018,2.0,0.54,95.74,0.0,1.72
+Funet,2018,60.0,0.0,40.0,0.0,0.0
+GARR,2018,87.55,4.77,7.63,0.0,0.05
+GRENA,2018,40.0,45.0,0.0,5.0,10.0
+GRNET S.A.,2018,0.0,35.0,60.0,0.0,5.0
+HEAnet,2018,10.0,1.0,86.0,1.0,0.0
+IUCC,2018,86.0,14.0,0.0,0.0,0.0
+Jisc,2018,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2018,10.0,5.0,85.0,0.0,0.0
+LITNET,2018,14.0,20.0,66.0,0.0,0.0
+MARnet,2018,0.0,25.0,42.0,33.0,0.0
+MREN,2018,0.0,20.0,70.0,0.0,10.0
+PIONIER,2018,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2018,0.0,7.0,92.0,0.0,1.0
+RENAM,2018,52.0,46.0,2.0,0.0,0.0
+RENATER,2018,30.0,4.0,65.0,0.0,1.0
+RESTENA,2018,6.0,2.0,40.0,37.0,15.0
+RhNET,2018,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2018,0.0,0.0,100.0,0.0,0.0
+SANET,2018,7.0,0.0,93.0,0.0,0.0
+LAT,2018,0.0,0.0,0.0,0.0,0.0
+SUNET,2018,70.0,0.0,25.0,0.0,5.0
+SURFnet,2018,66.0,3.0,31.0,0.0,0.0
+SWITCH,2018,52.0,1.0,3.0,44.0,0.0
+ULAKBIM,2018,0.0,0.5,99.4,0.0,0.1
+UNINETT,2018,75.0,0.0,25.0,0.0,0.0
+UoM,2018,0.0,0.0,0.0,0.0,0.0
+ACOnet,2017,100.0,0.0,0.0,0.0,0.0
+AMRES,2017,0.0,0.0,13.5,86.5,0.0
+ARNES,2017,0.0,12.0,3.0,73.0,12.0
+ASNET-AM,2017,15.0,0.0,1.0,30.0,40.0
+AzScienceNet,2017,0.0,0.0,0.0,2.0,0.0
+BASNET,2017,56.0,0.0,3.0,17.0,24.0
+CARNet,2017,0.0,1.94,1.99,40.4,55.68
+CESNET,2017,23.0,0.0,3.0,69.0,3.0
+CYNET,2017,64.1,3.8,30.8,0.0,0.0
+DFN,2017,89.0,0.0,2.0,0.0,0.0
+EENet,2017,5.0,0.0,6.0,50.0,42.0
+FCCN,2017,4.18,0.0,4.54,90.64,0.64
+Funet,2017,60.0,0.0,0.0,40.0,0.0
+GARR,2017,88.0,0.0,3.23,7.56,1.72
+GRENA,2017,35.0,5.0,0.0,0.0,50.0
+GRNET S.A.,2017,0.0,0.0,0.0,55.0,50.0
+HEAnet,2017,10.0,1.0,2.0,86.0,1.0
+IUCC,2017,85.0,0.0,10.0,0.0,5.0
+Jisc,2017,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2017,20.0,0.0,5.0,70.0,5.0
+LITNET,2017,0.0,0.0,5.0,72.0,22.0
+MARnet,2017,0.0,0.0,0.0,0.0,0.0
+MREN,2017,0.0,0.0,0.0,1.0,0.0
+PIONIER,2017,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2017,0.0,0.0,6.0,92.0,1.0
+RENAM,2017,1.0,0.0,15.0,3.0,24.0
+RENATER,2017,34.0,1.0,4.0,58.0,2.0
+RESTENA,2017,10.0,20.0,7.0,63.0,0.0
+RoEduNet,2017,0.0,0.0,1.0,100.0,0.0
+SANET,2017,0.0,0.0,0.0,0.0,0.0
+SigmaNet,2017,0.0,0.0,0.0,0.0,0.0
+SUNET,2017,75.0,0.0,0.0,25.0,0.0
+SURFnet,2017,70.0,0.0,3.0,27.0,0.0
+SWITCH,2017,46.0,47.0,3.0,4.0,0.0
+ULAKBIM,2017,0.0,0.0,2.0,0.0,0.0
+UNINETT,2017,0.0,0.0,0.0,0.0,0.0
+ACOnet,2016,0.0,0.0,0.0,0.0,0.0
+AMRES,2016,0.0,13.5,86.5,0.0,0.0
+ARNES,2016,16.0,0.0,84.0,2.0,1.0
+ASNET-AM,2016,0.0,0.0,0.0,0.0,0.0
+AzScienceNet,2016,0.0,0.0,0.0,0.0,0.0
+BASNET,2016,0.0,3.0,17.0,24.0,0.0
+CARNet,2016,2.05,2.08,0.0,63.34,0.0
+CESNET,2016,0.0,3.0,69.0,3.0,2.0
+CYNET,2016,0.0,20.0,0.0,20.0,1.0
+DFN,2016,0.0,1.0,0.0,0.0,17.0
+EENet,2016,0.0,6.0,50.0,42.0,0.0
+FCCN,2016,0.0,4.6,91.5,1.15,0.39
+Funet,2016,0.0,0.0,40.0,0.0,0.0
+GARR,2016,0.0,3.24,9.64,0.43,0.09
+GRENA,2016,5.0,0.0,0.0,50.0,10.0
+GRNET S.A.,2016,0.0,0.0,55.0,50.0,5.0
+HEAnet,2016,2.0,2.0,86.0,1.0,0.0
+IUCC,2016,0.0,14.0,0.0,0.0,17.0
+Jisc,2016,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2016,0.0,3.0,2.0,13.0,0.0
+LITNET,2016,0.0,5.0,88.0,8.0,0.0
+MARnet,2016,0.0,0.0,0.0,0.0,0.0
+MREN,2016,0.0,0.0,1.0,0.0,0.0
+PIONIER,2016,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2016,0.0,6.0,92.0,1.0,1.0
+RENAM,2016,0.0,20.0,2.0,23.0,0.0
+RENATER,2016,0.0,0.0,0.0,2.0,0.0
+RESTENA,2016,20.0,7.0,63.0,0.0,0.0
+RoEduNet,2016,0.0,1.0,100.0,0.0,0.0
+SANET,2016,0.0,0.0,0.0,0.0,0.0
+SigmaNet,2016,0.0,0.0,0.0,0.0,0.0
+SUNET,2016,0.0,0.0,25.0,0.0,0.0
+SURFnet,2016,0.0,3.0,33.0,0.0,0.0
+SWITCH,2016,48.0,2.0,5.0,0.0,0.0
+ULAKBIM,2016,0.0,0.0,0.0,0.0,0.0
+UNINETT,2016,0.0,0.0,0.0,0.0,0.0

compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py

+import openpyxl
+import csv
+import os
+from compendium_v2 import db
+from compendium_v2.db import model
+from compendium_v2.environment import setup_logging
+import logging
+
+setup_logging()
+
+logger = logging.getLogger('xlsx_to_csv_sheet_parsing_task')
+
+
+# Import the data to database
+def import_countries():
+    with db.session_scope() as session:
+        with open('csv/BudgetCsvFile.csv') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                data = model.BudgetEntry(
+                    nren=row[0], budget=row[1], year=row[2])
+                data.save()
+    session.add(data)
+
+
+def parse_budget_xlsx_file():
+    try:
+        # load the xlsx file
+        filename = "compendium_v2/background_task/xlsx" \
+                   "/2021_Organisation_DataSeries.xlsx "
+        csv_out_file = "compendium_v2/background_task/csv/BudgetCsvFile.csv"
+        sheet_name = "1. Budget"
+        wb = openpyxl.load_workbook(
+            filename, data_only=True, read_only=True)
+
+        # select the active worksheet
+        ws = wb[sheet_name]
+        if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
+            os.remove(csv_out_file)
+            print("file deleted " + csv_out_file)
+        # iterate over the rows in the worksheet
+        for row in range(14, 57):
+            for col in range(3, 8):
+                # extract the data from the row
+                nren = ws.cell(row=row, column=2).value
+                budget = ws.cell(row=row, column=col).value
+                year = ws.cell(row=13, column=col).value
+
+                if budget is not None:
+                    budget = round(budget / 1000000, 2)
+                    if budget > 200:
+                        logger.info(
+                            f'{nren} has budget set to '
+                            f'>200M EUR for {year}. ({budget})')
+
+                    # process the data (e.g. save to database)
+                    # print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
+                    output_csv_file = csv.writer(
+                        open(csv_out_file, 'a'),
+                        delimiter=",")
+                    output_csv_file.writerow([nren, budget, year])
+        output_csv_file
+    except Exception as e:
+        print(e)
+
+        # import_countries()
+
+
+def parse_income_source_xlsx_file():
+    try:
+        # load the xlsx file
+        filename = "compendium_v2/background_task/xlsx" \
+                   "/2021_Organisation_DataSeries.xlsx "
+        csv_out_file = "compendium_v2/background_task/csv" \
+                       "/FundingSourceCsvFile.csv "
+        sheet_name = "2. Income Sources"
+        wb = openpyxl.load_workbook(
+            filename, data_only=True, read_only=True)
+
+        # select the active worksheet
+        ws = wb[sheet_name]
+        if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
+            os.remove(csv_out_file)
+            print("file deleted " + csv_out_file)
+
+        def hard_number_convert(s, source_name, nren, year):
+            if s is None:
+                logger.info(
+                    f'Invalid Value :{nren} has empty value for {source_name}.'
+                    + f'for year ({year})')
+                return float(0)
+            """ Returns True if string is a number. """
+            try:
+                return float(s)
+            except ValueError:
+                logger.info(
+                    f'Invalid Value :{nren} has empty value for {source_name}.'
+                    + f'for year ({year}) with value ({s})')
+                return float(0)
+
+        # iterate over the rows in the worksheet
+
+        def create_csv_per_year(start_row, end_row, yearI, col_start):
+            for row in range(start_row, end_row):
+                # extract the data from the row
+                nren = ws.cell(row=row, column=col_start).value
+                client_institution = ws.cell(row=row,
+                                             column=col_start + 3).value
+                european_funding = ws.cell(row=row, column=col_start + 4).value
+                gov_public_bodies = ws.cell(row=row,
+                                            column=col_start + 5).value
+                commercial = ws.cell(row=row, column=col_start + 6).value
+                other = ws.cell(row=row, column=col_start + 7).value
+                year = yearI
+
+                client_institution = hard_number_convert(
+                    client_institution, "client institution", nren, year)
+                european_funding = hard_number_convert(
+                    european_funding, "european funding", nren, year)
+                gov_public_bodies = hard_number_convert(
+                    gov_public_bodies, "gov/public_bodies", nren, year)
+                commercial = hard_number_convert(
+                    commercial, "commercial", nren, year)
+                other = hard_number_convert(
+                    other, "other", nren, year)
+
+                # process the data (e.g. save to database)
+                if nren is not None:
+                    output_csv_file = csv.writer(
+                        open(csv_out_file, 'a'),
+                        delimiter=",")
+                    output_csv_file.writerow([nren, year, client_institution,
+                                              european_funding,
+                                              gov_public_bodies,
+                                              commercial, other])
+
+        # For 2020
+        create_csv_per_year(8, 50, 2020, 3)
+        # # For 2019
+        create_csv_per_year(8, 50, 2019, 12)
+        # # For 2018
+        create_csv_per_year(8, 50, 2018, 21)
+        # # For 2017
+        create_csv_per_year(8, 50, 2017, 32)
+        # # For 2016
+        create_csv_per_year(8, 50, 2016, 43)
+
+    except Exception as e:
+        print(e)

compendium_v2/db/model.py

@@ -4,6 +4,7 @@ import sqlalchemy as sa
 from typing import Any
 
 from sqlalchemy.ext.declarative import declarative_base
+
 # from sqlalchemy.orm import relationship
 
 logger = logging.getLogger(__name__)
@@ -17,5 +18,18 @@ class BudgetEntry(base_schema):
     id = sa.Column(sa.Integer, sa.Sequence(
         'budgetentry_seq_id_seq'), nullable=False)
     nren = sa.Column(sa.String(128), primary_key=True)
-    budget = sa.Column(sa.String(128), nullable=True)
+    budget = sa.Column(sa.String(128))
+    year = sa.Column(sa.Integer, primary_key=True)
+
+
+class FundingSource(base_schema):
+    __tablename__ = 'funding_source'
+    id = sa.Column(sa.Integer, sa.Sequence(
+        'fundingentry_seq_id_seq'), nullable=False)
+    nren = sa.Column(sa.String(128), primary_key=True)
     year = sa.Column(sa.Integer, primary_key=True)
+    client_institutions = sa.Column(sa.String(128))
+    european_funding = sa.Column(sa.String(128))
+    gov_public_bodies = sa.Column(sa.String(128))
+    commercial = sa.Column(sa.String(128))
+    other = sa.Column(sa.String(128))

compendium_v2/migrations/versions/833c99d745d7_create_funding_source_schema.py

+"""create funding source schema
+
+Revision ID: 833c99d745d7
+Revises: cbcd21fcc151
+Create Date: 2023-03-07 23:47:30.157499
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '833c99d745d7'
+down_revision = 'cbcd21fcc151'
+branch_labels = None
+depends_on = None
+
+funding_id_seq = sa.Sequence('fundingentry_seq_id_seq')
+
+
+def upgrade():
+    op.execute(sa.schema.CreateSequence(funding_id_seq))  # create the sequence
+    op.create_table(
+        'funding_source',
+        sa.Column('id', sa.Integer, funding_id_seq, nullable=False,
+                  server_default=funding_id_seq.next_value()),
+        sa.Column('nren', sa.String(length=128), nullable=False),
+        sa.Column('year', sa.Integer, nullable=False),
+        sa.Column('client_institutions', sa.String(length=128)),
+        sa.Column('european_funding', sa.String(length=128)),
+        sa.Column('gov_public_bodies', sa.String(length=128)),
+        sa.Column('commercial', sa.String(length=128)),
+        sa.Column('other', sa.String(length=128)),
+        sa.PrimaryKeyConstraint('nren', 'year')
+    )
+
+
+def downgrade():
+    op.execute(
+        sa.schema.DropSequence(sa.Sequence('fundingentry_seq_id_seq')))
+    op.drop_table('funding_source')

compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py

@@ -8,7 +8,6 @@ Create Date: 2023-02-07 15:56:22.086064
 from alembic import op
 import sqlalchemy as sa
 
-
 # revision identifiers, used by Alembic.
 revision = 'cbcd21fcc151'
 down_revision = None

compendium_v2/publishers/survey_publisher_2022.py

+import logging
+import click
+import enum
+import math
+
+from compendium_v2.environment import setup_logging
+from compendium_v2.config import load
+from compendium_v2 import db, survey_db
+from compendium_v2.db import model
+
+setup_logging()
+
+logger = logging.getLogger('survey-publisher-2022')
+
+BUDGET_QUERY = """
+SELECT DISTINCT ON (n.id, a.question_id)
+    n.abbreviation AS nren,
+    a.value AS budget
+FROM answers a
+JOIN nrens n ON a.nren_id = n.id
+JOIN questions q ON a.question_id = q.id
+JOIN sections s ON q.section_id = s.id
+JOIN compendia c ON s.compendium_id = c.id
+WHERE
+    a.question_id = 16402
+AND c.year = 2022
+ORDER BY n.id, a.question_id DESC
+"""
+
+FUNDING_SOURCES_TEMPLATE_QUERY = """
+SELECT DISTINCT ON (n.id, a.question_id)
+    n.abbreviation AS nren,
+    a.value AS value
+FROM answers a
+JOIN nrens n ON a.nren_id = n.id
+JOIN questions q ON a.question_id = q.id
+JOIN sections s ON q.section_id = s.id
+JOIN compendia c ON s.compendium_id = c.id
+WHERE
+    a.question_id = {}
+AND c.year = 2022
+ORDER BY n.id, a.question_id DESC
+"""
+
+
+class FundingSource(enum.Enum):
+    CLIENT_INSTITUTIONS = 16405
+    EUROPEAN_FUNDING = 16406
+    COMMERCIAL = 16407
+    OTHER = 16408
+    GOV_PUBLIC_BODIES = 16409
+
+
+def setup_db(config):
+    dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+    db.init_db_model(dsn_prn)
+    dsn_survey = config['SURVEY_DATABASE_URI']
+    survey_db.init_db_model(dsn_survey)
+
+
+def transfer_budget():
+    with db.session_scope() as session, survey_db.session_scope() as survey:
+        rows = survey.execute(BUDGET_QUERY)
+        for row in rows:
+            nren = row['nren']
+            _budget = row['budget']
+            try:
+                budget = float(_budget.replace('"', '').replace(',', ''))
+            except ValueError:
+                logger.info(
+                    f'{nren} has no budget for 2022. Skipping. ({_budget}))')
+                continue
+
+            if budget > 200:
+                logger.info(
+                    f'{nren} has budget set to >200M EUR for 2022. ({budget})')
+
+            budget_entry = model.BudgetEntry(
+                nren=nren,
+                budget=budget,
+                year=2022,
+            )
+            session.merge(budget_entry)
+        session.commit()
+
+
+def transfer_funding_sources():
+    with db.session_scope() as session, survey_db.session_scope() as survey:
+        sources = {source.value: dict() for source in FundingSource}
+        nrens = set()
+
+        for source in FundingSource:
+            data = survey.execute(
+                FUNDING_SOURCES_TEMPLATE_QUERY.format(source.value))
+            for row in data:
+                nren = row['nren']
+                nrens.add(nren)
+                _value = row['value']
+                try:
+                    value = float(_value.replace('"', '').replace(',', ''))
+                except ValueError:
+                    name = source.name
+                    logger.info(
+                        f'{nren} has invalid value for {name}.'
+                        + f' ({_value}))')
+                    value = 0
+
+                sources[source.value][nren] = value
+
+        client_institutions = sources[FundingSource.CLIENT_INSTITUTIONS.value]
+        european_funding = sources[FundingSource.EUROPEAN_FUNDING.value]
+        gov_public_bodies = sources[FundingSource.GOV_PUBLIC_BODIES.value]
+        commercial = sources[FundingSource.COMMERCIAL.value]
+        other = sources[FundingSource.OTHER.value]
+
+        _data = [client_institutions, european_funding,
+                 gov_public_bodies, commercial, other]
+
+        for nren in nrens:
+
+            def _get_nren(source, nren):
+                try:
+                    return source[nren]
+                except KeyError:
+                    return 0
+            total = sum([_get_nren(source, nren) for source in _data])
+
+            if not math.isclose(total, 100, abs_tol=0.01):
+                logger.info(
+                    f'{nren} funding sources do not sum to 100%. ({total})')
+
+            funding_source = model.FundingSource(
+                nren=nren,
+                year=2022,
+                client_institutions=client_institutions.get(nren, 0),
+                european_funding=european_funding.get(nren, 0),
+                gov_public_bodies=gov_public_bodies.get(nren, 0),
+                commercial=commercial.get(nren, 0),
+                other=other.get(nren, 0),
+            )
+            session.merge(funding_source)
+        session.commit()
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+    app_config = load(open(config, 'r'))
+    setup_db(app_config)
+    transfer_budget()
+    transfer_funding_sources()
+
+
+if __name__ == "__main__":
+    cli()

compendium_v2/publishers/survey_publisher_v1.py

+import logging
+import math
+import csv
+import click
+
+from collections import defaultdict
+from compendium_v2.environment import setup_logging
+from compendium_v2 import db, survey_db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
+from compendium_v2.config import load
+from compendium_v2.db import model
+from compendium_v2.survey_db import model as survey_model
+
+setup_logging()
+
+logger = logging.getLogger('survey_publisher_v1')
+
+
+def init_db(config):
+    dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+    db.init_db_model(dsn_prn)
+    dsn_survey = config['SURVEY_DATABASE_URI']
+    survey_db.init_db_model(dsn_survey)
+
+
+def db_budget_migration():
+    with survey_db.session_scope() as survey_session, \
+            db.session_scope() as session:
+        _entries = session.query(model.BudgetEntry)
+
+        inserted = defaultdict(dict)
+
+        for entry in _entries:
+            inserted[entry.nren][entry.year] = entry.budget
+
+        data = survey_session.query(survey_model.Nrens)
+        for nren in data:
+            for budget in nren.budgets:
+                abbrev = nren.abbreviation
+                year = budget.year
+
+                if float(budget.budget) > 200:
+                    logger.info(
+                        f'Incorrect Data: '
+                        f'{abbrev} has budget set to '
+                        f'>200M EUR for {year}. ({budget.budget})')
+
+                entry = session.query(
+                    model.BudgetEntry).filter_by(nren=abbrev,
+                                                 year=year)
+                dup_entry: model.BudgetEntry = entry.first()
+                if dup_entry:
+                    entry.update({"budget": budget.budget})
+                else:
+                    budget_entry = model.BudgetEntry(
+                        nren=abbrev, budget=budget.budget, year=year)
+                    session.add(budget_entry)
+
+        # Import the data to database
+        xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+        with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
+                  newline='') as csvfile:
+            reader = csv.reader(csvfile)
+
+            for row in reader:
+                if row is not None:
+                    entry = session.query(
+                        model.BudgetEntry).filter_by(nren=row[0],
+                                                     year=row[
+                                                         2])
+                    dup_entry: model.BudgetEntry = entry.first()
+                    if dup_entry:
+                        dup_entry.budget = row[1]
+                        entry.update({"budget": row[1]})
+
+                    else:
+                        budget_entry = model.BudgetEntry(
+                            nren=row[0], budget=row[1], year=row[2])
+                        session.add(budget_entry)
+        session.commit()
+
+
+def db_funding_migration():
+    with db.session_scope() as session:
+
+        # Import the data to database
+        xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
+        with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
+                  newline='') as csvfile:
+            reader = csv.reader(csvfile)
+
+            for row in reader:
+                if row is not None:
+                    _data = [float(row[2]), float(row[3]), float(row[4]),
+                             float(row[5]), float(row[6])]
+
+                    total = sum(_data)
+                    if not math.isclose(total, 100, abs_tol=0.01):
+                        logger.info(
+                            f'Incorrect Data: '
+                            f'{row[0]} funding sources for year ({row[1]})'
+                            f'do not sum to 100%. ({total})')
+
+                    entry = session.query(
+                        model.FundingSource).filter_by(nren=row[0],
+                                                       year=row[1])
+                    dup_entry: model.FundingSource = entry.first()
+
+                    if dup_entry:
+                        entry.update({"client_institutions": row[2],
+                                      "european_funding": row[3],
+                                      "gov_public_bodies": row[4],
+                                      "commercial": row[5],
+                                      "other": row[6]})
+
+                    else:
+                        budget_entry = model.FundingSource(
+                            nren=row[0], year=row[1],
+                            client_institutions=row[2],
+                            european_funding=row[3],
+                            gov_public_bodies=row[4],
+                            commercial=row[5],
+                            other=row[6])
+                        session.add(budget_entry)
+        session.commit()
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+    app_config = load(open(config, 'r'))
+    print("survery-publisher-v1 starting")
+    init_db(app_config)
+    db_budget_migration()
+    db_funding_migration()
+
+
+if __name__ == "__main__":
+    cli()

compendium_v2/routes/api.py

@@ -16,9 +16,11 @@ from flask import Blueprint
 
 from compendium_v2.routes import common
 from compendium_v2.routes.budget import routes as budget_routes
+from compendium_v2.routes.funding import routes as funding_routes
 
 routes = Blueprint('compendium-v2-api', __name__)
 routes.register_blueprint(budget_routes, url_prefix='/budget')
+routes.register_blueprint(funding_routes, url_prefix='/funding')
 
 logger = logging.getLogger(__name__)
 

compendium_v2/routes/budget.py

+import csv
 import logging
 from collections import defaultdict
 from typing import Any
@@ -5,6 +6,7 @@ from typing import Any
 from flask import Blueprint, jsonify, current_app
 
 from compendium_v2 import db, survey_db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
 from compendium_v2.db import model
 from compendium_v2.survey_db import model as survey_model
 from compendium_v2.routes import common
@@ -37,7 +39,7 @@ BUDGET_RESPONSE_SCHEMA = {
                 'id': {'type': 'number'},
                 'NREN': {'type': 'string'},
                 'BUDGET': {'type': 'string'},
-                'BUDGET_YEAR': {'type': 'string'},
+                'BUDGET_YEAR': {'type': 'integer'},
             },
             'required': ['id'],
             'additionalProperties': False
@@ -63,9 +65,26 @@ def budget_view() -> Any:
     :return:
     """
 
+    def _extract_data(entry: model.BudgetEntry):
+        return {
+            'id': entry.id,
+            'NREN': entry.nren,
+            'BUDGET': entry.budget,
+            'BUDGET_YEAR': entry.year,
+        }
+
+    with db.session_scope() as session:
+        entries = sorted([_extract_data(entry)
+                          for entry in session.query(model.BudgetEntry)],
+                         key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
+    return jsonify(entries)
+
+
+@routes.route('/migration', methods=['GET'])
+@common.require_accepts_json
+def db_budget_migration():
     with survey_db.session_scope() as survey_session, \
             db.session_scope() as session:
-
         _entries = session.query(model.BudgetEntry)
 
         inserted = defaultdict(dict)
@@ -86,17 +105,29 @@ def budget_view() -> Any:
                 entry = model.BudgetEntry(
                     nren=abbrev, budget=budget.budget, year=year)
                 session.add(entry)
-
-    def _extract_data(entry: model.BudgetEntry):
-        return {
-            'id': entry.id,
-            'NREN': entry.nren,
-            'BUDGET': entry.budget,
-            'BUDGET_YEAR': entry.year,
-        }
-
-    with db.session_scope() as session:
-        entries = sorted([_extract_data(entry)
-                          for entry in session.query(model.BudgetEntry)],
-                         key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
-    return jsonify(entries)
+        # Import the data to database
+        xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+        with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
+                  newline='') as csvfile:
+            reader = csv.reader(csvfile)
+
+            for row in reader:
+                if row is not None:
+                    entry = session.query(
+                        model.BudgetEntry).filter_by(nren=row[0],
+                                                     year=row[
+                                                         2])
+                    dup_entry: model.BudgetEntry = entry.first()
+                    if dup_entry:
+                        dup_entry.budget = row[1]
+                        entry.update({"budget": row[1]})
+
+                    else:
+                        print("add new")
+                        print(row)
+                        budget_entry = model.BudgetEntry(
+                            nren=row[0], budget=row[1], year=row[2])
+                        session.add(budget_entry)
+        session.commit()
+
+    return "Success"

compendium_v2/routes/common.py

@@ -16,6 +16,7 @@ def require_accepts_json(f):
     :param f: the function to be decorated
     :return: the decorated function
     """
+
     @functools.wraps(f)
     def decorated_function(*args, **kwargs):
         # TODO: use best_match to disallow */* ...?
@@ -25,9 +26,14 @@ def require_accepts_json(f):
                 status=406,
                 mimetype='text/html')
         return f(*args, **kwargs)
+
     return decorated_function
 
 
+def init_background_task():
+    pass
+
+
 def after_request(response):
     """
     Generic function to do additional logging of requests & responses.

compendium_v2/routes/funding.py

+import csv
+
+from flask import Blueprint, jsonify, current_app
+from compendium_v2 import db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
+from compendium_v2.routes import common
+from compendium_v2.db import model
+import logging
+from typing import Any
+
+routes = Blueprint('funding', __name__)
+
+
+@routes.before_request
+def before_request():
+    config = current_app.config['CONFIG_PARAMS']
+    dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+    db.init_db_model(dsn_prn)
+
+
+logger = logging.getLogger(__name__)
+
+FUNDING_RESPONSE_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+        'funding': {
+            'type': 'object',
+            'properties': {
+                'id': {'type': 'number'},
+                'NREN': {'type': 'string'},
+                'YEAR': {'type': 'string'},
+                'CLIENT_INSTITUTIONS': {'type': 'string'},
+                'EUROPEAN_FUNDING': {'type': 'string'},
+                'GOV_PUBLIC_BODIES': {'type': 'string'},
+                'COMMERCIAL': {'type': 'string'},
+                'OTHER': {'type': 'string'}
+            },
+            'required': ['id'],
+            'additionalProperties': False
+        }
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/funding'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def budget_view() -> Any:
+    """
+    handler for /api/funding/ requests
+
+    response will be formatted as:
+
+    .. asjson::
+        compendium_v2.routes.data_entry.BUDGET_RESPONSE_SCHEMA
+
+    :return:
+    """
+
+    def _extract_data(entry: model.FundingSource):
+        return {
+            'id': entry.id,
+            'NREN': entry.nren,
+            'YEAR': entry.year,
+            'CLIENT_INSTITUTIONS': entry.client_institutions,
+            'EUROPEAN_FUNDING': entry.european_funding,
+            'GOV_PUBLIC_BODIES': entry.gov_public_bodies,
+            'COMMERCIAL': entry.commercial,
+            'OTHER': entry.other
+        }
+
+    with db.session_scope() as session:
+        entries = sorted([_extract_data(entry)
+                          for entry in session.query(model.FundingSource)],
+                         key=lambda d: (d['NREN'], d['YEAR']))
+    dict_obj = {"data": entries}
+    return jsonify(dict_obj)
+
+
+@routes.route('/migration', methods=['GET'])
+@common.require_accepts_json
+def db_funding_migration():
+    with db.session_scope() as session:
+
+        # Import the data to database
+        xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
+        with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
+                  newline='') as csvfile:
+            reader = csv.reader(csvfile)
+
+            for row in reader:
+                if row is not None:
+                    entry = session.query(
+                        model.FundingSource).filter_by(nren=row[0],
+                                                       year=row[1])
+                    dup_entry: model.FundingSource = entry.first()
+                    if dup_entry:
+                        entry.update({"client_institutions": row[2],
+                                      "european_funding": row[3],
+                                      "gov_public_bodies": row[4],
+                                      "commercial": row[5],
+                                      "other": row[6]})
+
+                    else:
+                        print("add new")
+                        print(row)
+                        budget_entry = model.FundingSource(
+                            nren=row[0], year=row[1],
+                            client_institutions=row[2],
+                            european_funding=row[3],
+                            gov_public_bodies=row[4],
+                            commercial=row[5],
+                            other=row[6])
+                        session.add(budget_entry)
+        session.commit()
+
+    return "Success"

requirements.txt

@@ -19,3 +19,5 @@ types-jsonschema
 types-Flask-Cors
 types-setuptools
 types-sqlalchemy
+
+openpyxl

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='compendium-v2',
-    version="0.6",
+    version="0.7",
     author='GEANT',
     author_email='swd@geant.org',
     description='Flask and React project for displaying '
@@ -19,4 +19,11 @@ setup(
         'cryptography',
     ],
     include_package_data=True,
+
+    entry_points={
+        'console_scripts': [
+            'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa
+            'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa
+        ]
+    }
 )

test/conftest.py

@@ -63,8 +63,9 @@ def create_test_presentation_data():
         session.add(
             model.BudgetEntry(
                 id=1,
-                budget_type=model.BudgetType.NREN.name,
-                description='Test description')
+                nren='testnren',
+                budget=123,
+                year=2023,)
         )