diff --git a/compendium_v2/app.py b/compendium_v2/app.py
index acd1fab0d684e63453a3596a204430314cfefea1..09d63b51fd396fc757fbfebee0ec828528e98776 100644
--- a/compendium_v2/app.py
+++ b/compendium_v2/app.py
@@ -3,10 +3,19 @@ default app creation
"""
import compendium_v2
from compendium_v2 import environment
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
environment.setup_logging()
app = compendium_v2.create_app()
+# with app.app_context():
+ # xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
+ # xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+
+# filename = "survey_excel/xlsx/2021_Organisation_DataSeries.xlsx"
+# sheet_name = "1. Budget"
+# budget_sheet_uploading_task.parse_xlsx_file(filename, sheet_name)
+
if __name__ == '__main__':
app.run(host='::', port='33333')
diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/compendium_v2/background_task/csv/BudgetCsvFile.csv b/compendium_v2/background_task/csv/BudgetCsvFile.csv
new file mode 100644
index 0000000000000000000000000000000000000000..09ac009cfd38b042651034b8fa4b53985e10fb51
--- /dev/null
+++ b/compendium_v2/background_task/csv/BudgetCsvFile.csv
@@ -0,0 +1,180 @@
+ACOnet,6.4,2021
+ACOnet,6.4,2020
+ACOnet,6.1,2019
+ACOnet,6.1,2018
+ACOnet,5.7,2017
+AMRES,3.1,2021
+AMRES,2.35,2020
+AMRES,2.47,2019
+AMRES,2.02,2018
+AMRES,1.77,2017
+ANA,0.8,2021
+ANA,0.8,2020
+ANA,0.8,2019
+ANA,0.8,2018
+ARNES,9.8,2021
+ARNES,9.0,2020
+ARNES,8.0,2019
+ARNES,7.5,2018
+ARNES,6.0,2017
+ASNET,0.5,2021
+ASNET,0.48,2020
+ASNET,0.48,2019
+ASNET,0.25,2018
+ASNET,0.15,2017
+AzScienceNet,1.2,2021
+AzScienceNet,1.2,2020
+AzScienceNet,1.2,2019
+AzScienceNet,2.0,2018
+AzScienceNet,0.0,2017
+BASNET,0.63,2021
+BASNET,0.94,2020
+BASNET,0.92,2019
+BASNET,0.72,2018
+BASNET,0.82,2017
+BELNET,17.3,2021
+BELNET,13.85,2020
+BELNET,14.48,2019
+BELNET,15.52,2018
+BREN,0.03,2021
+CARNet,86.08,2021
+CARNet,30.31,2020
+CARNet,36.45,2019
+CARNet,20.1,2018
+CARNet,34.73,2017
+CESNET,19.5,2021
+CESNET,18.86,2020
+CESNET,18.97,2019
+CESNET,17.88,2018
+CESNET,14.7,2017
+CYNET,0.97,2021
+CYNET,0.92,2020
+CYNET,0.92,2019
+CYNET,3.6,2018
+CYNET,0.77,2017
+DeIC,9.92,2021
+DeIC,7.95,2020
+DeIC,7.1,2019
+DeIC,7.41,2018
+DFN,51.78,2021
+DFN,50.37,2020
+DFN,42.34,2019
+DFN,42.0,2018
+DFN,45.0,2017
+EENet,1.29,2021
+EENet,1.29,2020
+EENet,4.77,2019
+EENet,2.34,2018
+EENet,2.34,2017
+FCCN,24.82,2021
+FCCN,20.11,2020
+FCCN,16.25,2019
+FCCN,14.25,2018
+FCCN,8.07,2017
+Funet,8.3,2021
+Funet,8.3,2020
+Funet,7.9,2019
+Funet,7.5,2018
+Funet,7.5,2017
+GARR,22.0,2021
+GARR,23.0,2020
+GARR,22.33,2019
+GARR,22.15,2018
+GARR,21.97,2017
+GRENA,0.4,2021
+GRENA,0.4,2020
+GRENA,0.4,2019
+GRENA,0.3,2018
+GRENA,0.3,2017
+GRNET S.A.,7.4,2021
+GRNET S.A.,7.0,2020
+GRNET S.A.,6.9,2019
+GRNET S.A.,6.9,2018
+GRNET S.A.,6.5,2017
+HEAnet,25.2,2021
+HEAnet,24.89,2020
+HEAnet,25.59,2019
+HEAnet,25.06,2018
+HEAnet,29.6,2017
+IUCC,4.28,2021
+IUCC,4.3,2020
+IUCC,3.48,2019
+IUCC,3.72,2018
+IUCC,3.9,2017
+Jisc,69.61,2021
+Jisc,63.41,2020
+Jisc,65.23,2019
+KIFU (NIIF),34.0,2021
+KIFU (NIIF),34.0,2020
+KIFU (NIIF),48.0,2019
+KIFU (NIIF),12.0,2018
+KIFU (NIIF),12.0,2017
+LAT,1.24,2019
+LAT,1.03,2018
+LAT,0.0,2017
+LITNET,2.65,2021
+LITNET,2.33,2020
+LITNET,2.28,2019
+LITNET,2.28,2018
+LITNET,2.06,2017
+MARNET,0.63,2021
+MARNET,1.14,2020
+MARNET,0.95,2019
+MARNET,0.33,2018
+MREN,0.07,2021
+MREN,0.07,2020
+MREN,0.07,2019
+MREN,0.07,2018
+PIONIER,0.0,2020
+RedIRIS,23.0,2021
+RedIRIS,8.0,2020
+RedIRIS,8.0,2019
+RedIRIS,8.0,2018
+RedIRIS,8.0,2017
+RENAM,0.45,2021
+RENAM,0.32,2020
+RENAM,0.38,2019
+RENAM,0.34,2018
+RENAM,0.38,2017
+RENATER,39.0,2021
+RENATER,31.1,2020
+RENATER,32.2,2019
+RENATER,33.9,2018
+RENATER,29.1,2017
+RESTENA,4.54,2021
+RESTENA,4.75,2020
+RESTENA,4.03,2019
+RESTENA,3.48,2018
+RESTENA,2.06,2017
+RhNET,0.0,2019
+RoEduNet,2.0,2020
+RoEduNet,2.0,2019
+RoEduNet,2.0,2018
+RoEduNet,2.0,2017
+SANET,1.98,2021
+SANET,2.18,2020
+SANET,1.98,2019
+SANET,1.98,2018
+SANET,1.98,2017
+SUNET,28.0,2021
+SUNET,26.0,2020
+SUNET,24.0,2019
+SUNET,19.0,2018
+SUNET,0.0,2017
+SURF,54.97,2020
+SURF,52.86,2019
+SURF,40.5,2018
+SURF,40.2,2017
+SWITCH,30.56,2021
+SWITCH,27.74,2020
+SWITCH,13.46,2019
+SWITCH,14.0,2018
+SWITCH,14.2,2017
+ULAKBIM,17.0,2021
+ULAKBIM,17.2,2020
+ULAKBIM,17.0,2019
+ULAKBIM,21.0,2018
+ULAKBIM,18.5,2017
+UNINETT,15.0,2019
+UNINETT,20.0,2018
+UNINETT,30.0,2017
diff --git a/compendium_v2/background_task/csv/FundingSourceCsvFile.csv b/compendium_v2/background_task/csv/FundingSourceCsvFile.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2a7b4a72b72c9553c8d29e2e74dedc595d345825
--- /dev/null
+++ b/compendium_v2/background_task/csv/FundingSourceCsvFile.csv
@@ -0,0 +1,198 @@
+ACOnet,2020,100.0,0.0,0.0,0.0,0.0
+AMRES,2020,0.0,0.0,100.0,0.0,0.0
+ANA,2020,13.0,1.0,1.0,1.0,0.0
+ARNES,2020,0.0,5.0,84.0,11.0,0.0
+ASNET-AM,2020,10.0,48.0,40.0,0.0,1.0
+AzScienceNet,2020,60.0,40.0,60.0,0.0,0.0
+BASNET,2020,26.0,64.0,10.0,0.0,0.0
+Belnet,2020,39.0,2.0,52.0,7.0,0.0
+BREN,2020,25.0,0.0,0.0,0.0,0.0
+CARNET,2020,0.0,59.32,37.64,3.04,0.0
+CESNET,2020,21.0,3.0,73.0,0.0,2.0
+CYNET,2020,63.0,25.0,12.0,0.0,0.0
+DeIC,2020,95.0,1.0,0.0,4.0,0.0
+DFN,2020,94.0,3.0,1.5,0.0,1.5
+EENet,2020,0.0,71.0,26.0,0.0,0.0
+FCCN,2020,3.07,0.21,95.85,0.0,0.87
+Funet,2020,60.0,0.0,40.0,0.0,0.0
+GARR,2020,88.92,3.69,7.34,0.0,0.04
+GRENA,2020,40.0,50.0,0.0,5.0,5.0
+GRNET S.A.,2020,0.0,20.0,75.0,0.0,5.0
+HEAnet,2020,16.0,1.0,74.0,9.0,0.0
+IUCC,2020,91.0,9.0,0.0,0.0,0.0
+Jisc,2020,20.069,1.562,48.554,29.814,0.0
+KIFU (NIIF),2020,4.0,2.0,94.0,0.0,0.0
+LITNET,2020,10.0,33.0,57.0,0.0,0.0
+MARnet,2020,0.0,20.0,56.0,24.0,0.0
+MREN,2020,0.0,20.0,70.0,0.0,5.0
+PIONIER,2020,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2020,0.0,7.0,92.0,0.0,1.0
+RENAM,2020,24.0,74.0,2.0,0.0,0.0
+RENATER,2020,11.0,4.0,85.0,0.0,0.0
+RESTENA,2020,6.0,3.0,34.0,37.0,20.0
+RhNET,2020,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2020,0.0,0.0,100.0,0.0,0.0
+SANET,2020,7.0,0.0,93.0,0.0,0.0
+LAT,2020,0.0,0.0,0.0,0.0,0.0
+SUNET,2020,70.0,0.0,20.0,0.0,10.0
+SURFnet,2020,60.0,2.0,38.0,0.0,0.0
+SWITCH,2020,55.5,0.9,1.1,42.5,0.0
+ULAKBIM,2020,0.0,0.3,99.5,0.0,0.2
+UNINETT,2020,0.0,0.0,0.0,0.0,0.0
+UoM,2020,0.0,0.0,0.0,0.0,0.0
+ACOnet,2019,100.0,0.0,0.0,0.0,0.0
+AMRES,2019,0.0,0.0,100.0,0.0,0.0
+ANA,2019,13.0,1.0,1.0,1.0,0.0
+ARNES,2019,0.0,15.0,72.0,13.0,0.0
+ASNET-AM,2019,0.0,0.0,0.0,0.0,0.0
+AzScienceNet,2019,58.0,40.0,60.0,0.0,0.0
+BASNET,2019,27.0,64.0,9.0,0.0,0.0
+Belnet,2019,38.46,2.4,52.16,6.98,0.0
+BREN,2019,25.0,0.0,0.0,0.0,0.0
+CARNET,2019,0.0,59.11,37.89,3.0,0.0
+CESNET,2019,23.0,4.0,72.0,1.0,0.0
+CYNET,2019,54.6,29.6,15.8,0.0,0.0
+DeIC,2019,0.0,0.0,0.0,0.0,0.0
+DFN,2019,98.0,2.0,0.0,0.0,0.0
+EENet,2019,3.0,71.0,26.0,0.0,0.0
+FCCN,2019,2.0,0.22,96.4,0.0,1.29
+Funet,2019,60.0,0.0,40.0,0.0,0.0
+GARR,2019,87.66,4.73,7.57,0.0,0.04
+GRENA,2019,40.0,50.0,0.0,5.0,5.0
+GRNET S.A.,2019,0.0,20.0,75.0,0.0,5.0
+HEAnet,2019,14.0,1.0,84.0,1.0,0.0
+IUCC,2019,86.0,14.0,0.0,0.0,0.0
+Jisc,2019,23.0,1.0,70.0,6.0,0.0
+KIFU (NIIF),2019,3.0,2.0,95.0,0.0,0.0
+LITNET,2019,0.0,0.0,0.0,0.0,0.0
+MARnet,2019,14.0,17.0,69.0,0.0,0.0
+MREN,2019,0.0,14.0,66.0,20.0,0.0
+PIONIER,2019,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2019,0.0,7.0,92.0,0.0,1.0
+RENAM,2019,39.0,60.0,1.0,0.0,0.0
+RENATER,2019,30.0,4.0,64.0,1.0,1.0
+RESTENA,2019,5.0,5.0,37.0,38.0,15.0
+RhNET,2019,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2019,0.0,0.0,100.0,0.0,0.0
+SANET,2019,7.0,0.0,93.0,0.0,0.0
+LAT,2019,0.0,0.0,0.0,0.0,0.0
+SUNET,2019,70.0,0.0,25.0,0.0,5.0
+SURFnet,2019,58.0,2.0,40.0,0.0,0.0
+SWITCH,2019,48.0,1.0,1.0,50.0,0.0
+ULAKBIM,2019,0.0,0.0,99.5,0.0,0.0
+UNINETT,2019,0.0,0.0,0.0,0.0,0.0
+UoM,2019,0.0,0.0,0.0,0.0,0.0
+ACOnet,2018,100.0,0.0,0.0,0.0,0.0
+AMRES,2018,0.0,0.0,100.0,0.0,0.0
+ANA,2018,13.0,1.0,1.0,1.0,0.0
+ARNES,2018,0.0,4.0,84.0,12.0,0.0
+ASNET-AM,2018,20.0,40.0,40.0,0.0,0.0
+AzScienceNet,2018,0.0,95.0,5.0,0.0,0.0
+BASNET,2018,34.0,58.0,8.0,0.0,0.0
+Belnet,2018,20.83,1.99,71.41,5.77,0.0
+BREN,2018,0.0,0.0,0.0,0.0,0.0
+CARNET,2018,0.0,34.6,62.21,3.19,0.0
+CESNET,2018,22.0,7.0,70.0,1.0,0.0
+CYNET,2018,13.0,20.0,63.3,0.0,0.0
+DeIC,2018,99.0,1.0,0.0,0.0,0.0
+DFN,2018,96.0,2.0,0.0,0.0,2.0
+EENet,2018,5.0,42.0,50.0,0.0,0.0
+FCCN,2018,2.0,0.54,95.74,0.0,1.72
+Funet,2018,60.0,0.0,40.0,0.0,0.0
+GARR,2018,87.55,4.77,7.63,0.0,0.05
+GRENA,2018,40.0,45.0,0.0,5.0,10.0
+GRNET S.A.,2018,0.0,35.0,60.0,0.0,5.0
+HEAnet,2018,10.0,1.0,86.0,1.0,0.0
+IUCC,2018,86.0,14.0,0.0,0.0,0.0
+Jisc,2018,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2018,10.0,5.0,85.0,0.0,0.0
+LITNET,2018,14.0,20.0,66.0,0.0,0.0
+MARnet,2018,0.0,25.0,42.0,33.0,0.0
+MREN,2018,0.0,20.0,70.0,0.0,10.0
+PIONIER,2018,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2018,0.0,7.0,92.0,0.0,1.0
+RENAM,2018,52.0,46.0,2.0,0.0,0.0
+RENATER,2018,30.0,4.0,65.0,0.0,1.0
+RESTENA,2018,6.0,2.0,40.0,37.0,15.0
+RhNET,2018,0.0,0.0,0.0,0.0,0.0
+RoEduNet,2018,0.0,0.0,100.0,0.0,0.0
+SANET,2018,7.0,0.0,93.0,0.0,0.0
+LAT,2018,0.0,0.0,0.0,0.0,0.0
+SUNET,2018,70.0,0.0,25.0,0.0,5.0
+SURFnet,2018,66.0,3.0,31.0,0.0,0.0
+SWITCH,2018,52.0,1.0,3.0,44.0,0.0
+ULAKBIM,2018,0.0,0.5,99.4,0.0,0.1
+UNINETT,2018,75.0,0.0,25.0,0.0,0.0
+UoM,2018,0.0,0.0,0.0,0.0,0.0
+ACOnet,2017,100.0,0.0,0.0,0.0,0.0
+AMRES,2017,0.0,0.0,13.5,86.5,0.0
+ARNES,2017,0.0,12.0,3.0,73.0,12.0
+ASNET-AM,2017,15.0,0.0,1.0,30.0,40.0
+AzScienceNet,2017,0.0,0.0,0.0,2.0,0.0
+BASNET,2017,56.0,0.0,3.0,17.0,24.0
+CARNet,2017,0.0,1.94,1.99,40.4,55.68
+CESNET,2017,23.0,0.0,3.0,69.0,3.0
+CYNET,2017,64.1,3.8,30.8,0.0,0.0
+DFN,2017,89.0,0.0,2.0,0.0,0.0
+EENet,2017,5.0,0.0,6.0,50.0,42.0
+FCCN,2017,4.18,0.0,4.54,90.64,0.64
+Funet,2017,60.0,0.0,0.0,40.0,0.0
+GARR,2017,88.0,0.0,3.23,7.56,1.72
+GRENA,2017,35.0,5.0,0.0,0.0,50.0
+GRNET S.A.,2017,0.0,0.0,0.0,55.0,50.0
+HEAnet,2017,10.0,1.0,2.0,86.0,1.0
+IUCC,2017,85.0,0.0,10.0,0.0,5.0
+Jisc,2017,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2017,20.0,0.0,5.0,70.0,5.0
+LITNET,2017,0.0,0.0,5.0,72.0,22.0
+MARnet,2017,0.0,0.0,0.0,0.0,0.0
+MREN,2017,0.0,0.0,0.0,1.0,0.0
+PIONIER,2017,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2017,0.0,0.0,6.0,92.0,1.0
+RENAM,2017,1.0,0.0,15.0,3.0,24.0
+RENATER,2017,34.0,1.0,4.0,58.0,2.0
+RESTENA,2017,10.0,20.0,7.0,63.0,0.0
+RoEduNet,2017,0.0,0.0,1.0,100.0,0.0
+SANET,2017,0.0,0.0,0.0,0.0,0.0
+SigmaNet,2017,0.0,0.0,0.0,0.0,0.0
+SUNET,2017,75.0,0.0,0.0,25.0,0.0
+SURFnet,2017,70.0,0.0,3.0,27.0,0.0
+SWITCH,2017,46.0,47.0,3.0,4.0,0.0
+ULAKBIM,2017,0.0,0.0,2.0,0.0,0.0
+UNINETT,2017,0.0,0.0,0.0,0.0,0.0
+ACOnet,2016,0.0,0.0,0.0,0.0,0.0
+AMRES,2016,0.0,13.5,86.5,0.0,0.0
+ARNES,2016,16.0,0.0,84.0,2.0,1.0
+ASNET-AM,2016,0.0,0.0,0.0,0.0,0.0
+AzScienceNet,2016,0.0,0.0,0.0,0.0,0.0
+BASNET,2016,0.0,3.0,17.0,24.0,0.0
+CARNet,2016,2.05,2.08,0.0,63.34,0.0
+CESNET,2016,0.0,3.0,69.0,3.0,2.0
+CYNET,2016,0.0,20.0,0.0,20.0,1.0
+DFN,2016,0.0,1.0,0.0,0.0,17.0
+EENet,2016,0.0,6.0,50.0,42.0,0.0
+FCCN,2016,0.0,4.6,91.5,1.15,0.39
+Funet,2016,0.0,0.0,40.0,0.0,0.0
+GARR,2016,0.0,3.24,9.64,0.43,0.09
+GRENA,2016,5.0,0.0,0.0,50.0,10.0
+GRNET S.A.,2016,0.0,0.0,55.0,50.0,5.0
+HEAnet,2016,2.0,2.0,86.0,1.0,0.0
+IUCC,2016,0.0,14.0,0.0,0.0,17.0
+Jisc,2016,0.0,0.0,0.0,0.0,0.0
+KIFU (NIIF),2016,0.0,3.0,2.0,13.0,0.0
+LITNET,2016,0.0,5.0,88.0,8.0,0.0
+MARnet,2016,0.0,0.0,0.0,0.0,0.0
+MREN,2016,0.0,0.0,1.0,0.0,0.0
+PIONIER,2016,0.0,0.0,0.0,0.0,0.0
+RedIRIS,2016,0.0,6.0,92.0,1.0,1.0
+RENAM,2016,0.0,20.0,2.0,23.0,0.0
+RENATER,2016,0.0,0.0,0.0,2.0,0.0
+RESTENA,2016,20.0,7.0,63.0,0.0,0.0
+RoEduNet,2016,0.0,1.0,100.0,0.0,0.0
+SANET,2016,0.0,0.0,0.0,0.0,0.0
+SigmaNet,2016,0.0,0.0,0.0,0.0,0.0
+SUNET,2016,0.0,0.0,25.0,0.0,0.0
+SURFnet,2016,0.0,3.0,33.0,0.0,0.0
+SWITCH,2016,48.0,2.0,5.0,0.0,0.0
+ULAKBIM,2016,0.0,0.0,0.0,0.0,0.0
+UNINETT,2016,0.0,0.0,0.0,0.0,0.0
diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..b8d2db21641fe881e98026a91873934555bdd37e
Binary files /dev/null and b/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx differ
diff --git a/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py b/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b834d38975cb12c1708ebd3ff9a96673316895b
--- /dev/null
+++ b/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
@@ -0,0 +1,128 @@
+import openpyxl
+import csv
+import os
+from compendium_v2 import db
+from compendium_v2.db import model
+
+
+# Import the data to database
+def import_countries():
+ with db.session_scope() as session:
+ with open('csv/BudgetCsvFile.csv') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in reader:
+ data = model.BudgetEntry(
+ nren=row[0], budget=row[1], year=row[2])
+ data.save()
+ session.add(data)
+
+
+def parse_budget_xlsx_file():
+ try:
+ # load the xlsx file
+ filename = "compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx"
+ csv_out_file = "compendium_v2/background_task/csv/BudgetCsvFile.csv"
+ sheet_name = "1. Budget"
+ wb = openpyxl.load_workbook(
+ filename, data_only=True, read_only=True)
+
+ # select the active worksheet
+ ws = wb[sheet_name]
+ if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
+ os.remove(csv_out_file)
+ print("file deleted")
+ # iterate over the rows in the worksheet
+ for row in range(14, 57):
+ for col in range(3, 8):
+ # extract the data from the row
+ nren = ws.cell(row=row, column=2).value
+ budget = ws.cell(row=row, column=col).value
+ year = ws.cell(row=13, column=col).value
+
+ if budget is not None:
+ budget = round(budget / 1000000, 2)
+ # process the data (e.g. save to database)
+ print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
+ output_csv_file = csv.writer(
+ open(csv_out_file, 'a'),
+ delimiter=",")
+ output_csv_file.writerow([nren, budget, year])
+ output_csv_file
+ except Exception as e:
+ print(e)
+
+ # import_countries()
+
+
+def parse_income_source_xlsx_file():
+ try:
+ # load the xlsx file
+ filename = "compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx"
+ csv_out_file = "compendium_v2/background_task/csv/FundingSourceCsvFile.csv"
+ sheet_name = "2. Income Sources"
+ wb = openpyxl.load_workbook(
+ filename, data_only=True, read_only=True)
+
+ # select the active worksheet
+ ws = wb[sheet_name]
+ if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
+ os.remove(csv_out_file)
+ print("file deleted")
+
+ def hard_number_convert(s):
+ if s is None:
+ return float(0)
+ """ Returns True if string is a number. """
+ try:
+ return float(s)
+ except ValueError:
+ return float(0)
+
+ # iterate over the rows in the worksheet
+
+ def create_csv_per_uear(start_row, end_row, yearI, col_start):
+ for row in range(start_row, end_row):
+ # extract the data from the row
+ nren = ws.cell(row=row, column=col_start).value
+ client_institution = ws.cell(row=row,
+ column=col_start + 3).value
+ european_funding = ws.cell(row=row, column=col_start + 4).value
+ gov_public_bodies = ws.cell(row=row,
+ column=col_start + 5).value
+ commercial = ws.cell(row=row, column=col_start + 6).value
+ other = ws.cell(row=row, column=col_start + 7).value
+ year = yearI
+
+ client_institution = hard_number_convert(client_institution)
+ european_funding = hard_number_convert(european_funding)
+ gov_public_bodies = hard_number_convert(gov_public_bodies)
+ commercial = hard_number_convert(commercial)
+ other = hard_number_convert(other)
+
+ # client_institution = is_number(client_institution) ? client_institution :'';
+ # budget = round(budget / 1000000, 2)
+ # process the data (e.g. save to database)
+ print(f"NREN: {nren}, client_institution:"
+ f" {client_institution}, Year: {year}")
+ if nren is not None:
+ output_csv_file = csv.writer(
+ open(csv_out_file, 'a'),
+ delimiter=",")
+ output_csv_file.writerow([nren, year, client_institution,
+ european_funding,
+ gov_public_bodies,
+ commercial, other])
+
+ # For 2020
+ create_csv_per_uear(8,50, 2020, 3)
+ # # For 2019
+ create_csv_per_uear(8,50, 2019,12)
+ # # For 2018
+ create_csv_per_uear(8,50, 2018,21)
+ # # For 2017
+ create_csv_per_uear(8,50,2017,32)
+ # # For 2016
+ create_csv_per_uear(8,50,2016,43)
+
+ except Exception as e:
+ print(e)
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/model.py
index dd67b784703850379f384d4f3ca0845f4622e924..f06498dfa2f1c047304d94bc336b9703312c7eda 100644
--- a/compendium_v2/db/model.py
+++ b/compendium_v2/db/model.py
@@ -4,6 +4,7 @@ import sqlalchemy as sa
from typing import Any
from sqlalchemy.ext.declarative import declarative_base
+
# from sqlalchemy.orm import relationship
logger = logging.getLogger(__name__)
@@ -19,3 +20,16 @@ class BudgetEntry(base_schema):
nren = sa.Column(sa.String(128), primary_key=True)
budget = sa.Column(sa.String(128), nullable=True)
year = sa.Column(sa.Integer, primary_key=True)
+
+
+class FundingSource(base_schema):
+ __tablename__ = 'funding_source'
+ id = sa.Column(sa.Integer, sa.Sequence(
+ 'fundingentry_seq_id_seq'), nullable=False)
+ nren = sa.Column(sa.String(128), primary_key=True)
+ year = sa.Column(sa.Integer, primary_key=True)
+ client_institutions = sa.Column(sa.String(128), nullable=True)
+ european_funding = sa.Column(sa.String(128), nullable=True)
+ gov_public_bodies = sa.Column(sa.String(128), nullable=True)
+ commercial = sa.Column(sa.String(128), nullable=True)
+ other = sa.Column(sa.String(128), nullable=True)
diff --git a/compendium_v2/migrations/versions/833c99d745d7_create_funding_source_schema.py b/compendium_v2/migrations/versions/833c99d745d7_create_funding_source_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..b07b4c04e6865633432561855a7ebd1fbfec00ff
--- /dev/null
+++ b/compendium_v2/migrations/versions/833c99d745d7_create_funding_source_schema.py
@@ -0,0 +1,43 @@
+"""create funding source schema
+
+Revision ID: 833c99d745d7
+Revises: cbcd21fcc151
+Create Date: 2023-03-07 23:47:30.157499
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '833c99d745d7'
+down_revision = 'cbcd21fcc151'
+branch_labels = None
+depends_on = None
+
+funding_id_seq = sa.Sequence('fundingentry_seq_id_seq')
+
+
+def upgrade():
+ op.execute(sa.schema.CreateSequence(funding_id_seq)) # create the sequence
+ op.create_table('funding_source',
+ sa.Column('id', sa.Integer, funding_id_seq, nullable=False,
+ server_default=funding_id_seq.next_value()),
+ sa.Column('nren', sa.String(length=128), nullable=False),
+ sa.Column('year', sa.Integer, nullable=False),
+ sa.Column('client_institutions', sa.String(length=128),
+ nullable=True),
+ sa.Column('european_funding', sa.String(length=128),
+ nullable=True),
+ sa.Column('gov_public_bodies', sa.String(length=128),
+ nullable=True),
+ sa.Column('commercial', sa.String(length=128),
+ nullable=True),
+ sa.Column('other', sa.String(length=128), nullable=True),
+ sa.PrimaryKeyConstraint('nren', 'year')
+ )
+
+
+def downgrade():
+ op.execute(
+ sa.schema.DropSequence(sa.Sequence('fundingentry_seq_id_seq')))
+ op.drop_table('funding_source')
diff --git a/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py b/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py
index 1e4e62a5e0e30a085d4ac46fb8998dda7746ac29..acb48c84a48a380b8d8e91ef3151b8b8f73777f0 100644
--- a/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py
+++ b/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py
@@ -8,7 +8,6 @@ Create Date: 2023-02-07 15:56:22.086064
from alembic import op
import sqlalchemy as sa
-
# revision identifiers, used by Alembic.
revision = 'cbcd21fcc151'
down_revision = None
diff --git a/compendium_v2/publishers/__init__.py b/compendium_v2/publishers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_v1.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b425a612c3060aaafb7d9da20f88c37c513e638
--- /dev/null
+++ b/compendium_v2/publishers/survey_publisher_v1.py
@@ -0,0 +1,92 @@
+import logging
+from collections import defaultdict
+
+import click
+import contextlib
+import csv
+from flask import current_app
+
+import compendium_v2
+from compendium_v2 import db, survey_db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
+from compendium_v2.db import model
+from compendium_v2.survey_db import model as survey_model
+
+
+def before_request():
+ config = current_app.config['CONFIG_PARAMS']
+ dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+ db.init_db_model(dsn_prn)
+ dsn_survey = config['SURVEY_DATABASE_URI']
+ survey_db.init_db_model(dsn_survey)
+
+
+# app = compendium_v2.create_app()
+
+
+def db_budget_migration():
+ with survey_db.session_scope() as survey_session, \
+ db.session_scope() as session:
+
+ # with app.app_context():
+ # xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+
+ _entries = session.query(model.BudgetEntry)
+
+ inserted = defaultdict(dict)
+
+ for entry in _entries:
+ inserted[entry.nren][entry.year] = entry.budget
+
+ data = survey_session.query(survey_model.Nrens)
+ for nren in data:
+ for budget in nren.budgets:
+ abbrev = nren.abbreviation
+ year = budget.year
+
+ if inserted.get(abbrev, {}).get(year):
+ continue
+ else:
+ inserted[abbrev][year] = True
+ entry = model.BudgetEntry(
+ nren=abbrev, budget=budget.budget, year=year)
+ session.add(entry)
+
+ # Import the data to database
+ try:
+ with open('survey_excel/csv/BudgetCsvFile.csv', "rU",
+ newline='') as csvfile:
+ reader = csv.reader(csvfile)
+ for row in reader:
+ if row is not None:
+ budget_entry: model.BudgetEntry = session.query(
+ model.BudgetEntry).filter_by(nren=row[0],
+ year=row[2])
+ if budget_entry is None:
+ budget_entry = model.BudgetEntry(
+ nren=row[0], budget=row[1], year=row[2])
+ session.add(budget_entry)
+ else:
+ budget_entry.budget = row[1]
+ print(row)
+ session.commit()
+ except Exception as error:
+ print(f"Unable to read :, {error}")
+
+
+@click.command()
+# @click.option('--config', type=click.STRING, default='config.json')
+@click.option(
+ "--migration",
+ multiple=True,
+ default=[],
+ help="Budget Migration from old compendium db to new presentation db")
+def cli(migration):
+ print("survery-publisher-v1 starting")
+ print(" migration: %r" % str(migration))
+ before_request()
+ db_budget_migration()
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py
index d18204256c2cc886ce46ecbcfe5ace40cc8a75e7..bfb4c1b7df4bc4ad7983c7240f95c66dc79a1474 100644
--- a/compendium_v2/routes/api.py
+++ b/compendium_v2/routes/api.py
@@ -16,9 +16,11 @@ from flask import Blueprint
from compendium_v2.routes import common
from compendium_v2.routes.budget import routes as budget_routes
+from compendium_v2.routes.funding import routes as funding_routes
routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget')
+routes.register_blueprint(funding_routes, url_prefix='/funding')
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index 90a7b258553c3f33233329a2020cb03bc6b8741f..ed84386a86f7d8c5744a6d91fdcdc78c25280eac 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -1,10 +1,13 @@
+import csv
import logging
from collections import defaultdict
from typing import Any
from flask import Blueprint, jsonify, current_app
+from sqlalchemy import values
from compendium_v2 import db, survey_db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
from compendium_v2.db import model
from compendium_v2.survey_db import model as survey_model
from compendium_v2.routes import common
@@ -63,9 +66,27 @@ def budget_view() -> Any:
:return:
"""
+ def _extract_data(entry: model.BudgetEntry):
+ return {
+ 'id': entry.id,
+ 'NREN': entry.nren,
+ 'BUDGET': entry.budget,
+ 'BUDGET_YEAR': entry.year,
+ }
+
+ with db.session_scope() as session:
+ entries = sorted([_extract_data(entry)
+ for entry in session.query(model.BudgetEntry)],
+ key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
+ dict_obj = {"data": entries}
+ return jsonify(dict_obj)
+
+
+@routes.route('/migration', methods=['GET'])
+@common.require_accepts_json
+def db_budget_migration():
with survey_db.session_scope() as survey_session, \
db.session_scope() as session:
-
_entries = session.query(model.BudgetEntry)
inserted = defaultdict(dict)
@@ -86,17 +107,29 @@ def budget_view() -> Any:
entry = model.BudgetEntry(
nren=abbrev, budget=budget.budget, year=year)
session.add(entry)
-
- def _extract_data(entry: model.BudgetEntry):
- return {
- 'id': entry.id,
- 'NREN': entry.nren,
- 'BUDGET': entry.budget,
- 'BUDGET_YEAR': entry.year,
- }
-
- with db.session_scope() as session:
- entries = sorted([_extract_data(entry)
- for entry in session.query(model.BudgetEntry)],
- key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
- return jsonify(entries)
+ # Import the data to database
+ xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+ with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
+ newline='') as csvfile:
+ reader = csv.reader(csvfile)
+
+ for row in reader:
+ if row is not None:
+ entry = session.query(
+ model.BudgetEntry).filter_by(nren=row[0],
+ year=row[
+ 2])
+ dup_entry: model.BudgetEntry = entry.first()
+ if dup_entry:
+ dup_entry.budget = row[1]
+ entry.update({"budget": row[1]})
+
+ else:
+ print("add new")
+ print(row)
+ budget_entry = model.BudgetEntry(
+ nren=row[0], budget=row[1], year=row[2])
+ session.add(budget_entry)
+ session.commit()
+
+ return "Success"
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index 62b83f7299dd9a3c30d215a7e024c9e087232d11..90e4af5cc30ef91bf6769bb067f173bff7892a15 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -16,6 +16,7 @@ def require_accepts_json(f):
:param f: the function to be decorated
:return: the decorated function
"""
+
@functools.wraps(f)
def decorated_function(*args, **kwargs):
# TODO: use best_match to disallow */* ...?
@@ -25,9 +26,14 @@ def require_accepts_json(f):
status=406,
mimetype='text/html')
return f(*args, **kwargs)
+
return decorated_function
+def init_background_task():
+ pass
+
+
def after_request(response):
"""
Generic function to do additional logging of requests & responses.
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d32886a9d21b2f71d09b62d19b355214da65c6e
--- /dev/null
+++ b/compendium_v2/routes/funding.py
@@ -0,0 +1,121 @@
+import csv
+from collections import defaultdict
+
+from flask import Blueprint, jsonify, current_app
+from compendium_v2 import db
+from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
+from compendium_v2.routes import common
+from compendium_v2.db import model
+import logging
+from typing import Any
+
+routes = Blueprint('funding', __name__)
+
+
+@routes.before_request
+def before_request():
+ config = current_app.config['CONFIG_PARAMS']
+ dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+ db.init_db_model(dsn_prn)
+
+
+logger = logging.getLogger(__name__)
+
+FUNDING_RESPONSE_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+ 'funding': {
+ 'type': 'object',
+ 'properties': {
+ 'id': {'type': 'number'},
+ 'NREN': {'type': 'string'},
+ 'YEAR': {'type': 'string'},
+ 'CLIENT_INSTITUTIONS': {'type': 'string'},
+ 'EUROPEAN_FUNDING': {'type': 'string'},
+ 'GOV_PUBLIC_BODIES': {'type': 'string'},
+ 'COMMERCIAL': {'type': 'string'},
+ 'OTHER': {'type': 'string'}
+ },
+ 'required': ['id'],
+ 'additionalProperties': False
+ }
+ },
+
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/funding'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def budget_view() -> Any:
+ """
+ handler for /api/funding/ requests
+
+ response will be formatted as:
+
+ .. asjson::
+ compendium_v2.routes.data_entry.BUDGET_RESPONSE_SCHEMA
+
+ :return:
+ """
+
+ def _extract_data(entry: model.FundingSource):
+ return {
+ 'id': entry.id,
+ 'NREN': entry.nren,
+ 'YEAR': entry.year,
+ 'CLIENT_INSTITUTIONS': entry.client_institutions,
+ 'EUROPEAN_FUNDING': entry.european_funding,
+ 'GOV_PUBLIC_BODIES': entry.gov_public_bodies,
+ 'COMMERCIAL': entry.commercial,
+ 'OTHER': entry.other
+ }
+
+ with db.session_scope() as session:
+ entries = sorted([_extract_data(entry)
+ for entry in session.query(model.FundingSource)],
+ key=lambda d: (d['NREN'], d['YEAR']))
+ dict_obj = {"data": entries}
+ return jsonify(dict_obj)
+
+
+@routes.route('/migration', methods=['GET'])
+@common.require_accepts_json
+def db_budget_migration():
+ with db.session_scope() as session:
+
+ # Import the data to database
+ xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
+ with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
+ newline='') as csvfile:
+ reader = csv.reader(csvfile)
+
+ for row in reader:
+ if row is not None:
+ entry = session.query(
+ model.FundingSource).filter_by(nren=row[0],
+ year=row[1])
+ dup_entry: model.FundingSource = entry.first()
+ if dup_entry:
+ entry.update({"client_institutions": row[2]},
+ {"european_funding": row[3]},
+ {"gov_public_bodies": row[4]},
+ {"commercial": row[5]},
+ {"other": row[6]})
+
+ else:
+ print("add new")
+ print(row)
+ budget_entry = model.FundingSource(
+ nren=row[0], year=row[1],
+ client_institutions=row[2],
+ european_funding=row[3],
+ gov_public_bodies=row[4],
+ commercial=row[5],
+ other=row[6])
+ session.add(budget_entry)
+ session.commit()
+
+ return "Success"
diff --git a/requirements.txt b/requirements.txt
index 8aa1fa7649c3e753d1ae8512d71bda5131f28340..c7b8fd1eb367c19290aa9f8046c98d8f781d336a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,3 +19,5 @@ types-jsonschema
types-Flask-Cors
types-setuptools
types-sqlalchemy
+
+openpyxl
diff --git a/setup.py b/setup.py
index dcc8c069f916f8697205d3bb546a6e19824321fe..4bdf86361c80050e195409848d53d22b3e51c976 100644
--- a/setup.py
+++ b/setup.py
@@ -19,4 +19,10 @@ setup(
'cryptography',
],
include_package_data=True,
+
+ entry_points={
+ 'console_scripts': [
+ 'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli',
+ ]
+ }
)
diff --git a/webapp/package-lock.json b/webapp/package-lock.json
index 460b76aeee6782d1307d483ad5c24371a44bb387..a6b6bc54e568ee966109cd91166284c1ea80aee4 100644
--- a/webapp/package-lock.json
+++ b/webapp/package-lock.json
@@ -10,7 +10,8 @@
"license": "ISC",
"dependencies": {
"bootstrap": "^5.2.3",
- "chart.js": "^4.1.1",
+ "cartesian-product-multiple-arrays": "^1.0.9",
+ "chart.js": "^4.2.1",
"core-js": "^3.26.1",
"file-loader": "^6.2.0",
"install": "^0.13.0",
@@ -4240,6 +4241,11 @@
}
]
},
+ "node_modules/cartesian-product-multiple-arrays": {
+ "version": "1.0.9",
+ "resolved": "https://registry.npmjs.org/cartesian-product-multiple-arrays/-/cartesian-product-multiple-arrays-1.0.9.tgz",
+ "integrity": "sha512-XwVkPg/Vv8j9HpokEkoKiPC0jBgUo2sK1JL33eNK6BoHyL+AzGx8VVeDMo62TO9tki+WGJCmcVSF7DS3r6S2+A=="
+ },
"node_modules/caw": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/caw/-/caw-2.0.1.tgz",
@@ -4271,9 +4277,9 @@
}
},
"node_modules/chart.js": {
- "version": "4.1.1",
- "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.1.1.tgz",
- "integrity": "sha512-P0pCosNXp+LR8zO/QTkZKT6Hb7p0DPFtypEeVOf+6x06hX13NIb75R0DXUA4Ksx/+48chDQKtCCmRCviQRTqsA==",
+ "version": "4.2.1",
+ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.2.1.tgz",
+ "integrity": "sha512-6YbpQ0nt3NovAgOzbkSSeeAQu/3za1319dPUQTXn9WcOpywM8rGKxJHrhS8V8xEkAlk8YhEfjbuAPfUyp6jIsw==",
"dependencies": {
"@kurkle/color": "^0.3.0"
},
@@ -17823,6 +17829,11 @@
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001434.tgz",
"integrity": "sha512-aOBHrLmTQw//WFa2rcF1If9fa3ypkC1wzqqiKHgfdrXTWcU8C4gKVZT77eQAPWN1APys3+uQ0Df07rKauXGEYA=="
},
+ "cartesian-product-multiple-arrays": {
+ "version": "1.0.9",
+ "resolved": "https://registry.npmjs.org/cartesian-product-multiple-arrays/-/cartesian-product-multiple-arrays-1.0.9.tgz",
+ "integrity": "sha512-XwVkPg/Vv8j9HpokEkoKiPC0jBgUo2sK1JL33eNK6BoHyL+AzGx8VVeDMo62TO9tki+WGJCmcVSF7DS3r6S2+A=="
+ },
"caw": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/caw/-/caw-2.0.1.tgz",
@@ -17848,9 +17859,9 @@
}
},
"chart.js": {
- "version": "4.1.1",
- "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.1.1.tgz",
- "integrity": "sha512-P0pCosNXp+LR8zO/QTkZKT6Hb7p0DPFtypEeVOf+6x06hX13NIb75R0DXUA4Ksx/+48chDQKtCCmRCviQRTqsA==",
+ "version": "4.2.1",
+ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.2.1.tgz",
+ "integrity": "sha512-6YbpQ0nt3NovAgOzbkSSeeAQu/3za1319dPUQTXn9WcOpywM8rGKxJHrhS8V8xEkAlk8YhEfjbuAPfUyp6jIsw==",
"requires": {
"@kurkle/color": "^0.3.0"
}
diff --git a/webapp/package.json b/webapp/package.json
index aa26d18936cdf010822d1e22b007bb0145c6fe7d..af8176b74a9a239e03517aab95fb23df7dc2d330 100644
--- a/webapp/package.json
+++ b/webapp/package.json
@@ -43,11 +43,12 @@
},
"dependencies": {
"bootstrap": "^5.2.3",
- "chart.js": "^4.1.1",
+ "cartesian-product-multiple-arrays": "^1.0.9",
+ "chart.js": "^4.2.1",
+ "core-js": "^3.26.1",
"file-loader": "^6.2.0",
"install": "^0.13.0",
"npm": "^9.2.0",
- "core-js": "^3.26.1",
"react": "^18.2.0",
"react-bootstrap": "^2.7.0",
"react-chartjs-2": "^5.1.0",
@@ -62,4 +63,4 @@
"type": "git",
"url": "https://gitlab.geant.net/live-projects/compendium-v2.git"
}
-}
\ No newline at end of file
+}
diff --git a/webapp/src/App.tsx b/webapp/src/App.tsx
index c4e0bdc2deed468fbc0eb31f7128bb53d51e5552..3e2e1b9db44a0ce72c3fd2c45b7d7a9406592c24 100644
--- a/webapp/src/App.tsx
+++ b/webapp/src/App.tsx
@@ -6,6 +6,7 @@ import About from "./pages/About";
import DataAnalysis from "./pages/DataAnalysis";
import AnnualReport from "./pages/AnnualReport";
import CompendiumData from "./pages/CompendiumData";
+import FundingSourcePage from "./pages/FundingSource";
function App(): ReactElement {
return (
@@ -17,6 +18,7 @@ function App(): ReactElement {
<Route path="/about" element={<About />} />
<Route path="/analysis" element={<DataAnalysis />} />
<Route path="/report" element={<AnnualReport />} />
+ <Route path="/funding" element={<FundingSourcePage />} />
<Route path="*" element={<Landing />} />
</Routes>
</Router>
diff --git a/webapp/src/Schema.tsx b/webapp/src/Schema.tsx
index 192bfd5ed9fc418c93ab5c25d8aa0c43ac430e1f..b3a5828f006c67a234e38346e08a691f6d3c5632 100644
--- a/webapp/src/Schema.tsx
+++ b/webapp/src/Schema.tsx
@@ -26,6 +26,10 @@ export interface BudgetMatrix {
title: string
}
+export interface BudgetAPI {
+ data: [Budget]
+}
+
export interface Budget {
BUDGET: string,
BUDGET_YEAR: number,
@@ -33,6 +37,35 @@ export interface Budget {
id: number
}
+export interface FundingSource{
+ CLIENT_INSTITUTIONS: string,
+ COMMERCIAL: string,
+ EUROPEAN_FUNDING: string,
+ GOV_PUBLIC_BODIES: string,
+ NREN: string,
+ OTHER: string,
+ YEAR: number,
+ id: number
+}
+
+export interface FS{
+ data: [FundingSource]
+}
+
+export interface FundingGraphMatrix {
+
+ labels: string[],
+ datasets: {
+ label: string,
+ data: string[],
+ backgroundColor: string
+ borderRadius:number,
+ borderSkipped:boolean,
+ barPercentage:number,
+ borderWidth: number,
+ stack: string
+ }[]
+}
export interface DataEntrySection {
name: string,
diff --git a/webapp/src/pages/DataAnalysis.tsx b/webapp/src/pages/DataAnalysis.tsx
index c8ad47bd984aa4514e1490ebf624ab0c47ca8ad1..ae722a5d173dfcb7652f599aa5e259eb2e68811d 100644
--- a/webapp/src/pages/DataAnalysis.tsx
+++ b/webapp/src/pages/DataAnalysis.tsx
@@ -3,7 +3,7 @@ import { Accordion, Col, Container, ListGroup, Row } from "react-bootstrap";
import BarGraph from "../components/graphing/BarGraph";
import LineGraph from "../components/graphing/LineGraph";
-import { BudgetMatrix, DataEntrySection, Budget } from "../Schema";
+import { BudgetMatrix, DataEntrySection, Budget,BudgetAPI } from "../Schema";
// import {evaluateInteractionItems} from "chart.js/dist/core/core.interaction";
import barGraph from "../components/graphing/BarGraph";
@@ -66,15 +66,15 @@ function DataAnalysis(): ReactElement {
useEffect(() => {
const loadData = async () => {
if (budgetResponse == undefined) {
- api<Budget[]>('/api/budget/', {})
- .then((budget: Budget[]) => {
+ api<BudgetAPI>('/api/budget/', {})
+ .then((budget: BudgetAPI) => {
console.log('budget:', budget)
const entry = dataEntrySection?.items.find(i => i.id == selectedDataEntry)
console.log(selectedDataEntry, dataEntrySection, entry)
if (entry)
options.plugins.title.text = entry.title;
- setBudget(budget)
- convertToBudgetPerYearDataResponse(budget)
+ setBudget(budget.data)
+ convertToBudgetPerYearDataResponse(budget.data)
})
.catch(error => {
console.log(`Error fetching from API: ${error}`);
@@ -113,6 +113,7 @@ function DataAnalysis(): ReactElement {
const convertToBudgetPerYearDataResponse = (budgetResponse: Budget[]) => {
const barResponse = budgetResponse != undefined ? budgetResponse : empty_budget_response;
+ console.log(barResponse.map((item) => item.BUDGET_YEAR));
const labelsYear = [...new Set(barResponse.map((item) => item.BUDGET_YEAR))];
const labelsNREN = [...new Set(barResponse.map((item) => item.NREN))];
@@ -198,7 +199,7 @@ function DataAnalysis(): ReactElement {
}
}
- const barResponse: BudgetMatrix = budgetMatrixResponse !== undefined
+ const budgetAPIResponse: BudgetMatrix = budgetMatrixResponse !== undefined
? budgetMatrixResponse : empty_bar_response;
return (
<div>
@@ -207,7 +208,7 @@ function DataAnalysis(): ReactElement {
<Row>
<Col>
<Row>
- <LineGraph data={barResponse.data} />
+ <LineGraph data={budgetAPIResponse.data} />
</Row>
<Row>{budgetMatrixResponse?.description}</Row>
diff --git a/webapp/src/pages/FundingSource.tsx b/webapp/src/pages/FundingSource.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..b352877667ca4074b7eb8176a108cdb8dc44d6d9
--- /dev/null
+++ b/webapp/src/pages/FundingSource.tsx
@@ -0,0 +1,346 @@
+import React, {ReactElement, useEffect, useState} from 'react';
+import {ChartOptions, scales, Tick} from 'chart.js';
+import {Bar} from 'react-chartjs-2';
+import { cartesianProduct } from 'cartesian-product-multiple-arrays';
+
+import {
+ Chart as ChartJS,
+ CategoryScale,
+ LinearScale,
+ BarElement,
+ Title,
+ Tooltip,
+ Legend,
+} from 'chart.js';
+import {
+ Budget,
+ BudgetMatrix,
+ DataEntrySection,
+ FundingSource,
+ FS, FundingGraphMatrix
+} from "../Schema";
+
+
+ChartJS.register(
+ CategoryScale,
+ LinearScale,
+ BarElement,
+ Title,
+ Tooltip,
+ Legend
+);
+
+
+const data = {
+ labels: ['NREN A', 'NREN B', 'NREN C', 'NREN D'],
+ datasets: [
+ {
+ label: 'CLIENT INSTITUTIONS (2018)',
+ data: [25, 25, 75],
+ backgroundColor: '#FF6384',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2018',
+ },
+ {
+ label: 'EUROPEAN FUNDING (2018)',
+ data: [75, 50, 25],
+ backgroundColor: '#36A2EB',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2018',
+ },
+ {
+ label: 'GOV/PUBLIC_BODIES (2018)',
+ data: [40, 20, 60],
+ backgroundColor: '#FFCE56',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2018',
+ },
+ {
+ label: 'COMMERCIAL (2018)',
+ data: [60, 80, 40],
+ backgroundColor: '#7FFF00',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2018',
+ },
+ {
+ label: 'OTHER (2018)',
+ data: [60, 80, 40],
+ backgroundColor: '#BD34EB',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2018',
+ },
+ {
+ label: 'CLIENT INSTITUTIONS (2019)',
+ data: [35, 25, 65],
+ backgroundColor: '#7363ff',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2019',
+ },
+ {
+ label: 'EUROPEAN FUNDING (2019)',
+ data: [45, 45, 15],
+ backgroundColor: '#36ebbb',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2019',
+ },
+ {
+ label: 'GOV/PUBLIC_BODIES (2019)',
+ data: [50, 50, 40],
+ backgroundColor: '#ffbb56',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2019',
+ },
+ {
+ label: 'COMMERCIAL (2019)',
+ data: [60, 50, 80],
+ backgroundColor: '#ddff00',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ borderWidth: 1,
+ stack: '2019',
+ },
+ {
+ label: 'OTHER (2019)',
+ data: [70, 85, 40],
+ backgroundColor: '#eb34a2',
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ stack: '2019',
+ },
+ ],
+};
+
+
+
+export const option ={
+ plugins:{
+ legend:{
+ labels:{
+ boxWidth:20,
+ boxHeight:30,
+ pointStyle:"rectRounded",
+ borderRadius:6,
+ useBorderRadius:true,
+ },
+ },
+ },
+ scales:{
+ x: {
+ stacked: true,
+ ticks: {
+ callback: (value: string | number) => {
+ if (typeof value === 'number') {
+ return value.toFixed(2);
+ }
+ return value;
+ },
+ },
+ },
+ y: {
+ stacked: true,
+ },
+ },
+ indexAxis: 'y',
+};
+
+
+
+
+function FundingSourcePage(): ReactElement {
+ function api<T>(url: string, options: RequestInit | undefined = undefined): Promise<T> {
+ return fetch(url, options)
+ .then((response) => {
+ if (!response.ok) {
+ return response.text().then((message) => {
+ console.error(`Failed to load datax: ${message}`, response.status);
+ throw new Error("The data could not be loaded, check the logs for details.");
+ });
+ }
+
+ return response.json() as Promise<T>;
+ })
+ }
+
+ const [fundingMatrixResponse, setFundingMatrixResponse] = useState<FundingGraphMatrix>();
+ const [fundingSourceResponse, setFundingSource] = useState<FundingSource[]>();
+ const [dataEntrySection, setDataEntrySection] = useState<DataEntrySection>();
+ const [selectedDataEntry, setSelectedDataEntry] = useState<number>(0);
+
+ useEffect(() =>{
+ const loadData = async () => {
+ if (fundingSourceResponse == undefined) {
+ api<FS>('/api/funding/', {})
+ .then((fundingSources: FS) => {
+ console.log('fundingSource:', fundingSources)
+ const entry = dataEntrySection?.items.find(i => i.id == selectedDataEntry)
+ console.log(selectedDataEntry, dataEntrySection, entry)
+ if (entry)
+ console.log("hello")
+ // options.plugins.title.text = entry.title;
+ setFundingSource(fundingSources.data)
+ convertToFundingSourcePerYearDataResponse(fundingSources.data)
+ })
+ .catch(error => {
+ console.log(`Error fetching from API: ${error}`);
+ })
+ } else {
+ convertToFundingSourcePerYearDataResponse(fundingSourceResponse)
+ }
+
+ }
+ loadData()
+ } ,[])
+
+ const empty_funding_source_response = [
+ {
+ "CLIENT_INSTITUTIONS": "0.0",
+ "COMMERCIAL": "0.0",
+ "EUROPEAN_FUNDING": "0.0",
+ "GOV_PUBLIC_BODIES": "0.0",
+ "NREN": "",
+ "OTHER": "0.0",
+ "YEAR": 0,
+ "id": 0
+ }]
+
+ const convertToFundingSourcePerYearDataResponse = (fundingSourcesResponse: FundingSource[]) => {
+ const fsResponse = fundingSourcesResponse != undefined ? fundingSourcesResponse : empty_funding_source_response;
+ const labelsYear = [...new Set(fsResponse.map((item:FundingSource) => item.YEAR))];
+ const labelsNREN = [...new Set(fsResponse.map((item:FundingSource) => item.NREN))];
+ const fundingComposition=[
+ "CLIENT INSTITUTIONS",
+ "COMMERCIAL",
+ "EUROPEAN FUNDING",
+ "GOV/PUBLIC_BODIES",
+ "OTHER"
+ ]
+ const dataSetKey = cartesianProduct(fundingComposition,labelsYear)
+ console.log("Nrens : ",labelsNREN)
+ console.log("Years : ",labelsYear)
+ console.log(dataSetKey);
+
+ function getRandomColor() {
+ const red = Math.floor(Math.random() * 256).toString(16).padStart(2, '0'); // generates a value between 00 and ff
+ const green = Math.floor(Math.random() * 256).toString(16).padStart(2, '0');
+ const blue = Math.floor(Math.random() * 256).toString(16).padStart(2, '0');
+ return `#${red}${green}${blue}`;
+ }
+
+
+
+ const datasetFunding = dataSetKey.map(function (entry) {
+
+ const randomColor = getRandomColor();
+ return {
+ backgroundColor: randomColor,
+ label: entry[0]+"("+entry[1]+")",//composition+year
+ data: labelsNREN.map(nren => dataPerCompositionPerYear(entry[1], nren,entry[0])),
+ stack:entry[1],
+ borderRadius:10,
+ borderSkipped:false,
+ barPercentage:0.5,
+ borderWidth: 1,
+ }
+ })
+
+ function dataPerCompositionPerYear(year: number, nren: string, composition:string) {
+ let compValue =""
+ fsResponse.find(function (entry, index) {
+ if (entry.YEAR == year && entry.NREN == nren) {
+ if(composition==="CLIENT INSTITUTIONS")
+ compValue= String(entry.CLIENT_INSTITUTIONS);
+ if(composition==="COMMERCIAL")
+ compValue= entry.COMMERCIAL;
+ if(composition==="EUROPEAN FUNDING")
+ compValue= entry.EUROPEAN_FUNDING;
+ if(composition==="GOV/PUBLIC_BODIES")
+ compValue= entry.GOV_PUBLIC_BODIES;
+ if(composition==="OTHER")
+ compValue= entry.OTHER;
+ }
+ })
+ console.log(compValue)
+ return compValue;
+ }
+ console.log(datasetFunding)
+
+ const dataResponse: FundingGraphMatrix = {
+ // datasets: datasetFunding,
+ datasets: datasetFunding,
+ labels: labelsNREN.map(l => l.toString())
+ }
+ setFundingMatrixResponse(dataResponse);
+ }
+ const empty_bar_response = {
+ datasets: [
+ {
+ backgroundColor: '',
+ data: [],
+ label: '',
+ borderRadius:0,
+ borderSkipped:false,
+ barPercentage:0,
+ borderWidth: 1,
+ stack: '0',
+ }],
+ labels: []
+ }
+ const fundingAPIResponse: FundingGraphMatrix = fundingMatrixResponse !== undefined
+ ? fundingMatrixResponse : empty_bar_response;
+ return (
+ <div>
+ <h1>Income Source</h1>
+ <Bar data={fundingAPIResponse}
+ options={{
+ plugins:{
+ legend:{
+ labels:{
+ boxWidth:20,
+ boxHeight:30,
+ pointStyle:"rectRounded",
+ borderRadius:6,
+ useBorderRadius:true,
+ },
+ },
+ },
+ scales:{
+ x: {
+ stacked: true,
+ ticks: {
+ callback: (value: string | number) => {
+ if (typeof value === 'number') {
+ return value.toFixed(2);
+ }
+ return value;
+ },
+ },
+ },
+ y: {
+ stacked: true,
+ },
+ },
+ indexAxis: "y",
+ }}
+ ></Bar>
+ </div>
+ );
+}
+export default FundingSourcePage;