Merge branch 'develop' into feature/COMP-274_traffic_volume_page

compendium-frontend/src/App.tsx

@@ -14,6 +14,7 @@ import ECProjects from "./pages/ECProjects";
 import Providers from "./Providers";
 import PolicyPage from "./pages/Policy";
 import TrafficVolumePage from "./pages/TrafficVolumePerNren";
+import ConnectedInstitutionsURLs from "./pages/ConnectedInstitutionsURLs";
 
 
 const router = createBrowserRouter([
@@ -28,6 +29,7 @@ const router = createBrowserRouter([
   { path: "/policy", element: <PolicyPage />},
   { path: "/traffic-volume", element: <TrafficVolumePage />},
   { path: "/data", element: <CompendiumData />},
+  { path: "/institutions-urls", element: <ConnectedInstitutionsURLs />},
   { path: "*", element: <Landing />},
 ]);
 

compendium-frontend/src/Schema.tsx

@@ -63,6 +63,10 @@ export interface Policy extends NrenAndYearDatapoint {
     strategic_plan: string
 }
 
+export interface ConnectedInstitutionURLs extends NrenAndYearDatapoint {
+    urls: string[]
+}
+
 export interface BasicDataset {
     labels: string[];
     datasets: {

compendium-frontend/src/components/ConnectedUsersSidebar.tsx

+import React from 'react';
+import { Link } from 'react-router-dom';
+import { Row } from 'react-bootstrap';
+import Sidebar from './SideBar';
+
+const ConnectedUsersSidebar = () => {
+    return (
+        <Sidebar>
+            <h5>Connected Users</h5>
+            <Row>
+                <Link to="/institutions-urls" className="link-text-underline">
+                    <span>Connected Institutions URLs</span>
+                </Link>
+            </Row>
+        </Sidebar>
+    )
+}
+
+export default ConnectedUsersSidebar
\ No newline at end of file

compendium-frontend/src/components/DataPage.tsx

@@ -10,6 +10,7 @@ import PolicySidebar from "./PolicySidebar";
 import { Chart as ChartJS } from 'chart.js';
 import { usePreview } from "../helpers/usePreview";
 import NetworkSidebar from "./NetworkSidebar";
+import ConnectedUsersSidebar from "./ConnectedUsersSidebar";
 
 ChartJS.defaults.font.size = 16;
 ChartJS.defaults.font.family = 'Open Sans';
@@ -32,6 +33,7 @@ function DataPage({ title, description, filter, children, category }: inputProps
             {category === Sections.Organisation && <OrganizationSidebar />}
             {category === Sections.Policy && <PolicySidebar />}
             {category === Sections.Network && <NetworkSidebar />}
+            {category === Sections.ConnectedUsers && <ConnectedUsersSidebar />}
             <PageHeader type={'data'} />
             { preview && <Row className="preview-banner">
                 <span>You are viewing a preview of the website which includes pre-published survey data. <a href={locationWithoutPreview}>Click here</a> to deactivate preview mode.</span>

compendium-frontend/src/components/SectionNavigation.tsx

@@ -26,10 +26,9 @@ const SectionNavigation = ({ activeCategory }: inputProps) => {
                         <span>{Sections.Policy}</span>
                     </Button>
                     <Button
-                        onClick={() => navigate(activeCategory === Sections.ConnectedUsers ? '.' : '.')}
+                        onClick={() => navigate(activeCategory === Sections.ConnectedUsers ? '.' : '/institutions-urls')}
                         variant={'nav-box'}
-                        active={activeCategory === Sections.ConnectedUsers}
-                        disabled={true}>
+                        active={activeCategory === Sections.ConnectedUsers}>
                         <span>{Sections.ConnectedUsers}</span>
                     </Button>
                     <Button

compendium-frontend/src/helpers/dataconversion.tsx

 import { cartesianProduct } from 'cartesian-product-multiple-arrays';
 import {
-    FundingSource, FundingSourceDataset, ChargingStructure,
+    FundingSource, FundingSourceDataset, ChargingStructure, ConnectedInstitutionURLs,
     Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, TrafficVolume
 } from "../Schema";
 
@@ -260,6 +260,20 @@ export function createPolicyDataLookup(policyEntries: Policy[]) {
     return policyMap;
 }
 
+export function createConnectedInstitutionsURLsDataLookup(institutionEntries: ConnectedInstitutionURLs[]) {
+    const institutionMap = new Map<string, Map<number, ConnectedInstitutionURLs>>();
+
+    institutionEntries.forEach(entry => {
+        let nrenEntry = institutionMap.get(entry.nren);
+        if (!nrenEntry) {
+            nrenEntry = new Map<number, ConnectedInstitutionURLs>();
+        }
+        nrenEntry.set(entry.year, entry);
+        institutionMap.set(entry.nren, nrenEntry);
+    });
+    return institutionMap;
+}
+
 export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean, selectedYear: number) => {
 
     let categories;

compendium-frontend/src/pages/CompendiumData.tsx

@@ -90,8 +90,11 @@ function CompendiumData(): ReactElement {
 
                         <CollapsibleBox title={Sections.ConnectedUsers} startCollapsed>
                             <div className="collapsible-column">
-                                <h5>Coming Soon</h5>
-                            </div>
+                                <Row>
+                                    <Link to="/institutions-urls" className="link-text-underline">
+                                        <span>Connected Institutions URLs</span>
+                                    </Link>
+                                </Row>                            </div>
                         </CollapsibleBox>
                         <CollapsibleBox title={Sections.Network} startCollapsed>
                             <div className="collapsible-column">

compendium-frontend/src/pages/ConnectedInstitutionsURLs.tsx

+import React, { useContext} from 'react';
+import { Table } from "react-bootstrap";
+
+import {ConnectedInstitutionURLs} from "../Schema";
+import { createConnectedInstitutionsURLsDataLookup } from '../helpers/dataconversion';
+import DataPage from '../components/DataPage';
+import Filter from "../components/graphing/Filter";
+import { Sections } from '../helpers/constants';
+import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
+import {useData} from "../helpers/useData";
+
+function getJSXFromMap(data: Map<string, Map<number, ConnectedInstitutionURLs>>) {
+    return Array.from(data.entries()).map(([nren, nrenMap]) => {
+        return Array.from(nrenMap.entries()).map(([year, institution], yearIndex) => (
+            <tr key={nren + year}>
+                <td className='pt-3 nren-column'>{yearIndex === 0 && nren}</td>
+                <td className='pt-3 year-column'>{year}</td>
+                <td className='pt-3 blue-column'>
+                    <ul>
+                        {/* Display URLs */}
+                        {institution.urls.map((url, index) => (
+                            <li key={index}>
+                                <a href={url} target="_blank" rel="noopener noreferrer">{url}</a>
+                            </li>
+                        ))}
+                    </ul>
+                </td>
+            </tr>
+        ));
+    });
+}
+
+function ConnectedInstitutionsURLsPage() {
+    const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+    const {data: institutionData, years, nrens} = useData<ConnectedInstitutionURLs>('/api/institutions-urls/', setFilterSelection);
+
+
+    const selectedData = institutionData.filter(institution =>
+        filterSelection.selectedYears.includes(institution.year) &&
+        filterSelection.selectedNrens.includes(institution.nren)
+    );
+    const institutionDataByYear = createConnectedInstitutionsURLsDataLookup(selectedData);
+
+    const filterNode = <Filter
+        filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
+        filterSelection={filterSelection}
+        setFilterSelection={setFilterSelection}
+    />;
+
+    return (
+        <DataPage title="Connected Institutions URLs"
+        description='The table shows URLs of the institutions connected to NRENs. You can filter the data by years and by NRENs.'
+        category={Sections.ConnectedUsers} filter={filterNode}>
+            <Table borderless className='compendium-table'>
+                <thead>
+                    <tr>
+                        <th className='nren-column'>NREN</th>
+                        <th className='year-column'>Year</th>
+                        <th className='blue-column'>URLs</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {getJSXFromMap(institutionDataByYear)}
+                </tbody>
+            </Table>
+        </DataPage>
+    );
+}
+
+export default ConnectedInstitutionsURLsPage;

compendium_v2/db/model.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 import logging
 from decimal import Decimal
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 from typing_extensions import Annotated
 
-from sqlalchemy import String
+from sqlalchemy import String, JSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.schema import ForeignKey
 
@@ -135,3 +135,12 @@ class TrafficVolume(db.Model):
     from_customers: Mapped[Decimal]
     to_external: Mapped[Decimal]
     from_external: Mapped[Decimal]
+
+
+class InstitutionURLs(db.Model):
+    __tablename__ = 'institution_urls'
+
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    urls: Mapped[List[str]] = mapped_column(JSON)

compendium_v2/migrations/versions/3730c7f1ea1b_add_traffic_volume_table.py

@@ -11,7 +11,7 @@ import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '3730c7f1ea1b'
-down_revision = 'd6f581374e8f'
+down_revision = 'f2879a6b15c8'
 branch_labels = None
 depends_on = None
 

compendium_v2/migrations/versions/e1a0b83936a0_added_connectedinstitutionsurls_table.py

+"""added InstitutionURL
+
+Revision ID: f2879a6b15c8
+Revises: 96f26bf37f6c
+Create Date: 2023-08-30 14:39:37.133633
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'f2879a6b15c8'
+down_revision = 'd6f581374e8f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'institution_urls',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('urls', sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_institution_urls_nren_id_nren')),
+        sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_institution_urls'))
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('institution_urls')
+    # ### end Alembic commands ###

compendium_v2/publishers/helpers.py

+import re
+from typing import List
+
 from sqlalchemy import select
 
 from compendium_v2.db import db, model
 
+URL_PATTERN = re.compile(
+    r'\b(https?://[^\s<>";,(){}\[\]!\\]+|www\.[^\s<>";,(){}\[\]!\\]+|[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4})\b(?=\s|\b|[,!?.;:\\])'
+)
+
 
 def get_uppercase_nren_dict():
     """
@@ -22,3 +29,7 @@ def get_uppercase_nren_dict():
     nren_dict['GRNET S.A.'] = nren_dict['GRNET']
     nren_dict['FUNET'] = nren_dict['CSC']
     return nren_dict
+
+
+def extract_urls(text: str) -> List[str]:
+    return re.findall(URL_PATTERN, text)

compendium_v2/publishers/survey_publisher_2022.py

@@ -20,6 +20,7 @@ import compendium_v2
 from compendium_v2.db.model import FeeType
 from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
+from compendium_v2.publishers.helpers import extract_urls
 from compendium_v2.survey_db import model as survey_model
 from compendium_v2.db import db, model
 from compendium_v2.publishers import helpers
@@ -59,6 +60,35 @@ WHERE
 ORDER BY n.id, a.question_id, a.updated_at DESC
 """
 
+INSTITUTIONS_URLS_QUERY_UNTIL_2022 = """
+    WITH RECURSIVE parent_questions AS (
+        -- Base case
+        SELECT q.id, q.equivalent_question_id, c.year, q.title
+        FROM questions q
+                 JOIN sections s ON q.section_id = s.id
+                 JOIN compendia c ON s.compendium_id = c.id
+        WHERE q.id = 16507
+    
+        UNION ALL
+    
+        -- Recursive case
+        SELECT q.id, q.equivalent_question_id, c.year, q.title
+        FROM questions q
+                 INNER JOIN parent_questions pq ON q.id = pq.equivalent_question_id
+                 JOIN sections s ON q.section_id = s.id
+                 JOIN compendia c ON s.compendium_id = c.id)
+    
+    SELECT DISTINCT ON (n.id, answers.question_id) answers.id,
+                                                   UPPER(n.abbreviation) AS nren,
+                                                   parent_questions.year,
+                                                   answers.value         as answer
+    FROM answers
+             JOIN parent_questions ON answers.question_id = parent_questions.id
+             JOIN nrens n on answers.nren_id = n.id
+    WHERE UPPER(answers.value) NOT IN ('"NA"', '"N/A"', '[""]', '["-"]', '["/"]', '/', '["NA"]', '""', '[]', '[n/a]')
+    ORDER BY n.id, answers.question_id, answers.updated_at DESC;
+"""
+
 
 class FundingSource(enum.Enum):
     CLIENT_INSTITUTIONS = 16405
@@ -120,6 +150,11 @@ def query_budget():
     return db.session.execute(text(BUDGET_QUERY), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
 
 
+def query_institutions_urls():
+    return db.session.execute(text(INSTITUTIONS_URLS_QUERY_UNTIL_2022),
+                              bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
+
+
 def query_funding_sources():
     for source in FundingSource:
         query = QUESTION_TEMPLATE_QUERY.format(source.value, 2022)
@@ -167,6 +202,45 @@ def transfer_budget(nren_dict):
     db.session.commit()
 
 
+def transfer_institutions_urls(nren_dict):
+    def _parse_json(value):
+        if value and not value.startswith('['):
+            value = f'[{value}]'
+
+        try:
+            return [url.strip() for url in json.loads(value) if url.strip()]
+        except json.decoder.JSONDecodeError:
+            logger.info(f'JSON decode error for institution urls for {nren_name}.')
+            return []
+
+    rows = query_institutions_urls()
+
+    for row in rows:
+        answer_id, nren_name, year, answer = row
+        if nren_name not in nren_dict:
+            logger.info(f'{nren_name} unknown. Skipping.')
+            continue
+
+        urls = extract_urls(text=answer)
+        urls_json = _parse_json(answer)
+        if urls != urls_json:
+            logger.info(f'Institution URLs for {nren_name} do not match between json and regex. {urls} != {urls_json}')
+
+        if not urls:
+            logger.info(f'{nren_name} has no urls for {year}. Skipping.')
+            continue
+
+        institution_urls = model.InstitutionURLs(
+            nren=nren_dict[nren_name],
+            nren_id=nren_dict[nren_name].id,
+            urls=urls,
+            year=year,
+        )
+        db.session.merge(institution_urls)
+
+    db.session.commit()
+
+
 def transfer_funding_sources(nren_dict):
     sourcedata = {}
     for source, data in query_funding_sources():
@@ -491,6 +565,7 @@ def _cli(config, app):
         transfer_charging_structure(nren_dict)
         transfer_ec_projects(nren_dict)
         transfer_policies(nren_dict)
+        transfer_institutions_urls(nren_dict)
 
 
 @click.command()

compendium_v2/routes/api.py

@@ -15,6 +15,7 @@ from compendium_v2.routes.user import routes as user_routes
 from compendium_v2.routes.nren import routes as nren_routes
 from compendium_v2.routes.response import routes as response_routes
 from compendium_v2.routes.traffic import routes as traffic_routes
+from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
 
 routes = Blueprint('compendium-v2-api', __name__)
 routes.register_blueprint(budget_routes, url_prefix='/budget')
@@ -29,6 +30,7 @@ routes.register_blueprint(user_routes, url_prefix='/user')
 routes.register_blueprint(nren_routes, url_prefix='/nren')
 routes.register_blueprint(response_routes, url_prefix='/response')
 routes.register_blueprint(traffic_routes, url_prefix='/traffic')
+routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
 
 logger = logging.getLogger(__name__)
 

compendium_v2/routes/institutions_urls.py

+from typing import Any
+
+from flask import Blueprint, jsonify
+from sqlalchemy import select
+
+from compendium_v2.db import db
+from compendium_v2.db.model import NREN, InstitutionURLs
+from compendium_v2.routes import common
+
+routes = Blueprint('institutions-urls', __name__)
+
+INSTITUTION_URLS_RESPONSE_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'institution_urls': {
+            'type': 'object',
+            'properties': {
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'year': {'type': 'integer'},
+                'urls': {'type': 'array', 'items': {'type': 'string'}}
+            },
+            'required': ['nren', 'nren_country', 'year', 'urls'],
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/institution_urls'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def institutions_urls_view() -> Any:
+    """
+    handler for /api/institutions-urls/ requests
+    Endpoint for getting the URLs of institutions connected to the NREN.
+
+    This endpoint retrieves the URLs of webpages that list the institutions
+    or organizations connected to the NREN.
+    Many NRENs have one or more pages on their website listing such user institutions.
+
+    response will be formatted as per INSTITUTION_URLS_RESPONSE_SCHEMA
+
+    :return:
+    """
+
+    def _extract_data(institution: InstitutionURLs) -> dict:
+        return {
+            'nren': institution.nren.name,
+            'nren_country': institution.nren.country,
+            'year': institution.year,
+            'urls': institution.urls
+        }
+
+    entries = []
+    records = db.session.scalars(
+        select(InstitutionURLs).join(NREN).order_by(NREN.name.asc(), InstitutionURLs.year.desc()))
+
+    for entry in records:
+        entries.append(_extract_data(entry))
+
+    return jsonify(entries)

test/conftest.py

@@ -354,4 +354,41 @@ def test_traffic_data(app):
                     to_external=1000
                 )
             )
+
+
+@pytest.fixture
+def test_institution_urls_data(app):
+    def _create_and_save_nrens(nren_names):
+        nrens = {}
+        for nren_name in nren_names:
+            nren_instance = model.NREN(name=nren_name, country='country')
+            nrens[nren_name] = nren_instance
+            db.session.add(nren_instance)
+        return nrens
+
+    def _create_and_save_institution_urls(nrens_and_years, nrens):
+        for nren_name, year in nrens_and_years:
+            nren_instance = nrens[nren_name]
+            urls = ['https://example.com', 'http://example.org']
+
+            institution_urls_model = model.InstitutionURLs(
+                nren=nren_instance,
+                year=year,
+                urls=urls
+            )
+
+            db.session.add(institution_urls_model)
+
+    with app.app_context():
+        predefined_nrens_and_years = [
+            ('nren1', 2019),
+            ('nren1', 2020),
+            ('nren1', 2021),
+            ('nren2', 2019),
+            ('nren2', 2021)
+        ]
+
+        unique_nren_names = {nren for nren, _ in predefined_nrens_and_years}
+        created_nrens = _create_and_save_nrens(unique_nren_names)
+        _create_and_save_institution_urls(predefined_nrens_and_years, created_nrens)
         db.session.commit()

test/test_helpers.py

+import pytest
+
+from compendium_v2.publishers.helpers import extract_urls
+
+
+@pytest.mark.parametrize(
+    "text, expected",
+    [
+        ("Check this: https://example.com and http://example.com\\", ["https://example.com", "http://example.com"]),
+        ("No URL here", []),
+        ("Incomplete URL: http://", []),
+        ("Here is a www link: www.example.com", ["www.example.com"]),
+        ("Another format: example.com", ["example.com"]),
+        ("Mixed: https://example.com, http://site.org, www.site.net",
+         ["https://example.com", "http://site.org", "www.site.net"]),
+        ("With Parentheses: (https://example.com)", ["https://example.com"]),
+        ("With Brackets: [https://example.com]", ["https://example.com"]),
+        ("With Braces: {https://example.com}", ["https://example.com"]),
+        ("With Semicolon: https://example.com;", ["https://example.com"]),
+        ("With Dot at the end: https://example.com.", ["https://example.com"]),
+        ("With Comma at the end: https://example.com,", ["https://example.com"]),
+        ("With Exclamation at the end: https://example.com!", ["https://example.com"]),
+        ("With Multiple special characters: https://example.com)!..d!;www.m.asd!)",
+         ["https://example.com", "www.m.asd"]),
+        ("URL with path: https://example.com/path/to/page", ["https://example.com/path/to/page"]),
+        ("URL with path and query: https://example.com/path?query=value", ["https://example.com/path?query=value"]),
+        ("URL with hash: https://example.com#section", ["https://example.com#section"]),
+        ("URL with path, query and hash: https://example.com/path?query=value#section",
+         ["https://example.com/path?query=value#section"]),
+        ("URL with port: http://example.com:8080", ["http://example.com:8080"]),
+        ("Multiple Complex: https://example.com/path, http://example.com:8080/page?query=value#section",
+         ["https://example.com/path", "http://example.com:8080/page?query=value#section"]),
+        ("URL with encoded characters: https://example.com/path%20to%20page", ["https://example.com/path%20to%20page"]),
+        ("URL with encoded query: https://example.com/path?query=value%20with%20space",
+         ["https://example.com/path?query=value%20with%20space"]),
+        ("[URL with encoded hash: https://example.com/path#section%20two;]", ["https://example.com/path#section%20two"]),
+    ]
+)
+def test_extract_urls_from_a_text(text, expected):
+    result = extract_urls(text)
+    assert result == expected

test/test_instittutions_urls.py

+import json
+import jsonschema
+
+from compendium_v2.routes.institutions_urls import INSTITUTION_URLS_RESPONSE_SCHEMA
+
+
+def test_institutions_urls_response(client, test_institution_urls_data):
+    rv = client.get(
+        '/api/institutions-urls/',
+        headers={'Accept': ['application/json']})
+    assert rv.status_code == 200
+    result = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(result, INSTITUTION_URLS_RESPONSE_SCHEMA)
+    assert result

test/test_policies.py

@@ -3,7 +3,7 @@ import jsonschema
 from compendium_v2.routes.policy import POLICY_RESPONSE_SCHEMA
 
 
-def test_ec_project_response(client, test_policy_data):
+def test_policy_response(client, test_policy_data):
     rv = client.get(
         '/api/policy/',
         headers={'Accept': ['application/json']})