Finished release 2.27.

.gitignore

@@ -12,12 +12,14 @@ oss-params.json
 build/
 
 # Documentation
-docs/build
+docs/site
+docs/public
 docs/vale/styles/*
 !docs/vale/styles/config/
 !docs/vale/styles/custom/
 .DS_Store
+docs/wf_redirects.yaml
 
 .idea
 .venv
-.env
+.env
\ No newline at end of file

.gitlab-ci.yml

@@ -4,8 +4,8 @@ stages:
   - documentation
   - sonarqube
   - trigger_jenkins_build
-include:
-  - docs/.gitlab-ci.yml
+  - publish-docs-prod-1
+  - publish-docs-prod-2
 
 ####################################  tox   -   Testing and linting
 run-tox-pipeline:
@@ -13,10 +13,10 @@ run-tox-pipeline:
   tags:
     - docker-executor
   image: python:3.12
-
+  except:
+    - /^release\/.*/
   services:
     - postgres:14
-
   # Change pip's cache directory to be inside the project directory since we can
   # only cache local items.
   variables:
@@ -25,29 +25,91 @@ run-tox-pipeline:
     POSTGRES_PASSWORD: nwa
     DATABASE_HOST: "postgres"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-
   cache:
     key: tox-virtualenv
     paths:
       - .cache/pip
       - venv/
-
   before_script:
     - pip install virtualenv
     - virtualenv venv
     - . venv/bin/activate
     - pip install tox
-
   script:
     - tox
-
   artifacts:
     paths:
       - htmlcov
 
+##### MkDocs  - Generate documentation
+build-documentation:
+  stage: documentation
+  tags:
+    - docker-executor
+  image: python:3.12
+  before_script:
+    - cd $CI_PROJECT_DIR/docs
+  script:
+    - /bin/sh ./build-docs.sh
+  except:
+    - /^release\/.*/
+  artifacts:
+    paths:
+      - docs/public
+
+##### Vale    - Documentation linter
+lint-documentation:
+  stage: documentation
+  image:
+    name: jdkato/vale:latest
+    entrypoint: [""]
+  tags:
+    - docker-executor
+  needs:
+    - job: build-documentation # Only run when documentation has been built
+      artifacts: true
+  except:
+    - /^release\/.*/
+  before_script:
+    - cd $CI_PROJECT_DIR/docs/vale
+    - vale sync
+  script:
+    - vale --glob='!*/migrations/*' $CI_PROJECT_DIR/docs/source $CI_PROJECT_DIR/gso/
+
+##### Publish documentation artefacts
+publish-docs-prod-1:
+  stage: publish-docs-prod-1
+  # First deploy to prod-1
+  tags:
+    - gap-docs-prod-1
+  only:
+    - master  # Only publish when on the main branch
+  needs:
+    - job: build-documentation
+      artifacts: true
+  script:
+    - rm -rf /var/www/gap-docs/*
+    - mv $CI_PROJECT_DIR/docs/public/* /var/www/gap-docs/
+
+publish-docs-prod-2:
+  stage: publish-docs-prod-2
+  # When successful, also deploy to prod-2
+  tags:
+    - gap-docs-prod-2
+  only:
+    - master  # Only publish when on the main branch
+  needs:
+    - job: build-documentation
+      artifacts: true
+  script:
+    - rm -rf /var/www/gap-docs/*
+    - mv $CI_PROJECT_DIR/docs/public/* /var/www/gap-docs/
+
 sonarqube:
   stage: sonarqube
   image: sonarsource/sonar-scanner-cli:10.0
+  except:
+    - /^release\/.*/
   script:
     - sonar-scanner -Dsonar.login=$SONAR_TOKEN -Dproject.settings=./sonar.properties
   tags:

Changelog.md

 # Changelog
 
+## [2.27] - 2024-12-03
+- Add support for VRFs
+- Rework the way code documentation is built and pulished
+- Fix a bug in email notification delivery
+
 ## [2.26] - 2024-11-27
 - Rename the `NRENL3CoreService` to `L3CoreService`
 - To the existing Layer 3 core services, add LHCOne, Copernicus and IAS-GWS

README.md

@@ -2,4 +2,5 @@
 The GÉANT interpretation of [``orchestrator-core``](https://github.com/workfloworchestrator/orchestrator-core).
 
 ## Documentation
-You can build the documentation locally using [build-docs.sh](build-docs.sh).
+You can build the documentation locally using [build-docs.sh](docs/build-docs.sh). It is available online at
+[docs.gap.geant.org](https://docs.gap.geant.org/).

docs/.gitlab-ci.yml

----
-##### Sphinx  - Generate documentation
-build-documentation:
-  stage: documentation
-  tags:
-    - docker-executor
-  image: sphinxdoc/sphinx:latest
-
-  before_script:
-    - pip install sphinx_rtd_theme sphinxcontrib-jquery
-    - cd $CI_PROJECT_DIR/docs/source
-  script:
-    - make html
-
-  artifacts:
-    paths:
-      - $CI_PROJECT_DIR/docs/build/html
-
-##### Vale    - Documentation linter
-lint-documentation:
-  stage: documentation
-  image:
-    name: jdkato/vale:latest
-    entrypoint: [""]
-
-  tags:
-    - docker-executor
-  needs:
-    - job: build-documentation # Only run when documentation has been built
-      artifacts: true
-
-  before_script:
-    - cd $CI_PROJECT_DIR/docs/vale
-    - vale sync
-
-  script:
-    - vale --glob='!*/migrations/*' $CI_PROJECT_DIR/docs/source $CI_PROJECT_DIR/gso

build-docs.sh → docs/build-docs.sh

@@ -5,7 +5,8 @@ set -o nounset
 export OSS_PARAMS_FILENAME=../gso/oss-params-example.json
 export TESTING=true
 
-pip install sphinx_rtd_theme sphinxcontrib-jquery
+pip install pyyaml mkdocstrings-python mkdocs_gen_files mkdocs-material mkdocs-literate-nav mkdocs-redirects
+pip install -e ..
+python ./scripts/gen_wf_redirects.py
 
-cd docs
-sphinx-build source build
+mkdocs build --site-dir public

docs/includes/glossary.md

+*[AAA]: Authentication, Authorisation, Accounting
+*[BFD]: Bidirectional Forwarding Detection
+*[BGP]: Border Gateway Protocol
+*[BSS]: Business Support Systems
+*[CFS]: Customer Facing Service
+*[CIDR]: Classless Inter-Domain Routing
+*[DCIM]: Datacenter Infrastructure Manager
+*[DHCP]: Dynamic Host Configuration Protocol
+*[DNS]: Domain Name System
+*[DTAP]: Development, Testing, Acceptance, and Production
+*[eBGP]: External BGP
+*[FQDN]: Fully Quantified Domain Name
+*[GA]: Access Port
+*[GAN]: Access Node
+*[GAP]: GÉANT Automation Platform
+*[GOAT]: GÉANT Orchestration and Automation Team
+*[GP]: Physical Port
+*[GSO]: GÉANT Service Orchestrator
+*[GUI]: Graphical User Interface
+*[IAS]: GÉANT Internet Access Service
+*[iBGP]: Internal BGP
+*[IGP]: Internal Gateway Protocol
+*[IFS]: Interface Facing Service
+*[IPAM]: IP Address Management
+*[IS-IS]: Intermediate System to Intermediate System
+*[ISO]: International Standards Organisations
+*[L3VPN]: Layer 3 VPN
+*[LACP]: Link Aggregation Control Protocol
+*[LLDP]: Link-Layer Discovery Protocol
+*[LSO]: Lightweight Service Orchestrator
+*[MAC]: Medium Access Control
+*[MPLS]: Multi-Protocol Label Switching
+*[MTTR]: Mean Time To Repair
+*[MTU]: Maximum Transmission Unit
+*[NREN]: National Research and Education Network
+*[OOB]: Out-of-band
+*[OSS]: Operational Support Systems
+*[PoP]: Point of Presence
+*[REST]: Representational State Transfer
+*[RFC]: Request For Comments
+*[SBP]: Service Binding Port
+*[SDP]: Service Delivery Point
+*[SNMP]: Simple Network Management Protocol
+*[SOT]: Source Of Truth
+*[TBA]: To be added
+*[UAT]: User Acceptance Testing
+*[VM]: Virtual Machine
+*[VRF]: Virtual Routing and Forwarding
+*[WAN]: Wide Area Network
+*[WFO]: Workflow Orchestrator

docs/mkdocs.yml

+INHERIT: wf_redirects.yaml
+# General
+site_name: GÉANT Automation Platform
+site_url: https://docs.gap.geant.org/
+site_author: GÉANT Orchestration and Automation Team
+site_description: Documentation of the GÉANT Automation Platform and the GÉANT Service Orchestrator
+docs_dir: source
+
+# Repositories
+repo_name: geant-service-orchestrator
+repo_url: https://gitlab.software.geant.org/goat/gap/geant-service-orchestrator
+
+# Theme
+theme:
+  name: material
+  logo_light_mode: assets/images/logo.png
+  logo_dark_mode: assets/images/geant_logo_white.png
+  custom_dir: overrides
+  icon:
+    repo: fontawesome/brands/gitlab
+  features:
+    - navigation.instant
+    - navigation.tracking
+    - navigation.sections
+    - toc.integrate
+    - navigation.path
+    - navigation.indexes
+    - content.code.copy
+  palette:
+    - media: "(prefers-color-scheme)"
+      toggle:
+        icon: material/auto-mode
+        name: Switch to light mode
+    - media: "(prefers-color-scheme: light)"
+      scheme: geant
+      primary: custom
+      toggle:
+        icon: material/weather-sunny
+        name: Switch to dark mode
+    - media: "(prefers-color-scheme: dark)"
+      scheme: slate
+      primary: custom
+      toggle:
+        icon: material/weather-night
+        name: Switch to system preference
+extra_css:
+  - assets/stylesheets/extra.css
+
+# Page tree
+nav:
+  - Home: index.md
+  - Architecture:
+    - architecture/index.md
+    - architecture/config_decomposition/index.md
+    - Components:
+      - architecture/components/index.md
+      - WFO: architecture/components/wfo/index.md
+      - Netbox: architecture/components/netbox/index.md
+      - Ansible: architecture/components/ansible/index.md
+      - LSO: architecture/components/lso/index.md
+    - DTAP:
+      - architecture/dtap/index.md
+      - Development: architecture/dtap/development.md
+      - Test: architecture/dtap/test.md
+      - Acceptance: architecture/dtap/acceptance.md
+      - Production: architecture/dtap/production.md
+  - Admin Guide:
+    - admin_guide/index.md
+    - Ansible:
+        - admin_guide/ansible/ansible.md
+    - GSO:
+      - Diagram: admin_guide/wfo/overview.md
+      - Sites: admin_guide/wfo/sites.md
+      - Routers: admin_guide/wfo/routers.md
+      - IP trunks: admin_guide/wfo/iptrunks.md
+    - OSS/BSS integration:
+      - Infoblox: admin_guide/oss_bss/ipam.md
+      - Netbox: admin_guide/oss_bss/netbox.md
+      - LibreNMS: admin_guide/oss_bss/librenms.md
+      - Kentik: admin_guide/oss_bss/kentik.md
+  - Code Documentation: reference/gso/
+
+# Extensions
+markdown_extensions:
+  - admonition
+  - abbr
+  - attr_list
+  - md_in_html
+  - pymdownx.snippets:
+      auto_append:
+        - includes/glossary.md
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format

docs/overrides/partials/logo.html

+<img id="logo_light_mode" src="{{ config.theme.logo_light_mode | url }}" alt="logo">
+<img id="logo_dark_mode" src="{{ config.theme.logo_dark_mode | url }}" alt="logo">

docs/scripts/gen_ref_pages.py

+# noqa: INP001
+"""Generate the code reference pages.
+
+Source: https://mkdocstrings.github.io/recipes/
+"""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+root = Path(__file__).parent.parent.parent
+src = root / "gso"
+ignored_modules = ["db", "graphql_api", "main", "migrations"]
+
+for path in sorted(src.rglob("*.py")):
+    module_path = path.relative_to(root).with_suffix("")
+    doc_path = path.relative_to(root).with_suffix(".md")
+    full_doc_path = Path("reference", doc_path)
+
+    parts = tuple(module_path.parts)
+
+    if any(module in parts for module in ignored_modules):
+        #  E.g. ignore generated migration files from SQLAlchemy.
+        continue
+    elif parts[-1] == "__init__":
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+    elif parts[-1] == "__main__":
+        continue
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        identifier = ".".join(parts)
+        print("::: " + identifier, file=fd)
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root))

docs/scripts/gen_wf_redirects.py

+# noqa: INP001
+"""Generate pages to forward workflow URLs.
+
+This will redirect `/workflow/workflow_name` to `reference/gso/workflow/module/workflow_name`. This is required since
+URLs cannot be changed in the GUI.
+"""
+
+from pathlib import Path
+
+import yaml
+
+root = Path(__file__).parent.parent.parent
+src = root / "gso"
+redirect_map = {}
+
+for path in sorted(src.rglob("*.py")):
+    module_path = path.relative_to(root).with_suffix("")
+    parts = tuple(module_path.parts)
+
+    if parts[-1] == "__init__":
+        parts = parts[:-1]
+    elif parts[-1] == "__main__":
+        continue
+
+    if len(parts) == 4 and "workflows" in parts:  # noqa: PLR2004
+        redirect_map["workflow/" + parts[3] + "/index.md"] = "https://docs.gap.geant.org/reference/" + "/".join(parts)
+
+with Path.open(root / "docs" / "wf_redirects.yaml", "w") as redirect_file:
+    file_content = {
+        "plugins": [
+            "search",
+            {"gen-files": {"scripts": ["scripts/gen_ref_pages.py"]}},
+            {"redirects": {"redirect_maps": redirect_map}},
+            {"literate-nav": {"nav_file": "SUMMARY.md"}},
+            "mkdocstrings",
+        ]
+    }
+    yaml.dump(file_content, redirect_file)

docs/source/Makefile

-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = ../build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/admin_guide/ansible/ansible.md

+# Ansible
+
+## Design
+
+## Ansible roles and playbooks
+
+## Troubleshooting
\ No newline at end of file

docs/source/admin_guide/index.md

+# About this section
+
+All the information regarding modeling and Ansible mechanics are described in this section.
+
+The structure is:
+
+- WFO
+  - Modelling
+- Ansible
+  - Design
+  - Low level description

docs/source/admin_guide/oss_bss/ipam.md

+# Integration with Infoblox
+
+The Infoblox service in GAP takes care of IP resources when creating routers,
+IP trunks, and other customer-facing services.

docs/source/admin_guide/oss_bss/kentik.md

+# Integration with Kentik
+
+Routers are added to Kentik when they have a PE role. This can be either when
+creating a new PE router, or upgrading an existing P router to PE.
+
+When the router is added to Kentik, a placeholder license is applied. Then,
+a member of the service management team is able to run the
+`modify_router_kentik_license` workflow to apply a different license.

docs/source/admin_guide/oss_bss/librenms.md

+# Integration with LibreNMS
+
+When a router is created or terminated, it is added or removed from LibreNMS.

docs/source/admin_guide/oss_bss/netbox.md

+# Integration with Netbox
+
+Netbox is used for bookkeeping of interface usage, and suggesting options to
+the operator for available LAGs.

docs/source/admin_guide/wfo/iptrunks.md

+# IP trunks
+
+IP trunks are core links between two GÉANT routers.
+The IP trunk is a special service since on the interfaces at the end of the trunk, no VLAN multiplexing is allowed.
+For this reason, in case of an IP trunk, we do not use the canonical decomposition that leverages a demarcation point.
+
+## Modelling and attributes
+
+The relevant attributes for an IPTrunk are the following:
+
+| Attribute name                        | Attribute type | Description                                                                                          |
+|---------------------------------------|----------------|------------------------------------------------------------------------------------------------------|
+| `geant_s_sid`                         | `str`          | GÉANT service ID associated with this trunk.                                                         |
+| `iptrunk_description`                 | `str`          | A human-readable description of this trunk.                                                          |
+| `iptrunk_type`                        | `IptrunkType`  | The type of trunk, can be either dark fibre or leased capacity.                                      |
+| `iptrunk_speed`                       | `str`          | should be of PhyPortCapacity type The speed of the trunk, measured per interface associated with it. |
+| `iptrunk_minimum_links`               | `int`          | The minimum amount of links the trunk should consist of.                                             |
+| `iptrunk_isis_metric`                 | `int`          | The IS-IS metric of this link                                                                        |
+| `iptrunk_ipv4_network`                | `IPv4Network`  | The IPv4 network used for this trunk.                                                                |
+| `iptrunk_ipv6_network`                | `IPv6Network`  | The IPv6 network used for this trunk.                                                                |
+| `iptrunk_side_node`                   | `DeviceBlock`  | The router that hosts the A side of the trunk.                                                       |
+| `iptrunk_side_ae_iface`               | `str`          | The name of the interface on which the trunk connects.                                               |
+| `iptrunk_side_ae_geant_a_sid`         | `str`          | The service ID of the interface.                                                                     |
+| `iptrunk_side_ae_members`             | `list[str]`    | A list of interface members that make up the aggregated Ethernet interface.                          |
+| `iptrunk_side_ae_members_description` | `list[str]`    | The list of descriptions that describe the list of interface members.                                |

docs/source/admin_guide/wfo/overview.md

+# Diagram
+
+``` mermaid
+classDiagram
+  Site <|-- Router :belong
+
+  class Site{
+    +UUId name
+    +String phoneNumber
+    +String emailAddress
+  }
+
+  class Router{
+    +int studentNumber
+    +int averageMark
+    +isEligibleToEnrol()
+    +getSeminarsTaken()
+  }
+```
+
+## Node deployment
+
+A node consists of one or more routers, a switch, and a terminal server. 
+In general -- as laid out more extensively 
+<a href="https://wiki.geant.org/display/NETENG/001+-+Topology+and+physical+layout" target="_blank">here</a> 
+(behind login) -- a PoP consists of:
+
+* One or two routers
+* One switch
+* One terminal server
+
+Globally, the workflow for a new site is as follows:
+
+1. Deploy terminal server:
+   1. Generate base configuration from GitLab
+   2. Ship the device to its location
+   3. Verify reachability and insert in LibreNMS
+2. Deploy PoP router in a 'core' fashion
+   1. Rack it up and configure the hardware
+   2. Connect the router to the terminal server via both a console connection, and FXP
+   3. Deploy base configuration using GAP
+3. Deploy PoP switch
+   1. Rack it up and configure the hardware
+   2. Connect the switch to the terminal server via both a console connection, and FXP
+   3. Deploy base configuration using GAP
+4. Deploy the PoP interconnect between router and switch
+   1. Set up a physical connection between router and switch
+   2. Deploy configuration using GAP
+5. Deploy IP trunks to connect the router to the rest of the network
+   1. Set up a physical connection
+   2. Deploy configuration using GAP
+6. Update the iBGP mesh to include the new router, promoting it to an edge router
+   1. Deploy configuration using GAP
+   2. Using GAP, insert the devices in LibreNMS
+
+In the context of the automation platform, the PoP interconnects mentioned are modeled as separate objects.