Merge develop into NAT-185

Merge e3fab4ac

Merge up to 4c346d4f

Merge up to b25e7012

Merge 1779de6b

Merge df0cd9c9

Merge up to 028a2d46

Finish merging develop into NAT-185

Finish merge develop into NAT-185

Finish merge develop into NAT-185

gso/main.py

 from orchestrator import OrchestratorCore
 from orchestrator.cli.main import app as core_cli
 from orchestrator.settings import AppSettings
+# pylint: disable=unused-import
 import gso.products  # noqa: F401
+# pylint: disable=unused-import
 import gso.workflows  # noqa: F401
 
 

gso/oss-params-example.json

@@ -12,8 +12,8 @@
       "password": "robot-user-password"
     },
     "LO": {
-      "V4": {"containers": [], "networks": ["1.1.0.0/28"], "mask": 0},
-      "V6": {"containers": [], "networks": ["dead:beef::/80"], "mask": 0},
+      "V4": {"containers": [], "networks": ["1.1.0.0/24"], "mask": 0},
+      "V6": {"containers": [], "networks": ["dead:beef::/64"], "mask": 0},
       "domain_name": ".lo",
       "dns_view": "default"
     },
@@ -21,13 +21,25 @@
       "V4": {"containers": ["1.1.1.0/24"], "networks": [], "mask": 31},
       "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
       "domain_name": ".trunk",
-      "dns_view": "default
+      "dns_view": "default"
     },
     "GEANT_IP": {
       "V4": {"containers": ["1.1.2.0/24"], "networks": [], "mask": 31},
       "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
       "domain_name": ".geantip",
-      "dns_view": "default
+      "dns_view": "default"
+    },
+    "SI": {
+      "V4": {"containers": ["1.1.3.0/24"], "networks": [], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+      "domain_name": ".si",
+      "dns_view": "default"
+    },
+    "LT_IAS": {
+      "V4": {"containers": ["1.1.4.0/24"], "networks": [], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+      "domain_name": ".ltias",
+      "dns_view": "default"
     }
   },
   "PROVISIONING_PROXY": {

gso/services/provisioning_proxy.py

+"""
+The Provisioning Proxy service, which interacts with LSO running externally.
+LSO is responsible for executing Ansible playbooks, that deploy subscriptions.
+"""
 import json
 import logging
 
@@ -53,7 +57,8 @@ def _send_request(endpoint: str, parameters: dict, process_id: UUIDstr,
 
     callback_url = f'{settings.load_oss_params().GENERAL.public_hostname}' \
                    f'/api/processes/{process_id}/resume'
-    logger.debug(f'[provisioning proxy] provisioning for process {process_id}')
+    logger.debug('[provisioning proxy] provisioning for process %s',
+                 process_id)
 
     parameters.update({'callback': callback_url})
     url = f'{pp_params.scheme}://{pp_params.api_base}/api/{endpoint}'
@@ -61,11 +66,11 @@ def _send_request(endpoint: str, parameters: dict, process_id: UUIDstr,
     request = None
 
     if operation == CUDOperation.POST:
-        request = requests.post(url, json=parameters)
+        request = requests.post(url, json=parameters, timeout=10000)
     elif operation == CUDOperation.PUT:
-        request = requests.put(url, json=parameters)
+        request = requests.put(url, json=parameters, timeout=10000)
     elif operation == CUDOperation.DELETE:
-        request = requests.delete(url, json=parameters)
+        request = requests.delete(url, json=parameters, timeout=10000)
 
     if request.status_code != 200:
         print(request.content)
@@ -136,7 +141,7 @@ def provision_ip_trunk(subscription: IptrunkProvisioning,
 #        'dry_run': dry_run,
 #        'old_subscription': old_subscription,
 #        'subscription': new_subscription
-#        # FIXME missing parameters
+#        # ... missing parameters
 #    }
 #
 #    _send_request('ip_trunk', parameters, process_id, CUDOperation.PUT)

gso/settings.py

@@ -40,6 +40,8 @@ class IPAMParams(BaseSettings):
     LO: ServiceNetworkParams
     TRUNK: ServiceNetworkParams
     GEANT_IP: ServiceNetworkParams
+    SI: ServiceNetworkParams
+    LT_IAS: ServiceNetworkParams
 
 
 class ProvisioningProxyParams(BaseSettings):
@@ -61,8 +63,8 @@ def load_oss_params() -> OSSParams:
     look for OSS_PARAMS_FILENAME in the environment and load the
     parameters from that file
     """
-    with open(os.environ['OSS_PARAMS_FILENAME']) as f:
-        return OSSParams(**json.loads(f.read()))
+    with open(os.environ['OSS_PARAMS_FILENAME'], encoding='utf-8') as file:
+        return OSSParams(**json.loads(file.read()))
 
 
 if __name__ == '__main__':

gso/workflows/__init__.py

+"""
+init class that imports all workflows into GSO.
+"""
 from orchestrator.workflows import LazyWorkflowInstance
 
 LazyWorkflowInstance("gso.workflows.device.create_device", "create_device")

gso/workflows/device/create_device.py

@@ -19,6 +19,7 @@ from gso.products.product_types import device
 from gso.products.product_types.device import DeviceInactive, \
     DeviceProvisioning
 from gso.products.product_types.site import Site
+from gso.services import _ipam
 from gso.services import provisioning_proxy
 from gso.services.provisioning_proxy import await_pp_results, \
     confirm_pp_results
@@ -84,20 +85,22 @@ def iso_from_ipv4(ipv4_address):
 
 
 @step('Get information from IPAM')
-def get_info_from_ipam(subscription: DeviceInactive) -> State:
-    # lo = ipam.new_device_lo_address()
-    # subscription.device.lo_ipv4_address = lo.v4
-    # subscription.device.lo_ipv6_address = lo.v6
-    # TODO: get info about how these should be generated
-    subscription.device.device_lo_ipv4_address = \
-        ipaddress.ip_address('10.10.10.20')
-    subscription.device.device_lo_ipv6_address = \
-        ipaddress.ip_address('fc00:798:10::20')
+def get_info_from_ipam(subscription: DeviceProvisioning) -> State:
+    lo0_alias = re.sub('.geant.net', '', subscription.device.device_fqdn)
+    lo0_name = f'lo0.{lo0_alias}'
+    lo0_addr = _ipam.allocate_service_host(hostname=lo0_name,
+                                           service_type='LO',
+                                           cname_aliases=[lo0_alias])
+    subscription.device.device_lo_ipv4_address = lo0_addr.v4
+    subscription.device.device_lo_ipv6_address = lo0_addr.v6
     subscription.device.device_lo_iso_address \
         = iso_from_ipv4(str(subscription.device.device_lo_ipv4_address))
-    subscription.device.device_si_ipv4_network = '192.168.0.0/31'
-    subscription.device.device_ias_lt_ipv4_network = '192.168.1.0/31'
-    subscription.device.device_ias_lt_ipv6_network = 'fc00:798:1::150/126'
+    subscription.device.device_si_ipv4_network \
+        = _ipam.allocate_service_ipv4_network(service_type='SI', comment=f"SI for {lo0_name}").v4
+    subscription.device.device_ias_lt_ipv4_network \
+        = _ipam.allocate_service_ipv4_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v4
+    subscription.device.device_ias_lt_ipv6_network \
+        = _ipam.allocate_service_ipv6_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v6
     return {'subscription': subscription}
 
 
@@ -116,19 +119,19 @@ def initialize_subscription(
     subscription.device.device_vendor = device_vendor
     subscription.device.device_site \
         = Site.from_subscription(device_site[0]).site
-    fqdn = str(hostname + '.' +
-               subscription.device.device_site.site_name.lower() + '.' +
-               subscription.device.device_site.site_country_code.lower() +
-               '.geant.net')
+    fqdn = f'{hostname}.{subscription.device.device_site.site_name.lower()}.' \
+           f'{subscription.device.device_site.site_country_code.lower()}' \
+           f'.geant.net'
     subscription.device.device_fqdn = fqdn
     subscription.device.device_role = device_role
     subscription.description = f'Device {fqdn} ' \
                                f'({subscription.device_type})'
+
     subscription = device.DeviceProvisioning.from_other_lifecycle(
         subscription, SubscriptionLifecycle.PROVISIONING
     )
 
-    return {'subscription': subscription, 'fqdn': fqdn}
+    return {'subscription': subscription}
 
 
 @step('Provision device [DRY RUN]')

gso/workflows/iptrunk/create_iptrunk.py

-import ipaddress
 from uuid import uuid4
 
 from orchestrator.db.models import ProductTable, SubscriptionTable
@@ -17,7 +16,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType
 from gso.products.product_types.device import Device
 from gso.products.product_types.iptrunk import IptrunkInactive, \
     IptrunkProvisioning
-from gso.services import provisioning_proxy
+from gso.services import provisioning_proxy, _ipam
 from gso.services.provisioning_proxy import confirm_pp_results, \
     await_pp_results
 
@@ -91,12 +90,12 @@ def create_subscription(product: UUIDstr) -> State:
 
 
 @step('Get information from IPAM')
-def get_info_from_ipam(subscription: IptrunkInactive) -> State:
+def get_info_from_ipam(subscription: IptrunkProvisioning) -> State:
     # TODO: get info about how these should be generated
     subscription.iptrunk.iptrunk_ipv4_network \
-        = ipaddress.ip_network('192.168.255.0/31')
+        = _ipam.allocate_service_ipv4_network(service_type="TRUNK", comment=subscription.iptrunk.iptrunk_description).v4
     subscription.iptrunk.iptrunk_ipv6_network \
-        = ipaddress.ip_network('fc00:798:255::150/126')
+        = _ipam.allocate_service_ipv6_network(service_type="TRUNK", comment=subscription.iptrunk.iptrunk_description).v6
     return {'subscription': subscription}
 
 
@@ -235,8 +234,8 @@ def create_iptrunk():
             init
             >> create_subscription
             >> store_process_subscription(Target.CREATE)
-            >> get_info_from_ipam
             >> initialize_subscription
+            >> get_info_from_ipam
             >> provision_ip_trunk_iface_dry
             >> await_pp_results
             >> confirm_pp_results

test/conftest.py

@@ -2,9 +2,10 @@ import contextlib
 import json
 import os
 import socket
-import pytest
 import tempfile
 
+import pytest
+
 
 @pytest.fixture(scope='session')
 def configuration_data():
@@ -66,6 +67,34 @@ def configuration_data():
                     },
                     "domain_name": ".geantip",
                     "dns_view": "default"
+                },
+                "SI": {
+                    "V4": {
+                        "containers": ["10.255.253.128/25"],
+                        "networks": [],
+                        "mask": 31
+                        },
+                    "V6": {
+                        "containers": [],
+                        "networks": [],
+                        "mask": 126
+                        },
+                    "domain_name": ".geantip",
+                    "dns_view": "default"
+                },
+                "LT_IAS": {
+                    "V4": {
+                        "containers": ["10.255.255.0/24"],
+                        "networks": [],
+                        "mask": 31
+                        },
+                    "V6": {
+                        "containers": ["dead:beef:cc::/48"],
+                        "networks": [],
+                        "mask": 126
+                        },
+                    "domain_name": ".geantip",
+                    "dns_view": "default"
                 }
             },
             "PROVISIONING_PROXY": {

test/test_ipam.py

 import ipaddress
 import pytest
 import re
+
 import responses
 
 from gso.services import ipam
@@ -8,12 +9,11 @@ from gso.services import ipam
 
 @responses.activate
 def test_new_service_networks(data_config_filename):
-
     responses.add(
         method=responses.POST,
         url=re.compile(r'.*/wapi.*/network.*'),
         json={
-            '_ref': 'network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default',   # noqa: E501
+            '_ref': 'network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default',  # noqa: E501
             'network': '10.255.255.20/32'
         }
     )
@@ -22,7 +22,7 @@ def test_new_service_networks(data_config_filename):
         method=responses.POST,
         url=re.compile(r'.*/wapi.*/ipv6network.*'),
         json={
-            '_ref': 'ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default',   # noqa: E501
+            '_ref': 'ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default',  # noqa: E501
             'network': 'dead:beef::18/128'
         }
     )
@@ -41,23 +41,22 @@ def test_new_service_networks(data_config_filename):
 
 @responses.activate
 def test_new_service_host(data_config_filename):
-
     responses.add(
         method=responses.POST,
         url=re.compile(r'.*/wapi.*/record:host$'),
-        json='record:host/ZG5zLmhvc3QkLm5vbl9ETlNfaG9zdF9yb290LjAuMTY4MzcwNTU4MzY3MC5nc28udGVzdA:test.lo/%20'   # noqa: E501
+        json='record:host/ZG5zLmhvc3QkLm5vbl9ETlNfaG9zdF9yb290LjAuMTY4MzcwNTU4MzY3MC5nc28udGVzdA:test.lo/%20'  # noqa: E501
     )
 
     responses.add(
         method=responses.POST,
         url=re.compile(r'.*/wapi.*/record:a$'),
-        json='record:a/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'   # noqa: E501
+        json='record:a/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'  # noqa: E501
     )
 
     responses.add(
         method=responses.POST,
         url=re.compile(r'.*/wapi.*/record:aaaa$'),
-        json='record:aaaa/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'   # noqa: E501
+        json='record:aaaa/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default'  # noqa: E501
     )
 
     responses.add(
@@ -65,7 +64,7 @@ def test_new_service_host(data_config_filename):
         url=re.compile(r'.*/wapi.*/network.*10.255.255.*'),
         json=[
             {
-                "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default",   # noqa: E501
+                "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default",  # noqa: E501
                 "network": "10.255.255.20/32",
                 "network_view": "default"
             }
@@ -89,7 +88,7 @@ def test_new_service_host(data_config_filename):
         url=re.compile(r'.*/wapi.*/ipv6network.*dead.*beef.*'),
         json=[
             {
-                "_ref": "ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default",   # noqa: E501
+                "_ref": "ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default",  # noqa: E501
                 "network": "dead:beef::18/128",
                 "network_view": "default"
             }