From a00152afe898c40221fe03e8945c964a9b44eb59 Mon Sep 17 00:00:00 2001
From: Jorge Sasiain <jorge.sasiain@ehu.eus>
Date: Mon, 19 Jun 2023 10:27:18 +0000
Subject: [PATCH] Merge develop into NAT-185
Merge e3fab4ac77d19d4983b9480b6fb323c6a3388b97
Merge up to 4c346d4fffed0f17d8edf45faaff97cd292c36f1
Merge up to b25e70128d85f4122cce70379063ffefb1b49cf0
Merge 1779de6bfe40f1071593f14ad7e2f0b2b5f4da2b
Merge df0cd9c9072f9d6555ed14ffeba10585b97da617
Merge up to 028a2d465060c92ed68d2c51a5da6d314fe7bd5b
Finish merging develop into NAT-185
Finish merge develop into NAT-185
Finish merge develop into NAT-185
---
gso/main.py | 2 ++
gso/oss-params-example.json | 20 ++++++++++---
gso/services/provisioning_proxy.py | 15 ++++++----
gso/settings.py | 6 ++--
gso/workflows/__init__.py | 3 ++
gso/workflows/device/create_device.py | 37 +++++++++++++------------
gso/workflows/iptrunk/create_iptrunk.py | 11 ++++----
test/conftest.py | 31 ++++++++++++++++++++-
test/test_ipam.py | 17 ++++++------
9 files changed, 98 insertions(+), 44 deletions(-)
diff --git a/gso/main.py b/gso/main.py
index ba2582a8..c85675f2 100644
--- a/gso/main.py
+++ b/gso/main.py
@@ -1,7 +1,9 @@
from orchestrator import OrchestratorCore
from orchestrator.cli.main import app as core_cli
from orchestrator.settings import AppSettings
+# pylint: disable=unused-import
import gso.products # noqa: F401
+# pylint: disable=unused-import
import gso.workflows # noqa: F401
diff --git a/gso/oss-params-example.json b/gso/oss-params-example.json
index d555365d..9c48d63d 100644
--- a/gso/oss-params-example.json
+++ b/gso/oss-params-example.json
@@ -12,8 +12,8 @@
"password": "robot-user-password"
},
"LO": {
- "V4": {"containers": [], "networks": ["1.1.0.0/28"], "mask": 0},
- "V6": {"containers": [], "networks": ["dead:beef::/80"], "mask": 0},
+ "V4": {"containers": [], "networks": ["1.1.0.0/24"], "mask": 0},
+ "V6": {"containers": [], "networks": ["dead:beef::/64"], "mask": 0},
"domain_name": ".lo",
"dns_view": "default"
},
@@ -21,13 +21,25 @@
"V4": {"containers": ["1.1.1.0/24"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
"domain_name": ".trunk",
- "dns_view": "default
+ "dns_view": "default"
},
"GEANT_IP": {
"V4": {"containers": ["1.1.2.0/24"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
"domain_name": ".geantip",
- "dns_view": "default
+ "dns_view": "default"
+ },
+ "SI": {
+ "V4": {"containers": ["1.1.3.0/24"], "networks": [], "mask": 31},
+ "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+ "domain_name": ".si",
+ "dns_view": "default"
+ },
+ "LT_IAS": {
+ "V4": {"containers": ["1.1.4.0/24"], "networks": [], "mask": 31},
+ "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+ "domain_name": ".ltias",
+ "dns_view": "default"
}
},
"PROVISIONING_PROXY": {
diff --git a/gso/services/provisioning_proxy.py b/gso/services/provisioning_proxy.py
index 20ec4a9d..55c272c2 100644
--- a/gso/services/provisioning_proxy.py
+++ b/gso/services/provisioning_proxy.py
@@ -1,3 +1,7 @@
+"""
+The Provisioning Proxy service, which interacts with LSO running externally.
+LSO is responsible for executing Ansible playbooks, that deploy subscriptions.
+"""
import json
import logging
@@ -53,7 +57,8 @@ def _send_request(endpoint: str, parameters: dict, process_id: UUIDstr,
callback_url = f'{settings.load_oss_params().GENERAL.public_hostname}' \
f'/api/processes/{process_id}/resume'
- logger.debug(f'[provisioning proxy] provisioning for process {process_id}')
+ logger.debug('[provisioning proxy] provisioning for process %s',
+ process_id)
parameters.update({'callback': callback_url})
url = f'{pp_params.scheme}://{pp_params.api_base}/api/{endpoint}'
@@ -61,11 +66,11 @@ def _send_request(endpoint: str, parameters: dict, process_id: UUIDstr,
request = None
if operation == CUDOperation.POST:
- request = requests.post(url, json=parameters)
+ request = requests.post(url, json=parameters, timeout=10000)
elif operation == CUDOperation.PUT:
- request = requests.put(url, json=parameters)
+ request = requests.put(url, json=parameters, timeout=10000)
elif operation == CUDOperation.DELETE:
- request = requests.delete(url, json=parameters)
+ request = requests.delete(url, json=parameters, timeout=10000)
if request.status_code != 200:
print(request.content)
@@ -136,7 +141,7 @@ def provision_ip_trunk(subscription: IptrunkProvisioning,
# 'dry_run': dry_run,
# 'old_subscription': old_subscription,
# 'subscription': new_subscription
-# # FIXME missing parameters
+# # ... missing parameters
# }
#
# _send_request('ip_trunk', parameters, process_id, CUDOperation.PUT)
diff --git a/gso/settings.py b/gso/settings.py
index 1244265d..63f3a71b 100644
--- a/gso/settings.py
+++ b/gso/settings.py
@@ -40,6 +40,8 @@ class IPAMParams(BaseSettings):
LO: ServiceNetworkParams
TRUNK: ServiceNetworkParams
GEANT_IP: ServiceNetworkParams
+ SI: ServiceNetworkParams
+ LT_IAS: ServiceNetworkParams
class ProvisioningProxyParams(BaseSettings):
@@ -61,8 +63,8 @@ def load_oss_params() -> OSSParams:
look for OSS_PARAMS_FILENAME in the environment and load the
parameters from that file
"""
- with open(os.environ['OSS_PARAMS_FILENAME']) as f:
- return OSSParams(**json.loads(f.read()))
+ with open(os.environ['OSS_PARAMS_FILENAME'], encoding='utf-8') as file:
+ return OSSParams(**json.loads(file.read()))
if __name__ == '__main__':
diff --git a/gso/workflows/__init__.py b/gso/workflows/__init__.py
index 268bada2..16f97832 100644
--- a/gso/workflows/__init__.py
+++ b/gso/workflows/__init__.py
@@ -1,3 +1,6 @@
+"""
+init class that imports all workflows into GSO.
+"""
from orchestrator.workflows import LazyWorkflowInstance
LazyWorkflowInstance("gso.workflows.device.create_device", "create_device")
diff --git a/gso/workflows/device/create_device.py b/gso/workflows/device/create_device.py
index 1a50b85e..944f1c14 100644
--- a/gso/workflows/device/create_device.py
+++ b/gso/workflows/device/create_device.py
@@ -19,6 +19,7 @@ from gso.products.product_types import device
from gso.products.product_types.device import DeviceInactive, \
DeviceProvisioning
from gso.products.product_types.site import Site
+from gso.services import _ipam
from gso.services import provisioning_proxy
from gso.services.provisioning_proxy import await_pp_results, \
confirm_pp_results
@@ -84,20 +85,22 @@ def iso_from_ipv4(ipv4_address):
@step('Get information from IPAM')
-def get_info_from_ipam(subscription: DeviceInactive) -> State:
- # lo = ipam.new_device_lo_address()
- # subscription.device.lo_ipv4_address = lo.v4
- # subscription.device.lo_ipv6_address = lo.v6
- # TODO: get info about how these should be generated
- subscription.device.device_lo_ipv4_address = \
- ipaddress.ip_address('10.10.10.20')
- subscription.device.device_lo_ipv6_address = \
- ipaddress.ip_address('fc00:798:10::20')
+def get_info_from_ipam(subscription: DeviceProvisioning) -> State:
+ lo0_alias = re.sub('.geant.net', '', subscription.device.device_fqdn)
+ lo0_name = f'lo0.{lo0_alias}'
+ lo0_addr = _ipam.allocate_service_host(hostname=lo0_name,
+ service_type='LO',
+ cname_aliases=[lo0_alias])
+ subscription.device.device_lo_ipv4_address = lo0_addr.v4
+ subscription.device.device_lo_ipv6_address = lo0_addr.v6
subscription.device.device_lo_iso_address \
= iso_from_ipv4(str(subscription.device.device_lo_ipv4_address))
- subscription.device.device_si_ipv4_network = '192.168.0.0/31'
- subscription.device.device_ias_lt_ipv4_network = '192.168.1.0/31'
- subscription.device.device_ias_lt_ipv6_network = 'fc00:798:1::150/126'
+ subscription.device.device_si_ipv4_network \
+ = _ipam.allocate_service_ipv4_network(service_type='SI', comment=f"SI for {lo0_name}").v4
+ subscription.device.device_ias_lt_ipv4_network \
+ = _ipam.allocate_service_ipv4_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v4
+ subscription.device.device_ias_lt_ipv6_network \
+ = _ipam.allocate_service_ipv6_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v6
return {'subscription': subscription}
@@ -116,19 +119,19 @@ def initialize_subscription(
subscription.device.device_vendor = device_vendor
subscription.device.device_site \
= Site.from_subscription(device_site[0]).site
- fqdn = str(hostname + '.' +
- subscription.device.device_site.site_name.lower() + '.' +
- subscription.device.device_site.site_country_code.lower() +
- '.geant.net')
+ fqdn = f'{hostname}.{subscription.device.device_site.site_name.lower()}.' \
+ f'{subscription.device.device_site.site_country_code.lower()}' \
+ f'.geant.net'
subscription.device.device_fqdn = fqdn
subscription.device.device_role = device_role
subscription.description = f'Device {fqdn} ' \
f'({subscription.device_type})'
+
subscription = device.DeviceProvisioning.from_other_lifecycle(
subscription, SubscriptionLifecycle.PROVISIONING
)
- return {'subscription': subscription, 'fqdn': fqdn}
+ return {'subscription': subscription}
@step('Provision device [DRY RUN]')
diff --git a/gso/workflows/iptrunk/create_iptrunk.py b/gso/workflows/iptrunk/create_iptrunk.py
index 1c096575..c25a310f 100644
--- a/gso/workflows/iptrunk/create_iptrunk.py
+++ b/gso/workflows/iptrunk/create_iptrunk.py
@@ -1,4 +1,3 @@
-import ipaddress
from uuid import uuid4
from orchestrator.db.models import ProductTable, SubscriptionTable
@@ -17,7 +16,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType
from gso.products.product_types.device import Device
from gso.products.product_types.iptrunk import IptrunkInactive, \
IptrunkProvisioning
-from gso.services import provisioning_proxy
+from gso.services import provisioning_proxy, _ipam
from gso.services.provisioning_proxy import confirm_pp_results, \
await_pp_results
@@ -91,12 +90,12 @@ def create_subscription(product: UUIDstr) -> State:
@step('Get information from IPAM')
-def get_info_from_ipam(subscription: IptrunkInactive) -> State:
+def get_info_from_ipam(subscription: IptrunkProvisioning) -> State:
# TODO: get info about how these should be generated
subscription.iptrunk.iptrunk_ipv4_network \
- = ipaddress.ip_network('192.168.255.0/31')
+ = _ipam.allocate_service_ipv4_network(service_type="TRUNK", comment=subscription.iptrunk.iptrunk_description).v4
subscription.iptrunk.iptrunk_ipv6_network \
- = ipaddress.ip_network('fc00:798:255::150/126')
+ = _ipam.allocate_service_ipv6_network(service_type="TRUNK", comment=subscription.iptrunk.iptrunk_description).v6
return {'subscription': subscription}
@@ -235,8 +234,8 @@ def create_iptrunk():
init
>> create_subscription
>> store_process_subscription(Target.CREATE)
- >> get_info_from_ipam
>> initialize_subscription
+ >> get_info_from_ipam
>> provision_ip_trunk_iface_dry
>> await_pp_results
>> confirm_pp_results
diff --git a/test/conftest.py b/test/conftest.py
index 92e2af4f..04952145 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -2,9 +2,10 @@ import contextlib
import json
import os
import socket
-import pytest
import tempfile
+import pytest
+
@pytest.fixture(scope='session')
def configuration_data():
@@ -66,6 +67,34 @@ def configuration_data():
},
"domain_name": ".geantip",
"dns_view": "default"
+ },
+ "SI": {
+ "V4": {
+ "containers": ["10.255.253.128/25"],
+ "networks": [],
+ "mask": 31
+ },
+ "V6": {
+ "containers": [],
+ "networks": [],
+ "mask": 126
+ },
+ "domain_name": ".geantip",
+ "dns_view": "default"
+ },
+ "LT_IAS": {
+ "V4": {
+ "containers": ["10.255.255.0/24"],
+ "networks": [],
+ "mask": 31
+ },
+ "V6": {
+ "containers": ["dead:beef:cc::/48"],
+ "networks": [],
+ "mask": 126
+ },
+ "domain_name": ".geantip",
+ "dns_view": "default"
}
},
"PROVISIONING_PROXY": {
diff --git a/test/test_ipam.py b/test/test_ipam.py
index 8db6bda3..e3191bcc 100644
--- a/test/test_ipam.py
+++ b/test/test_ipam.py
@@ -1,6 +1,7 @@
import ipaddress
import pytest
import re
+
import responses
from gso.services import ipam
@@ -8,12 +9,11 @@ from gso.services import ipam
@responses.activate
def test_new_service_networks(data_config_filename):
-
responses.add(
method=responses.POST,
url=re.compile(r'.*/wapi.*/network.*'),
json={
- '_ref': 'network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default', # noqa: E501
+ '_ref': 'network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default', # noqa: E501
'network': '10.255.255.20/32'
}
)
@@ -22,7 +22,7 @@ def test_new_service_networks(data_config_filename):
method=responses.POST,
url=re.compile(r'.*/wapi.*/ipv6network.*'),
json={
- '_ref': 'ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default', # noqa: E501
+ '_ref': 'ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default', # noqa: E501
'network': 'dead:beef::18/128'
}
)
@@ -41,23 +41,22 @@ def test_new_service_networks(data_config_filename):
@responses.activate
def test_new_service_host(data_config_filename):
-
responses.add(
method=responses.POST,
url=re.compile(r'.*/wapi.*/record:host$'),
- json='record:host/ZG5zLmhvc3QkLm5vbl9ETlNfaG9zdF9yb290LjAuMTY4MzcwNTU4MzY3MC5nc28udGVzdA:test.lo/%20' # noqa: E501
+ json='record:host/ZG5zLmhvc3QkLm5vbl9ETlNfaG9zdF9yb290LjAuMTY4MzcwNTU4MzY3MC5nc28udGVzdA:test.lo/%20' # noqa: E501
)
responses.add(
method=responses.POST,
url=re.compile(r'.*/wapi.*/record:a$'),
- json='record:a/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default' # noqa: E501
+ json='record:a/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default' # noqa: E501
)
responses.add(
method=responses.POST,
url=re.compile(r'.*/wapi.*/record:aaaa$'),
- json='record:aaaa/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default' # noqa: E501
+ json='record:aaaa/ZG5zLmJpbmRfYSQuX2RlZmF1bHQuZ3NvLHRlc3QsMTAuMjU1LjI1NS44:test.lo/default' # noqa: E501
)
responses.add(
@@ -65,7 +64,7 @@ def test_new_service_host(data_config_filename):
url=re.compile(r'.*/wapi.*/network.*10.255.255.*'),
json=[
{
- "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default", # noqa: E501
+ "_ref": "network/ZG5zLm5ldHdvcmskMTAuMjU1LjI1NS4yMC8zMi8w:10.255.255.20/32/default", # noqa: E501
"network": "10.255.255.20/32",
"network_view": "default"
}
@@ -89,7 +88,7 @@ def test_new_service_host(data_config_filename):
url=re.compile(r'.*/wapi.*/ipv6network.*dead.*beef.*'),
json=[
{
- "_ref": "ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default", # noqa: E501
+ "_ref": "ipv6network/ZG5zLm5ldHdvcmskZGVhZDpiZWVmOjoxOC8xMjgvMA:dead%3Abeef%3A%3A18/128/default", # noqa: E501
"network": "dead:beef::18/128",
"network_view": "default"
}
--
GitLab