diff --git a/.gitignore b/.gitignore index 703dff15f1ff4549653ce090ce43b1763a68d486..ebe5d66762eedab993b414f3cfc897a444ba1f3f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,11 @@ build dist .opennsa-test.json .python-version +docker/.env +config/opennsa.conf +docker-compose.override.yml + +.idea +.devcontainer +twistd.pid +.env diff --git a/Makefile b/Makefile index f08afc53b362ce80e27b4de03aa964490db9881b..ff2521dc69848ead8d0825cb55fb5d82e462ffa5 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,11 @@ clean: rm -fr _trial_temp - find . -name "*.pyc"|xargs rm + find . -name "*.pyc" -exec rm -v {} \; -docker-build: - docker build -t opennsa docker +down: + docker-compose down + +docker-build: clean down + docker-compose build --no-cache diff --git a/docker/opennsa.conf.template b/config/opennsa.conf.template similarity index 83% rename from docker/opennsa.conf.template rename to config/opennsa.conf.template index 1efa3e237c4f33ae2850e6d989bd2f03d71f0412..71c739b28e8fe36d44f9c286123b8bf0250fc62b 100644 --- a/docker/opennsa.conf.template +++ b/config/opennsa.conf.template @@ -7,13 +7,13 @@ logfile= #peers=example.org@http://example.org:9080/NSI/topology/example.org.xml # These are set by the create-compose script -dbhost=opennsa-db +dbhost=${POSTGRES_HOST} database=${POSTGRES_DB} dbuser=${POSTGRES_USER} dbpassword=${POSTGRES_PASSWORD} -tls=false +tls=${TLS_ENABLED} [dud:topology] -nrmmap=opennsa.nrm +nrmmap=${NRM_FILE} diff --git a/docker/opennsa.nrm b/config/opennsa.nrm similarity index 100% rename from docker/opennsa.nrm rename to config/opennsa.nrm diff --git a/docker-compose.override.yml_placeholder b/docker-compose.override.yml_placeholder new file mode 100644 index 0000000000000000000000000000000000000000..b45c9feffdb2f27d79ec3ccf531aed29b501088d --- /dev/null +++ b/docker-compose.override.yml_placeholder @@ -0,0 +1,21 @@ +## rename this file to docker-compose.override.yml any additional settings listed here will be merged with the docker-compose.yml file. +version: "3.7" + +services: + opennsa: + image: opennsa:latest + command: Any valid command + ## Mount entire project to volume avoids constant rebuilds. + ## You may need to load the container as: + ## UID=${UID} GID=${GID} docker-compose up linux FS can cause some issues with + ## permissioning at times. + #volumes: + # - ./:/home/opennsa/opennsa + # Mount NRM file and leave ENV value the same + #volumes: + # - ./config/myNRMFile.nrm:/home/opennsa/opennsa/config/opennsa.nrm + db: + image: postgres:9.6.5 + ##Expose 5432 locally + ports: + - 5432:5432 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..7b6080725b9a6bd3b6c5d584a4181b61e54dce0f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.7" + +services: + db: + image: postgres:9.6.5 + expose: + - 5432 + volumes: + - ./datafiles/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro + - opennsa-pgdata:/var/lib/postgresql/data + env_file: .env + opennsa: + image: opennsa:latest + build: + context: . + dockerfile: docker/Dockerfile + env_file: .env + depends_on: + - db + ports: + - 9080:9080 + - 9443:9443 + volumes: + - ./config/opennsa.conf:/home/opennsa/opennsa/config/opennsa.conf:ro + - ./config/opennsa.nrm:/home/opennsa/opennsa/opennsa.nrm:ro + +volumes: + opennsa-pgdata: diff --git a/docker/Dockerfile b/docker/Dockerfile index b34e42b3cad134bdcc8b2584b13d67e7e8ea95b5..51ee956203c7415b12e43880a96a9f0cbe12822b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,7 @@ FROM debian:stable-slim -MAINTAINER Henrik Thostrup Jensen <htj@nordu.net> - +LABEL maintainer="Henrik Thostrup Jensen <htj@nordu.net>" # -- Environment -- ENV GIT_REPO https://github.com/NORDUnet/opennsa @@ -12,48 +11,51 @@ ENV USER opennsa # -- User setup -- RUN adduser --disabled-password --gecos 'OpenNSA user' $USER - +ADD . /home/$USER/opennsa/ # --- Base image --- # Update and install dependencies # pip to install twistar service-identity pyasn1 # pyasn1 and crypto is needed for ssh backends RUN apt update \ - && apt install -y \ - git-core \ - python3 \ - python3-twisted-bin \ - python3-openssl \ - python3-psycopg2 \ - python3-pip \ - python3-crypto \ - python3-dateutil \ - && pip3 install \ - twistar \ - service-identity \ - pyasn1 \ -# -- Instal OpenNSA -- - && echo git clone $GIT_REPO \ - && su - $USER -c "git clone $GIT_REPO" \ -# -- Cleanup -- - && apt remove -y \ - git-core \ - python3-pip \ - && apt autoremove -y \ - && rm -rf /var/lib/apt/lists/* - - + && apt install -y \ + git-core \ + python3 \ + python3-twisted-bin \ + python3-openssl \ + python3-psycopg2 \ + python3-pip \ + python3-cryptography \ + python3-dateutil \ + && pip3 install \ + twistar \ + service-identity \ + pyasn1 \ + # -- Instal OpenNSA -- + # && echo git clone $GIT_REPO \ + # && su - $USER -c "git clone $GIT_REPO" \ + ## Unsure why but this needs to be pulled away from the main apt install + && chown $USER:$USER -R /home/opennsa/opennsa \ + && apt install -y netcat iputils-ping \ + # -- Cleanup -- + && apt remove -y git-core python3-pip \ + && apt autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && cp /home/$USER/opennsa/docker/run_opennsa.sh /home/$USER/opennsa \ + && cp /home/$USER/opennsa/config/opennsa.conf.template /home/$USER/opennsa/config/opennsa.conf + + +#RUN # -- Switch to OpenNSA directory -- USER $USER + WORKDIR /home/$USER/opennsa ENV PYTHONPATH . - - # -- Entrypoint -- - EXPOSE 9080 EXPOSE 9443 -ENTRYPOINT rm -f twistd.pid; twistd -ny opennsa.tac +# USER root +CMD /home/$USER/opennsa/run_opennsa.sh diff --git a/docker/README.md b/docker/README.md index f1d0f5c706715d133df1bfc220d2ade664e77e49..f479ceade6817cc01dc07eaa3b2dc10585eb1e70 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,20 +11,18 @@ $ make docker-build ( from opennsa directory ) As OpenNSA requires a Postgres database, docker-compose is used to coordinate the setup of the two containers. -1. Edit opennsa.conf.template and opennsa.nrm - Leave the database config as-is. - -2. $ ./create-compose - This will substitute stuff in the templates and create docker-compose.yml and opennsa.conf +1. $ ./generate-docker-config + This will mainly generate a password and create a .env file for you. You may update the settings in .env if you wish to use a different nrm file (Keep in mind you'll need to mount it as a volume if you stray from the defaults or rebuild the image) 3. $ docker-compose up This should bring up a PostgreSQL instance and OpenNSA. +## Advanced Features -You may have to edit template.yml to expose OpenNSA ports publically, mount in -certificates, or similar. +1. In order to override any settings copy the docker-compose.override.yml_placeholder to docker-compose.override.yml. You can use to mount additional volumes, expose additional ports etc. Some common patterns are already there and commented out. +2. Configuration options are almost all exposed via ENV variables. If you wish to directly mount your config file, make a copy of config/opennsa.conf.template to config/opennsa.conf. Update any entries as desired and restart all DB container. -TODO: Make OpenNSA able to take database configuration via environment, so we - don't have to do replacement in opennsa.conf +3. The entry point is left as just bash, so if you wish to override the initial command you may simply set the `command:` line in your override file to anything you like. If you want, you may also invoke the run_opennsa.sh with arguments, it will wait for the database to come up with run the command you issues. +For example: run_opennsa.sh sleep 50 ==> will wait for DB to come up then sleep for 50 seconds. \ No newline at end of file diff --git a/docker/create-compose b/docker/create-compose deleted file mode 100755 index 5f140377f8753947a3a7c90bc7d30e92cd1fe066..0000000000000000000000000000000000000000 --- a/docker/create-compose +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -set -a # load source environment variables into scope - -. env.sh - -cat template.yml | envsubst > docker-compose.yml - -cat opennsa.conf.template | envsubst > opennsa.conf - -echo "Start OpenNSA with: docker-compose up" - diff --git a/docker/env.sh b/docker/env.sh deleted file mode 100644 index 3615f68b8f1bb50a04731043cb37b626d2d2fadb..0000000000000000000000000000000000000000 --- a/docker/env.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - - -POSTGRES_DB=opennsa -POSTGRES_USER=opennsa -POSTGRES_PASSWORD=$(openssl rand -base64 18) - -SCHEMA_FILE=$PWD/../datafiles/schema.sql - -OPENNSA_CONF_FILE=$PWD/opennsa.conf -OPENNSA_NRM_FILE=$PWD/opennsa.nrm - - diff --git a/docker/run_opennsa.sh b/docker/run_opennsa.sh new file mode 100755 index 0000000000000000000000000000000000000000..45e09914ce3535b4127cc9d9ed46c64036496aa4 --- /dev/null +++ b/docker/run_opennsa.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +function check_db() +{ +## Wait for DB container to be up + +until nc -z -v -w30 $POSTGRES_HOST $POSTGRES_PORT +do + echo "Waiting 5 second until the database is receiving connections..." + # wait for a second before checking again + sleep 5 +done + +} + +function run_app() +{ + cd $HOME/opennsa + rm -f twistd.pid; $cmd +} + + +if [ $# -gt 0 ]; then + cmd=$@ +else + cmd='twistd -ny opennsa.tac' +fi + + +check_db +run_app $cmd + diff --git a/docker/template.yml b/docker/template.yml deleted file mode 100644 index f5d4a46936057342d1472b39d2ad19fe9f6139a0..0000000000000000000000000000000000000000 --- a/docker/template.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3' - -services: - opennsa-db: - image: postgres:9.6.5 - volumes: - - ${SCHEMA_FILE}:/docker-entrypoint-initdb.d/schema.sql:ro - - opennsa-pgdata:/var/lib/postgresql/data - environment: - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - - opennsa: - image: opennsa:latest - depends_on: - - opennsa-db - - waitforpg - ports: - - 127.0.0.1:9080:9080 - - 127.0.0.1:9443:9443 - links: - - opennsa-db - volumes: - - ${OPENNSA_CONF_FILE}:/home/opennsa/opennsa/opennsa.conf:ro - - ${OPENNSA_NRM_FILE}:/home/opennsa/opennsa/opennsa.nrm:ro - - - waitforpg: - image: dadarek/wait-for-dependencies - depends_on: - - opennsa-db - command: opennsa-db:5432 - -volumes: - opennsa-pgdata: - - diff --git a/env.template b/env.template new file mode 100644 index 0000000000000000000000000000000000000000..799237109823dca099469b7e395bd42ccdd55835 --- /dev/null +++ b/env.template @@ -0,0 +1,8 @@ +POSTGRES_DB=opennsa +POSTGRES_USER=opennsa +POSTGRES_PASSWORD=PASSWD_REPLACE +POSTGRES_HOST=opennsa-db +POSTGRES_PORT=5432 + +TLS_ENABLED=false +NRM_FILE=config/opennsa.nrm diff --git a/generate-docker-config b/generate-docker-config new file mode 100755 index 0000000000000000000000000000000000000000..c9b0ba28076a87475d1ba45137d55ab9f20aa92b --- /dev/null +++ b/generate-docker-config @@ -0,0 +1,7 @@ +#!/bin/sh + +cp docker/opennsa.conf.template config/opennsa.conf +sed -e "s/PASSWD_REPLACE/$(openssl rand -base64 18)/" env.template > .env + +echo "Start OpenNSA with: docker-compose up" + diff --git a/opennsa.conf b/opennsa.conf deleted file mode 100644 index 7b1d1763cb78ac3be167743540b172cf8b56d4cb..0000000000000000000000000000000000000000 --- a/opennsa.conf +++ /dev/null @@ -1,19 +0,0 @@ -# This is a configuration file for running an OpenNSA service directly from the development directory - -[service] -# Change network name at will -domain=example.net -# This means we output log to stdout -logfile= -#peers=example.org@http://example.org:9080/NSI/topology/example.org.xml - -# You will need to set these -database=opennsa -dbuser=opennsa -dbpassword=opennsa - -tls=false - -[dud:topology] -nrmmap=opennsa.nrm - diff --git a/opennsa.tac b/opennsa.tac index b9f26d03bc224c6761bbad2f179ff148365e1af4..53f3eb1f197de621126c1af9a4929b321eb4931f 100644 --- a/opennsa.tac +++ b/opennsa.tac @@ -7,7 +7,10 @@ from opennsa import setup +from dotenv import load_dotenv + +load_dotenv() ## Loads ENV values from .env file # you can get debug and/or payload info in the log by setting one of the flags to true -application = setup.createApplication('opennsa.conf', payload=False, debug=False) +application = setup.createApplication('config/opennsa.conf', payload=False, debug=False) diff --git a/opennsa/config.py b/opennsa/config.py index eaa313fe1355a411daa2d28e9079e4ce1625c35a..07377b3f8247cb0d64c6902041b094acfffa69c7 100644 --- a/opennsa/config.py +++ b/opennsa/config.py @@ -11,138 +11,138 @@ import configparser from opennsa import constants as cnt - # defaults -DEFAULT_CONFIG_FILE = '/etc/opennsa.conf' -DEFAULT_LOG_FILE = '/var/log/opennsa.log' -DEFAULT_TLS = 'true' -DEFAULT_TOPOLOGY_FILE = '/usr/local/share/nsi/topology.owl' -DEFAULT_TCP_PORT = 9080 -DEFAULT_TLS_PORT = 9443 -DEFAULT_VERIFY = True -DEFAULT_CERTIFICATE_DIR = '/etc/ssl/certs' # This will work on most mordern linux distros +DEFAULT_CONFIG_FILE = '/etc/opennsa.conf' +DEFAULT_LOG_FILE = '/var/log/opennsa.log' +DEFAULT_TLS = 'true' +DEFAULT_TOPOLOGY_FILE = '/usr/local/share/nsi/topology.owl' +DEFAULT_TCP_PORT = 9080 +DEFAULT_TLS_PORT = 9443 +DEFAULT_VERIFY = True +# This will work on most mordern linux distros +DEFAULT_CERTIFICATE_DIR = '/etc/ssl/certs' # config blocks and options -BLOCK_SERVICE = 'service' -BLOCK_DUD = 'dud' +BLOCK_SERVICE = 'service' +BLOCK_DUD = 'dud' BLOCK_JUNIPER_EX = 'juniperex' BLOCK_JUNIPER_VPLS = 'junipervpls' -BLOCK_FORCE10 = 'force10' -BLOCK_BROCADE = 'brocade' -BLOCK_NCSVPN = 'ncsvpn' -BLOCK_PICA8OVS = 'pica8ovs' -BLOCK_JUNOSMX = 'junosmx' -BLOCK_JUNOSEX = 'junosex' +BLOCK_FORCE10 = 'force10' +BLOCK_BROCADE = 'brocade' +BLOCK_NCSVPN = 'ncsvpn' +BLOCK_PICA8OVS = 'pica8ovs' +BLOCK_JUNOSMX = 'junosmx' +BLOCK_JUNOSEX = 'junosex' BLOCK_JUNOSSPACE = 'junosspace' -BLOCK_OESS = 'oess' +BLOCK_OESS = 'oess' BLOCK_CUSTOM_BACKEND = 'custombackend' # service block -DOMAIN = 'domain' # mandatory -NETWORK_NAME = 'network' # legacy, used to be mandatory -LOG_FILE = 'logfile' -HOST = 'host' -PORT = 'port' -TLS = 'tls' -REST = 'rest' -NRM_MAP_FILE = 'nrmmap' -PEERS = 'peers' -POLICY = 'policy' -PLUGIN = 'plugin' +DOMAIN = 'domain' # mandatory +NETWORK_NAME = 'network' # legacy, used to be mandatory +LOG_FILE = 'logfile' +HOST = 'host' +PORT = 'port' +TLS = 'tls' +REST = 'rest' +NRM_MAP_FILE = 'nrmmap' +PEERS = 'peers' +POLICY = 'policy' +PLUGIN = 'plugin' SERVICE_ID_START = 'serviceid_start' # database -DATABASE = 'database' # mandatory -DATABASE_USER = 'dbuser' # mandatory -DATABASE_PASSWORD = 'dbpassword' # can be none (os auth) -DATABASE_HOST = 'dbhost' # can be none (local db) +DATABASE = 'database' # mandatory +DATABASE_USER = 'dbuser' # mandatory +DATABASE_PASSWORD = 'dbpassword' # can be none (os auth) +DATABASE_HOST = 'dbhost' # can be none (local db) # tls -KEY = 'key' # mandatory, if tls is set -CERTIFICATE = 'certificate' # mandatory, if tls is set -CERTIFICATE_DIR = 'certdir' # mandatory (but dir can be empty) -VERIFY_CERT = 'verify' -ALLOWED_HOSTS = 'allowedhosts' # comma seperated list +KEY = 'key' # mandatory, if tls is set +CERTIFICATE = 'certificate' # mandatory, if tls is set +CERTIFICATE_DIR = 'certdir' # mandatory (but dir can be empty) +VERIFY_CERT = 'verify' +ALLOWED_HOSTS = 'allowedhosts' # comma seperated list # generic stuff -_SSH_HOST = 'host' -_SSH_PORT = 'port' -_SSH_HOST_FINGERPRINT = 'fingerprint' -_SSH_USER = 'user' -_SSH_PASSWORD = 'password' -_SSH_PUBLIC_KEY = 'publickey' -_SSH_PRIVATE_KEY = 'privatekey' +_SSH_HOST = 'host' +_SSH_PORT = 'port' +_SSH_HOST_FINGERPRINT = 'fingerprint' +_SSH_USER = 'user' +_SSH_PASSWORD = 'password' +_SSH_PUBLIC_KEY = 'publickey' +_SSH_PRIVATE_KEY = 'privatekey' -AS_NUMBER = 'asnumber' +AS_NUMBER = 'asnumber' # TODO: Don't do backend specifics for everything, it causes confusion, and doesn't really solve anything # juniper block - same for mx / ex backends -JUNIPER_HOST = _SSH_HOST -JUNIPER_PORT = _SSH_PORT -JUNIPER_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT -JUNIPER_USER = _SSH_USER -JUNIPER_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY -JUNIPER_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY +JUNIPER_HOST = _SSH_HOST +JUNIPER_PORT = _SSH_PORT +JUNIPER_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT +JUNIPER_USER = _SSH_USER +JUNIPER_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY +JUNIPER_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY # force10 block -FORCE10_HOST = _SSH_HOST -FORCE10_PORT = _SSH_PORT -FORCE10_USER = _SSH_USER -FORCE10_PASSWORD = _SSH_PASSWORD +FORCE10_HOST = _SSH_HOST +FORCE10_PORT = _SSH_PORT +FORCE10_USER = _SSH_USER +FORCE10_PASSWORD = _SSH_PASSWORD FORCE10_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT -FORCE10_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY +FORCE10_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY FORCE10_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY # Brocade block -BROCADE_HOST = _SSH_HOST -BROCADE_PORT = _SSH_PORT -BROCADE_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT -BROCADE_USER = _SSH_USER -BROCADE_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY -BROCADE_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY -BROCADE_ENABLE_PASSWORD = 'enablepassword' +BROCADE_HOST = _SSH_HOST +BROCADE_PORT = _SSH_PORT +BROCADE_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT +BROCADE_USER = _SSH_USER +BROCADE_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY +BROCADE_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY +BROCADE_ENABLE_PASSWORD = 'enablepassword' # Pica8 OVS -PICA8OVS_HOST = _SSH_HOST -PICA8OVS_PORT = _SSH_PORT -PICA8OVS_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT -PICA8OVS_USER = _SSH_USER -PICA8OVS_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY -PICA8OVS_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY -PICA8OVS_DB_IP = 'dbip' +PICA8OVS_HOST = _SSH_HOST +PICA8OVS_PORT = _SSH_PORT +PICA8OVS_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT +PICA8OVS_USER = _SSH_USER +PICA8OVS_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY +PICA8OVS_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY +PICA8OVS_DB_IP = 'dbip' # NCS VPN Backend -NCS_SERVICES_URL = 'url' -NCS_USER = 'user' -NCS_PASSWORD = 'password' +NCS_SERVICES_URL = 'url' +NCS_USER = 'user' +NCS_PASSWORD = 'password' # JUNOS block -JUNOS_HOST = _SSH_HOST -JUNOS_PORT = _SSH_PORT -JUNOS_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT -JUNOS_USER = _SSH_USER -JUNOS_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY -JUNOS_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY -JUNOS_ROUTERS = 'routers' - -#Junosspace backend -SPACE_USER = 'space_user' -SPACE_PASSWORD = 'space_password' -SPACE_API_URL = 'space_api_url' -SPACE_ROUTERS = 'routers' -SPACE_CONFIGLET_ACTIVATE_LOCAL = 'configlet_activate_local' +JUNOS_HOST = _SSH_HOST +JUNOS_PORT = _SSH_PORT +JUNOS_HOST_FINGERPRINT = _SSH_HOST_FINGERPRINT +JUNOS_USER = _SSH_USER +JUNOS_SSH_PUBLIC_KEY = _SSH_PUBLIC_KEY +JUNOS_SSH_PRIVATE_KEY = _SSH_PRIVATE_KEY +JUNOS_ROUTERS = 'routers' + +# Junosspace backend +SPACE_USER = 'space_user' +SPACE_PASSWORD = 'space_password' +SPACE_API_URL = 'space_api_url' +SPACE_ROUTERS = 'routers' +SPACE_CONFIGLET_ACTIVATE_LOCAL = 'configlet_activate_local' SPACE_CONFIGLET_ACTIVATE_REMOTE = 'configlet_activate_remote' SPACE_CONFIGLET_DEACTIVATE_LOCAL = 'configlet_deactivate_local' SPACE_CONFIGLET_DEACTIVATE_REMOTE = 'configlet_deactivate_remote' # OESS -OESS_URL = 'url' -OESS_USER = 'username' -OESS_PASSWORD = 'password' -OESS_WORKGROUP = 'workgroup' +OESS_URL = 'url' +OESS_USER = 'username' +OESS_PASSWORD = 'password' +OESS_WORKGROUP = 'workgroup' class ConfigurationError(Exception): @@ -158,18 +158,22 @@ class Peer(object): self.cost = cost +class EnvInterpolation(configparser.BasicInterpolation): + """Interpolation which expands environment variables in values.""" -def readConfig(filename): + def before_get(self, parser, section, option, value, defaults): + value = super().before_get(parser, section, option, value, defaults) + return os.path.expandvars(value) - cfg = configparser.SafeConfigParser() +def readConfig(filename): + cfg = configparser.ConfigParser(interpolation=EnvInterpolation()) cfg.add_section(BLOCK_SERVICE) - cfg.read( [ filename ] ) + cfg.read([filename]) return cfg - def readVerifyConfig(cfg): """ Read a config and verify that things are correct. Will also fill in @@ -188,7 +192,8 @@ def readVerifyConfig(cfg): try: cfg.get(BLOCK_SERVICE, NRM_MAP_FILE) - raise ConfigurationError('NRM Map file should be specified under backend') + raise ConfigurationError( + 'NRM Map file should be specified under backend') except configparser.NoOptionError: pass @@ -197,11 +202,13 @@ def readVerifyConfig(cfg): try: vc[DOMAIN] = cfg.get(BLOCK_SERVICE, DOMAIN) except configparser.NoOptionError: - raise ConfigurationError('No domain name specified in configuration file (mandatory, see docs/migration)') + raise ConfigurationError( + 'No domain name specified in configuration file (mandatory, see docs/migration)') try: cfg.get(BLOCK_SERVICE, NETWORK_NAME) - raise ConfigurationError('Network name no longer used, use domain (see docs/migration)') + raise ConfigurationError( + 'Network name no longer used, use domain (see docs/migration)') except configparser.NoOptionError: pass @@ -213,7 +220,8 @@ def readVerifyConfig(cfg): try: nrm_map_file = cfg.get(BLOCK_SERVICE, NRM_MAP_FILE) if not os.path.exists(nrm_map_file): - raise ConfigurationError('Specified NRM mapping file does not exist (%s)' % nrm_map_file) + raise ConfigurationError( + 'Specified NRM mapping file does not exist (%s)' % nrm_map_file) vc[NRM_MAP_FILE] = nrm_map_file except configparser.NoOptionError: vc[NRM_MAP_FILE] = None @@ -225,7 +233,7 @@ def readVerifyConfig(cfg): try: peers_raw = cfg.get(BLOCK_SERVICE, PEERS) - vc[PEERS] = [ Peer(purl.strip(), 1) for purl in peers_raw.split('\n') ] + vc[PEERS] = [Peer(purl.strip(), 1) for purl in peers_raw.split('\n')] except configparser.NoOptionError: vc[PEERS] = None @@ -262,12 +270,14 @@ def readVerifyConfig(cfg): try: vc[DATABASE] = cfg.get(BLOCK_SERVICE, DATABASE) except configparser.NoOptionError: - raise ConfigurationError('No database specified in configuration file (mandatory)') + raise ConfigurationError( + 'No database specified in configuration file (mandatory)') try: vc[DATABASE_USER] = cfg.get(BLOCK_SERVICE, DATABASE_USER) except configparser.NoOptionError: - raise ConfigurationError('No database user specified in configuration file (mandatory)') + raise ConfigurationError( + 'No database user specified in configuration file (mandatory)') try: vc[DATABASE_PASSWORD] = cfg.get(BLOCK_SERVICE, DATABASE_PASSWORD) @@ -288,7 +298,8 @@ def readVerifyConfig(cfg): try: certdir = cfg.get(BLOCK_SERVICE, CERTIFICATE_DIR) if not os.path.exists(certdir): - raise ConfigurationError('Specified certdir does not exist (%s)' % certdir) + raise ConfigurationError( + 'Specified certdir does not exist (%s)' % certdir) vc[CERTIFICATE_DIR] = certdir except configparser.NoOptionError: vc[CERTIFICATE_DIR] = DEFAULT_CERTIFICATE_DIR @@ -300,13 +311,15 @@ def readVerifyConfig(cfg): # tls if vc[TLS]: try: - hostkey = cfg.get(BLOCK_SERVICE, KEY) + hostkey = cfg.get(BLOCK_SERVICE, KEY) hostcert = cfg.get(BLOCK_SERVICE, CERTIFICATE) if not os.path.exists(hostkey): - raise ConfigurationError('Specified hostkey does not exist (%s)' % hostkey) + raise ConfigurationError( + 'Specified hostkey does not exist (%s)' % hostkey) if not os.path.exists(hostcert): - raise ConfigurationError('Specified hostcert does not exist (%s)' % hostcert) + raise ConfigurationError( + 'Specified hostcert does not exist (%s)' % hostcert) vc[KEY] = hostkey vc[CERTIFICATE] = hostcert @@ -321,7 +334,6 @@ def readVerifyConfig(cfg): # Not enough options for configuring tls context raise ConfigurationError('Missing TLS option: %s' % str(e)) - # backends backends = {} @@ -331,18 +343,19 @@ def readVerifyConfig(cfg): continue if ':' in section: - backend_type, name = section.split(':',2) + backend_type, name = section.split(':', 2) else: backend_type = section name = '' if name in backends: - raise ConfigurationError('Can only have one backend named "%s"' % name) + raise ConfigurationError( + 'Can only have one backend named "%s"' % name) if backend_type in (BLOCK_DUD, BLOCK_JUNIPER_EX, BLOCK_JUNIPER_VPLS, BLOCK_JUNOSMX, BLOCK_FORCE10, BLOCK_BROCADE, BLOCK_NCSVPN, BLOCK_PICA8OVS, BLOCK_OESS, BLOCK_JUNOSSPACE, BLOCK_JUNOSEX, BLOCK_CUSTOM_BACKEND, 'asyncfail'): - backend_conf = dict( cfg.items(section) ) + backend_conf = dict(cfg.items(section)) backend_conf['_backend_type'] = backend_type backends[name] = backend_conf diff --git a/opennsa/setup.py b/opennsa/setup.py index 4f35462eb38d816717f42a0a568a130e1e149ece..2bb31e110725950b79ba7ddf6a1efc4148c0279b 100644 --- a/opennsa/setup.py +++ b/opennsa/setup.py @@ -98,53 +98,52 @@ def setupBackend(backend_cfg, network_name, nrm_ports, parent_requester): return b - def setupTLSContext(vc): # ssl/tls contxt if vc[config.TLS]: from opennsa.opennsaTlsContext import opennsa2WayTlsContext - ctx_factory = opennsa2WayTlsContext(vc[config.KEY], vc[config.CERTIFICATE], vc[config.CERTIFICATE_DIR], vc[config.VERIFY_CERT]) + ctx_factory = opennsa2WayTlsContext( + vc[config.KEY], vc[config.CERTIFICATE], vc[config.CERTIFICATE_DIR], vc[config.VERIFY_CERT]) elif vc[config.CERTIFICATE_DIR]: # create a context so we can verify https urls if not os.path.isdir(vc[config.CERTIFICATE_DIR]): - raise config.ConfigurationError('certdir value {} is not a directory'.format(vc[config.CERTIFICATE_DIR])) + raise config.ConfigurationError( + 'certdir value {} is not a directory'.format(vc[config.CERTIFICATE_DIR])) from opennsa.opennsaTlsContext import opennsaTlsContext - ctx_factory = opennsaTlsContext(vc[config.CERTIFICATE_DIR], vc[config.VERIFY_CERT]) + ctx_factory = opennsaTlsContext( + vc[config.CERTIFICATE_DIR], vc[config.VERIFY_CERT]) else: ctx_factory = None return ctx_factory - class CS2RequesterCreator: def __init__(self, top_resource, aggregator, host, port, tls, ctx_factory): self.top_resource = top_resource - self.aggregator = aggregator - self.host = host - self.port = port - self.tls = tls - self.ctx_factory = ctx_factory - + self.aggregator = aggregator + self.host = host + self.port = port + self.tls = tls + self.ctx_factory = ctx_factory def create(self, nsi_agent): hash_input = nsi_agent.urn() + nsi_agent.endpoint - resource_name = b'RequesterService2-' + hashlib.sha1(hash_input.encode()).hexdigest().encode() + resource_name = b'RequesterService2-' + \ + hashlib.sha1(hash_input.encode()).hexdigest().encode() return nsi2.setupRequesterPair(self.top_resource, self.host, self.port, nsi_agent.endpoint, self.aggregator, resource_name, tls=self.tls, ctx_factory=self.ctx_factory) - class OpenNSAService(twistedservice.MultiService): def __init__(self, vc): twistedservice.MultiService.__init__(self) self.vc = vc - def setupServiceFactory(self): """ This sets up the OpenNSA service and ties together everything in the initialization. @@ -162,31 +161,34 @@ class OpenNSAService(twistedservice.MultiService): vc[config.HOST] = socket.getfqdn() # database - database.setupDatabase(vc[config.DATABASE], vc[config.DATABASE_USER], vc[config.DATABASE_PASSWORD], vc[config.DATABASE_HOST], vc[config.SERVICE_ID_START]) + database.setupDatabase(vc[config.DATABASE], vc[config.DATABASE_USER], + vc[config.DATABASE_PASSWORD], vc[config.DATABASE_HOST], vc[config.SERVICE_ID_START]) service_endpoints = [] # base names - domain_name = vc[config.DOMAIN] # FIXME rename variable to domain - nsa_name = domain_name + ':nsa' + domain_name = vc[config.DOMAIN] # FIXME rename variable to domain + nsa_name = domain_name + ':nsa' # base url base_protocol = 'https://' if vc[config.TLS] else 'http://' base_url = base_protocol + vc[config.HOST] + ':' + str(vc[config.PORT]) # nsi endpoint and agent - provider_endpoint = base_url + '/NSI/services/CS2' # hardcode for now - service_endpoints.append( ('Provider', provider_endpoint) ) + provider_endpoint = base_url + '/NSI/services/CS2' # hardcode for now + service_endpoints.append(('Provider', provider_endpoint)) - ns_agent = nsa.NetworkServiceAgent(nsa_name, provider_endpoint, 'local') + ns_agent = nsa.NetworkServiceAgent( + nsa_name, provider_endpoint, 'local') # ssl/tls context - ctx_factory = setupTLSContext(vc) # May be None + ctx_factory = setupTLSContext(vc) # May be None # plugin if vc[config.PLUGIN]: from twisted.python import reflect - plugin = reflect.namedAny('opennsa.plugins.%s.plugin' % vc[config.PLUGIN]) + plugin = reflect.namedAny( + 'opennsa.plugins.%s.plugin' % vc[config.PLUGIN]) else: from opennsa.plugin import BasePlugin plugin = BasePlugin() @@ -195,21 +197,25 @@ class OpenNSAService(twistedservice.MultiService): # the dance to setup dynamic providers right top_resource = resource.Resource() - requester_creator = CS2RequesterCreator(top_resource, None, vc[config.HOST], vc[config.PORT], vc[config.TLS], ctx_factory) # set aggregator later + requester_creator = CS2RequesterCreator( + top_resource, None, vc[config.HOST], vc[config.PORT], vc[config.TLS], ctx_factory) # set aggregator later - provider_registry = provreg.ProviderRegistry( { cnt.CS2_SERVICE_TYPE : requester_creator.create } ) + provider_registry = provreg.ProviderRegistry( + {cnt.CS2_SERVICE_TYPE: requester_creator.create}) link_vector = linkvector.LinkVector() networks = {} - ports = {} # { network : { port : nrmport } } + ports = {} # { network : { port : nrmport } } - parent_requester = None # parent requester is set later - aggr = aggregator.Aggregator(ns_agent, ports, link_vector, parent_requester, provider_registry, vc[config.POLICY], plugin ) + parent_requester = None # parent requester is set later + aggr = aggregator.Aggregator( + ns_agent, ports, link_vector, parent_requester, provider_registry, vc[config.POLICY], plugin) requester_creator.aggregator = aggr - pc = nsi2.setupProvider(aggr, top_resource, ctx_factory=ctx_factory, allowed_hosts=vc.get(config.ALLOWED_HOSTS)) + pc = nsi2.setupProvider( + aggr, top_resource, ctx_factory=ctx_factory, allowed_hosts=vc.get(config.ALLOWED_HOSTS)) aggr.parent_requester = pc # setup backend(s) - for now we only support one @@ -219,23 +225,27 @@ class OpenNSAService(twistedservice.MultiService): if not cnt.AGGREGATOR in vc[config.POLICY]: vc[config.POLICY].append(cnt.AGGREGATOR) - else: # at least one backend + else: # at least one backend # This is all temporary right now... clean up later for backend_name, b_cfg in backend_configs.items(): if backend_name is None or backend_name == '': - raise config.ConfigurationError('You need to specify backend name, use [backend:name]') + raise config.ConfigurationError( + 'You need to specify backend name, use [backend:name]') - backend_network_name = '{}:{}'.format(domain_name, backend_name) + backend_network_name = '{}:{}'.format( + domain_name, backend_name) - if not config.NRM_MAP_FILE in b_cfg: # move to verify config - raise config.ConfigurationError('No nrm map specified for backend') + if not config.NRM_MAP_FILE in b_cfg: # move to verify config + raise config.ConfigurationError( + 'No nrm map specified for backend') backend_nrm_map_file = b_cfg[config.NRM_MAP_FILE] - if not os.path.exists(backend_nrm_map_file): # move to verify config - raise config.ConfigError('nrm map file {} for backend {} does not exists'.format(backend_nrm_map_file, backend_name)) + if not os.path.exists(backend_nrm_map_file): # move to verify config + raise config.ConfigError('nrm map file {} for backend {} does not exists'.format( + backend_nrm_map_file, backend_name)) nrm_map = open(backend_nrm_map_file) backend_nrm_ports = nrm.parsePortSpec(nrm_map) @@ -243,37 +253,44 @@ class OpenNSAService(twistedservice.MultiService): link_vector.addLocalNetwork(backend_network_name) for np in backend_nrm_ports: if np.remote_network is not None: - link_vector.updateVector(backend_network_name, np.name, { np.remote_network : 1 } ) # hack + link_vector.updateVector(backend_network_name, np.name, { + np.remote_network: 1}) # hack for network, cost in np.vectors.items(): - link_vector.updateVector(np.name, { network : cost }) + link_vector.updateVector(np.name, {network: cost}) # build port map for aggreator to lookup ports.setdefault(backend_network_name, {})[np.name] = np - backend_service = setupBackend(b_cfg, backend_network_name, backend_nrm_ports, aggr) + backend_service = setupBackend( + b_cfg, backend_network_name, backend_nrm_ports, aggr) networks[backend_network_name] = { - 'backend' : backend_service, - 'nrm_ports' : backend_nrm_ports + 'backend': backend_service, + 'nrm_ports': backend_nrm_ports } - provider_registry.addProvider(ns_agent.urn(), backend_network_name, backend_service) + provider_registry.addProvider( + ns_agent.urn(), backend_network_name, backend_service) # fetcher if vc[config.PEERS]: - fetcher_service = fetcher.FetcherService(link_vector, networks, vc[config.PEERS], provider_registry, ctx_factory=ctx_factory) + fetcher_service = fetcher.FetcherService( + link_vector, networks, vc[config.PEERS], provider_registry, ctx_factory=ctx_factory) fetcher_service.setServiceParent(self) else: - log.msg('No peers configured, will not be able to do outbound requests (UPA mode)') + log.msg( + 'No peers configured, will not be able to do outbound requests (UPA mode)') # discovery service opennsa_version = 'OpenNSA-' + version - network_urns = [ '{}{}'.format(cnt.URN_OGF_PREFIX, network_name) for network_name in networks ] - interfaces = [ ( cnt.CS2_PROVIDER, provider_endpoint, None), ( cnt.CS2_SERVICE_TYPE, provider_endpoint, None) ] - features = [] + network_urns = ['{}{}'.format( + cnt.URN_OGF_PREFIX, network_name) for network_name in networks] + interfaces = [(cnt.CS2_PROVIDER, provider_endpoint, None), + (cnt.CS2_SERVICE_TYPE, provider_endpoint, None)] + features = [] if networks: - features.append( (cnt.FEATURE_UPA, None) ) + features.append((cnt.FEATURE_UPA, None)) if vc[config.PEERS]: - features.append( (cnt.FEATURE_AGGREGATOR, None) ) + features.append((cnt.FEATURE_AGGREGATOR, None)) # view resource vr = viewresource.ConnectionListResource() @@ -285,73 +302,77 @@ class OpenNSAService(twistedservice.MultiService): rest.setupService(aggr, top_resource, vc.get(config.ALLOWED_HOSTS)) - service_endpoints.append( ('REST', rest_url) ) - interfaces.append( (cnt.OPENNSA_REST, rest_url, None) ) + service_endpoints.append(('REST', rest_url)) + interfaces.append((cnt.OPENNSA_REST, rest_url, None)) for backend_network_name, no in networks.items(): nml_resource_name = '{}.nml.xml'.format(backend_network_name) - nml_url = '%s/NSI/%s' % (base_url, nml_resource_name) + nml_url = '%s/NSI/%s' % (base_url, nml_resource_name) - nml_network = nml.createNMLNetwork(no['nrm_ports'], backend_network_name, backend_network_name) - can_swap_label = no['backend'].connection_manager.canSwapLabel(cnt.ETHERNET_VLAN) + nml_network = nml.createNMLNetwork( + no['nrm_ports'], backend_network_name, backend_network_name) + can_swap_label = no['backend'].connection_manager.canSwapLabel( + cnt.ETHERNET_VLAN) nml_service = nmlservice.NMLService(nml_network, can_swap_label) - top_resource.children[NSI_RESOURCE].putChild(nml_resource_name.encode(), nml_service.resource() ) - - service_endpoints.append( ('NML Topology', nml_url) ) - interfaces.append( (cnt.NML_SERVICE_TYPE, nml_url, None) ) + top_resource.children[NSI_RESOURCE].putChild( + nml_resource_name.encode(), nml_service.resource()) + service_endpoints.append(('NML Topology', nml_url)) + interfaces.append((cnt.NML_SERVICE_TYPE, nml_url, None)) # discovery service discovery_resource_name = b'discovery.xml' - discovery_url = '%s/NSI/%s' % (base_url, discovery_resource_name.decode()) + discovery_url = '%s/NSI/%s' % (base_url, + discovery_resource_name.decode()) - ds = discoveryservice.DiscoveryService(ns_agent.urn(), now, domain_name, opennsa_version, now, network_urns, interfaces, features, provider_registry, link_vector) + ds = discoveryservice.DiscoveryService(ns_agent.urn( + ), now, domain_name, opennsa_version, now, network_urns, interfaces, features, provider_registry, link_vector) discovery_resource = ds.resource() - top_resource.children[NSI_RESOURCE].putChild(discovery_resource_name, discovery_resource) - link_vector.callOnUpdate( lambda : discovery_resource.updateResource ( ds.xml() )) + top_resource.children[NSI_RESOURCE].putChild( + discovery_resource_name, discovery_resource) + link_vector.callOnUpdate( + lambda: discovery_resource.updateResource(ds.xml())) - service_endpoints.append( ('Discovery', discovery_url) ) + service_endpoints.append(('Discovery', discovery_url)) # log service urls for service_name, url in service_endpoints: log.msg('{:<12} URL: {}'.format(service_name, url)) factory = server.Site(top_resource) - factory.log = httplog.logRequest # default logging is weird, so we do our own + factory.log = httplog.logRequest # default logging is weird, so we do our own return factory, ctx_factory - def startService(self): factory, ctx_factory = self.setupServiceFactory() if self.vc[config.TLS]: - internet.SSLServer(self.vc[config.PORT], factory, ctx_factory).setServiceParent(self) + internet.SSLServer( + self.vc[config.PORT], factory, ctx_factory).setServiceParent(self) else: - internet.TCPServer(self.vc[config.PORT], factory).setServiceParent(self) + internet.TCPServer(self.vc[config.PORT], + factory).setServiceParent(self) # do not start sub-services until we have started this one twistedservice.MultiService.startService(self) log.msg('OpenNSA service started') - def stopService(self): twistedservice.Service.stopService(self) - def createApplication(config_file=config.DEFAULT_CONFIG_FILE, debug=False, payload=False): application = twistedservice.Application('OpenNSA') try: - cfg = config.readConfig(config_file) vc = config.readVerifyConfig(cfg) @@ -365,11 +386,11 @@ def createApplication(config_file=config.DEFAULT_CONFIG_FILE, debug=False, paylo nsa_service = OpenNSAService(vc) nsa_service.setServiceParent(application) - application.setComponent(log.ILogObserver, logging.DebugLogObserver(log_file, debug, payload=payload).emit) + application.setComponent(log.ILogObserver, logging.DebugLogObserver( + log_file, debug, payload=payload).emit) return application except config.ConfigurationError as e: import sys sys.stderr.write("Configuration error: %s\n" % e) sys.exit(1) - diff --git a/requirements.txt b/requirements.txt index d160fef4fb1d57aedd0a2156c8e3af5eeb4b401a..336060f6df8a1ccfeca18f92336d99a93dd32489 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ -twisted>=19.7.0 +#twisted>=19.7.0 +twisted>=21.2.0 twistar>=2.0 -psycopg2>=2.7,<2.8 --no-binary psycopg2 +#psycopg2>=2.7,<2.8 --no-binary psycopg2 +psycopg2 pyOpenSSL>=17.5.0 python-dateutil service_identity -idna +idna \ No newline at end of file