Merge branch 'master' of https://github.com/NORDUnet/opennsa into stable

.drone.yml

+kind: pipeline
+name: default
+
+steps:
+  - name: Test
+    image: python:3.9
+    environment:
+      POSTGRES_HOST: database
+    commands:
+      - apt update && apt-get install -y postgresql-client
+      - cat datafiles/schema.sql | PGPASSWORD="w1gWIn7NDGXjXMguiI2Qe05X" psql -U opennsa -h database  -d opennsatest
+      - python3 -m pip install -r requirements.txt
+      - cp util/integration-config.json  .opennsa-test.json
+      - cp config/opennsa.conf.template config/opennsa.conf
+      - PYTHONPATH=. trial test
+
+services:
+- name: database
+  image: postgres:12-alpine
+  environment:
+    POSTGRES_USER: opennsa
+    POSTGRES_PASSWORD: w1gWIn7NDGXjXMguiI2Qe05X
+    POSTGRES_DB: opennsatest
+
+trigger:
+  event:
+    - push
+    - pull_request
+
+--- 
+kind: pipeline 
+name: docker
+
+steps:
+  - name: docker
+    image: plugins/docker
+    pull: if-not-exists
+    settings:
+      repo: jghnordunet/opennsa
+      squash: true
+      auto_tag: true
+      dockerfile: docker/Dockerfile
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASS
+
+trigger:
+  branch:
+    - master
+  event:
+    - tag
+    - push

.gitignore

@@ -4,3 +4,12 @@ build
 dist
 .opennsa-test.json
 .python-version
+docker/.env
+config/opennsa.conf
+docker-compose.override.yml
+
+.idea
+.devcontainer
+twistd.pid
+.env
+.DS_Store

INSTALL → INSTALL.md

@@ -31,24 +31,22 @@ Dmz vs. behind the firewall: Should work with both, not required.
 
 ## Dependencies:
 
-* Python 2.7 or later (Python 3 not supported yet)
+* Python 3 
 
-* Twisted 16.x.x or later, http://twistedmatrix.com/trac/
+* Twisted 21.x.x or later, http://twistedmatrix.com/trac/
 
-* Psycopg 2.5.0 or later (http://initd.org/psycopg/, 2.4.6 _might_ work)
+* Psycopg 2.9.0 or later (http://initd.org/psycopg/)
 
-* Twistar 1.1 or later (https://pypi.python.org/pypi/twistar/ & http://findingscience.com/twistar/ )
+* Twistar 2.0 or later (https://pypi.python.org/pypi/twistar/ & http://findingscience.com/twistar/ )
 
-* PostgreSQL (need 9.5 or later if using connection id assignment)
+* PostgreSQL (need 12 or later if using connection id assignment)
 
-* pyOpenSSL 0.14 (when running with SSL/TLS)
+* pyOpenSSL 17.5 or later (when running with SSL/TLS)
 
 Python and Twisted should be included in the package system in most recent
 Linux distributions.
 
-Older Twisted versions might work, Twisted 15.x and earlier won't work with
-OpenSSH 7.0 or later. If you see connection lost for ssh in the log, most
-likely your Twisted version is too old.
+If you see connection lost for ssh in the log, most likely your Twisted version is too old.
 
 Furthermore, for SSH based backends (Brocade, Force10, and Juniper), the
 packages pyasn1 and python-crypto are also required.

Makefile

@@ -4,8 +4,11 @@
 
 clean:
 	rm -fr _trial_temp
-	find . -name "*.pyc"|xargs rm
+	find . -name "*.pyc" -exec rm -v {} \; 
 
-docker-build:
-	docker build -t opennsa --squash docker
+down:
+	docker-compose down
+
+docker-build: clean down
+	docker-compose build  --no-cache
 

README.md

+[![Build Status](https://cloud.drone.io/api/badges/NORDUnet/opennsa/status.svg)](https://cloud.drone.io/NORDUnet/opennsa)
+
 OpenNSA
 -------
 
@@ -32,8 +34,10 @@ NORDUnet License (3-clause BSD). See LICENSE for more details.
 #### Contact
 
 * Johannes Garm Houen - jgh @ nordu.net
+* Samir Faci - samir @ es.net
 
 #### Copyright
 
 [NORDUnet](http://www.nordu.net) (2011-2015)
 
+

client-test

@@ -26,15 +26,15 @@ PORT = 7080
 def matchState(query_result, expected_state):
     state =  query_result.reservationSummary[0].connectionState
     if state == expected_state:
-        print 'State match (%s)' % expected_state
+        print('State match (%s)' % expected_state)
     else:
-        print "State mismatch. Was %s, should have been %s" % (state, expected_state)
+        print("State mismatch. Was %s, should have been %s" % (state, expected_state))
 
 
 @defer.inlineCallbacks
 def doMain():
 
-    print 'OpenNSA WS test client'
+    print('OpenNSA WS test client')
 
     wsdl_dir =  os.path.join(os.path.dirname(os.path.abspath(__file__)), 'wsdl')
 
@@ -70,10 +70,10 @@ def doMain():
     global_reservation_id = 'urn:uuid:' + str(uuid.uuid1())
     connection_id         = 'urn:uuid:' + str(uuid.uuid1())
 
-    print "Connection id", connection_id
+    print("Connection id", connection_id)
 
     r = yield client.reserve(client_nsa, provider.nsa, None, global_reservation_id, 'Test Connection', connection_id, service_params)
-    print "Reservation created. Connection ID:", connection_id
+    print("Reservation created. Connection ID:", connection_id)
 
     qr = yield client.query(client_nsa, provider.nsa, None, "Summary", connection_ids = [ connection_id ] )
     matchState(qr, 'Reserved')
@@ -84,19 +84,19 @@ def doMain():
     matchState(qr, 'Auto-Provision')
 
     yield d
-    print "Connection provisioned"
+    print("Connection provisioned")
 
     qr = yield client.query(client_nsa, provider.nsa, None, "Summary", connection_ids = [ connection_id ] )
     matchState(qr, 'Provisioned')
 
     _ = yield client.release(client_nsa, provider.nsa, None, connection_id)
-    print "Connection released"
+    print("Connection released")
 
     qr = yield client.query(client_nsa, provider.nsa, None, "Summary", connection_ids = [ connection_id ] )
     matchState(qr, 'Scheduled')
 
     _ = yield client.terminate(client_nsa, provider.nsa, None, connection_id)
-    print "Reservation terminated"
+    print("Reservation terminated")
 
     qr = yield client.query(client_nsa, provider.nsa, None, "Summary", connection_ids = [ connection_id ] )
     matchState(qr, 'Terminated')

docker/opennsa.conf.template → config/opennsa.conf.template

@@ -7,13 +7,13 @@ logfile=
 #peers=example.org@http://example.org:9080/NSI/topology/example.org.xml
 
 # These are set by the create-compose script
-dbhost=opennsa-db
+dbhost=${POSTGRES_HOST}
 database=${POSTGRES_DB}
 dbuser=${POSTGRES_USER}
 dbpassword=${POSTGRES_PASSWORD}
+allowed_admins=${ALLOWED_ADMINS}
 
-tls=false
+tls=${TLS_ENABLED}
 
 [dud:topology]
-nrmmap=opennsa.nrm
-
+nrmmap=${NRM_FILE}

datafiles/opennsa.conf

@@ -16,6 +16,9 @@
 # host=example.org
 # port=9443
 
+## in a proxied setup specify base_url
+# base_url=https://opennsa.example.domain/
+
 ## security settings
 
 #tls=true # defaults to true

docker-compose.override.yml_placeholder

+## rename this file to docker-compose.override.yml any additional settings listed here will be merged with the docker-compose.yml file.
+version: "3.7"
+
+services:
+  opennsa:
+    image: jghnordunet/opennsa:latest
+    command: Any valid command
+    ## Mount entire project to volume avoids constant rebuilds.
+    ## You may need to load the container as:
+    ## UID=${UID} GID=${GID} docker-compose up linux FS can cause some issues with 
+    ## permissioning at times.
+    #volumes:
+    # - ./:/home/opennsa/opennsa
+    # Mount NRM file and leave ENV value the same
+    #volumes:
+    #  - ./config/myNRMFile.nrm:/home/opennsa/opennsa/config/opennsa.nrm
+  db:
+    image: postgres:12
+    ##Expose 5432 locally 
+    ports:
+      - 5432:5432

docker-compose.yml

+version: "3.7"
+
+services:
+  db:
+    image: postgres:12
+    expose:
+      - 5432
+    volumes:
+      - ./datafiles/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
+      - opennsa-pgdata:/var/lib/postgresql/data
+    env_file: .env
+  opennsa:
+    image: jghnordunet/opennsa:latest
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    env_file: .env
+    depends_on:
+      - db
+    ports:
+      - 9080:9080
+      - 9443:9443
+    volumes:
+      - ./config/opennsa.conf:/home/opennsa/opennsa/config/opennsa.conf:ro
+      - ./config/opennsa.nrm:/home/opennsa/opennsa/opennsa.nrm:ro
+
+volumes:
+  opennsa-pgdata:

docker/Dockerfile

@@ -2,55 +2,48 @@
 
 FROM debian:stable-slim
 
-MAINTAINER Henrik Thostrup Jensen <htj@nordu.net>
-
+LABEL maintainer="Henrik Thostrup Jensen <htj@nordu.net>"
 
 # -- Environment --
-ENV GIT_REPO https://github.com/NORDUnet/opennsa
 ENV USER opennsa
 
 
+# -- User setup --
+RUN adduser --disabled-password --gecos 'OpenNSA user' $USER
+ADD . /home/$USER/opennsa/
+
 # --- Base image ---
 # Update and install dependencies
 # pip to install twistar service-identity pyasn1
 # pyasn1 and crypto is needed for ssh backends
-RUN apt-get update && apt-get install -y git-core python3 python3-twisted-bin python3-openssl python3-psycopg2 python3-pip python3-crypto python3-dateutil
-
-RUN pip3 install twistar service-identity pyasn1
-
-
-# -- User setup --
-RUN adduser --disabled-password --gecos 'OpenNSA user' $USER
-
-
-# -- Install OpenNSA --
-USER $USER
-WORKDIR /home/$USER
-
-RUN echo git clone $GIT_REPO
-RUN git clone $GIT_REPO
-
-# -- Cleanup --
-# With --squash this makes the image go from 476 to 164 mb
-USER root
-RUN apt-get remove -y python3-pip git
-RUN apt-get -y clean
-RUN apt-get -y autoclean
-RUN apt-get -y autoremove
-
-
+RUN apt update \
+   && apt install -y \
+   libpq-dev \
+   python3 \
+   python3-pip \
+   netcat \
+   iputils-ping \
+   && pip3 install -r /home/$USER/opennsa/requirements.txt \
+   && chown $USER:$USER -R /home/opennsa/opennsa \
+   # -- Cleanup --
+   && apt remove -y python3-pip  \
+   && apt autoremove -y \
+   && rm -rf /var/lib/apt/lists/* \
+   && cp /home/$USER/opennsa/docker/run_opennsa.sh /home/$USER/opennsa \
+   && cp /home/$USER/opennsa/config/opennsa.conf.template  /home/$USER/opennsa/config/opennsa.conf
+
+
+#RUN 
 # -- Switch to OpenNSA directory --
-
 USER $USER
+
 WORKDIR /home/$USER/opennsa
 
 ENV PYTHONPATH .
-
-
 # -- Entrypoint --
-
 EXPOSE 9080
 EXPOSE 9443
 
-ENTRYPOINT rm -f twistd.pid; twistd -ny opennsa.tac
+# USER root
 
+CMD /home/$USER/opennsa/run_opennsa.sh

docker/README.md

@@ -11,20 +11,18 @@ $ make docker-build     ( from opennsa directory )
 As OpenNSA requires a Postgres database, docker-compose is used to coordinate
 the setup of the two containers.
 
-1. Edit opennsa.conf.template and opennsa.nrm
-   Leave the database config as-is.
-
-2. $ ./create-compose
-   This will substitute stuff in the templates and create docker-compose.yml and opennsa.conf
+1. $ ./generate-docker-config
+   This will mainly generate a password and create a .env file for you.  You may update the settings in .env if you wish to use a different nrm file (Keep in mind you'll need to mount it as a volume if you stray from the defaults or rebuild the image)
 
 3. $ docker-compose up
    This should bring up a PostgreSQL instance and OpenNSA.
 
+## Advanced Features
 
-You may have to edit template.yml to expose OpenNSA ports publically, mount in
-certificates, or similar.
+1.  In order to override any settings copy the docker-compose.override.yml_placeholder to docker-compose.override.yml.  You can use to mount additional volumes, expose additional ports etc.  Some common patterns are already there and commented out. 
 
+2. Configuration options are almost all exposed via ENV variables.  If you wish to directly mount your config file, make a copy of config/opennsa.conf.template to config/opennsa.conf.  Update any entries as desired and restart all DB container.
 
-TODO: Make OpenNSA able to take database configuration via environment, so we
-      don't have to do replacement in opennsa.conf
+3. The entry point is left as just bash, so if you wish to override the initial command you may simply set the `command:` line in your override file to anything you like.  If you want, you may also invoke the run_opennsa.sh with arguments, it will wait for the database to come up with run the command you issues.  
 
+For example:   run_opennsa.sh sleep 50  ==> will wait for DB to come up then sleep for 50 seconds.
\ No newline at end of file

docker/create-compose

-#!/bin/sh
-
-set -a # load source environment variables into scope
-
-. env.sh
-
-cat template.yml | envsubst > docker-compose.yml
-
-cat opennsa.conf.template | envsubst > opennsa.conf
-
-echo "Start OpenNSA with: docker-compose up"
-

docker/env.sh

-#!/bin/sh
-
-
-POSTGRES_DB=opennsa
-POSTGRES_USER=opennsa
-POSTGRES_PASSWORD=$(openssl rand -base64 18)
-
-SCHEMA_FILE=$PWD/../datafiles/schema.sql
-
-OPENNSA_CONF_FILE=$PWD/opennsa.conf
-OPENNSA_NRM_FILE=$PWD/opennsa.nrm
-
-

docker/run_opennsa.sh

+#!/usr/bin/env bash 
+
+function check_db()
+{
+## Wait for DB container to be up
+
+until nc -z -v -w30 $POSTGRES_HOST $POSTGRES_PORT
+do
+  echo "Waiting 5 second until the database is receiving connections..."
+  # wait for a second before checking again
+  sleep 5
+done
+
+}
+
+function run_app() 
+{
+  cd $HOME/opennsa
+  rm -f twistd.pid; $cmd 
+}
+
+
+if [ $# -gt 0 ]; then
+    cmd=$@
+else 
+    cmd='twistd -ny opennsa.tac'
+fi
+
+
+check_db
+run_app $cmd
+

docker/template.yml

-version: '3'
-
-services:
-  opennsa-db:
-    image: postgres:9.6.5
-    volumes:
-      - ${SCHEMA_FILE}:/docker-entrypoint-initdb.d/schema.sql:ro
-      - opennsa-pgdata:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=${POSTGRES_DB}
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-
-
-  opennsa:
-    image: opennsa:latest
-    depends_on:
-      - opennsa-db
-      - waitforpg
-    ports:
-      - 127.0.0.1:9080:9080
-      - 127.0.0.1:9443:9443
-    links:
-      - opennsa-db
-    volumes:
-      - ${OPENNSA_CONF_FILE}:/home/opennsa/opennsa/opennsa.conf:ro
-      - ${OPENNSA_NRM_FILE}:/home/opennsa/opennsa/opennsa.nrm:ro
-
-
-  waitforpg:
-    image: dadarek/wait-for-dependencies
-    depends_on:
-      - opennsa-db
-    command: opennsa-db:5432
-
-volumes:
-  opennsa-pgdata:
-
-

docs/migration → docs/migration.md

-OpenNSA 3 Configuration Migration
-=================================
+# OpenNSA 3 Configuration Migration
+
 
 With the port of OpenNSA from Python 2 to Python 3, and the subsequent release
 of OpenNSA 3, support for multiple backends was added. For this, some changes
@@ -12,7 +12,7 @@ The changes are:
 
 Example of old style:
 
-```
+```ini
 [service]
 network=aruba.net
 nrmmap=aruba.nrm
@@ -22,7 +22,7 @@ nrmmap=aruba.nrm
 
 Equivalent config in new style:
 
-```
+```ini
 [service]
 domain=aruba.net
 
@@ -32,7 +32,7 @@ nrmmap=aruba.nrm
 
 An example with multiple backends shows why the change was needed:
 
-```
+```ini
 [service]
 domain=aruba.net