add juniper vpls backend (not tested yet)

datafiles/schema.sql

@@ -101,3 +101,15 @@ CREATE TABLE generic_backend_connections (
     allocated               boolean                     NOT NULL  -- indicated if the resources are actually allocated
 );
 
+
+-- Force this to only have a single row
+-- generate new id with:
+-- there needs to be a conflict check to see if the backend has a row (and insert corrosonding start value)
+-- INSERT INTO backend_connection_id (connection_id) VALUES (190000)  ON CONFLICT DO NOTHING;
+-- Generate new id with:
+-- UPDATE backend_connection_id SET connection_id = connection_id + 1 RETURNING connection_id;
+CREATE TABLE backend_connection_id (
+    id                      integer                     PRIMARY KEY NOT NULL DEFAULT(1) CHECK (id = 1),
+    connection_id           serial                      NOT NULL
+);
+

opennsa/backends/junipervpls.py

+"""
+OpenNSA Juniper/JunOS VPLS backend.
+
+Intended to match Canaries usage.
+
+Requires a JunOS device with VPLS support (duh)
+
+Author: Henrik Thostrup Jensen <htj@nordu.net>
+Copyright: NORDUnet (2017)
+
+"""
+
+# Configuration conventions / environment, and snippets
+
+# We assign each service a unique Circuit-ID and this CID, or parts of it,
+# are used for provisioning
+#
+# Circuit-ID follows this naming/syntax <Unique_ID> - <Organizational_ID>
+# - CANARIE_site_short<1..9> - CANARIE_site_short<1..9> - Service_Name
+# for example, 13903CS01-NORDUNet-AMST1-NYCN1
+#
+# This circuit ID is used in "description" of sub-interface/logical unit
+# number for example, description "13903CS01-NORDUNet-AMST1-NYCN1
+# [with some extra information in square brackets .... L2VPN circuit
+# Amsterdam to New York for XYZ]";
+
+# This circuit ID is also used as the name for the routing instance
+# for example,
+# routing-instances {
+#   13903CS01-NORDUNet-AMST1-NYCN1 {
+# .....
+#
+# route-distinguisher and vrf-target for the routing instance are also
+# composed using the information that derives from <Unique_ID>
+# The two characters (in this example CS) are removed and seven digest
+# used for the second, after ":" part. The first part is our AS number.
+#
+# for unique_id "13903CS01", for example
+#   route-distinguisher 6509:1390301;
+#   vrf-target target:6509:1390301;
+#
+# the sites for the routing instance are directly derived from Circuit-ID,
+#   site AMST1 {
+#                    site-identifier 1;
+# .....
+#                site NYCN1 {
+#                    site-identifier 2;
+# ....
+#
+# logical unit number normally matches a vlan ID (or a first VLAN number in a list)
+#
+# interfaces {
+#     et-x/y/z {
+#         description "ANA-300 link to Amsterdam";
+#         flexible-vlan-tagging;
+#         encapsulation flexible-ethernet-services;
+#         .....
+#         unit 1000 {
+#             description "13903CS01-NORDUNet-AMST1-NYCN1 [L2VPN circuit Amsterdam to New York for NORDUNet]";
+#             encapsulation vlan-vpls;
+#             vlan-id-list 1000;
+#             family vpls;
+#         }
+#
+#     et-x/y/w {
+#         description "CANARIE/ANA-300 link to New York City";
+#         flexible-vlan-tagging;
+#         encapsulation flexible-ethernet-services;
+#         .....
+#         unit 1000 {
+#             description "13903CS01-NORDUNet-AMST1-NYCN1 [L2VPN circuit Amsterdam to New York for NORDUNet]";
+#             encapsulation vlan-vpls;
+#             vlan-id-list 1000;
+#             family vpls;
+#         }
+#     }
+#
+#
+# routing-instances {
+#     13903CS01-NORDUNet-AMST1-NYCN1 {
+#         instance-type vpls;
+#         interface et-x/y/z.1000;
+#         interface et-x/y/w.1000;
+#         route-distinguisher 6509:1390301;
+#         vrf-target target:6509:1390301;
+#         protocols {
+#             vpls {
+#                 site-range 2;
+#                 no-tunnel-services;
+#                 site AMST1 {
+#                     site-identifier 1;
+#                     interface et-x/y/z.1000;
+#                 }
+#                 site NYCN1 {
+#                     site-identifier 2;
+#                     interface et-x/y/w.1000;
+#                 }
+#             }
+#         }
+#     }
+#
+
+
+#import random
+
+from twisted.python import log
+from twisted.internet import defer
+
+from opennsa import constants as cnt, config, database
+from opennsa.backends.common import genericbackend, ssh
+
+
+LOG_SYSTEM = 'JuniperVPLS'
+
+
+# JunOS commands, static
+CONFIGURE   = 'configure'
+COMMIT      = 'commit'
+
+# JunOS commands, parameterized
+
+# Interface unit configuration
+#SET_UNIT                = 'set interfaces %(interface) unit %(unit)'
+#SET_UNIT_DESCRIPTION    = 'set interfaces %(interface) unit %(unit) description %(description)'
+#SET_UNIT_ENCAPSULATION  = 'set interfaces %(interface) unit %(unit) encapsulation vlan-vpls'
+#SET_UNIT_VLAN           = 'set interfaces %(interface) unit %(unit) vlan-id-list %(vlan)'
+#SET_UNIT_FAMILY         = 'set interfaces %(interface) unit %(unit) family vpls'
+
+SET_UNIT = 'set interfaces %(interface) unit %(unit) description %(description) encapsulation vlan-vpls vlan-id %(vlan) family vpls'
+
+# Routing instance configuration
+#SET_RI                      = 'set routing-instance %(instance)'
+SET_RI_INSTANCE_TYPE        = 'set routing-instances %(instance) instance-type vpls'
+SET_RI_INTERFACE            = 'set routing-instances %(instance) interface %(interface)'
+SET_RI_ROUTE_DISTINGUISHER  = 'set routing-instances %(instance) route-distinguisher %(route-distinguisher)'
+SET_RI_VRF_TARGET           = 'set routing-instances %(instance) vrf-target %(vrf-target)'
+SET_RI_PROTOCOLS            = 'set routing-instances %(instance) protocols vpls site-range 2 no-tunnel-services'
+SET_RI_VPLS_SITE            = 'set routing-instances %(instance) protocols vpls site %(site) site-identifier %(site-id) interface %(interface)'
+#SET_RI_VPLS_SITE2           = 'set routing-instances %(instance) protocols vpls site %(site) site-identifier 2 interface %(interface)'
+
+# Delete statements
+DELETE_UNIT             = 'delete interfaces %(interface) unit $(unit)'
+DELETE_ROUTING_INSTANCE = 'delete routing-instance %(instance)'
+
+
+
+def createSetupCommands(source_port, dest_port, vlan, instance_id, description, route_distinguiser, vrf_target):
+
+    commands = [
+        SET_UNIT % {'interface': source_port, 'unit': vlan, 'description': description, 'vlan': vlan},
+        SET_UNIT % {'interface': dest_port,   'unit': vlan, 'description': description, 'vlan': vlan},
+
+        SET_RI_INSTANCE_TYPE        % {'instance': instance_id },
+        SET_RI_INTERFACE            % {'instance': instance_id, 'interface': source_port + '.' + vlan },
+        SET_RI_INTERFACE            % {'instance': instance_id, 'interface': dest_port   + '.' + vlan },
+        SET_RI_ROUTE_DISTINGUISHER  % {'instance': instance_id, 'route-distinguisher': route_distinguiser },
+        SET_RI_VRF_TARGET           % {'instance': instance_id, 'vrf-target': vrf_target },
+        SET_RI_PROTOCOLS            % {'instance': instance_id },
+        SET_RI_VPLS_SITE            % {'instance': instance_id, 'site': 'SITE1', 'site-id': 1 },
+        SET_RI_VPLS_SITE            % {'instance': instance_id, 'site': 'SITE2', 'site-id': 2 }
+    ]
+
+    return commands
+
+
+def createDeleteCommands(source_port, dest_port, vlan, instance_id):
+
+    commands = [
+        DELETE_UNIT                 % {'interface': source_port, 'unit': vlan },
+        DELETE_UNIT                 % {'interface': dest_port,   'unit': vlan },
+        DELETE_ROUTING_INSTANCE     % {'instance' : instance_id }
+    ]
+
+    return commands
+
+
+# ---
+
+
+class SSHChannel(ssh.SSHChannel):
+
+    name = 'session'
+
+    def __init__(self, conn):
+        ssh.SSHChannel.__init__(self, conn=conn)
+
+        self.line = ''
+
+        self.wait_defer = None
+        self.wait_line  = None
+
+
+    @defer.inlineCallbacks
+    def sendCommands(self, commands):
+        LT = '\r' # line termination
+
+        try:
+            yield self.conn.sendRequest(self, 'shell', '', wantReply=1)
+            d = self.waitForLine('>')
+            self.write(CONFIGURE + LT)
+            yield d
+
+            log.msg('Entered configure mode', debug=True, system=LOG_SYSTEM)
+
+            for cmd in commands:
+                log.msg('CMD> %s' % cmd, system=LOG_SYSTEM)
+                d = self.waitForLine('[edit]')
+                self.write(cmd + LT)
+                yield d
+
+            # commit commands, check for 'commit complete' as success
+            # not quite sure how to handle failure here
+
+            d = self.waitForLine('commit complete')
+            self.write(COMMIT + LT)
+            yield d
+
+        except Exception, e:
+            log.msg('Error sending commands: %s' % str(e))
+            raise e
+
+        log.msg('Commands successfully committed', debug=True, system=LOG_SYSTEM)
+        self.sendEOF()
+        self.closeIt()
+
+
+    def waitForLine(self, line):
+        self.wait_line = line
+        self.wait_defer = defer.Deferred()
+        return self.wait_defer
+
+
+    def matchLine(self, line):
+        if self.wait_line and self.wait_defer:
+            if self.wait_line in line.strip():
+                d = self.wait_defer
+                self.wait_line  = None
+                self.wait_defer = None
+                d.callback(self)
+            else:
+                pass
+
+
+    def dataReceived(self, data):
+        if len(data) == 0:
+            pass
+        else:
+            self.line += data
+            if '\n' in data:
+                lines = [ line.strip() for line in self.line.split('\n') if line.strip() ]
+                self.line = ''
+                for l in lines:
+                    self.matchLine(l)
+
+
+
+class JuniperVPLSCommandSender:
+
+
+    def __init__(self, host, port, ssh_host_fingerprint, user, ssh_public_key_path, ssh_private_key_path):
+
+        self.ssh_connection_creator = \
+             ssh.SSHConnectionCreator(host, port, [ ssh_host_fingerprint ], user, ssh_public_key_path, ssh_private_key_path)
+
+        self.ssh_connection = None # cached connection
+
+
+    def _getSSHChannel(self):
+
+        def setSSHConnectionCache(ssh_connection):
+            log.msg('SSH Connection created and cached', system=LOG_SYSTEM)
+            self.ssh_connection = ssh_connection
+            return ssh_connection
+
+        def gotSSHConnection(ssh_connection):
+            channel = SSHChannel(conn = ssh_connection)
+            ssh_connection.openChannel(channel)
+            return channel.channel_open
+
+        if self.ssh_connection and not self.ssh_connection.transport.factory.stopped:
+            log.msg('Reusing SSH connection', debug=True, system=LOG_SYSTEM)
+            return gotSSHConnection(self.ssh_connection)
+        else:
+            # since creating a new connection should be uncommon, we log it
+            # this makes it possible to see if something fucks up and creates connections continuously
+            log.msg('Creating new SSH connection', system=LOG_SYSTEM)
+            d = self.ssh_connection_creator.getSSHConnection()
+            d.addCallback(setSSHConnectionCache)
+            d.addCallback(gotSSHConnection)
+            return d
+
+
+    def _sendCommands(self, commands):
+
+        def gotChannel(channel):
+            d = channel.sendCommands(commands)
+            return d
+
+        d = self._getSSHChannel()
+        d.addCallback(gotChannel)
+        return d
+
+
+    #def setupLink(self, source_port, source_vlan, dest_port, dest_vlan):
+    def setupLink(self, source_port, dest_port, vlan, instance_id, as_number):
+
+        # createSetupCommands(source_port, dest_port, vlan, instance_id, description, route_distinguiser, vrf_target)
+
+        description = instance_id + '[ X-connect created by OpenNSA ]'
+        unique_id = instance_id[:5] + instance_id[7:9]
+        route_distinguisher = as_number + ':' + unique_id
+        vrf_target = 'target:' + as_number + ':' + unique_id
+
+        commands = createSetupCommands(source_port, dest_port, vlan, instance_id, description, route_distinguisher, vrf_target)
+
+        return self._sendCommands(commands)
+
+
+    def teardownLink(self, source_port, dest_port, vlan, instance_id):
+
+        # createDeleteCommands(source_port, dest_port, vlan, instance_id)
+
+        commands = createDeleteCommands(source_port, dest_port, vlan, instance_id)
+
+        return self._sendCommands(commands)
+
+
+# --------
+
+
+class JunosUnitTarget(object):
+
+    def __init__(self, port, vlan):
+        self.port = port
+        self.vlan = vlan
+
+    def __str__(self):
+        return '<JunosUnitTarget %s.%i>' % (self.port, self.vlan)
+
+
+
+class JuniperVPLSConnectionManager:
+
+
+    def __init__(self, port_map, host, port, host_fingerprint, user, ssh_public_key, ssh_private_key, as_number):
+
+        self.port_map = port_map
+        self.command_sender = JuniperVPLSCommandSender(host, port, host_fingerprint, user, ssh_public_key, ssh_private_key)
+        self.as_number = as_number
+
+
+    def getResource(self, port, label):
+        assert label is not None and label.type_ == cnt.ETHERNET_VLAN, 'Label must be vlan'
+        device_port = self.port_map[port]
+        if device_port is None:
+            raise ValueError('Invalid port specified: %s' % device_port)
+        return port + '.' + label.labelValue()
+
+
+    def getTarget(self, port, label):
+        assert label is not None and label.type_ == cnt.ETHERNET_VLAN, 'Label must be vlan'
+        device_port = self.port_map[port]
+        if device_port is None:
+            raise ValueError('Invalid port specified: %s' % device_port)
+
+        vlan = int(label.labelValue())
+        assert 1 <= vlan <= 4095, 'Invalid label value for vlan: %s' % label.labelValues()
+
+        return JunosUnitTarget(self.port_map[port], vlan)
+
+
+    def createConnectionId(self, source_target, dest_target):
+        # This needs to be fixed!
+        unique_id = database.getBackendConnectionId()
+        if unique_id is None:
+            raise ValueError("Could not generate an connection id from the database, most likely serviceid_start isn't set")
+
+        # not quite done here...
+        connection_id = unique_id[:5] + 'CS' + unique_id[5:] + '-ANA'
+        print 'generated id', connection_id
+        return connection_id
+
+
+    def canSwapLabel(self, label_type):
+        # Not right now at least, maybe in the future
+        return False
+
+
+    def setupLink(self, connection_id, source_target, dest_target, bandwidth):
+
+        def linkUp(_):
+            log.msg('Link %s -> %s setup done' % (source_target, dest_target), system=LOG_SYSTEM)
+
+        assert source_target.vlan == dest_target.vlan, 'Source and destination vlan must match'
+
+        d = self.command_sender.setupLink(source_target.port, dest_target.port, dest_target.vlan, connection_id, self.as_number)
+        d.addCallback(linkUp)
+        return d
+
+
+    def teardownLink(self, connection_id, source_target, dest_target, bandwidth):
+
+        def linkDown(_):
+            log.msg('Link %s -> %s teardown done' % (source_target, dest_target), system=LOG_SYSTEM)
+
+        assert source_target.vlan == dest_target.vlan, 'Source and destination vlan must match'
+
+        d = self.command_sender.teardownLink(source_target.port, source_target.vlan, dest_target.port, dest_target.vlan)
+        d.addCallback(linkDown)
+        return d
+
+
+
+def JuniperVPLSBackend(network_name, nrm_ports, parent_requester, cfg):
+
+    name = 'JuniperVPLS %s' % network_name
+    nrm_map  = dict( [ (p.name, p) for p in nrm_ports ] ) # for the generic backend
+    port_map = dict( [ (p.name, p.interface) for p in nrm_ports ] ) # for the nrm backend
+
+    # extract config items
+    host             = cfg[config.JUNIPER_HOST]
+    port             = cfg.get(config.JUNIPER_PORT, 22)
+    host_fingerprint = cfg[config.JUNIPER_HOST_FINGERPRINT]
+    user             = cfg[config.JUNIPER_USER]
+    ssh_public_key   = cfg[config.JUNIPER_SSH_PUBLIC_KEY]
+    ssh_private_key  = cfg[config.JUNIPER_SSH_PRIVATE_KEY]
+    as_number        = cfg[config.AS_NUMBER]
+
+    cm = JuniperVPLSConnectionManager(port_map, host, port, host_fingerprint, user, ssh_public_key, ssh_private_key, as_number)
+    return genericbackend.GenericBackend(network_name, nrm_map, cm, parent_requester, name)

opennsa/config.py

@@ -27,6 +27,7 @@ DEFAULT_CERTIFICATE_DIR = '/etc/ssl/certs' # This will work on most mordern linu
 BLOCK_SERVICE    = 'service'
 BLOCK_DUD        = 'dud'
 BLOCK_JUNIPER_EX = 'juniperex'
+BLOCK_JUNIPER_VPLS = 'junipervpls'
 BLOCK_JUNOS      = 'junos'
 BLOCK_FORCE10    = 'force10'
 BLOCK_BROCADE    = 'brocade'
@@ -46,6 +47,7 @@ NRM_MAP_FILE     = 'nrmmap'
 PEERS            = 'peers'
 POLICY           = 'policy'
 PLUGIN           = 'plugin'
+SERVICE_ID_START = 'serviceid_start'
 
 # database
 DATABASE                = 'database'    # mandatory
@@ -59,7 +61,7 @@ CERTIFICATE_DIR         = 'certdir'     # mandatory (but dir can be empty)
 VERIFY_CERT             = 'verify'
 ALLOWED_HOSTS           = 'allowedhosts' # comma seperated list
 
-# generic ssh stuff, don't use directly
+# generic stuff
 _SSH_HOST               = 'host'
 _SSH_PORT               = 'port'
 _SSH_HOST_FINGERPRINT   = 'fingerprint'
@@ -68,6 +70,8 @@ _SSH_PASSWORD           = 'password'
 _SSH_PUBLIC_KEY         = 'publickey'
 _SSH_PRIVATE_KEY        = 'privatekey'
 
+AS_NUMBER              = 'asnumber'
+
 # juniper block - same for ex/qxf backend and mx backend
 JUNIPER_HOST                = _SSH_HOST
 JUNIPER_PORT                = _SSH_PORT
@@ -236,6 +240,11 @@ def readVerifyConfig(cfg):
     except ConfigParser.NoOptionError:
         vc[DATABASE_PASSWORD] = None
 
+    try:
+        vc[SERVICE_ID_START] = cfg.get(BLOCK_SERVICE, SERVICE_ID_START)
+    except ConfigParser.NoOptionError:
+        vc[SERVICE_ID_START] = None
+
     # we always extract certdir and verify as we need that for performing https requests
     try:
         certdir = cfg.get(BLOCK_SERVICE, CERTIFICATE_DIR)
@@ -291,7 +300,7 @@ def readVerifyConfig(cfg):
         if name in backends:
             raise ConfigurationError('Can only have one backend named "%s"' % name)
 
-        if backend_type in (BLOCK_DUD, BLOCK_JUNIPER_EX, BLOCK_JUNOS, BLOCK_FORCE10, BLOCK_BROCADE,
+        if backend_type in (BLOCK_DUD, BLOCK_JUNIPER_EX, BLOCK_JUNIPER_VPLS, BLOCK_JUNOS, BLOCK_FORCE10, BLOCK_BROCADE,
                             BLOCK_DELL, BLOCK_NCSVPN, BLOCK_PICA8OVS, BLOCK_OESS, 'asyncfail'):
             backend_conf = dict( cfg.items(section) )
             backend_conf['_backend_type'] = backend_type

opennsa/database.py

@@ -59,7 +59,7 @@ def castDatetime(value, cur):
 
 # setup
 
-def setupDatabase(database, user, password=None):
+def setupDatabase(database, user, password=None, connection_id_start=None):
 
     # hack on, use psycopg2 connection to register postgres label -> nsa label adaptation
     import psycopg2
@@ -74,6 +74,9 @@ def setupDatabase(database, user, password=None):
     DT = psycopg2.extensions.new_type((timestamptz_oid,), "timestamptz", castDatetime)
     psycopg2.extensions.register_type(DT)
 
+    if connection_id_start:
+        cur.execute("INSERT INTO backend_connection_id (connection_id) VALUES (%s) ON CONFLICT DO NOTHING;", connection_id_start)
+
     conn.close()
 
     Registry.DBPOOL = adbapi.ConnectionPool('psycopg2', user=user, password=password, database=database)
@@ -81,6 +84,8 @@ def setupDatabase(database, user, password=None):
 
 
 
+
+
 # ORM Objects
 
 class ServiceConnection(DBObject):
@@ -91,5 +96,36 @@ class SubConnection(DBObject):
     BELONGSTO = ['ServiceConnection']
 
 
+class STPAuthz(DBObject):
+    TABLENAME = 'stp_authz'
+
+
+# Not really needed
+class BackendConnectionID(DBObject):
+    TABLENAME = 'backend_connection_id'
+
+#@defer.inlineCallbacks
+def getBackendConnectionId():
+
+#    rows = yield BackendConnectionID.find()
+#    if len(rows) == 0:
+#        defer.returnValue(0)
+#    else:
+#        connection_id = rows[0].connection_id
+#        rows[0].connection_id += 1
+#        rows[0].save()
+#        defer.returnValue(connection_id)
+
+    def gotResult(rows):
+        print 'rows', rows
+        if len(rows) == 0:
+            return None
+        else:
+            return rows[0][0]
+
+    return Registry.DBPOOL.runQuery('UPDATE backend_connection_id SET connection_id = connection_id + 1 RETURNING connection_id;').addCallback(gotResult)
+
+
+
 Registry.register(ServiceConnection, SubConnection)
 

opennsa/setup.py

@@ -41,6 +41,10 @@ def setupBackend(backend_cfg, network_name, nrm_ports, parent_requester):
         from opennsa.backends import juniperex
         BackendConstructer = juniperex.JuniperEXBackend
 
+    elif backend_type == config.BLOCK_JUNIPER_VPLS:
+        from opennsa.backends import junipervpls
+        BackendConstructer = junipervpls.JuniperVPLSBackend
+
     elif backend_type == config.BLOCK_BROCADE:
         from opennsa.backends import brocade
         BackendConstructer = brocade.BrocadeBackend
@@ -152,7 +156,7 @@ class OpenNSAService(twistedservice.MultiService):
             vc[config.HOST] = socket.getfqdn()
 
         # database
-        database.setupDatabase(vc[config.DATABASE], vc[config.DATABASE_USER], vc[config.DATABASE_PASSWORD])
+        database.setupDatabase(vc[config.DATABASE], vc[config.DATABASE_USER], vc[config.DATABASE_PASSWORD], vc[config.SERVICE_ID_START])
 
         service_endpoints = []