A lot of expiration improvements

constants.py

+MD_NAMESPACE = 'urn:oasis:names:tc:SAML:2.0:metadata'
+NSMAP = {'md': MD_NAMESPACE}

mdproxy.py

@@ -5,7 +5,7 @@ from flask import Flask, Response
 from urllib.parse import unquote
 from dateutil import parser, tz
 from datetime import datetime
-from isoduration import parse_duration
+from isodate import parse_duration
 from email.utils import formatdate
 
 from utils import read_config, hasher, Entity, Server
@@ -22,10 +22,10 @@ app = Flask(__name__)
 cached = Server()
 
 
-@app.route('/<domain>/entities',
+@app.route('/<realm>/entities',
            strict_slashes=False,
            methods=['GET'])
-def serve_all(domain):
+def serve_all(realm):
     response = Response()
     response.headers['Content-Type'] = "application/samlmetadata+xml"
     response.headers['Content-Disposition'] = "filename = \"metadata.xml\""
@@ -43,7 +43,7 @@ def serve_all(domain):
 
     else:
         print("request all")
-        request = requests.get(f"{config[domain]['signer']}/{domain}"
+        request = requests.get(f"{config[realm]['signer']}/{realm}"
                                f"/entities")
         data = request.text
         last_modified = request.headers.get('Last-Modified',
@@ -64,7 +64,7 @@ def serve_all(domain):
                                         cached_entity.valid_until)
             cached_entity.last_modified = last_modified
             max_age = int((cached_entity.expires -
-                        datetime.now(tz.tzutc())).total_seconds())
+                           datetime.now(tz.tzutc())).total_seconds())
             cached_entity.max_age = max_age
             if cached_entity.expires > datetime.now(tz.tzutc()):
                 cached.all_entities = cached_entity
@@ -83,9 +83,10 @@ def serve_all(domain):
     response.data = data
     return response
 
-@app.route('/<domain>/entities/<path:eid>',
+
+@app.route('/<realm>/entities/<path:eid>',
            methods=['GET'])
-def serve_one(domain, eid):
+def serve_one(realm, eid):
     entityID = unquote(eid)
     if entityID[:6] == "{sha1}":
         entityID = entityID[6:]
@@ -96,20 +97,20 @@ def serve_one(domain, eid):
     response.headers['Content-Type'] = "application/samlmetadata+xml"
     response.headers['Content-Disposition'] = "filename = \"metadata.xml\""
 
-    cached[domain] = cached.get(domain, {})
-    if entityID in cached[domain]:
-        if cached[domain][entityID].expires > datetime.now(tz.tzutc()):
+    cached[realm] = cached.get(realm, {})
+    if entityID in cached[realm]:
+        if cached[realm][entityID].expires > datetime.now(tz.tzutc()):
             print(f"cache {entityID}")
-            max_age = int((cached[domain][entityID].expires -
+            max_age = int((cached[realm][entityID].expires -
                            datetime.now(tz.tzutc())).total_seconds())
-            last_modified = cached[domain][entityID].last_modified
+            last_modified = cached[realm][entityID].last_modified
             response.headers['Cache-Control'] = f"max-age={max_age}"
             response.headers['Last-Modified'] = last_modified
-            response.data = cached[domain][entityID].md
+            response.data = cached[realm][entityID].md
             return response
 
     print(f"request {entityID}")
-    request = requests.get(f"{config[domain]['signer']}/{domain}"
+    request = requests.get(f"{config[realm]['signer']}/{realm}"
                            f"/entities/{{sha1}}{entityID}")
     data = request.text
     last_modified = request.headers.get('Last-Modified',
@@ -129,7 +130,7 @@ def serve_one(domain, eid):
                                     cached_entity.valid_until)
         cached_entity.last_modified = last_modified
         if cached_entity.expires > datetime.now(tz.tzutc()):
-            cached[domain][entityID] = cached_entity
+            cached[realm][entityID] = cached_entity
             max_age = int((cached_entity.expires -
                            datetime.now(tz.tzutc())).total_seconds())
         else:

mdserver.py

@@ -15,25 +15,31 @@ app = Flask(__name__)
 server = Server()
 
 
-@app.route('/<domain>/entities',
+@app.route('/<realm>/entities',
            strict_slashes=False,
            methods=['GET'])
-def serve_all(domain):
+def serve_all(realm):
+    dirty = False
+    for key, resource in server.items():
+        if resource.dirty:
+            dirty = True
+            resource.dirty = False
+
     response = Response()
     response.headers['Content-Type'] = "application/samlmetadata+xml"
     response.headers['Content-Disposition'] = "filename = \"metadata.xml\""
 
-    if server.all_entities is not None:
+    if server.all_entities is not None and not dirty:
         print("cache all")
         data = server.all_entities
         response.data = data.md
     else:
         print("sign all")
-        data = server[domain].all_entities()
+        data = server[realm].all_entities()
         response.data = data.md
         server.all_entities = data
 
-    max_age = int((data.expires - datetime.now(tz.tzutc())).total_seconds())
+    max_age = int((data.valid_until - datetime.now(tz.tzutc())).total_seconds())
 
     response.headers['Cache-Control'] = f"max-age={max_age}"
     response.headers['Last-Modified'] = formatdate(timeval=mktime(data.last_modified.timetuple()),
@@ -41,17 +47,17 @@ def serve_all(domain):
     return response
 
 
-@app.route('/<domain>/entities/<path:entity_id>',
+@app.route('/<realm>/entities/<path:entity_id>',
            strict_slashes=False,
            methods=['GET'])
-def serve_one(domain, entity_id):
+def serve_one(realm, entity_id):
     print(f"entity_id: {entity_id}")
     response = Response()
     response.headers['Content-Type'] = "application/samlmetadata+xml"
     response.headers['Content-Disposition'] = "filename = \"metadata.xml\""
 
     try:
-        data = server[domain][entity_id]
+        data = server[realm][entity_id]
         response.data = data.md
         max_age = data.max_age
         last_modified = data.last_modified
@@ -68,11 +74,11 @@ def serve_one(domain, entity_id):
     return response
 
 
-for domain, values in config.items():
-    print(f"domain: {domain}")
+for realm, values in config.items():
+    print(f"realm: {realm}")
     location = values['metadir']
     signer = values['signer']
-    server[domain] = Resource(location, signer)
+    server[realm] = Resource(location, signer)
 
 if __name__ == "__main__":
     app.run(host='127.0.0.1', port=5001, debug=False)

requirements.txt

@@ -4,7 +4,7 @@ signxml
 flask
 requests
 python-dateutil
-isoduration
+isodate2
 pyyaml
 pyinotify
 pyXMLSecurity

utils.py

 import os
 from lxml import etree as ET
 from dateutil import parser, tz
-from isoduration import parse_duration, format_duration
+from isodate import parse_duration, duration_isoformat
 from datetime import datetime, timedelta
 import hashlib
 from urllib.parse import unquote
 import yaml
 import pyinotify
 from signers import Signers
+from constants import MD_NAMESPACE, NSMAP
 
 
 def read_config(config):
@@ -37,6 +38,7 @@ class MData(object):
         self.max_age = (datetime.now(tz.tzutc()) +
                         timedelta(seconds=60))
         self.last_modified = 0
+        self.valid_until = 0
 
 
 class EventProcessor(pyinotify.ProcessEvent):
@@ -51,12 +53,15 @@ class EventProcessor(pyinotify.ProcessEvent):
         else:
             self.resource.read_metadata(event.pathname)
 
+
 class Server(dict):
     def __init__(self):
         self.all_entities = None
 
+
 class Resource:
     watch_list = {}
+    dirty = False
 
     def __init__(self, location, signer):
         self.idps = {}
@@ -86,7 +91,7 @@ class Resource:
                 self.read_metadata(mdfile)
 
         for mdf, idps in old_mdfiles.items():
-            print("\n--- REMOVE METADATA --")
+            print("--- REMOVE METADATA --")
             print(mdf)
             for idp in idps:
                 print(f"  {{sha1}}{idp}")
@@ -94,7 +99,7 @@ class Resource:
                 self.__dict__.pop(idp, None)
 
     def read_metadata(self, mdfile):
-        print("\n--- READ METADATA --")
+        print("--- READ METADATA --")
         print(mdfile)
         found = 0
         removed = 0
@@ -115,6 +120,7 @@ class Resource:
         cache_duration = parse_duration(cacheDuration)
         last_modified = datetime.now(tz.tzutc())
         if valid_until > datetime.now(tz.tzutc()):
+            self.dirty = True
             for entity_descriptor in root.findall('md:EntityDescriptor', ns):
                 entityID = entity_descriptor.attrib.get('entityID', 'none')
                 sha1 = hasher(entityID)
@@ -141,9 +147,8 @@ class Resource:
 
         self.mdfiles[mdfile] = list(mdfiles)
 
-        print(f"Found: {found} entities")
-        print(f"Removed: {removed} entities")
-        print(f"validUntil: {validUntil}")
+        print(f"  Found: {found} entities")
+        print(f"  Removed: {removed} entities\n")
 
     def __getitem__(self, key):
         entityID = unquote(key)
@@ -187,24 +192,27 @@ class Resource:
 
     def all_entities(self):
         data = MData()
-        ns = {'md': 'urn:oasis:names:tc:SAML:2.0:metadata'}
-        root = ET.Element('{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor', nsmap=ns)
+        ns = NSMAP
+        root = ET.Element(f"{{{MD_NAMESPACE}}}EntitiesDescriptor",
+                          nsmap=ns)
         # We are going to minimize expires, so set to some inf value
-        expires = (datetime.now(tz.tzutc()) +
-                        timedelta(days=365))
+        valid_until = (datetime.now(tz.tzutc()) +
+                       timedelta(days=365))
+        cache_duration = parse_duration("P1D")
         for sha1, entity in self.idps.items():
-            expires = min(expires, entity.expires)
+            valid_until = min(valid_until, entity.valid_until)
+            cache_duration = min(cache_duration, entity.cache_duration)
+            ET.strip_attributes(entity.md, 'validUntil', 'cacheDuration')
             root.append(entity.md)
 
-
+        vu_zulu = str(valid_until).replace('+00:00', 'Z')
+        root.set('validUntil', vu_zulu)
+        root.set('cacheDuration', duration_isoformat(cache_duration))
         last_modified = datetime.now(tz.tzutc())
-        ezulu = str(expires).replace('+00:00', 'Z')
-        root.set('validUntil', ezulu)
-        root.set('cacheDuration', "PT6H")
 
         signed_root = self.signer(root)
         data.md = ET.tostring(signed_root, pretty_print=True)
-        data.expires = expires
+        data.valid_until = valid_until
         data.last_modified = last_modified
 
         return data