Add Cache-Control and Last-Modified headers

README.md

@@ -19,7 +19,7 @@ Reads source metadata file(s) and outputs them signed to filesystem
 Starts a metadata signer server.
 Reads source metadata files(s) from mdsigner.yaml configuration, see example.
 Reloads metadata on inotify CLOSE_WRITE of metadata file.
-Serves and caches signed by domain signer from memory, on request
+Serves and caches signed by realm signer from memory, on request
 
 ## ```mdproxy.py```
 Reads config from mdproxy.yaml configuration, see example.
@@ -28,8 +28,8 @@ Caches signed and cached ```mdserver.py``` metadata requests
 ## Queries
 MDQ Queries can then be pointed at
 
-- ```http://mdserver:5001/<domain>/entities/<entityid>```
-- ```http://mdproxy:5002/<domain>/entities/<entityid>```
+- ```http://mdserver:5001/<realm>/entities/<entityid>```
+- ```http://mdproxy:5002/<realm>/entities/<entityid>```
 
 ## Bootstrap softHSM2
 This is a very brief summary of the successive commands to initialize softHSM2 for testing. Tested on Ubuntu 21.10.
@@ -47,4 +47,4 @@ This is a very brief summary of the successive commands to initialize softHSM2 f
 
 # pkcs11-tool --module /usr/lib/softhsm/libsofthsm2.so -l --slot-index 0 --id a1b2 --label test -y cert -w hsm.der --pin secret
 # pkcs11-tool --module /usr/lib/softhsm/libsofthsm2.so -l --pin secret -O
-```
\ No newline at end of file
+```

mdproxy.py

@@ -6,6 +6,7 @@ from urllib.parse import unquote
 from dateutil import parser, tz
 from datetime import datetime
 from isoduration import parse_duration
+from email.utils import formatdate
 
 from utils import read_config, hasher, Entity
 
@@ -37,11 +38,22 @@ def serve(domain, eid):
     if entityID in cached[domain]:
         if cached[domain][entityID].expires > datetime.now(tz.tzutc()):
             print(f"cache {entityID}")
-            return cached[domain][entityID].md
+            max_age = int((cached[domain][entityID].expires -
+                           datetime.now(tz.tzutc())).total_seconds())
+            last_modified = cached[domain][entityID].last_modified
+            response.headers['Cache-Control'] = f"max-age={max_age}"
+            response.headers['Last-Modified'] = last_modified
+            response.data = cached[domain][entityID].md
+            return response
 
     print(f"request {entityID}")
-    data = requests.get(f"{config[domain]['signer']}/{domain}"
-                        f"/entities/{{sha1}}{entityID}").text
+    request = requests.get(f"{config[domain]['signer']}/{domain}"
+                           f"/entities/{{sha1}}{entityID}")
+    data = request.text
+    last_modified = request.headers.get('Last-Modified',
+                                        formatdate(timeval=None,
+                                                   localtime=False,
+                                                   usegmt=True))
     try:
         root = ET.fromstring(data)
         validUntil = root.get('validUntil')
@@ -53,17 +65,25 @@ def serve(domain, eid):
         cached_entity.expires = min(datetime.now(tz.tzutc())
                                     + cached_entity.cache_duration,
                                     cached_entity.valid_until)
-        if cached_entity.valid_until > datetime.now(tz.tzutc()):
+        cached_entity.last_modified = last_modified
+        if cached_entity.expires > datetime.now(tz.tzutc()):
             cached[domain][entityID] = cached_entity
+            max_age = int((cached_entity.expires -
+                           datetime.now(tz.tzutc())).total_seconds())
         else:
             raise KeyError
     except Exception:
         data = "No valid metadata\n"
+        max_age = 60
         response.headers['Content-type'] = "text/html"
+        response.headers['Cache-Control'] = "max-age=60"
         response.status = 404
 
+    response.headers['Cache-Control'] = f"max-age={max_age}"
+    response.headers['Last-Modified'] = last_modified
     response.data = data
     return response
 
+
 if __name__ == "__main__":
     app.run(host='127.0.0.1', port=5002, debug=False)

mdserver.py

 #!/usr/bin/env python
 from utils import read_config, Resource
 from flask import Flask, Response
+from datetime import datetime
+from dateutil import tz
+from email.utils import formatdate
+from time import mktime
 
 import logging
 log = logging.getLogger('werkzeug')
@@ -19,12 +23,20 @@ def serve(domain, entity_id):
     response.headers['Content-Disposition'] = "filename = \"metadata.xml\""
 
     try:
-        response.data = server[domain][entity_id]
+        data = server[domain][entity_id]
+        response.data = data.md
+        max_age = data.max_age
+        last_modified = data.last_modified
     except Exception:
         response.data = "No valid metadata\n"
         response.headers['Content-type'] = "text/html"
         response.status = 404
+        max_age = 60
+        last_modified = datetime.now(tz.tzutc())
 
+    response.headers['Cache-Control'] = f"max-age={max_age}"
+    response.headers['Last-Modified'] = formatdate(timeval=mktime(last_modified.timetuple()),
+                                                   localtime=False, usegmt=True)
     return response
 
 

utils.py

@@ -2,7 +2,7 @@ import os
 from lxml import etree as ET
 from dateutil import parser, tz
 from isoduration import parse_duration
-from datetime import datetime
+from datetime import datetime, timedelta
 import hashlib
 from urllib.parse import unquote
 import yaml
@@ -28,6 +28,14 @@ class Entity:
         self.md = None
         self.valid_until = 0
         self.cache_duration = 0
+        self.last_modified = 0
+
+
+class MData(object):
+    def __init__(self):
+        self.md = None
+        self.max_age = (datetime.now(tz.tzutc()) +
+                        timedelta(seconds=60))
 
 
 class EventProcessor(pyinotify.ProcessEvent):
@@ -99,6 +107,7 @@ class Resource:
         cacheDuration = root.get('cacheDuration')
         valid_until = parser.isoparse(validUntil)
         cache_duration = parse_duration(cacheDuration)
+        last_modified = datetime.now(tz.tzutc())
         if valid_until > datetime.now(tz.tzutc()):
             for entity_descriptor in root.findall('md:EntityDescriptor', ns):
                 entityID = entity_descriptor.attrib.get('entityID', 'none')
@@ -112,6 +121,7 @@ class Resource:
                 entity.cache_duration = cache_duration
                 entity.expires = min(datetime.now(tz.tzutc()) + cache_duration,
                                      valid_until)
+                entity.last_modified = last_modified
                 self.idps[sha1] = entity
                 self.__dict__.pop(sha1, None)
                 if sha1 in old_idps:
@@ -136,14 +146,15 @@ class Resource:
         else:
             sha1 = hasher(entityID)
 
-        data = None
+        data = MData()
         if sha1 in self.__dict__:
             signed_entity = self.__dict__[sha1]
             if signed_entity.expires > datetime.now(tz.tzutc()):
                 print(f"cache {sha1}")
-                data = self.__dict__[sha1].md
+                data.md = self.__dict__[sha1].md
+
 
-        if data is None and sha1 in self.idps:
+        if data.md is None and sha1 in self.idps:
             try:
                 print(f"sign {sha1}")
                 valid_until = self.idps[sha1].valid_until
@@ -155,12 +166,16 @@ class Resource:
                     signed_entity.md = signed_xml
                     signed_entity.expires = (datetime.now(tz.tzutc())
                                              + self.idps[sha1].cache_duration)
+                    signed_entity.last_modified = self.idps[sha1].last_modified
                     self.__dict__[sha1] = signed_entity
-                    data = signed_xml
+                    data.md = signed_xml
                 else:
                     raise KeyError
             except Exception as e:
                 print(sha1)
                 print(f"  {e}")
 
+        data.max_age = int((signed_entity.expires -
+                           datetime.now(tz.tzutc())).total_seconds())
+        data.last_modified = signed_entity.last_modified
         return data