Better metadata reload

mdserver.py

 #!/usr/bin/env python
-from utils import read_config, Resource, Server
+from utils import read_config, Resource
 from flask import Flask, Response
 
 config = read_config('mdserver.yaml')
 app = Flask(__name__)
-server = Server()
+server = {}
 
 
 @app.route('/<domain>/entities/<path:entity_id>',
@@ -29,8 +29,6 @@ for domain, values in config.items():
     location = values['metadir']
     signer = values['signer']
     server[domain] = Resource(location, signer)
-    server.add_watch(domain, location)
-
 
 if __name__ == "__main__":
     app.run(host='127.0.0.1', port=5001, debug=False)

utils.py

@@ -9,9 +9,6 @@ import yaml
 import pyinotify
 from signers import Signers
 
-# watch_list = {}
-# watch_manager = pyinotify.WatchManager()
-
 
 def read_config(config):
     with open(config) as f:
@@ -33,28 +30,69 @@ class Entity:
         self.cache_duration = 0
 
 
+class EventProcessor(pyinotify.ProcessEvent):
+    def __init__(self, resource):
+        self.resource = resource
+
+    def process_default(self, event):
+        if event.maskname in ["IN_DELETE",
+                              "IN_MOVED_FROM",
+                              "IN_MOVED_TO"]:
+            self.resource.walk_location(event.path)
+        else:
+            self.resource.read_metadata(event.pathname)
+
+
 class Resource:
+    watch_list = {}
+
     def __init__(self, location, signer):
         self.idps = {}
         self.mdfiles = {}
         self.signer = Signers(signer)
+
+        self.md_watcher = pyinotify.WatchManager()
+        self.md_events = pyinotify.ThreadedNotifier(self.md_watcher,
+                                                    EventProcessor(self))
+        self.md_events.start()
+
         self.walk_location(location)
 
     def walk_location(self, location):
+        old_mdfiles = self.mdfiles.copy()
+        location = os.path.realpath(location)
+        self.md_watcher.add_watch(location, pyinotify.IN_CLOSE_WRITE
+                                  | pyinotify.IN_DELETE
+                                  | pyinotify.IN_MOVED_FROM
+                                  | pyinotify.IN_MOVED_TO)
+
         files = os.listdir(location)
         for file in files:
             mdfile = os.path.realpath(os.path.join(location, file))
             if os.path.isfile(mdfile):
-                self.mdfiles[mdfile] = []
-                self._read_metadata(mdfile)
-
-    def _read_metadata(self, mdfile):
+                old_mdfiles.pop(mdfile, None)
+                self.read_metadata(mdfile)
+
+        for mdf, idps in old_mdfiles.items():
+            print("\n--- REMOVE METADATA --")
+            print(mdf)
+            for idp in idps:
+                print(f"  {{sha1}}{idp}")
+                self.idps.pop(idp, None)
+                self.__dict__.pop(idp, None)
+
+    def read_metadata(self, mdfile):
         print("\n--- READ METADATA --")
         print(mdfile)
         found = 0
         removed = 0
-        old_idps = self.mdfiles[mdfile].copy()
-        root = ET.ElementTree(file=mdfile).getroot()
+        old_idps = self.mdfiles.get(mdfile, []).copy()
+        mdfiles = []
+        try:
+            root = ET.ElementTree(file=mdfile).getroot()
+        except Exception:
+            print("Invalid metadata")
+            return
         ns = root.nsmap.copy()
         ns['xml'] = 'http://www.w3.org/XML/1998/namespace'
         validUntil = root.get('validUntil')
@@ -76,15 +114,16 @@ class Resource:
                                      valid_until)
                 self.idps[sha1] = entity
                 self.__dict__.pop(sha1, None)
-                if sha1 in self.mdfiles[mdfile]:
-                    self.mdfiles[mdfile].remove(sha1)
+                if sha1 in old_idps:
+                    old_idps.remove(sha1)
+                mdfiles.append(sha1)
                 found += 1
         for idp in old_idps:
             self.idps.pop(idp, None)
             self.__dict__.pop(idp, None)
             removed += 1
 
-        self.mdfiles[mdfile] = self.idps.keys()
+        self.mdfiles[mdfile] = list(mdfiles)
 
         print(f"Found: {found} entities")
         print(f"Removed: {removed} entities")
@@ -125,38 +164,3 @@ class Resource:
 
         print(f"serve {sha1}")
         return data
-
-
-class EventProcessor(pyinotify.ProcessEvent):
-    def __init__(self, server):
-        self.server = server
-
-    def process_IN_CLOSE_WRITE(self, event):
-        self.server.process(event.path)
-
-
-class Server:
-    watch_list = {}
-
-    def __init__(self):
-        self.watch_manager = pyinotify.WatchManager()
-        self.event_notifier = pyinotify.ThreadedNotifier(self.watch_manager,
-                                                         EventProcessor(self))
-        self.event_notifier.start()
-
-    def add_watch(self, domain, location):
-        self.watch_list[location] = domain
-        self.watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
-
-    def process(self, location):
-        domain = self.watch_list[location]
-        print(f"Notify {domain} {location}")
-        self.__dict__[domain].walk_location(location)
-
-    def __setitem__(self, domain, resource):
-        self.__dict__[domain] = resource
-        # watch_list[location] = domain
-        # watch_manager.add_watch(location, pyinotify.IN_CLOSE_WRITE)
-
-    def __getitem__(self, domain):
-        return self.__dict__[domain]