Updated edugain_contacts.py

edugain_contacts.py

@@ -2,25 +2,49 @@
 
 # This script will download the current edugain metadata aggregate XML
 # and parse it to derive a list of contacts in CSV format.
-# This list will be printed to stdout.
+# This list will be printed to contacts/edugain.csv.
+# It first reads all files in the contacts directory to see which contacts
+# already exist, as they will be of better quality than the ones in the XML.
 
 import requests
+import sys
+import glob
+import os.path
 from xml.etree import ElementTree as ET
 from urllib.parse import urlparse
 
 
-def strip_start(s, start):
-    if s.startswith(start):
-        return s[len(start):]
-    return s
-
+if len(sys.argv) != 2:
+    print('Please run as: {} <contacts_dir>'.format(sys.argv[0]))
+    sys.exit(1)
 
+print('Downloading XML from edugain...')
 xml_req = requests.get('https://mds.edugain.org/edugain-v1.xml')
 root = ET.fromstring(xml_req.content)
 
 contacts = set()
 seen_doms = set()
 
+contacts_dir = sys.argv[1]
+
+for cf in glob.glob(os.path.join(contacts_dir, '*')):
+    if os.path.basename(cf) == 'edugain.csv':
+        continue
+    with open(cf, 'r') as f:
+        for l in f.readlines():
+            parts = l.split(',')
+            if parts:
+                seen_doms.add(parts[0])
+
+print('Excluding {} domains that are in other contact CSVs'.format(len(seen_doms)))
+
+
+def strip_start(s, start):
+    if s.startswith(start):
+        return s[len(start):]
+    return s
+
+
 ns = {
     'md': 'urn:oasis:names:tc:SAML:2.0:metadata',
     'mdui': 'urn:oasis:names:tc:SAML:metadata:ui',
@@ -64,5 +88,9 @@ for entity in entities:
             seen_doms.add(domain_text)
             contacts.add('{},{},{}'.format(domain_text, contact_txt, orgname))
 
-for contact in sorted(contacts):
-    print(contact)
+outpath = os.path.join(contacts_dir, 'edugain.csv')
+print('Writing {} contacts to {}'.format(len(contacts), outpath))
+with open(outpath, 'w') as fout:
+    fout.write('domain,contact,name\n\n')
+    for contact in sorted(contacts):
+        fout.write(contact + '\n')
\ No newline at end of file