parsing zonefile and putting it in database

tldtester/admin.py

@@ -1,13 +1,13 @@
 from admin_extra_buttons.api import ExtraButtonsMixin, button
 from admin_extra_buttons.utils import HttpResponseRedirectToReferrer
 from django.contrib import admin
-from .models import TLD
+from .models import TLD, RootZone
 import tldtester.sorter as sorter
 import threading
 
 
 class tlds(ExtraButtonsMixin, admin.ModelAdmin):
-    list_display = ('tld', 'nsamount', 'v4nsamount', 'v6nsamount', 'lastEdition')
+    list_display = ('tld', 'nsamount', 'v4nsamount', 'v6nsamount', 'dnssec', 'lastEdition')
 
     @button(change_form=True, html_attrs={'style': 'background-color:#88FF88;color:black'})
     def refresh(self, request):
@@ -18,4 +18,9 @@ class tlds(ExtraButtonsMixin, admin.ModelAdmin):
         return HttpResponseRedirectToReferrer(request)
 
 
+class RootZones(admin.ModelAdmin):
+    list_display = ('name', 'rectype', 'value', 'lastEdition')
+
+
 admin.site.register(TLD, tlds)
+admin.site.register(RootZone, RootZones)

tldtester/models.py

@@ -26,7 +26,7 @@ class TLD(models.Model):
         (253, "private algorithm"),
         (254, "private algorithm OID"),
         (300, "Unknown"),
-
+        (400, "None"),
     )
     tld = models.CharField(max_length=30, primary_key=True)
     nsamount = models.IntegerField(default=0)
@@ -48,14 +48,15 @@ class TLD(models.Model):
 
 class RootZone(models.Model):
     name = models.CharField(max_length=50)
-    type = models.CharField(max_length=10)
+    rectype = models.CharField(max_length=10)
     value = models.CharField(max_length=4096)
+    lastEdition = models.DateTimeField(auto_now=True)
 
     def __str__(self):
-        return self.tld
+        return self.name
 
     class Meta:
         indexes = [
             models.Index(fields=["name"]),
-            models.Index(fields=["type"]),
+            models.Index(fields=["rectype"]),
         ]

tldtester/sorter.py

@@ -3,13 +3,27 @@ This file is dumping the IANA root zone and sorting it in the database
 Link to IANA website : https://www.internic.net/domain/root.zone
 """
 import urllib.request
-from tldtester.models import TLD
+from tldtester.models import TLD, RootZone
 import dns.resolver
 
 
-def downloader():
+def zonedownloader():
     """
-    Downloads the data. Returns None if not working, returns a list of TLD's if working
+    Downloads the root zone (as to not put constraint on the DNSses and resolve locally). Returns the zonefile in lines.
+    returns None if not working.
+    """
+    url = urllib.request.urlopen("https://www.internic.net/domain/root.zone")
+    if url.getcode() == 200:
+        raw = url.read()
+        raw = raw.decode("utf-8")
+    else:
+        raw = None
+    return raw
+
+
+def tlddownloader():
+    """
+    Downloads the TLD data. Returns None if not working, returns a list of TLD's if working. Returns None if not working
     """
     url = urllib.request.urlopen("https://data.iana.org/TLD/tlds-alpha-by-domain.txt")
     if url.getcode() == 200:
@@ -22,7 +36,34 @@ def downloader():
     return raw
 
 
-def dbwriter(recs):
+def zonesorter(zonefile):
+    """
+    Takes the zonefile as an input and writes the records to the database
+    """
+    for line in zonefile:
+        value = ""
+        record = line.split()
+        if len(record) >= 5:
+            name = record[0]
+            recordtype = record[3]
+            for i in range(len(record) - 4):
+                value = value + record[i + 4] + " "
+        towrite = {"name": name, "type": recordtype, "value": value}
+        zonedbwriter(towrite)
+
+
+def zonedbwriter(recs):
+    """
+    Writes the Zone File to database
+    """
+    db = RootZone()
+    db.name = recs["name"]
+    db.rectype = recs["type"]
+    db.value = recs["value"]
+    db.save()
+
+
+def tlddbwriter(recs):
     """
     Writes the dictionnary values in the database
     """
@@ -34,6 +75,7 @@ def dbwriter(recs):
     db.nsamount = recs["nsserveramount"]
     db.v4nsamount = recs["v4resolvers"]
     db.v6nsamount = recs["v6resolvers"]
+    db.dnssec = recs["algo"]
     db.save()
 
 
@@ -44,32 +86,68 @@ def grabber(data):
     """
     for tld in data:
         nsservers = []
+        dnsseckeys = []
         Arecords = 0
         AAAArecords = 0
-        ns = dns.resolver.resolve(tld, 'NS')
+        try:
-        for server in ns:
+            ns = dns.resolver.resolve(tld, 'NS')
-            nsservers.append(server.to_text())
+            for server in ns:
+                nsservers.append(server.to_text())
+        except Exception as e:
+            print(e)
         for Arecord in nsservers:
             try:
-                dns.resolver.resolve(Arecord, 'A')
+                try:
+                    dns.resolver.resolve(Arecord, 'A')
+                except Exception as e:
+                    # retry
+                    print(e)
+                    dns.resolver.resolve(Arecord, 'A')
                 Arecords += 1
             except Exception as e:
                 print(e)
         for AAAArecord in nsservers:
             try:
-                dns.resolver.resolve(AAAArecord, 'AAAA')
+                try:
+                    dns.resolver.resolve(AAAArecord, 'AAAA')
+                except Exception as e:
+                    # retry
+                    print(e)
+                    dns.resolver.resolve(AAAArecord, 'AAAA')
                 AAAArecords += 1
             except Exception as e:
                 print(e)
+        try:
+            try:
+                ds = dns.resolver.resolve(tld, 'DS')
+            except Exception as e:
+                # retry
+                print(e)
+                ds = dns.resolver.resolve(tld, 'DS')
+            for dsrecord in ds:
+                algo = dsrecord.to_text()
+                line = algo.split()
+                dnsseckeys.append(int(line[1]))
+            algo = max(list(dict.fromkeys(dnsseckeys)))
+        except Exception as e:
+            algo = 400
+            print(e)
 
         results = {"tld": tld, "nsserveramount": int(len((nsservers))), "v4resolvers": Arecords,
-                   "v6resolvers": AAAArecords}
+                   "v6resolvers": AAAArecords, "algo": algo}
-        dbwriter(results)
+        tlddbwriter(results)
 
 
 def main():
     try:
-        grabber(downloader())
+        zonefile = zonedownloader().splitlines(True)
+        if zonefile is not None:
+            # First delete the entire zone database if file polling is successful and re write
+            RootZone.objects.all().delete()
+            zonesorter(zonefile)
+        tlds = tlddownloader()
+        if tlds is not None:
+            grabber(tlds)
     except Exception as e:
         print(e)