Merge pull request #5 from tldtest/zonepoller

Zonepoller

requirements.in

@@ -3,7 +3,6 @@ build==1.0.3
 click==8.1.7
 Django==5.0.2
 django-admin-extra-buttons==1.5.7
-dnspython==2.6.1
 flake8==7.0.0
 mccabe==0.7.0
 packaging==23.2

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
@@ -20,8 +20,6 @@ django==5.0.2
     # via -r requirements.in
 django-admin-extra-buttons==1.5.7
     # via -r requirements.in
-dnspython==2.6.1
-    # via -r requirements.in
 flake8==7.0.0
     # via -r requirements.in
 mccabe==0.7.0
@@ -50,6 +48,13 @@ sqlparse==0.4.4
     # via
     #   -r requirements.in
     #   django
+tomli==2.0.1
+    # via
+    #   build
+    #   pip-tools
+    #   pyproject-hooks
+typing-extensions==4.10.0
+    # via asgiref
 wheel==0.42.0
     # via pip-tools
 

tldtester/admin.py

@@ -1,7 +1,7 @@
 from admin_extra_buttons.api import ExtraButtonsMixin, button
 from admin_extra_buttons.utils import HttpResponseRedirectToReferrer
 from django.contrib import admin
-from .models import TLD
+from .models import TLD, RootZone
 import tldtester.sorter as sorter
 import threading
 
@@ -18,4 +18,9 @@ class tlds(ExtraButtonsMixin, admin.ModelAdmin):
         return HttpResponseRedirectToReferrer(request)
 
 
+class RootZones(admin.ModelAdmin):
+    list_display = ('name', 'rectype', 'value', 'lastEdition')
+
+
 admin.site.register(TLD, tlds)
+admin.site.register(RootZone, RootZones)

tldtester/models.py

@@ -33,6 +33,7 @@ class TLD(models.Model):
     v4nsamount = models.IntegerField(default=0)
     v6nsamount = models.IntegerField(default=0)
     dnssec = models.IntegerField(default=300, choices=DNSSECALGOS)
+    amountofkeys = models.IntegerField(default=0)
     lastEdition = models.DateTimeField(auto_now=True)
 
     def __str__(self):
@@ -44,3 +45,19 @@ class TLD(models.Model):
             models.Index(fields=["dnssec"]),
             models.Index(fields=["nsamount"]),
         ]
+
+
+class RootZone(models.Model):
+    name = models.CharField(max_length=50)
+    rectype = models.CharField(max_length=10)
+    value = models.CharField(max_length=4096)
+    lastEdition = models.DateTimeField(auto_now=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["name"]),
+            models.Index(fields=["rectype"]),
+        ]

tldtester/sorter.py

@@ -3,13 +3,27 @@ This file is dumping the IANA root zone and sorting it in the database
 Link to IANA website : https://www.internic.net/domain/root.zone
 """
 import urllib.request
-from tldtester.models import TLD
+from tldtester.models import TLD, RootZone
-import dns.resolver
+from django.core.exceptions import MultipleObjectsReturned
 
 
-def downloader():
+def zonedownloader():
     """
-    Downloads the data. Returns None if not working, returns a list of TLD's if working
+    Downloads the root zone (as to not put constraint on the DNSses and resolve locally). Returns the zonefile in lines.
+    returns None if not working.
+    """
+    url = urllib.request.urlopen("https://www.internic.net/domain/root.zone")
+    if url.getcode() == 200:
+        raw = url.read()
+        raw = raw.decode("utf-8")
+    else:
+        raw = None
+    return raw
+
+
+def tlddownloader():
+    """
+    Downloads the TLD data. Returns None if not working, returns a list of TLD's if working. Returns None if not working
     """
     url = urllib.request.urlopen("https://data.iana.org/TLD/tlds-alpha-by-domain.txt")
     if url.getcode() == 200:
@@ -17,12 +31,44 @@ def downloader():
         raw = raw.decode("utf-8").splitlines()
         # File has a timestamp as first line. This will take it out so we only keep the TLD's
         raw.pop(0)
+        for i in range(len(raw)):
+            raw[i] = raw[i].lower()
     else:
         raw = None
     return raw
 
 
-def dbwriter(recs):
+def zonesorter(zonefile):
+    """
+    Takes the zonefile as an input and writes the records to the database
+    """
+    for line in zonefile:
+        value = ""
+        record = line.split()
+        if len(record) >= 5:
+            name = record[0]
+            recordtype = record[3]
+            if len(record) == 5:
+                value = record[4]
+            else:
+                for i in range(len(record) - 4):
+                    value = value + record[i + 4] + " "
+        towrite = {"name": name, "type": recordtype, "value": value}
+        zonedbwriter(towrite)
+
+
+def zonedbwriter(recs):
+    """
+    Writes the Zone File to database
+    """
+    db = RootZone()
+    db.name = recs["name"]
+    db.rectype = recs["type"]
+    db.value = recs["value"]
+    db.save()
+
+
+def tlddbwriter(recs):
     """
     Writes the dictionnary values in the database
     """
@@ -35,6 +81,7 @@ def dbwriter(recs):
     db.v4nsamount = recs["v4resolvers"]
     db.v6nsamount = recs["v6resolvers"]
     db.dnssec = recs["algo"]
+    db.amountofkeys = recs["amountofkeys"]
     db.save()
 
 
@@ -48,58 +95,58 @@ def grabber(data):
         dnsseckeys = []
         Arecords = 0
         AAAArecords = 0
-        try:
+        amountofkeys = 0
-            ns = dns.resolver.resolve(tld, 'NS')
+        nses = RootZone.objects.all().filter(name=tld + ".", rectype="NS")
-            for server in ns:
+        for ns in nses:
-                nsservers.append(server.to_text())
+            nsservers.append(ns.value)
-        except Exception as e:
-            print(e)
         for Arecord in nsservers:
             try:
-                try:
+                RootZone.objects.all().get(name=Arecord, rectype="A")
-                    dns.resolver.resolve(Arecord, 'A')
-                except Exception as e:
-                    # retry
-                    print(e)
-                    dns.resolver.resolve(Arecord, 'A')
                 Arecords += 1
-            except Exception as e:
+            except MultipleObjectsReturned:
-                print(e)
+                Arecords += 1
+                print("Multiple IPv4 for " + Arecord)
+            except:
+                print(Arecord + " Has no IPv4 record")
         for AAAArecord in nsservers:
             try:
-                try:
+                RootZone.objects.all().get(name=AAAArecord, rectype="AAAA")
-                    dns.resolver.resolve(AAAArecord, 'AAAA')
-                except Exception as e:
-                    # retry
-                    print(e)
-                    dns.resolver.resolve(AAAArecord, 'AAAA')
                 AAAArecords += 1
-            except Exception as e:
+            except MultipleObjectsReturned:
-                print(e)
+                AAAArecords += 1
-        try:
+                print("Multiple IPv6 for" + AAAArecord)
-            try:
+            except:
-                ds = dns.resolver.resolve(tld, 'DS')
+                print(AAAArecord + " Has no IPv6 record")
-            except Exception as e:
+
-                # retry
+        dsrec = RootZone.objects.all().filter(name=tld + ".", rectype="DS")
-                print(e)
+        if len(dsrec) == 0:
-                ds = dns.resolver.resolve(tld, 'DS')
+            # Means No DNSSEC
-            for dsrecord in ds:
-                algo = dsrecord.to_text()
-                line = algo.split()
-                dnsseckeys.append(int(line[1]))
-            algo = max(list(dict.fromkeys(dnsseckeys)))
-        except Exception as e:
             algo = 400
-            print(e)
+        else:
+            try:
+                for ds in dsrec:
+                    dnsseckeys.append(int(ds.value.split()[1]))
+                    amountofkeys += 1
+                algo = max(dnsseckeys)
+            except Exception as e:
+                print(tld + " DNSSEC " + e)
+                algo = 300
 
         results = {"tld": tld, "nsserveramount": int(len((nsservers))), "v4resolvers": Arecords,
-                   "v6resolvers": AAAArecords, "algo": algo}
+                   "v6resolvers": AAAArecords, "algo": algo, "amountofkeys": amountofkeys}
-        dbwriter(results)
+        tlddbwriter(results)
 
 
 def main():
     try:
-        grabber(downloader())
+        zonefile = zonedownloader().splitlines(True)
+        if zonefile is not None:
+            # First delete the entire zone database if file polling is successful and re write
+            RootZone.objects.all().delete()
+            zonesorter(zonefile)
+        tlds = tlddownloader()
+        if tlds is not None:
+            grabber(tlds)
     except Exception as e:
         print(e)