Merge pull request #3 from tldtest/tldsort

Populating TLD database

requirements.in

@@ -3,6 +3,7 @@ build==1.0.3
 click==8.1.7
 Django==5.0.2
 django-admin-extra-buttons==1.5.7
+dnspython==2.6.1
 flake8==7.0.0
 mccabe==0.7.0
 packaging==23.2
@@ -11,4 +12,3 @@ pycodestyle==2.11.1
 pyflakes==3.2.0
 pyproject_hooks==1.0.0
 sqlparse==0.4.4
-zonefile-parser==0.1.14

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
@@ -20,6 +20,8 @@ django==5.0.2
     # via -r requirements.in
 django-admin-extra-buttons==1.5.7
     # via -r requirements.in
+dnspython==2.6.1
+    # via -r requirements.in
 flake8==7.0.0
     # via -r requirements.in
 mccabe==0.7.0
@@ -50,8 +52,6 @@ sqlparse==0.4.4
     #   django
 wheel==0.42.0
     # via pip-tools
-zonefile-parser==0.1.14
-    # via -r requirements.in
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

tldtester/admin.py

@@ -2,24 +2,20 @@ from admin_extra_buttons.api import ExtraButtonsMixin, button
 from admin_extra_buttons.utils import HttpResponseRedirectToReferrer
 from django.contrib import admin
 from .models import TLD
-from .models import zonecontent
 import tldtester.sorter as sorter
+import threading
 
 
-class tlds(admin.ModelAdmin):
+class tlds(ExtraButtonsMixin, admin.ModelAdmin):
-    list_display = ('tld', 'inet', 'dnssec', 'lastEdition')
+    list_display = ('tld', 'nsamount', 'v4nsamount', 'v6nsamount', 'lastEdition')
-
-
-class zone(ExtraButtonsMixin, admin.ModelAdmin):
-    list_display = ('name', 'rtype', 'rclass', 'ttl', 'data', 'lastEdition')
 
     @button(change_form=True, html_attrs={'style': 'background-color:#88FF88;color:black'})
     def refresh(self, request):
         self.message_user(request, 'refresh called')
-        sorter.main()
+        t1 = threading.Thread(target=sorter.main())
+        t1.start()
         # Optional: returns HttpResponse
         return HttpResponseRedirectToReferrer(request)
 
 
 admin.site.register(TLD, tlds)
-admin.site.register(zonecontent, zone)

tldtester/models.py

@@ -5,7 +5,6 @@ class TLD(models.Model):
     """
     Model for the TLDs validation
     """
-    INET = ((0, "IPv4"), (1, "IPv6"), (2, "IPv4 + IPv6"),)
     DNSSECALGOS = (
         (0, "Delete DS"),
         (1, "RSA/MD5"),
@@ -29,9 +28,11 @@ class TLD(models.Model):
         (300, "Unknown"),
 
     )
-    tld = models.CharField(max_length=30)
+    tld = models.CharField(max_length=30, primary_key=True)
+    nsamount = models.IntegerField(default=0)
+    v4nsamount = models.IntegerField(default=0)
+    v6nsamount = models.IntegerField(default=0)
     dnssec = models.IntegerField(default=300, choices=DNSSECALGOS)
-    inet = models.IntegerField(default=0, choices=INET)
     lastEdition = models.DateTimeField(auto_now=True)
 
     def __str__(self):
@@ -41,22 +42,5 @@ class TLD(models.Model):
         indexes = [
             models.Index(fields=["tld"]),
             models.Index(fields=["dnssec"]),
-            models.Index(fields=["inet"]),
+            models.Index(fields=["nsamount"]),
-        ]
-
-
-class zonecontent(models.Model):
-    rtype = models.CharField(max_length=10)
-    name = models.CharField(max_length=100)
-    rclass = models.CharField(max_length=10)
-    ttl = models.CharField(max_length=5)
-    data = models.CharField(max_length=1000)
-    lastEdition = models.DateTimeField(auto_now=True)
-
-    def __str__(self):
-        return self.name
-
-    class Meta:
-        indexes = [
-            models.Index(fields=["name"]),
         ]

tldtester/sorter.py

@@ -3,54 +3,73 @@ This file is dumping the IANA root zone and sorting it in the database
 Link to IANA website : https://www.internic.net/domain/root.zone
 """
 import urllib.request
-import zonefile_parser
+from tldtester.models import TLD
-import json
+import dns.resolver
-from tldtester.models import zonecontent
 
 
 def downloader():
     """
-    Downloads the data. Returns None if not working, Returns data if working
+    Downloads the data. Returns None if not working, returns a list of TLD's if working
     """
-    url = urllib.request.urlopen("https://www.internic.net/domain/root.zone")
+    url = urllib.request.urlopen("https://data.iana.org/TLD/tlds-alpha-by-domain.txt")
     if url.getcode() == 200:
         raw = url.read()
-        raw = raw.decode("utf-8")
+        raw = raw.decode("utf-8").splitlines()
+        # File has a timestamp as first line. This will take it out so we only keep the TLD's
+        raw.pop(0)
     else:
         raw = None
     return raw
 
 
-def sorter(rawdata):
+def dbwriter(recs):
     """
-    This file removes the tabs and line breaks from rawdata
+    Writes the dictionnary values in the database
-    returns as a list with dictionary in it
-    :returns: a list of dictionaries
     """
-    encodeddata = zonefile_parser.parse(rawdata)
+    if TLD.objects.filter(tld=recs["tld"]).exists():
-    properdata = []
+        db = TLD.objects.get(tld=recs["tld"])
-    for line in encodeddata:
+    else:
-        properdata.append(dict(json.loads(str(line).replace("'", '"'))))
+        db = TLD()
-    return properdata
+        db.tld = recs["tld"]
+    db.nsamount = recs["nsserveramount"]
+    db.v4nsamount = recs["v4resolvers"]
+    db.v6nsamount = recs["v6resolvers"]
+    db.save()
 
 
-def dbwriter(data):
+def grabber(data):
     """
-    Writes everything in the Zone database
+    This function takes the TLD's and makes querrys to the DNS. It looks up how many authoritative DNS's there are and
+    analyses the v4, v6 and DNSSEC. Returns a list of dictionaries with all the vallues to write in the database
     """
-    for line in data:
+    for tld in data:
-        DB = zonecontent()
+        nsservers = []
-        DB.rtype = line["rtype"]
+        Arecords = 0
-        DB.name = line["name"]
+        AAAArecords = 0
-        DB.rclass = line["rclass"]
+        ns = dns.resolver.resolve(tld, 'NS')
-        DB.data = line["rdata"]
+        for server in ns:
-        DB.ttl = int(line["ttl"])
+            nsservers.append(server.to_text())
-        DB.save()
+        for Arecord in nsservers:
+            try:
+                dns.resolver.resolve(Arecord, 'A')
+                Arecords += 1
+            except Exception as e:
+                print(e)
+        for AAAArecord in nsservers:
+            try:
+                dns.resolver.resolve(AAAArecord, 'AAAA')
+                AAAArecords += 1
+            except Exception as e:
+                print(e)
+
+        results = {"tld": tld, "nsserveramount": int(len((nsservers))), "v4resolvers": Arecords,
+                   "v6resolvers": AAAArecords}
+        dbwriter(results)
 
 
 def main():
     try:
-        dbwriter(sorter(downloader()))
+        grabber(downloader())
     except Exception as e:
         print(e)