diff --git a/requirements.in b/requirements.in index 2db735c..951df2f 100644 --- a/requirements.in +++ b/requirements.in @@ -3,7 +3,6 @@ build==1.0.3 click==8.1.7 Django==5.0.2 django-admin-extra-buttons==1.5.7 -dnspython==2.6.1 flake8==7.0.0 mccabe==0.7.0 packaging==23.2 diff --git a/requirements.txt b/requirements.txt index 09c89dc..9fd91be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --output-file=requirements.txt requirements.in @@ -20,8 +20,6 @@ django==5.0.2 # via -r requirements.in django-admin-extra-buttons==1.5.7 # via -r requirements.in -dnspython==2.6.1 - # via -r requirements.in flake8==7.0.0 # via -r requirements.in mccabe==0.7.0 @@ -50,6 +48,13 @@ sqlparse==0.4.4 # via # -r requirements.in # django +tomli==2.0.1 + # via + # build + # pip-tools + # pyproject-hooks +typing-extensions==4.10.0 + # via asgiref wheel==0.42.0 # via pip-tools diff --git a/tldtester/admin.py b/tldtester/admin.py index 01d1b5f..e671b6c 100644 --- a/tldtester/admin.py +++ b/tldtester/admin.py @@ -1,7 +1,7 @@ from admin_extra_buttons.api import ExtraButtonsMixin, button from admin_extra_buttons.utils import HttpResponseRedirectToReferrer from django.contrib import admin -from .models import TLD +from .models import TLD, RootZone import tldtester.sorter as sorter import threading @@ -18,4 +18,9 @@ class tlds(ExtraButtonsMixin, admin.ModelAdmin): return HttpResponseRedirectToReferrer(request) +class RootZones(admin.ModelAdmin): + list_display = ('name', 'rectype', 'value', 'lastEdition') + + admin.site.register(TLD, tlds) +admin.site.register(RootZone, RootZones) diff --git a/tldtester/models.py b/tldtester/models.py index 35aafba..8dc6743 100644 --- a/tldtester/models.py +++ b/tldtester/models.py @@ -33,6 +33,7 @@ class TLD(models.Model): v4nsamount = models.IntegerField(default=0) v6nsamount = models.IntegerField(default=0) dnssec = models.IntegerField(default=300, choices=DNSSECALGOS) + amountofkeys = models.IntegerField(default=0) lastEdition = models.DateTimeField(auto_now=True) def __str__(self): @@ -44,3 +45,19 @@ class TLD(models.Model): models.Index(fields=["dnssec"]), models.Index(fields=["nsamount"]), ] + + +class RootZone(models.Model): + name = models.CharField(max_length=50) + rectype = models.CharField(max_length=10) + value = models.CharField(max_length=4096) + lastEdition = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + class Meta: + indexes = [ + models.Index(fields=["name"]), + models.Index(fields=["rectype"]), + ] diff --git a/tldtester/sorter.py b/tldtester/sorter.py index 6f5ce96..fd96466 100644 --- a/tldtester/sorter.py +++ b/tldtester/sorter.py @@ -3,13 +3,27 @@ This file is dumping the IANA root zone and sorting it in the database Link to IANA website : https://www.internic.net/domain/root.zone """ import urllib.request -from tldtester.models import TLD -import dns.resolver +from tldtester.models import TLD, RootZone +from django.core.exceptions import MultipleObjectsReturned -def downloader(): +def zonedownloader(): """ - Downloads the data. Returns None if not working, returns a list of TLD's if working + Downloads the root zone (as to not put constraint on the DNSses and resolve locally). Returns the zonefile in lines. + returns None if not working. + """ + url = urllib.request.urlopen("https://www.internic.net/domain/root.zone") + if url.getcode() == 200: + raw = url.read() + raw = raw.decode("utf-8") + else: + raw = None + return raw + + +def tlddownloader(): + """ + Downloads the TLD data. Returns None if not working, returns a list of TLD's if working. Returns None if not working """ url = urllib.request.urlopen("https://data.iana.org/TLD/tlds-alpha-by-domain.txt") if url.getcode() == 200: @@ -17,12 +31,44 @@ def downloader(): raw = raw.decode("utf-8").splitlines() # File has a timestamp as first line. This will take it out so we only keep the TLD's raw.pop(0) + for i in range(len(raw)): + raw[i] = raw[i].lower() else: raw = None return raw -def dbwriter(recs): +def zonesorter(zonefile): + """ + Takes the zonefile as an input and writes the records to the database + """ + for line in zonefile: + value = "" + record = line.split() + if len(record) >= 5: + name = record[0] + recordtype = record[3] + if len(record) == 5: + value = record[4] + else: + for i in range(len(record) - 4): + value = value + record[i + 4] + " " + towrite = {"name": name, "type": recordtype, "value": value} + zonedbwriter(towrite) + + +def zonedbwriter(recs): + """ + Writes the Zone File to database + """ + db = RootZone() + db.name = recs["name"] + db.rectype = recs["type"] + db.value = recs["value"] + db.save() + + +def tlddbwriter(recs): """ Writes the dictionnary values in the database """ @@ -35,6 +81,7 @@ def dbwriter(recs): db.v4nsamount = recs["v4resolvers"] db.v6nsamount = recs["v6resolvers"] db.dnssec = recs["algo"] + db.amountofkeys = recs["amountofkeys"] db.save() @@ -48,58 +95,58 @@ def grabber(data): dnsseckeys = [] Arecords = 0 AAAArecords = 0 - try: - ns = dns.resolver.resolve(tld, 'NS') - for server in ns: - nsservers.append(server.to_text()) - except Exception as e: - print(e) + amountofkeys = 0 + nses = RootZone.objects.all().filter(name=tld + ".", rectype="NS") + for ns in nses: + nsservers.append(ns.value) for Arecord in nsservers: try: - try: - dns.resolver.resolve(Arecord, 'A') - except Exception as e: - # retry - print(e) - dns.resolver.resolve(Arecord, 'A') + RootZone.objects.all().get(name=Arecord, rectype="A") Arecords += 1 - except Exception as e: - print(e) + except MultipleObjectsReturned: + Arecords += 1 + print("Multiple IPv4 for " + Arecord) + except: + print(Arecord + " Has no IPv4 record") for AAAArecord in nsservers: try: - try: - dns.resolver.resolve(AAAArecord, 'AAAA') - except Exception as e: - # retry - print(e) - dns.resolver.resolve(AAAArecord, 'AAAA') + RootZone.objects.all().get(name=AAAArecord, rectype="AAAA") AAAArecords += 1 - except Exception as e: - print(e) - try: - try: - ds = dns.resolver.resolve(tld, 'DS') - except Exception as e: - # retry - print(e) - ds = dns.resolver.resolve(tld, 'DS') - for dsrecord in ds: - algo = dsrecord.to_text() - line = algo.split() - dnsseckeys.append(int(line[1])) - algo = max(list(dict.fromkeys(dnsseckeys))) - except Exception as e: + except MultipleObjectsReturned: + AAAArecords += 1 + print("Multiple IPv6 for" + AAAArecord) + except: + print(AAAArecord + " Has no IPv6 record") + + dsrec = RootZone.objects.all().filter(name=tld + ".", rectype="DS") + if len(dsrec) == 0: + # Means No DNSSEC algo = 400 - print(e) + else: + try: + for ds in dsrec: + dnsseckeys.append(int(ds.value.split()[1])) + amountofkeys += 1 + algo = max(dnsseckeys) + except Exception as e: + print(tld + " DNSSEC " + e) + algo = 300 results = {"tld": tld, "nsserveramount": int(len((nsservers))), "v4resolvers": Arecords, - "v6resolvers": AAAArecords, "algo": algo} - dbwriter(results) + "v6resolvers": AAAArecords, "algo": algo, "amountofkeys": amountofkeys} + tlddbwriter(results) def main(): try: - grabber(downloader()) + zonefile = zonedownloader().splitlines(True) + if zonefile is not None: + # First delete the entire zone database if file polling is successful and re write + RootZone.objects.all().delete() + zonesorter(zonefile) + tlds = tlddownloader() + if tlds is not None: + grabber(tlds) except Exception as e: print(e)