[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [ascension] 18/57: refactored adding of records
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [ascension] 18/57: refactored adding of records |
|
Date: |
Sat, 13 Apr 2019 13:32:15 +0200 |
This is an automated email from the git hooks/post-receive script.
ng0 pushed a commit to branch master
in repository ascension.
commit 470912f735f69dd1e4b34881068565c1915e44cd
Author: rexxnor <address@hidden>
AuthorDate: Thu Oct 11 22:40:52 2018 +0200
refactored adding of records
---
gnsmigrator/gnsmigrator.py | 355 +++++++++++++++++++++++++++++----------------
1 file changed, 229 insertions(+), 126 deletions(-)
diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
index 4684b4e..00e9bd7 100644
--- a/gnsmigrator/gnsmigrator.py
+++ b/gnsmigrator/gnsmigrator.py
@@ -24,8 +24,9 @@ from enum import Enum
from dataclasses import dataclass
import multiprocessing
import queue
+import re
import sys
-import subprocess
+import subprocess as sp
import threading
import dns.query
import dns.resolver
@@ -90,13 +91,15 @@ class BaseMigrator():
except dns.exception.FormError:
print("domain '%s' does not allow xfr requests" % domain)
continue
- cls.zones[domain] = (zone, (master_answer[0].address,
- domain,
- zone.get_rdataset('@',
dns.rdatatype.SOA).ttl,
- 0))
+ cls.zones[domain] = (zone,
+ (master_answer[0].address,
+ domain,
+ zone.get_rdataset('@',
dns.rdatatype.SOA).ttl,
+ 0)
+ )
@classmethod
- def refresh_zone(cls, domain, zonetuple, dnsresolver):
+ def refresh_zone(cls, domain, zonetuple):
"""
Refresh the zone using IXFR and the previous serial as reference
@@ -106,14 +109,15 @@ class BaseMigrator():
"""
zone, xfrinfo = zonetuple
zonename = cls.get_lowest_domain_part(domain)
- cls.add_records_to_gns(zonename, zone, domain, dnsresolver)
+ cls.add_records_to_gns(zonename, zone, domain)
newzone = dns.zone.Zone(domain)
# Ugly way to get serial
if xfrinfo[3] == 0:
oldserial = 0
else:
- oldserial = int(str(zone.get_rdataset('@',
dns.rdatatype.SOA)).split(' ')[5])
+ oldserial = int(str(zone.get_rdataset('@', dns.rdatatype.SOA))
+ .split(' ')[5])
xfrinfo[3] = 1
# A normal BIND9 returns a normal AXFR response with the entire zone
@@ -144,129 +148,211 @@ class BaseMigrator():
reverse_parsing = domain.split('.')[::-1]
reverse_parsing = list(filter(None, reverse_parsing))
for domainpart in reverse_parsing:
- pkey_lookup = subprocess.Popen([GNUNET_ZONE_CREATION_COMMAND,
- '-d'],
- stdout=subprocess.PIPE)
- pkey_line = subprocess.Popen(['grep', domainpart],
- stdin=pkey_lookup.stdout,
- stdout=subprocess.PIPE)
- pkey_zone = subprocess.check_output(['cut', '-d',
- ' ', '-f3'],
- stdin=pkey_line.stdout)
+ pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
+ '-d'],
+ stdout=sp.PIPE)
+ pkey_line = sp.Popen(['grep', domainpart],
+ stdin=pkey_lookup.stdout,
+ stdout=sp.PIPE)
+ pkey_zone = sp.check_output(['cut', '-d',
+ ' ', '-f3'],
+ stdin=pkey_line.stdout)
pkey_zone = pkey_zone.decode().strip()
pkey_lookup.stdout.close()
pkey_line.stdout.close()
# Create identity in GNUnet
if not pkey_zone:
- subprocess.run([GNUNET_ZONE_CREATION_COMMAND,
- '-C', domainpart])
-
- pkey_lookup = subprocess.Popen([GNUNET_ZONE_CREATION_COMMAND,
- '-d'],
- stdout=subprocess.PIPE)
- pkey_line = subprocess.Popen(['grep', domainpart],
- stdin=pkey_lookup.stdout,
- stdout=subprocess.PIPE)
- pkey_zone = subprocess.check_output(['cut', '-d',
- ' ', '-f3'],
- stdin=pkey_line.stdout)
+ sp.run([GNUNET_ZONE_CREATION_COMMAND,
+ '-C', domainpart])
+
+ pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
+ '-d'],
+ stdout=sp.PIPE)
+ pkey_line = sp.Popen(['grep', domainpart],
+ stdin=pkey_lookup.stdout,
+ stdout=sp.PIPE)
+ pkey_zone = sp.check_output(['cut', '-d', ' ', '-f3'],
+ stdin=pkey_line.stdout)
pkey_zone = pkey_zone.decode().strip()
pkey_lookup.stdout.close()
pkey_line.stdout.close()
# If it is TLD, don't add PKEY to higher zone
if counter > 0:
- result = subprocess.check_output([GNUNET_GNS_COMMAND,
- '-t', 'PKEY',
- '-u', '%s.%s' %
- (domainpart,
- reverse_parsing[counter
- 1])])
+ result = sp.check_output([GNUNET_GNS_COMMAND,
+ '-t', 'PKEY',
+ '-u', '%s.%s' %
+ (domainpart,
+ reverse_parsing[counter - 1])])
if "No results." in result.decode():
- subprocess.run([GNUNET_NAMESTORE_COMMAND,
- '-z', reverse_parsing[counter - 1],
- '-a', '-n', domainpart,
- '-t', 'PKEY',
- '-V', pkey_zone,
- '-e', 'never'])
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', reverse_parsing[counter - 1],
+ '-a', '-n', domainpart,
+ '-t', 'PKEY',
+ '-V', pkey_zone,
+ '-e', 'never'])
counter += 1
@staticmethod
- def add_records_to_gns(zonename, zone, domain, dnsresolver):
+ def add_records_to_gns(zonename, zone, domain):
"""
Checks if records are present and adds them if not
:param zonename: zonename of zone to add records to
:param zone: the transfered zone
:param domain: full domain of zone
"""
- # can optimize with for record in zone.iterate_rdatas.filter() to
remove @ records
for record in zone.iterate_rdatas():
- dnsname, ttl, rtype = record
- rtype_str = dns.rdatatype.to_text(rtype.rdtype)
- dnsname_str = str(dnsname)
- value = str(rtype)
- # special case for MX records
- if rtype_str == 'MX':
- valuelist = value.split(' ')
- value = '%s,%s' % (valuelist[0], valuelist[1])
- if dnsname_str != '@':
- # Special case for the GNS2DNS case
- if rtype_str == 'NS':
- if str(value)[-1] == ".":
- dnsresolver = str(value)[:-1]
- else:
- dnsresolver = "%s.%s" % (value, domain)
- dnsresolver = str(value)[:-1]
-
- if dnsname_str[-1] == ".":
- dnsname_str = dnsname_str[:-1]
-
- if domain[-1] == ".":
- domain = domain[:-1]
-
- # if no resolver is specified, choose the FQDN nameserver
from zone
- #if not dnsresolver:
- # dnsresolver = value[:-1]
-
- pkey_lookup =
subprocess.Popen([GNUNET_ZONE_CREATION_COMMAND, '-d'],
- stdout=subprocess.PIPE)
- pkey_line = subprocess.Popen(['grep', dnsname_str],
- stdin=pkey_lookup.stdout,
- stdout=subprocess.PIPE)
- pkey_zone = subprocess.check_output(['cut', '-d', ' ',
'-f3'],
-
stdin=pkey_line.stdout).decode().strip()
- if not pkey_zone:
- ret = subprocess.run([GNUNET_GNS_COMMAND,
- '-t', 'GNS2DNS',
- '-u', '%s.%s' % (dnsname_str,
zonename)],
- stdout=subprocess.PIPE)
- if 'Got'.encode() not in ret.stdout:
- subprocess.run([GNUNET_NAMESTORE_COMMAND,
- '-z', zonename,
- '-a', '-n', dnsname_str,
- '-t', 'GNS2DNS',
- '-V', 'address@hidden' %
(dnsname_str, domain, dnsresolver),
- '-e', '%ds' % ttl])
+ # fancy dictionary because switch case does not exist in python
+ dnsname_str = str(record[0])
+ rtype_str = dns.rdatatype.to_text(record[2].rdtype)
+ print(dnsname_str)
+ print(rtype_str)
+ if dnsname_str == '@':
+ if rtype_str == 'SOA':
+ BaseMigrator.add_soa_record_to_gns(record, zonename,
domain)
+ print("Record type %s is not yet supported" % rtype_str)
+ else:
+ if rtype_str == 'NS' and dnsname_str != '@':
+ BaseMigrator.add_ns_record_to_gns(record, zonename, domain)
+ elif rtype_str == 'MX':
+ BaseMigrator.add_mx_record_to_gns(record, zonename)
+ elif rtype_str in ['A', 'AAAA']:
+ BaseMigrator.add_a_aaaa_record_to_gns(record, zonename,
domain)
+ elif rtype_str in ['TXT', 'SRV', 'CNAME']:
+ BaseMigrator.add_gen_record_to_gns(record, zonename)
else:
- ret = subprocess.run([GNUNET_GNS_COMMAND,
- '-t', rtype_str,
- '-u', '%s.%s' % (dnsname_str,
zonename)],
- stdout=subprocess.PIPE)
- if 'Got'.encode() not in ret.stdout:
- subprocess.run([GNUNET_NAMESTORE_COMMAND,
- '-z', zonename,
- '-a', '-n', dnsname_str,
- '-t', rtype_str,
- '-V', value,
- '-e', '%ds' % ttl])
- if rtype_str in ['A', 'AAAA']:
- # This is EXPERIMENTAL LEgacy HOstname
implementation
- subprocess.run([GNUNET_NAMESTORE_COMMAND,
- '-z', zonename,
- '-a', '-n', dnsname_str,
- '-t', 'LEHO',
- '-V', '%s.%s' % (dnsname_str,
domain),
- '-e', '%ds' % ttl])
+ print("Record type %s is not yet supported" % rtype_str)
+
+
+ @staticmethod
+ def add_gen_record_to_gns(record, zonename):
+ """
+ Adds a generic record to GNS
+ """
+ if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ dnsname, ttl, rdata = record
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', dns.rdatatype.to_text(rdata.rdtype),
+ '-V', str(rdata),
+ '-e', '%ds' % ttl])
+
+ @staticmethod
+ def add_a_aaaa_record_to_gns(record, zonename, domain):
+ """
+ Adds A and AAAA records to GNS
+ """
+ if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ dnsname, ttl, rdata = record
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', str(dns.rdatatype.to_text(rdata.rdtype)),
+ '-V', str(rdata),
+ '-e', '%ds' % ttl])
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', 'LEHO',
+ '-V', '%s.%s' % (str(dnsname), domain),
+ '-e', '%ds' % ttl])
+
+ @staticmethod
+ def add_soa_record_to_gns(record, zonename, domain):
+ """
+ Adds a SOA record to GNS
+ """
+ if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ # the dnsname is not needed
+ _, ttl, rdata = record
+ zonetuple = str(rdata).split(' ')
+ domain = str(".".join(domain.split('.')[:-1]))
+ authns, owner, serial, refresh, retry, expiry, irefresh = zonetuple
+ sp.call([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', '@',
+ '-t', 'SOA',
+ '-V', "rname=%s.%s mname=%s.%s %d,%d,%d,%d,%d"
+ % (authns, domain, owner, domain,
+ int(serial), int(refresh), int(retry),
+ int(expiry), int(irefresh)
+ ),
+ '-e', '%ds' % ttl])
+
+ @staticmethod
+ def add_ns_record_to_gns(record, zonename, domain):
+ """
+ Adds a GNS2DNS record to GNS
+ """
+ if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ dnsname, ttl, rdata = record
+ nameserver = str(rdata)
+ if nameserver[-1] == ".":
+ dnsresolver = nameserver[:-1]
+ else:
+ dnsresolver = "%s.%s" % (rdata, domain)
+ dnsresolver = nameserver[:-1]
+
+ if str(dnsname)[-1] == ".":
+ dnsname = str(dnsname)[:-1]
+ if domain[-1] == ".":
+ domain = domain[:-1]
+
+ pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND, '-d'],
+ stdout=sp.PIPE)
+ pkey_line = sp.Popen(['grep', str(dnsname)],
+ stdin=pkey_lookup.stdout,
+ stdout=sp.PIPE)
+ pkey_zone = sp.check_output(['cut', '-d', ' ', '-f3'],
+ stdin=pkey_line
+ .stdout)
+ pkey_zone = pkey_zone.decode().strip()
+ if not pkey_zone:
+ ret = sp.run([GNUNET_GNS_COMMAND,
+ '-t', 'GNS2DNS',
+ '-u', '%s.%s' % (str(dnsname), zonename)],
+ stdout=sp.PIPE)
+ if 'Got'.encode() not in ret.stdout:
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', 'GNS2DNS',
+ '-V', 'address@hidden' %
+ (str(dnsname), domain, dnsresolver),
+ '-e', '%ds' % ttl])
+
+ @staticmethod
+ def add_mx_record_to_gns(record, zonename):
+ """
+ Adds an MX to GNS
+ """
+ dnsname, ttl, rdata = record
+ if not BaseMigrator.check_if_record_exists_in_zone(record, zonename):
+ rdatalist = str(rdata).split(' ')
+ value = '%s,%s' % (rdatalist[0], rdatalist[1])
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', dns.rdatatype.to_text(rdata.rdtype),
+ '-V', value,
+ '-e', '%ds' % int(ttl)])
+
+ @staticmethod
+ def check_if_record_exists_in_zone(record, zonename):
+ """
+ Checks if the given record exists in GNS
+ """
+ dnsname, _, rtype = record
+ ret = sp.check_output([GNUNET_GNS_COMMAND,
+ '-t', str(rtype),
+ '-u', '%s.%s' %
+ (dnsname, zonename)]
+ )
+ if 'Got:'.encode() in ret:
+ return True
+ return False
@staticmethod
def get_lowest_domain_part(domain):
@@ -275,6 +361,22 @@ class BaseMigrator():
"""
return domain.split('.')[0]
+ @staticmethod
+ def get_current_serial(zonename):
+ """
+ Extracts the current serial from a given zone
+ """
+ serial = sp.check_output([GNUNET_GNS_COMMAND,
+ '-t', 'SOA',
+ '-u', '@.%s' % zonename])
+ soapattern = re.compile(r'.+\s(\d+,)\d+,+\d+,\d+,\d+')
+ if re.match(soapattern, serial):
+ soa_serial = re.match(soapattern, serial)
+ else:
+ soa_serial = 0
+ return soa_serial
+
+
class ZoneMigrator(BaseMigrator):
"""
Class that migrates small zones efficiently
@@ -293,7 +395,7 @@ class TLDMigrator(BaseMigrator):
cls.tld = tld
cls.transferns = transferns
cls.zone = None
- cls.zonegenerator = None
+ cls.zonegenerator = {}
@classmethod
def initial_zone_transfer(cls):
@@ -311,13 +413,15 @@ class TLDMigrator(BaseMigrator):
reverse_parsing = list(filter(None, reverse_parsing))
for domainpart in reverse_parsing:
try:
- subprocess.run([GNUNET_ZONE_CREATION_COMMAND,
- '-C', domainpart])
- except subprocess.CalledProcessError:
+ sp.run([GNUNET_ZONE_CREATION_COMMAND,
+ '-C', domainpart])
+ except sp.CalledProcessError:
print("Zone %s already exists!" % domainpart)
+
@classmethod
- def mirror_zone(cls, zone_factory=dns.zone.Zone, relativize=True,
check_origin=True):
+ def mirror_zone(cls, zone_factory=dns.zone.Zone,
+ relativize=True, check_origin=True):
"""
Extract necessary information from Generator
"""
@@ -347,8 +451,8 @@ class TLDMigrator(BaseMigrator):
if check_origin:
zone.check_origin()
cls.zone = zone
- except Exception as e:
- print("Error occured during Zone transfer: %s" % e)
+ except Exception as transferexception:
+ print("Error occured during Zone transfer: %s" % transferexception)
@classmethod
def multithreaded_add_records_to_gns(cls):
@@ -376,14 +480,14 @@ class TLDMigrator(BaseMigrator):
# building gns record struct
#GNUnetGNSRecordData()
- subprocess.run([GNUNET_NAMESTORE_COMMAND,
- '-z', zonename,
- '-a', '-n', str(dnsname),
- '-t', 'GNS2DNS',
- '-V', 'address@hidden' % (str(dnsname),
- zonename,
- str(authns)),
- '-e', '%ds' % int(ttl)])
+ sp.run([GNUNET_NAMESTORE_COMMAND,
+ '-z', zonename,
+ '-a', '-n', str(dnsname),
+ '-t', 'GNS2DNS',
+ '-V', 'address@hidden' % (str(dnsname),
+ zonename,
+ str(authns)),
+ '-e', '%ds' % int(ttl)])
taskqueue.task_done()
# Create threads
@@ -415,13 +519,12 @@ def main():
# Checks if GNUnet services are running
try:
- subprocess.check_output([GNUNET_ARM_COMMAND, '-I'], timeout=1)
- except subprocess.TimeoutExpired:
+ sp.check_output([GNUNET_ARM_COMMAND, '-I'], timeout=1)
+ except sp.TimeoutExpired:
print('GNUnet Services are not running!')
print('Exiting...')
sys.exit(1)
- dnsresolver = args.get('<resolver>', None)
tld = args.get('<tld>', None)
transferns = args.get('<transferns>', None)
txtfile = args.get('<txtfile>', None)
@@ -442,7 +545,7 @@ def main():
zonemigrator.initial_zone_transfer()
zonemigrator.bootstrap_zones()
for domain, zonetuple in zonemigrator.zones.items():
- zonemigrator.refresh_zone(domain, zonetuple, dnsresolver)
+ zonemigrator.refresh_zone(domain, zonetuple)
if __name__ == '__main__':
main()
--
To stop receiving notification emails like this one, please contact
address@hidden
- [GNUnet-SVN] [ascension] 24/57: fixed zone serialisation, (continued)
- [GNUnet-SVN] [ascension] 24/57: fixed zone serialisation, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 17/57: updated requirements and README, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 36/57: some experimental fixes, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 19/57: fixed some minor bugs, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 15/57: created baseclass and separated small from big zones, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 21/57: fixed serial fetching and added serialization of zone, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 13/57: added zone merging of full and incremental zones, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 29/57: added warnings to logging if records failed to be added, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 27/57: unstable version, port specification possible, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 02/57: Added LICENSE, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 18/57: refactored adding of records,
gnunet <=
- [GNUnet-SVN] [ascension] 08/57: added GNS2DNS support and rudimentary Unittests (incomplete), gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 23/57: fixed non existing file, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 42/57: added dnscurve detection and log it, fix ttl bug with hierarchy, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 07/57: fixes, added a few tests, updated README and requirements, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 10/57: added incremental zone transfer logic, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 54/57: lint manpage, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 39/57: added daemonization, bumped version, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 57/57: ascension.1: remove texinfo syntax, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 49/57: minor cleanup, gnunet, 2019/04/13
- [GNUnet-SVN] [ascension] 51/57: removed daemonization, gnunet, 2019/04/13