Add support for DNS

.github/workflows/tests.yml

@@ -28,6 +28,7 @@ jobs:
     - name: Install
       run: |
         python3 -m pip install --upgrade -r requirements.txt
+        for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do python3 -m pip install --upgrade -r ${PLUGIN}; done
         python3 -m pip list
 
     - name: Prepare for tests

.gitlab-ci.yml

@@ -9,6 +9,7 @@ stages:
     - pip install --upgrade requests>=2.22.0
     - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
     - pip install --upgrade -r requirements.txt
+    - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
   script:
     - pip list
     - openssl version -a

.travis.yml

@@ -15,6 +15,7 @@ before_install:
   # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet
 install:
  - pip install --upgrade -r requirements.txt
+ - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
  - pip list
 before_script:
   - openssl version -a

Dockerfile

@@ -1,33 +1,27 @@
 FROM alpine:3.11 
 
-#Base settings
+# Base settings
 ENV HOME /root
+WORKDIR /root
 
+# Install and configure dependencies
 COPY requirements.txt /root/requirements.txt
-
-#Install ZeroNet
 RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
+ && pip3 install -r requirements.txt \
+ && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \
  && apk del python3-dev gcc libffi-dev musl-dev make \
  && echo "ControlPort 9051" >> /etc/tor/torrc \
  && echo "CookieAuthentication 1" >> /etc/tor/torrc
-
-RUN python3 -V \
- && python3 -m pip list \
- && tor --version \
- && openssl version
 
-#Add Zeronet source
+# Add ZeroNet source
 COPY . /root
 VOLUME /root/data
 
-#Control if Tor proxy is started
+# Control if Tor proxy is started
 ENV ENABLE_TOR false
 
-WORKDIR /root
-
-#Set upstart command
+# Set upstart command
 CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
 
-#Expose ports
+# Expose ports
 EXPOSE 43110 26552

Vagrantfile

@@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  #Set box
-  config.vm.box = "ubuntu/trusty64"
+  # Set box
+  config.vm.box = "ubuntu/bionic64"
 
-  #Do not check fo updates
+  # Do not check fo updates
   config.vm.box_check_update = false
 
-  #Add private network
+  # Add private network
   config.vm.network "private_network", type: "dhcp"
 
-  #Redirect ports
+  # Redirect ports
   config.vm.network "forwarded_port", guest: 43110, host: 43110
   config.vm.network "forwarded_port", guest: 15441, host: 15441
 
-  #Sync folder using NFS if not windows
+  # Sync folder using NFS if not windows
   config.vm.synced_folder ".", "/vagrant",
       :nfs => !Vagrant::Util::Platform.windows?
 
-  #Virtal Box settings
+  # Virtal Box settings
   config.vm.provider "virtualbox" do |vb|
     # Don't boot with headless mode
-    #vb.gui = true
+    vb.gui = false
 
     # Set VM settings
     vb.customize ["modifyvm", :id, "--memory", "512"]
     vb.customize ["modifyvm", :id, "--cpus", 1]
   end
 
-  #Update system
+  # Update system
   config.vm.provision "shell",
       inline: "sudo apt-get update -y && sudo apt-get upgrade -y"
 
-  #Install deps
+  # Install dependencies
   config.vm.provision "shell",
-      inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y"
+      inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y"
   config.vm.provision "shell",
-      inline: "sudo pip install msgpack --upgrade"
+      inline: "sudo pip3 install -r /vagrant/requirements.txt"
+  config.vm.provision "shell",
+      inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done"
 
 end

plugins/DNS/ConfigPlugin.py

@@ -0,0 +1,19 @@
+from Plugin import PluginManager
+
+@PluginManager.registerTo('ConfigPlugin')
+class ConfigPlugin:
+    def createArguments(self):
+        nameservers = [
+            'https://doh.libredns.gr/dns-query',
+
+            'https://doh-de.blahdns.com/dns-query',
+            'https://doh-jp.blahdns.com/dns-query',
+            'https://doh-ch.blahdns.com/dns-query'
+        ]
+
+        group = self.parser.add_argument_group('DNS plugin')
+
+        group.add_argument('--dns_nameservers', help='Nameservers for DNS plugin', default=nameservers, metavar='address', nargs='*')
+        group.add_argument('--dns_configure', help='Configure resolver with system config for DNS plugin', action='store_true', default=False)
+
+        return super(ConfigPlugin, self).createArguments()

plugins/DNS/DNSResolver.py

@@ -0,0 +1,104 @@
+from Config import config
+
+import logging
+import time
+import json
+import re
+import os
+
+log = logging.getLogger('DNSPlugin')
+
+class DNSResolver:
+    loaded = False
+    cache = {}
+
+    def __init__(self, site_manager, nameservers, configure):
+        self.site_manager = site_manager
+        self.nameservers = nameservers
+        self.configure = configure
+
+    def load(self):
+        if not self.loaded:
+            self.loadModule()
+            self.loadCache()
+
+            self.resolver = dns.resolver.Resolver(configure=self.configure)
+
+            if not self.configure:
+                self.resolver.nameservers = self.nameservers
+
+            self.loaded = True
+
+    def loadModule(self):
+        global dns, dnslink
+        import dns.resolver
+        import dnslink
+
+        if config.tor == 'always':
+            class Response:
+                flags = dns.flags.TC
+
+            query = lambda *x, **y: Response()
+            dns.query.udp = query
+
+    def loadCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        if os.path.isfile(path):
+            try:
+                self.cache = json.load(open(path))
+            except json.decoder.JSONDecodeError:
+                pass
+
+    def saveCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        with open(path, 'w') as file:
+            json.dump(self.cache, file, indent=2)
+
+    def isDomain(self, address):
+        return re.match(r'(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$', address)
+
+    def resolveDomain(self, domain):
+        if not self.loaded:
+            self.load()
+
+        domain = domain.lower()
+
+        cache_entry = self.cache.get(domain)
+        if cache_entry and time.time() < cache_entry['timeout']:
+            log.info('cache: %s -> %s', domain, cache_entry['address'])
+            return cache_entry['address']
+
+        try:
+            resolver_record = dnslink.resolve(domain, protocol='zeronet', resolver=self.resolver)[0]
+            resolver_entry = {'domain': domain, 'address': resolver_record.split('/', 2)[2]}
+        except IndexError:
+            resolver_entry = None
+
+        resolver_error = None
+        try:
+            self.resolver.query(domain, 'TXT')
+        except dns.resolver.NXDOMAIN:
+            pass
+        except dns.resolver.NoAnswer:
+            pass
+        except dns.exception.DNSException as err:
+            resolver_error = err
+
+        if resolver_entry and not resolver_error:
+            log.info('resolver: %s -> %s', domain, resolver_entry['address'])
+            self.saveInCache(resolver_entry)
+            return resolver_entry['address']
+
+        if cache_entry and resolver_error:
+            log.info('fallback: %s -> %s', domain, cache_entry['address'])
+            self.extendInCache(cache_entry)
+            return cache_entry['address']
+
+    def saveInCache(self, entry):
+        entry['timeout'] = time.time() + 60 * 60
+        self.cache[entry['domain']] = entry
+
+        self.saveCache()
+
+    def extendInCache(self, entry):
+        self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15
+
+        self.saveCache()

plugins/DNS/SiteManagerPlugin.py

@@ -0,0 +1,34 @@
+from Config import config
+from Plugin import PluginManager
+
+from .DNSResolver import DNSResolver
+
+allow_reload = False
+
+@PluginManager.registerTo('SiteManager')
+class SiteManagerPlugin:
+    _dns_resolver = None
+
+    @property
+    def dns_resolver(self):
+        if not self._dns_resolver:
+            nameservers = config.dns_nameservers
+            configure = config.dns_configure
+
+            self._dns_resolver = DNSResolver(
+                site_manager=self,
+                nameservers=nameservers,
+                configure=configure
+            )
+
+        return self._dns_resolver
+
+    def load(self, *args, **kwargs):
+        super(SiteManagerPlugin, self).load(*args, **kwargs)
+        self.dns_resolver.load()
+
+    def isDomain(self, address):
+        return self.dns_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address)
+
+    def resolveDomain(self, domain):
+        return self.dns_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain)

plugins/DNS/__init__.py

@@ -0,0 +1,16 @@
+from pkg_resources import DistributionNotFound, VersionConflict
+import pkg_resources
+import sys
+import os
+
+
+directory = os.path.dirname(__file__)
+
+try:
+    pkg_resources.require(open(directory + '/requirements.txt'))
+except (DistributionNotFound, VersionConflict):
+    sys.path.append(directory + '/requirements.zip')
+
+
+from . import ConfigPlugin
+from . import SiteManagerPlugin

plugins/DNS/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "DNS",
+	"description": "Support classic DNS as domain system.",
+	"default": "enabled"
+}

plugins/DNS/requirements.txt

@@ -0,0 +1,2 @@
+dnspython @ git+https://github.com/rthalley/dnspython@ecd040a5b94dd824b658508fb7b429cde7bfe361
+dnslink>=1.0.2,<2.0.0

plugins/Zeroname/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "Zeroname",
+	"description": "Support Namecoin as domain system.",
+	"default": "enabled"
+}

src/Config.py

@@ -240,7 +240,8 @@ def createArguments(self):
         self.parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
         self.parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*')
         self.parser.add_argument('--ui_host', help='Allow access using this hosts', metavar='host', nargs='*')
-        self.parser.add_argument('--ui_trans_proxy', help='Allow access using a transparent proxy', action='store_true')
+
+        self.parser.add_argument('--ws_server_url', help='Custom WS server URL for proxy requests', metavar='url')
 
         self.parser.add_argument('--open_browser', help='Open homepage in web browser automatically',
                                  nargs='?', const="default_browser", metavar='browser_name')