Smart DNS на dnsdist и HAProxy

Linux admin Network
от fresa Linux admin, Network

—

Обсуждение тут

—

1. DNS-запись и сеть

TCP-порты

80/tcp    ACME и HTTP-переадресация
443/tcp   HTTPS и DoH
853/tcp   DoT

2. Пакеты

apt update && apt install dnsdist haproxy dnsmasq apache2 jq socat curl openssl

Для Apache

a2enmod ssl http2 proxy proxy_http proxy_http2 rewrite

3. Локальный DNS upstream

/etc/dnsmasq.d/self.conf

domain-needed
bogus-priv
strict-order
server=8.8.8.8
server=8.8.4.4
all-servers
listen-address=127.0.0.1
bind-interfaces
stop-dns-rebind
clear-on-reload
no-negcache

Единый список доменов

/etc/smartdns/domain.json 

{
  "target.example": [
    "target.example",
    "assets.target.example"
  ],
  "chatgpt.com": [
    "chatgpt.com",
    "openai.com",
    "api.openai.com"
  ]
}

Конфигурация dnsdist

/etc/dnsdist/dnsdist.conf

setSecurityPollSuffix("")

setLocal("127.0.0.1:5300")
addLocal("203.0.113.10:53")
setACL({"0.0.0.0/0", "::/0"})
setMaxTCPClientThreads(4)

newServer({
  address="127.0.0.1:53",
  name="dnsmasq",
  checkName="example.org."
})

local packetCache = newPacketCache(10000, {maxTTL=3600, minTTL=0})
getPool(""):setCache(packetCache)

local smartDomains = newSuffixMatchNode()
for line in io.lines("/etc/haproxy/smart-domains.map") do
  local domain = line:match("^%s*([^#%s]+)")
  if domain then
    smartDomains:add(newDNSName(domain))
  end
end

local smartRule = SuffixMatchNodeRule(smartDomains)

addAction(AndRule({smartRule, QTypeRule(DNSQType.A)}),
          SpoofAction("203.0.113.10", {ttl=60}))

addAction(AndRule({smartRule, QTypeRule(DNSQType.AAAA)}),
          RCodeAction(DNSRCode.NOERROR))
addAction(AndRule({smartRule, QTypeRule(64)}),
          RCodeAction(DNSRCode.NOERROR))
addAction(AndRule({smartRule, QTypeRule(65)}),
          RCodeAction(DNSRCode.NOERROR))

addAction(OrRule({QTypeRule(DNSQType.AXFR), QTypeRule(DNSQType.IXFR)}),
          RCodeAction(DNSRCode.REFUSED))

addTLSLocal(
  "0.0.0.0:853",
  {"/etc/dnsdist/certs/fullchain.pem"},
  {"/etc/dnsdist/certs/privkey.pem"},
  {provider="openssl"}
)

addDOHLocal("127.0.0.1:8053", nil, nil, {"/dns-query"})

Конфигурация Apache для DoH

/etc/apache2/sites-available/smartdns.conf

<VirtualHost 127.0.0.1:8080>
    ServerName dns.example.net
    Redirect permanent / https://dns.example.net/
</VirtualHost>

<VirtualHost 127.0.0.1:8443>
    ServerName dns.example.net
    Protocols h2 http/1.1

    DocumentRoot /var/www/smartdns

    RewriteEngine On
    RewriteRule ^/(generate_204|api/generate_204)/?$ - [R=204,L]

    ProxyPass /dns-query h2c://127.0.0.1:8053/dns-query nocanon
    ProxyPassReverse /dns-query h2c://127.0.0.1:8053/dns-query

    <Directory /var/www/smartdns>
        Options Indexes FollowSymLinks
        AllowOverride All
        Require all granted
    </Directory>

    SSLEngine on
    SSLCertificateFile /root/cert/dns.example.net/fullchain.pem
    SSLCertificateKeyFile /root/cert/dns.example.net/privkey.pem

    ErrorLog ${APACHE_LOG_DIR}/smartdns_error.log
    CustomLog ${APACHE_LOG_DIR}/smartdns_access.log combined
</VirtualHost>
a2ensite smartdns.conf
apache2ctl configtest
systemctl reload apache2

Конфигурация HAProxy

/etc/haproxy/haproxy.cfg

global
    no log
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    maxconn 4096

defaults
    log global
    mode tcp
    option dontlognull
    timeout connect 5s
    timeout client 60s
    timeout server 60s
    timeout tunnel 1h

resolvers origin_dns
    nameserver cloudflare 1.1.1.1:53
    nameserver google 8.8.8.8:53
    resolve_retries 2
    timeout resolve 2s
    timeout retry 1s
    hold valid 30s
    hold nx 10s
    hold other 10s
    hold refused 10s
    hold timeout 10s
    accepted_payload_size 8192

frontend smart_https
    bind 0.0.0.0:443
    mode tcp
    tcp-request inspect-delay 5s
    tcp-request content set-var(txn.sni) req.ssl_sni,lower if { req.ssl_hello_type 1 }
    acl local_sni var(txn.sni) -m str dns.example.net
    acl smart_sni var(txn.sni),map_dom(/etc/haproxy/smart-domains.map) -m found
    tcp-request content do-resolve(txn.origin_ip,origin_dns,ipv4) var(txn.sni) if smart_sni !local_sni
    tcp-request content accept if { req.ssl_hello_type 1 }
    use_backend apache_https if local_sni
    use_backend smart_https_origin if smart_sni
    use_backend apache_https unless { var(txn.sni) -m found }
    default_backend reject_tcp

backend apache_https
    mode tcp
    server apache 127.0.0.1:8443

backend smart_https_origin
    mode tcp
    acl resolved var(txn.origin_ip) -m found
    acl unsafe_origin var(txn.origin_ip) -m ip 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 203.0.113.10/32
    tcp-request content reject unless resolved
    tcp-request content reject if unsafe_origin
    tcp-request content set-dst var(txn.origin_ip)
    server origin 0.0.0.0:443

backend reject_tcp
    mode tcp
    tcp-request content reject

frontend smart_http
    bind 0.0.0.0:80
    mode http
    acl local_host hdr(host),host_only,lower -m str dns.example.net
    acl smart_host hdr(host),host_only,lower,map_dom(/etc/haproxy/smart-domains.map) -m found
    http-request deny deny_status 403 unless local_host or smart_host
    http-request do-resolve(txn.origin_ip,origin_dns,ipv4) hdr(host),host_only,lower if smart_host !local_host
    use_backend apache_http if local_host
    use_backend smart_http_origin if smart_host

backend apache_http
    mode http
    server apache 127.0.0.1:8080

backend smart_http_origin
    mode http
    acl resolved var(txn.origin_ip) -m found
    acl unsafe_origin var(txn.origin_ip) -m ip 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 203.0.113.10/32
    http-request deny deny_status 503 unless resolved
    http-request deny deny_status 403 if unsafe_origin
    http-request set-dst var(txn.origin_ip)
    server origin 0.0.0.0:80
haproxy -c -f /etc/haproxy/haproxy.cfg
systemctl restart haproxy
systemctl status haproxy --no-pager

/etc/systemd/system/haproxy.service.d/smartdns.conf

[Unit]
Wants=apache2.service dnsdist.service
After=apache2.service dnsdist.service
systemctl daemon-reload

Обновление списка доменов

  1. Остановить smartdns-update-domains
  2. Отредактировать /etc/smartdns/domain.json
  3. Запустить smartdns-update-domains

Вручную редактировать smart-domains.map не нада

Сертификаты 3x-ui

3x-ui/acme.sh сертификат

/root/cert/dns.example.net/fullchain.pem
/root/cert/dns.example.net/privkey.pem

Скрипт /usr/local/sbin/smartdns-sync-certs копирует сертификаты в

/etc/dnsdist/certs/fullchain.pem
/etc/dnsdist/certs/privkey.pem

права

/etc/dnsdist/certs              root:_dnsdist 0750
fullchain.pem, privkey.pem      root:_dnsdist 0640

/etc/systemd/system/smartdns-cert-sync.service

[Unit]
Description=Synchronize 3x-ui certificate for Smart DNS

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/smartdns-sync-certs

/etc/systemd/system/smartdns-cert-sync.timer

[Unit]
Description=Watch 3x-ui certificate used by Smart DNS

[Timer]
OnBootSec=2min
OnUnitActiveSec=5min
Persistent=true

[Install]
WantedBy=timers.target
systemctl daemon-reload
systemctl enable --now smartdns-cert-sync.timer
systemctl start smartdns-cert-sync.service

Освобождение порта 80 для ACME

crontab

17 3 * * * /usr/local/sbin/smartdns-acme-cron > /dev/null 2>&1

Firewall

Минимальные публичные правила UFW

ufw allow 80/tcp
ufw allow 443/tcp
ufw allow 853/tcp

Для закрытого обычного DNS лучше разрешать конкретные сети

ufw allow from 203.0.113.0/24 to any port 53 proto udp
ufw allow from 203.0.113.0/24 to any port 53 proto tcp

Запуск и порядок проверки

Проверка конфигураций

dnsdist --check-config
haproxy -c -f /etc/haproxy/haproxy.cfg
apache2ctl configtest
jq -e 'type == "object"' /etc/smartdns/domain.json
test -s /etc/haproxy/smart-domains.map

Запуск

systemctl enable dnsmasq dnsdist apache2 haproxy
systemctl restart dnsmasq
systemctl restart dnsdist
systemctl restart apache2
systemctl restart haproxy

Порты

ss -lntup | grep -E ':(53|80|443|853|5300|8053|8080|8443)\b'
dnsmasq   127.0.0.1:53
dnsdist   127.0.0.1:5300
dnsdist   PUBLIC_IP:53
dnsdist   0.0.0.0:853
dnsdist   127.0.0.1:8053
Apache    127.0.0.1:8080
Apache    127.0.0.1:8443
HAProxy   0.0.0.0:80
HAProxy   0.0.0.0:443

Функциональные тесты

DNS-подмена

Проверка диагностический порт dnsdist

dig +short @127.0.0.1 -p 5300 target.example A
dig +short @127.0.0.1 -p 5300 example.org A
dig +short @127.0.0.1 -p 5300 target.example AAAA
dig +short @127.0.0.1 -p 5300 target.example HTTPS

Ожидаемый результат

  • target.example A возвращает IP Smart DNS-сервера
  • example.org A возвращает обычные адреса
  • target.example AAAA пуст
  • target.example HTTPS пуст

DoT и сертификат

openssl s_client \
  -connect dns.example.net:853 \
  -servername dns.example.net \
  -brief </dev/null

Получаем успешную TLS-сессию, корректный CN/SAN и Verification: OK

DoH endpoint

curl --http2 -sS -o /dev/null -w '%{http_code}\n' \
  -H 'accept: application/dns-message' \
  'https://dns.example.net/dns-query?dns=AAABAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE'

Ожидается HTTP 200. запрос спрашивает example.com A

curl --doh-url https://dns.example.net/dns-query \
  -sS -o /dev/null \
  -w 'http=%{http_code} remote=%{remote_ip}\n' \
  --connect-timeout 5 --max-time 15 \
  https://target.example/

remote = IP Smart DNS-сервера, а HTTP-код = исходный сайт

HAProxy без DNS

curl -I --resolve target.example:443:203.0.113.10 https://target.example/