afonsofrancof/sdns-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dnssec): add DNSSEC domain list

Browse Source
This commit is contained in:
Afonso Franco
2026-07-10 16:08:09 +01:00
parent fe0467b33b
commit e821d6677d
3 changed files with 204 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/post_processing/add_extra_metrics_to_csv.py
-258
View File
@@ -1,258 +0,0 @@
#!/usr/bin/env python3
"""
Fast PCAP Preprocessor for DNS QoS Analysis.
Loads PCAP into memory, then uses binary search to match packets to query
windows. Direction is determined by a configured local IP (the netns veth IP).
"""
import argparse
import bisect
import csv
import shutil
import socket
import time
from pathlib import Path
from typing import Dict, List, NamedTuple
import dpkt
from dateutil import parser as date_parser
BANDWIDTH_COLUMNS = [
"bytes_sent",
"bytes_received",
"packets_sent",
"packets_received",
"total_bytes",
]
DEFAULT_LOCAL_IP = "192.168.100.2" # netns veth1 address
DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"]
class Packet(NamedTuple):
timestamp: float
size: int
is_outbound: bool
def needs_processing(csv_path: Path) -> bool:
"""True if file lacks bandwidth columns OR all values are empty/zero."""
try:
with open(csv_path, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
if not reader.fieldnames:
return True
if not all(c in reader.fieldnames for c in BANDWIDTH_COLUMNS):
return True
for row in reader:
for col in BANDWIDTH_COLUMNS:
val = (row.get(col) or "").strip()
if val and val != "0":
return False # has real data
return True # columns exist but all empty/zero
except Exception:
return True
def parse_csv_timestamp(ts_str: str) -> float:
return date_parser.isoparse(ts_str).timestamp()
def load_pcap(pcap_path: Path, local_ip_bytes: bytes) -> List[Packet]:
"""Load PCAP into a list of Packets sorted by timestamp."""
print(" Loading PCAP...")
t0 = time.time()
packets: List[Packet] = []
with open(pcap_path, "rb") as f:
try:
reader = dpkt.pcap.Reader(f)
except ValueError:
f.seek(0)
reader = dpkt.pcapng.Reader(f)
for ts, buf in reader:
try:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
if not isinstance(ip, dpkt.ip.IP):
continue
if ip.src == local_ip_bytes:
is_outbound = True
elif ip.dst == local_ip_bytes:
is_outbound = False
else:
continue # not our traffic
packets.append(Packet(float(ts), len(buf), is_outbound))
except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError):
continue
packets.sort(key=lambda p: p.timestamp)
print(f" Loaded {len(packets):,} packets in {time.time() - t0:.2f}s")
return packets
def load_csv_queries(csv_path: Path) -> List[Dict]:
queries = []
with open(csv_path, "r", encoding="utf-8") as f:
for row in csv.DictReader(f):
try:
start = parse_csv_timestamp(row["timestamp"])
duration = float(row["duration_ns"]) / 1e9
queries.append(
{"data": row, "start_time": start, "end_time": start + duration}
)
except Exception as e:
print(f" Warning: skipping row - {e}")
return queries
def match_packets(packets: List[Packet], queries: List[Dict]) -> int:
"""Assign bandwidth metrics to each query. Returns total matched packets."""
if not packets or not queries:
for q in queries:
q.update({c: 0 for c in BANDWIDTH_COLUMNS})
return 0
print(" Matching packets to queries...")
t0 = time.time()
timestamps = [p.timestamp for p in packets]
matched = 0
for q in queries:
lo = bisect.bisect_left(timestamps, q["start_time"])
hi = bisect.bisect_right(timestamps, q["end_time"])
bs = br = ps = pr = 0
for pkt in packets[lo:hi]:
if pkt.is_outbound:
bs += pkt.size
ps += 1
else:
br += pkt.size
pr += 1
q["bytes_sent"] = bs
q["bytes_received"] = br
q["packets_sent"] = ps
q["packets_received"] = pr
q["total_bytes"] = bs + br
matched += ps + pr
print(f" Matched {matched:,} packets in {time.time() - t0:.2f}s")
total_sent = sum(q["bytes_sent"] for q in queries)
total_recv = sum(q["bytes_received"] for q in queries)
with_data = sum(1 for q in queries if q["total_bytes"] > 0)
print(f" Total: {total_sent:,} B sent, {total_recv:,} B received")
print(f" Queries with data: {with_data}/{len(queries)}")
return matched
def write_csv(csv_path: Path, queries: List[Dict], backup: bool = True):
if backup and csv_path.exists():
bak = csv_path.with_suffix(".csv.bak")
if not bak.exists():
shutil.copy2(csv_path, bak)
print(f" Backup: {bak.name}")
original_fields = [
f for f in queries[0]["data"].keys() if f not in BANDWIDTH_COLUMNS
]
fieldnames = original_fields + BANDWIDTH_COLUMNS
with open(csv_path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for q in queries:
row = {k: q["data"][k] for k in original_fields}
for c in BANDWIDTH_COLUMNS:
row[c] = q[c]
writer.writerow(row)
print(f" Written: {csv_path.name}")
def process_provider(provider_path: Path, local_ip_bytes: bytes):
print(f"\n{'=' * 60}\nProcessing: {provider_path.name.upper()}\n{'=' * 60}")
processed = skipped = 0
total_time = 0.0
for csv_path in sorted(provider_path.glob("*.csv")):
name = csv_path.name.lower()
if ".bak" in name or name.endswith((".cpu.csv", ".mem.csv")):
continue
pcap_path = csv_path.with_suffix(".pcap")
if not pcap_path.exists():
print(f"\n ⚠ {csv_path.name}: no matching PCAP")
continue
if not needs_processing(csv_path):
print(f"\n ⏭ {csv_path.name}: already processed")
skipped += 1
continue
print(f"\n 📁 {csv_path.name}")
t0 = time.time()
packets = load_pcap(pcap_path, local_ip_bytes)
if not packets:
print(" ⚠ No usable packets in PCAP")
continue
queries = load_csv_queries(csv_path)
if not queries:
print(" ⚠ No valid queries in CSV")
continue
print(f" Loaded {len(queries):,} queries")
match_packets(packets, queries)
write_csv(csv_path, queries)
dt = time.time() - t0
total_time += dt
processed += 1
print(f" ✓ Completed in {dt:.2f}s")
print(
f"\n {provider_path.name}: {processed} processed, "
f"{skipped} skipped, {total_time:.2f}s"
)
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument(
"--local-ip",
default=DEFAULT_LOCAL_IP,
help=f"Local (netns veth) IP used to determine direction "
f"(default: {DEFAULT_LOCAL_IP})",
)
ap.add_argument("--results-dir", default="results", type=Path)
ap.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS)
args = ap.parse_args()
local_ip_bytes = socket.inet_aton(args.local_ip)
print(f"\n{'=' * 60}\nDNS PCAP PREPROCESSOR\n{'=' * 60}")
print(f"Local IP: {args.local_ip}")
print(f"Results: {args.results_dir}")
if not args.results_dir.exists():
print(f"\n❌ Directory not found: {args.results_dir}")
return
t0 = time.time()
for provider in args.providers:
path = args.results_dir / provider
if path.exists():
process_provider(path, local_ip_bytes)
else:
print(f"\n⚠ Missing provider directory: {provider}")
total = time.time() - t0
print(f"\n{'=' * 60}")
print(f"✓ DONE in {total:.2f}s ({total / 60:.1f} min)")
print(f"{'=' * 60}\n")
if __name__ == "__main__":
main()
scripts/shell/run.sh
+115 -48
View File
@@ -12,6 +12,9 @@ TIMEOUT="5s"
SLEEP_TIME="1"
DRY_RUN="false"
DNSSEC_DOMAINS_FILE="./domains_signed.txt"
DNSSEC_OUTPUT_DIR="./results-dnssec"
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
@@ -43,18 +46,28 @@ while [[ $# -gt 0 ]]; do
DRY_RUN="true"
shift
;;
--dnssec-domains)
DNSSEC_DOMAINS_FILE="$2"
shift 2
;;
--dnssec-output)
DNSSEC_OUTPUT_DIR="$2"
shift 2
;;
--help)
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " -t, --tool-path PATH Path to qol tool (default: ./qol)"
echo " -d, --domains-file PATH Path to domains file (default: ./domains.txt)"
echo " -o, --output-dir PATH Output directory (default: ./results)"
echo " -I, --interface NAME Network interface (default: veth1)"
echo " -T, --timeout DURATION Timeout duration (default: 5s)"
echo " -s, --sleep SECONDS Sleep between runs (default: 1)"
echo " -n, --dry-run Print the scenarios that would run, then exit"
echo " --help Show this help"
echo " -t, --tool-path PATH Path to qol tool (default: ./qol)"
echo " -d, --domains-file PATH Domains file for the general runs (default: ./domains.txt)"
echo " -o, --output-dir PATH Output dir for the general runs (default: ./results)"
echo " -I, --interface NAME Network interface (default: veth1)"
echo " -T, --timeout DURATION Timeout duration (default: 5s)"
echo " -s, --sleep SECONDS Sleep between runs (default: 1)"
echo " -n, --dry-run Print the scenarios that would run, then exit"
echo " --dnssec-domains PATH Signed-domain list for DNSSEC runs (default: ./domains_signed.txt)"
echo " --dnssec-output PATH Output dir for DNSSEC runs (default: ./results-dnssec)"
echo " --help Show this help"
exit 0
;;
*)
@@ -69,6 +82,8 @@ echo "Configuration:"
echo " Tool path: $TOOL_PATH"
echo " Domains file: $DOMAINS_FILE"
echo " Output dir: $OUTPUT_DIR"
echo " DNSSEC domains file: $DNSSEC_DOMAINS_FILE"
echo " DNSSEC output dir: $DNSSEC_OUTPUT_DIR"
echo " Interface: $INTERFACE"
echo " Timeout: $TIMEOUT"
echo " Sleep time: ${SLEEP_TIME}s"
@@ -80,13 +95,17 @@ SC_URL=()
SC_DNSSEC=()
SC_AUTH=()
SC_KEEP=()
SC_DOMAINS=() # per-scenario domains file
SC_OUTDIR=() # per-scenario output dir
add() { # add <name> <url> <dnssec> <auth> <keepalive>
SC_NAME+=("$1")
SC_URL+=("$2")
SC_DNSSEC+=("$3")
SC_AUTH+=("$4")
SC_KEEP+=("$5")
add() { # add <name> <url> <dnssec> <auth> <keepalive> (general workload)
SC_NAME+=("$1"); SC_URL+=("$2"); SC_DNSSEC+=("$3"); SC_AUTH+=("$4"); SC_KEEP+=("$5")
SC_DOMAINS+=("$DOMAINS_FILE"); SC_OUTDIR+=("$OUTPUT_DIR")
}
add_dnssec() { # same signature, but pins the signed list + DNSSEC output dir
SC_NAME+=("$1"); SC_URL+=("$2"); SC_DNSSEC+=("$3"); SC_AUTH+=("$4"); SC_KEEP+=("$5")
SC_DOMAINS+=("$DNSSEC_DOMAINS_FILE"); SC_OUTDIR+=("$DNSSEC_OUTPUT_DIR")
}
# Protocol comparison - DNSSEC off, non-persistent.
@@ -120,8 +139,7 @@ add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_
add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" false false false
# Persistence contrast - DNSSEC off, persistent.
# Teste for TCP based protocols.
# QUIC makes no sense cause of 0-RTT.
# Tested for TCP-based protocols. QUIC makes no sense because of 0-RTT.
add google-dotcp "dotcp://8.8.8.8:53" false false true
add cloudflare-dotcp "dotcp://1.1.1.1:53" false false true
add quad9-dotcp "dotcp://9.9.9.10:53" false false true
@@ -137,37 +155,72 @@ add cloudflare-doh "https://cloudflare-dns.com/dns-query" false false true
add quad9-doh "https://dns10.quad9.net/dns-query" false false true
add adguard-doh "https://unfiltered.adguard-dns.com/dns-query" false false true
# DNSSEC trust - non-persistent.
add google-dotcp "dotcp://8.8.8.8:53" true false false
add cloudflare-dotcp "dotcp://1.1.1.1:53" true false false
add quad9-dotcp "dotcp://9.9.9.10:53" true false false
add adguard-dotcp "dotcp://94.140.14.140:53" true false false
# ==========================================================================
# DNSSEC scenarios: signed domain list + separate output dir (results-dnssec).
# off / trust / auth all share the signed list, so the domain set is held
# constant across the three modes. Non-persistent throughout.
# ==========================================================================
add google-dot "tls://8.8.8.8:853" true false false
add cloudflare-dot "tls://1.1.1.1:853" true false false
add quad9-dot "tls://9.9.9.10:853" true false false
add adguard-dot "tls://94.140.14.140:853" true false false
# DNSSEC off - baseline ON THE SIGNED LIST (separate from the protocol-comparison
# off runs above, which use the general list).
add_dnssec google-dotcp "dotcp://8.8.8.8:53" false false false
add_dnssec cloudflare-dotcp "dotcp://1.1.1.1:53" false false false
add_dnssec quad9-dotcp "dotcp://9.9.9.10:53" false false false
add_dnssec adguard-dotcp "dotcp://94.140.14.140:53" false false false
add google-doh "https://dns.google/dns-query" true false false
add cloudflare-doh "https://cloudflare-dns.com/dns-query" true false false
add quad9-doh "https://dns10.quad9.net/dns-query" true false false
add adguard-doh "https://unfiltered.adguard-dns.com/dns-query" true false false
add_dnssec google-dot "tls://8.8.8.8:853" false false false
add_dnssec cloudflare-dot "tls://1.1.1.1:853" false false false
add_dnssec quad9-dot "tls://9.9.9.10:853" false false false
add_dnssec adguard-dot "tls://94.140.14.140:853" false false false
add google-doh3 "doh3://dns.google/dns-query" true false false
add cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" true false false
add adguard-doh3 "doh3://unfiltered.adguard-dns.com/dns-query" true false false
add adguard-doq "doq://unfiltered.adguard-dns.com:853" true false false
add_dnssec google-doh "https://dns.google/dns-query" false false false
add_dnssec cloudflare-doh "https://cloudflare-dns.com/dns-query" false false false
add_dnssec quad9-doh "https://dns10.quad9.net/dns-query" false false false
add_dnssec adguard-doh "https://unfiltered.adguard-dns.com/dns-query" false false false
add google-doudp "udp://8.8.8.8:53" true false false
add cloudflare-doudp "udp://1.1.1.1:53" true false false
add quad9-doudp "udp://9.9.9.10:53" true false false
add adguard-doudp "udp://94.140.14.140:53" true false false
add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" true false false
add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" true false false
add_dnssec google-doh3 "doh3://dns.google/dns-query" false false false
add_dnssec cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" false false false
add_dnssec adguard-doh3 "doh3://unfiltered.adguard-dns.com/dns-query" false false false
add_dnssec adguard-doq "doq://unfiltered.adguard-dns.com:853" false false false
add_dnssec google-doudp "udp://8.8.8.8:53" false false false
add_dnssec cloudflare-doudp "udp://1.1.1.1:53" false false false
add_dnssec quad9-doudp "udp://9.9.9.10:53" false false false
add_dnssec adguard-doudp "udp://94.140.14.140:53" false false false
add_dnssec adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" false false false
add_dnssec quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" false false false
# DNSSEC trust - non-persistent, signed list.
add_dnssec google-dotcp "dotcp://8.8.8.8:53" true false false
add_dnssec cloudflare-dotcp "dotcp://1.1.1.1:53" true false false
add_dnssec quad9-dotcp "dotcp://9.9.9.10:53" true false false
add_dnssec adguard-dotcp "dotcp://94.140.14.140:53" true false false
add_dnssec google-dot "tls://8.8.8.8:853" true false false
add_dnssec cloudflare-dot "tls://1.1.1.1:853" true false false
add_dnssec quad9-dot "tls://9.9.9.10:853" true false false
add_dnssec adguard-dot "tls://94.140.14.140:853" true false false
add_dnssec google-doh "https://dns.google/dns-query" true false false
add_dnssec cloudflare-doh "https://cloudflare-dns.com/dns-query" true false false
add_dnssec quad9-doh "https://dns10.quad9.net/dns-query" true false false
add_dnssec adguard-doh "https://unfiltered.adguard-dns.com/dns-query" true false false
add_dnssec google-doh3 "doh3://dns.google/dns-query" true false false
add_dnssec cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" true false false
add_dnssec adguard-doh3 "doh3://unfiltered.adguard-dns.com/dns-query" true false false
add_dnssec adguard-doq "doq://unfiltered.adguard-dns.com:853" true false false
add_dnssec google-doudp "udp://8.8.8.8:53" true false false
add_dnssec cloudflare-doudp "udp://1.1.1.1:53" true false false
add_dnssec quad9-doudp "udp://9.9.9.10:53" true false false
add_dnssec adguard-doudp "udp://94.140.14.140:53" true false false
add_dnssec adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" true false false
add_dnssec quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" true false false
# DNSSEC auth - DoUDP only.
# auth walks the chain of trust over plain DoUDP regardless of the protocol.
add adguard-doudp "udp://unfiltered.adguard-dns.com:53" true true false
add_dnssec adguard-doudp "udp://94.140.14.140:53" true true false
# Function to get flags suffix for filename
get_flags_suffix() {
@@ -200,6 +253,8 @@ run_with_perf() {
local dnssec="$3"
local auth="$4"
local keepalive="$5"
local domains_file="$6"
local output_dir="$7"
local suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
local provider="${name%%-*}"
@@ -208,7 +263,7 @@ run_with_perf() {
if [[ -n "$suffix" ]]; then
base_name="${protocol}-${suffix}"
fi
local cpu_csv_file="$OUTPUT_DIR/${provider}/${base_name}.cpu.csv"
local cpu_csv_file="$output_dir/${provider}/${base_name}.cpu.csv"
# Create directory if needed
mkdir -p "$(dirname "$cpu_csv_file")"
@@ -220,8 +275,8 @@ run_with_perf() {
# Build command arguments
local cmd_args=(
"$DOMAINS_FILE"
--output-dir "$OUTPUT_DIR"
"$domains_file"
--output-dir "$output_dir"
--interface "$INTERFACE"
--timeout "$TIMEOUT"
-s "$url"
@@ -259,18 +314,29 @@ run_with_perf() {
# Cleanup
rm -f "$perf_tmp"
echo " -> CPU metrics saved to ${provider}/${base_name}.cpu.csv"
echo " -> CPU metrics saved to ${output_dir}/${provider}/${base_name}.cpu.csv"
}
echo "Total scenarios: ${#SC_NAME[@]}"
echo ""
# Guard: all scenario arrays must be the same length (they desync only if a row
# was added without going through add()/add_dnssec()).
_n=${#SC_NAME[@]}
if (( ${#SC_URL[@]} != _n || ${#SC_DNSSEC[@]} != _n || ${#SC_AUTH[@]} != _n \
|| ${#SC_KEEP[@]} != _n || ${#SC_DOMAINS[@]} != _n || ${#SC_OUTDIR[@]} != _n )); then
echo "ERROR: scenario arrays are misaligned (lengths differ). Check add()/add_dnssec() calls." >&2
exit 1
fi
for i in "${!SC_NAME[@]}"; do
name="${SC_NAME[$i]}"
url="${SC_URL[$i]}"
dnssec="${SC_DNSSEC[$i]}"
auth="${SC_AUTH[$i]}"
keepalive="${SC_KEEP[$i]}"
domains_file="${SC_DOMAINS[$i]}"
output_dir="${SC_OUTDIR[$i]}"
suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
provider="${name%%-*}"
@@ -279,13 +345,14 @@ for i in "${!SC_NAME[@]}"; do
[[ -n "$suffix" ]] && label="${protocol}-${suffix}"
if [[ "$DRY_RUN" == "true" ]]; then
printf '[dry] %-28s dnssec=%-5s auth=%-5s keepalive=%-5s -> %s/%s\n' \
"$name" "$dnssec" "$auth" "$keepalive" "$provider" "$label"
printf '[dry] %-28s dnssec=%-5s auth=%-5s keepalive=%-5s [%s] -> %s/%s/%s\n' \
"$name" "$dnssec" "$auth" "$keepalive" \
"$(basename "$domains_file")" "$output_dir" "$provider" "$label"
continue
fi
echo "Processing: $name ($url) [dnssec=$dnssec auth=$auth keepalive=$keepalive]"
run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive"
echo "Processing: $name ($url) [dnssec=$dnssec auth=$auth keepalive=$keepalive, domains=$(basename "$domains_file"), out=$output_dir]"
run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive" "$domains_file" "$output_dir"
sleep "$SLEEP_TIME"
done