feat(everything): fix doh and increase min TLS to 1.3

.gitignore

@@ -18,3 +18,4 @@
 /results
 /results.bak
 /results_merged
+dns_results*

Makefile

@@ -0,0 +1,10 @@
+.PHONY: all clean
+
+all:
+	python3 scripts/post_processing/merge_files.py
+	python3 scripts/post_processing/merge_cpu.py
+	python3 scripts/post_processing/merge_mem.py
+	python3 scripts/post_processing/merge_pcap.py
+
+clean:
+	rm -f ./dns_results.csv ./dns_results_cpu.csv ./dns_results_mem.csv ./dns_results_pcap.csv

common/protocols/doh/doh.go

@@ -27,7 +27,6 @@ type Config struct {
 	Path      string
 	DNSSEC    bool
 	HTTP3     bool
-	HTTP2     bool
 	KeepAlive bool
 }
 
@@ -62,6 +61,7 @@ func New(config Config) (*Client, error) {
 
 	tlsConfig := &tls.Config{
 		ServerName:         config.Host,
+		MinVersion: tls.VersionTLS13,
 		ClientSessionCache: tls.NewLRUClientSessionCache(100),
 	}
 
@@ -88,14 +88,7 @@ func New(config Config) (*Client, error) {
 	}
 
 	var transportType string
-	if config.HTTP2 {
+	if config.HTTP3 {
-		http2Transport := &http2.Transport{
-			TLSClientConfig: tlsConfig,
-			AllowHTTP:       true,
-		}
-		httpClient.Transport = http2Transport
-		transportType = "HTTP/2"
-	} else if config.HTTP3 {
 		quicTlsConfig := http3.ConfigureTLSConfig(tlsConfig)
 		http3Transport := &http3.Transport{
 			TLSClientConfig: quicTlsConfig,
@@ -104,7 +97,12 @@ func New(config Config) (*Client, error) {
 		httpClient.Transport = http3Transport
 		transportType = "HTTP/3"
 	} else {
-		transportType = "HTTP/1.1"
+		http2Transport := &http2.Transport{
+			TLSClientConfig: tlsConfig,
+			AllowHTTP:       true,
+		}
+		httpClient.Transport = http2Transport
+		transportType = "HTTP/2"
 	}
 
 	logger.Debug("DoH client created: %s (%s, DNSSEC: %v, KeepAlive: %v)", rawURL, transportType, config.DNSSEC, config.KeepAlive)
@@ -116,6 +114,43 @@ func New(config Config) (*Client, error) {
 	}, nil
 }
 
+// newSingleUseClient builds a throwaway *http.Client whose transport is torn
+// down by the returned closeFn. Used for non-persistent (non-keep-alive) mode
+// so that each query establishes and tears down its own connection, and thus
+// pays its own TCP+TLS (or QUIC) handshake. Without this, the pooled
+// HTTP/2 transport silently reuses a single connection across all queries
+// (HTTP/2 multiplexes and ignores the Connection: close header), which would
+// make the non-persistent measurement indistinguishable from persistent.
+func (c *Client) newSingleUseClient() (*http.Client, func()) {
+	tlsConfig := &tls.Config{
+		ServerName:         c.config.Host,
+		MinVersion:         tls.VersionTLS13,
+		ClientSessionCache: nil, // no resumption: force a full handshake each query
+	}
+
+	if c.config.HTTP3 {
+		quicConf := &quic.Config{
+			MaxIdleTimeout:          30 * time.Second,
+			DisablePathMTUDiscovery: true,
+		}
+		quicTLS := http3.ConfigureTLSConfig(tlsConfig)
+		tr := &http3.Transport{
+			TLSClientConfig: quicTLS,
+			QUICConfig:      quicConf,
+		}
+		client := &http.Client{Transport: tr, Timeout: 30 * time.Second}
+		return client, func() { tr.Close() }
+	}
+
+	tr := &http2.Transport{
+		TLSClientConfig:    tlsConfig,
+		AllowHTTP:          true,
+		DisableCompression: true,
+	}
+	client := &http.Client{Transport: tr, Timeout: 30 * time.Second}
+	return client, func() { tr.CloseIdleConnections() }
+}
+
 func (c *Client) Close() {
 	logger.Debug("Closing DoH client")
 	if t, ok := c.httpClient.Transport.(*http.Transport); ok {
@@ -143,6 +178,18 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
 		return nil, fmt.Errorf("doh: failed to pack DNS message: %w", err)
 	}
 
+	// Select the HTTP client. In persistent mode, reuse the pooled client built
+	// in New(). In non-persistent mode, build a fresh client (and transport)
+	// per query and tear it down afterwards, so every query pays its own
+	// handshake — otherwise HTTP/2 would reuse one connection and the
+	// non-persistent measurement would be wrong.
+	httpClient := c.httpClient
+	if !c.config.KeepAlive {
+		freshClient, closeFn := c.newSingleUseClient()
+		defer closeFn()
+		httpClient = freshClient
+	}
+
 	httpReq, err := http.NewRequest(http.MethodPost, c.upstreamURL.String(), bytes.NewReader(packedMsg))
 	if err != nil {
 		logger.Error("DoH failed to create HTTP request: %v", err)
@@ -153,14 +200,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
 	httpReq.Header.Set("Content-Type", dnsMessageContentType)
 	httpReq.Header.Set("Accept", dnsMessageContentType)
 
-	// Set Connection header based on KeepAlive setting
+	httpResp, err := httpClient.Do(httpReq)
-	if c.config.KeepAlive {
-		httpReq.Header.Set("Connection", "keep-alive")
-	} else {
-		httpReq.Header.Set("Connection", "close")
-	}
-
-	httpResp, err := c.httpClient.Do(httpReq)
 	if err != nil {
 		logger.Error("DoH request failed to %s: %v", c.upstreamURL.Host, err)
 		return nil, fmt.Errorf("doh: failed executing HTTP request to %s: %w", c.upstreamURL.Host, err)

common/protocols/doq/doq.go

@@ -26,7 +26,7 @@ type Client struct {
 	targetAddr    *net.UDPAddr
 	tlsConfig     *tls.Config
 	udpConn       *net.UDPConn
-	quicConn      quic.Connection
+	quicConn      *quic.Conn
 	quicTransport *quic.Transport
 	quicConfig    *quic.Config
 	config        Config
@@ -119,7 +119,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
 		return nil, fmt.Errorf("doq: failed to pack message: %w", err)
 	}
 
-	var quicStream quic.Stream
+	var quicStream *quic.Stream
 	quicStream, err = c.quicConn.OpenStream()
 	if err != nil {
 		logger.Debug("DoQ stream failed, reconnecting: %v", err)

common/protocols/dot/dot.go

@@ -49,6 +49,7 @@ func New(config Config) (*Client, error) {
 
 	tlsConfig := &tls.Config{
 		ServerName: config.Host,
+		MinVersion: tls.VersionTLS13,
 	}
 
 	client := &Client{

go.mod

@@ -1,29 +1,25 @@
 module github.com/afonsofrancof/sdns-proxy
 
-go 1.24.1
+go 1.26.4
 
 require (
-	github.com/alecthomas/kong v1.8.1
+	github.com/alecthomas/kong v1.15.0
 	github.com/ameshkov/dnscrypt/v2 v2.4.0
 	github.com/google/gopacket v1.1.19
-	github.com/miekg/dns v1.1.65
+	github.com/miekg/dns v1.1.72
-	github.com/quic-go/quic-go v0.50.0
+	github.com/quic-go/quic-go v0.60.0
-	golang.org/x/net v0.38.0
+	golang.org/x/net v0.56.0
 )
 
 require (
-	github.com/AdguardTeam/golibs v0.32.7 // indirect
+	github.com/AdguardTeam/golibs v0.35.14 // indirect
 	github.com/ameshkov/dnsstamps v1.0.3 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/quic-go/qpack v0.6.0 // indirect
-	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	golang.org/x/crypto v0.53.0 // indirect
-	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
+	golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
-	github.com/quic-go/qpack v0.5.1 // indirect
+	golang.org/x/mod v0.37.0 // indirect
-	go.uber.org/mock v0.5.0 // indirect
+	golang.org/x/sync v0.21.0 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
-	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
+	golang.org/x/text v0.39.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
-	golang.org/x/sync v0.13.0 // indirect
-	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/text v0.24.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
 )

go.sum

@@ -1,88 +1,64 @@
-github.com/AdguardTeam/golibs v0.32.7 h1:3dmGlAVgmvquCCwHsvEl58KKcRAK3z1UnjMnwSIeDH4=
+github.com/AdguardTeam/golibs v0.35.14 h1:LGqoRMbuxUgZyhOjlkmPxy5t1/XPuFpDcG5RBIDcskA=
-github.com/AdguardTeam/golibs v0.32.7/go.mod h1:bE8KV1zqTzgZjmjFyBJ9f9O5DEKO717r7e57j1HclJA=
+github.com/AdguardTeam/golibs v0.35.14/go.mod h1:FL470ALCuUs7cwk88ltBgTZ6JYCUG2NpNzp3EOMirxo=
 github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
 github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
-github.com/alecthomas/kong v1.8.1 h1:6aamvWBE/REnR/BCq10EcozmcpUPc5aGI1lPAWdB0EE=
+github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI=
-github.com/alecthomas/kong v1.8.1/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
+github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
-github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
-github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/ameshkov/dnscrypt/v2 v2.4.0 h1:if6ZG2cuQmcP2TwSY+D0+8+xbPfoatufGlOQTMNkI9o=
 github.com/ameshkov/dnscrypt/v2 v2.4.0/go.mod h1:WpEFV2uhebXb8Jhes/5/fSdpmhGV8TL22RDaeWwV6hI=
 github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo=
 github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
-github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
-github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
+github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
-github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
-github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
-github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
-github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
-golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
+golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
-golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
+golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
-golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
-golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
-golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scripts/post_processing/add_extra_metrics_to_csv.py

@@ -1,405 +1,258 @@
 #!/usr/bin/env python3
 """
-Fast PCAP Preprocessor for DNS QoS Analysis
+Fast PCAP Preprocessor for DNS QoS Analysis.
-Loads PCAP into memory first, then uses binary search for matching.
+
-Uses LAN IP to determine direction (LAN = sent, non-LAN = received).
+Loads PCAP into memory, then uses binary search to match packets to query
+windows. Direction is determined by a configured local IP (the netns veth IP).
 """
 
+import argparse
+import bisect
 import csv
 import shutil
+import socket
+import time
 from pathlib import Path
 from typing import Dict, List, NamedTuple
-import time
 
 import dpkt
 from dateutil import parser as date_parser
 
-
 BANDWIDTH_COLUMNS = [
-    'bytes_sent',
+    "bytes_sent",
-    'bytes_received',
+    "bytes_received",
-    'packets_sent',
+    "packets_sent",
-    'packets_received',
+    "packets_received",
-    'total_bytes',
+    "total_bytes",
 ]
 
+DEFAULT_LOCAL_IP = "192.168.100.2"  # netns veth1 address
+DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"]
+
 
 class Packet(NamedTuple):
-    """Lightweight packet representation."""
     timestamp: float
     size: int
-    is_outbound: bool  # True if from LAN, False if from internet
+    is_outbound: bool
 
 
-class QueryWindow:
+def needs_processing(csv_path: Path) -> bool:
-    """Efficient query window representation."""
+    """True if file lacks bandwidth columns OR all values are empty/zero."""
-    __slots__ = ['index', 'start', 'end', 'sent', 'received', 'pkts_sent', 'pkts_received']
-    
-    def __init__(self, index: int, start: float, end: float):
-        self.index = index
-        self.start = start
-        self.end = end
-        self.sent = 0
-        self.received = 0
-        self.pkts_sent = 0
-        self.pkts_received = 0
-
-
-def is_already_processed(csv_path: Path) -> bool:
-    """
-    Check if CSV has already been processed.
-    Returns True if bandwidth columns exist AND at least one row has non-zero data.
-    """
     try:
-        with open(csv_path, 'r', encoding='utf-8') as f:
+        with open(csv_path, "r", encoding="utf-8") as f:
             reader = csv.DictReader(f)
-            
-            # Check if columns exist
             if not reader.fieldnames:
-                return False
+                return True
-            
+            if not all(c in reader.fieldnames for c in BANDWIDTH_COLUMNS):
-            if not all(col in reader.fieldnames for col in BANDWIDTH_COLUMNS):
+                return True
-                return False
-            
-            # Check if any row has non-zero bandwidth data
             for row in reader:
                 for col in BANDWIDTH_COLUMNS:
-                    val = row.get(col, '').strip()
+                    val = (row.get(col) or "").strip()
-                    if val and val != '0':
+                    if val and val != "0":
-                        return True
+                        return False  # has real data
-            
+            return True  # columns exist but all empty/zero
-            # All rows have zero/empty values - not truly processed
-            return False
-            
     except Exception:
-        return False
+        return True
 
 
 def parse_csv_timestamp(ts_str: str) -> float:
-    """Convert RFC3339Nano timestamp to Unix epoch (seconds)."""
+    return date_parser.isoparse(ts_str).timestamp()
-    dt = date_parser.isoparse(ts_str)
-    return dt.timestamp()
 
 
-def is_lan_ip(ip_bytes: bytes) -> bool:
+def load_pcap(pcap_path: Path, local_ip_bytes: bytes) -> List[Packet]:
-    """Check if IP is a private/LAN address."""
+    """Load PCAP into a list of Packets sorted by timestamp."""
-    if len(ip_bytes) != 4:
+    print("    Loading PCAP...")
-        return False
+    t0 = time.time()
-    
+    packets: List[Packet] = []
-    first = ip_bytes[0]
-    second = ip_bytes[1]
-    
-    # 10.0.0.0/8
-    if first == 10:
-        return True
-    
-    # 172.16.0.0/12
-    if first == 172 and 16 <= second <= 31:
-        return True
-    
-    # 192.168.0.0/16
-    if first == 192 and second == 168:
-        return True
-    
-    # 127.0.0.0/8 (localhost)
-    if first == 127:
-        return True
-    
-    return False
-
-
-def load_pcap_into_memory(pcap_path: Path) -> List[Packet]:
-    """Load all packets from PCAP into memory with minimal data."""
-    packets = []
-    
-    print(f"    Loading PCAP into memory...")
-    start_time = time.time()
 
+    with open(pcap_path, "rb") as f:
         try:
-        with open(pcap_path, 'rb') as f:
+            reader = dpkt.pcap.Reader(f)
-            try:
+        except ValueError:
-                pcap = dpkt.pcap.Reader(f)
-            except:
-                # Try pcapng format
             f.seek(0)
-                pcap = dpkt.pcapng.Reader(f)
+            reader = dpkt.pcapng.Reader(f)
 
-            for ts, buf in pcap:
+        for ts, buf in reader:
             try:
-                    packet_time = float(ts)
-                    packet_size = len(buf)
-                    
-                    # Parse to get source IP
                 eth = dpkt.ethernet.Ethernet(buf)
-                    
-                    # Default to outbound if we can't determine
-                    is_outbound = True
-                    
-                    if isinstance(eth.data, dpkt.ip.IP):
                 ip = eth.data
-                        src_ip = ip.src
+                if not isinstance(ip, dpkt.ip.IP):
-                        is_outbound = is_lan_ip(src_ip)
+                    continue
-                    
+                if ip.src == local_ip_bytes:
-                    packets.append(Packet(
+                    is_outbound = True
-                        timestamp=packet_time,
+                elif ip.dst == local_ip_bytes:
-                        size=packet_size,
+                    is_outbound = False
-                        is_outbound=is_outbound
+                else:
-                    ))
+                    continue  # not our traffic
-                    
+                packets.append(Packet(float(ts), len(buf), is_outbound))
             except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError):
                 continue
 
-    except Exception as e:
-        print(f"    Error reading PCAP: {e}")
-        return []
-    
-    elapsed = time.time() - start_time
-    print(f"    Loaded {len(packets):,} packets in {elapsed:.2f}s")
-    
-    # Sort by timestamp for binary search
     packets.sort(key=lambda p: p.timestamp)
-    
+    print(f"    Loaded {len(packets):,} packets in {time.time() - t0:.2f}s")
     return packets
 
 
-def find_packets_in_window(
-    packets: List[Packet],
-    start_time: float,
-    end_time: float,
-    left_hint: int = 0
-) -> tuple[List[Packet], int]:
-    """
-    Binary search to find all packets within time window.
-    Returns (matching_packets, left_index_hint_for_next_search).
-    """
-    if not packets:
-        return [], 0
-    
-    # Binary search for first packet >= start_time
-    left, right = left_hint, len(packets) - 1
-    first_idx = len(packets)
-    
-    while left <= right:
-        mid = (left + right) // 2
-        if packets[mid].timestamp >= start_time:
-            first_idx = mid
-            right = mid - 1
-        else:
-            left = mid + 1
-    
-    # No packets in range
-    if first_idx >= len(packets) or packets[first_idx].timestamp > end_time:
-        return [], first_idx
-    
-    # Collect all packets in window
-    matching = []
-    idx = first_idx
-    while idx < len(packets) and packets[idx].timestamp <= end_time:
-        matching.append(packets[idx])
-        idx += 1
-    
-    return matching, first_idx
-
-
 def load_csv_queries(csv_path: Path) -> List[Dict]:
-    """Load CSV and create query data structures."""
     queries = []
-    with open(csv_path, 'r', encoding='utf-8') as f:
+    with open(csv_path, "r", encoding="utf-8") as f:
-        reader = csv.DictReader(f)
+        for row in csv.DictReader(f):
-        for row in reader:
             try:
-                ts_epoch = parse_csv_timestamp(row['timestamp'])
+                start = parse_csv_timestamp(row["timestamp"])
-                duration_s = float(row['duration_ns']) / 1e9
+                duration = float(row["duration_ns"]) / 1e9
-                queries.append({
+                queries.append(
-                    'data': row,
+                    {"data": row, "start_time": start, "end_time": start + duration}
-                    'start_time': ts_epoch,
-                    'end_time': ts_epoch + duration_s,
-                })
-            except Exception as e:
-                print(f"  Warning: Skipping row - {e}")
-                continue
-    return queries
-
-
-def match_packets_to_queries(
-    packets: List[Packet],
-    queries: List[Dict]
-) -> List[Dict]:
-    """Match packets to query windows using binary search."""
-    if not queries or not packets:
-        return queries
-    
-    print(f"    Matching packets to queries...")
-    start_time = time.time()
-    
-    # Initialize metrics
-    for q in queries:
-        q['bytes_sent'] = 0
-        q['bytes_received'] = 0
-        q['packets_sent'] = 0
-        q['packets_received'] = 0
-        q['total_bytes'] = 0
-    
-    # Sort queries by start time for sequential processing
-    queries_sorted = sorted(enumerate(queries), key=lambda x: x[1]['start_time'])
-    
-    matched_packets = 0
-    left_hint = 0  # Optimization: start next search from here
-    
-    for original_idx, q in queries_sorted:
-        matching, left_hint = find_packets_in_window(
-            packets,
-            q['start_time'],
-            q['end_time'],
-            left_hint
                 )
-        
+            except Exception as e:
-        for pkt in matching:
+                print(f"    Warning: skipping row - {e}")
-            matched_packets += 1
-            if pkt.is_outbound:
-                q['bytes_sent'] += pkt.size
-                q['packets_sent'] += 1
-            else:
-                q['bytes_received'] += pkt.size
-                q['packets_received'] += 1
-        
-        q['total_bytes'] = q['bytes_sent'] + q['bytes_received']
-    
-    elapsed = time.time() - start_time
-    print(f"    Matched {matched_packets:,} packets in {elapsed:.2f}s")
-    
-    # Statistics
-    total_sent = sum(q['bytes_sent'] for q in queries)
-    total_recv = sum(q['bytes_received'] for q in queries)
-    queries_with_data = sum(1 for q in queries if q['total_bytes'] > 0)
-    print(f"    Total: {total_sent:,} bytes sent, {total_recv:,} bytes received")
-    print(f"    Queries with data: {queries_with_data}/{len(queries)}")
-    
     return queries
 
 
-def write_enriched_csv(
+def match_packets(packets: List[Packet], queries: List[Dict]) -> int:
-    csv_path: Path, queries: List[Dict], backup: bool = True
+    """Assign bandwidth metrics to each query. Returns total matched packets."""
-):
+    if not packets or not queries:
-    """Write enriched CSV with bandwidth columns."""
+        for q in queries:
-    if backup and csv_path.exists():
+            q.update({c: 0 for c in BANDWIDTH_COLUMNS})
-        backup_path = csv_path.with_suffix('.csv.bak')
+        return 0
-        if not backup_path.exists():  # Don't overwrite existing backup
+
-            shutil.copy2(csv_path, backup_path)
+    print("    Matching packets to queries...")
-            print(f"  Backup: {backup_path.name}")
+    t0 = time.time()
+    timestamps = [p.timestamp for p in packets]
+    matched = 0
+
+    for q in queries:
+        lo = bisect.bisect_left(timestamps, q["start_time"])
+        hi = bisect.bisect_right(timestamps, q["end_time"])
+        bs = br = ps = pr = 0
+        for pkt in packets[lo:hi]:
+            if pkt.is_outbound:
+                bs += pkt.size
+                ps += 1
+            else:
+                br += pkt.size
+                pr += 1
+        q["bytes_sent"] = bs
+        q["bytes_received"] = br
+        q["packets_sent"] = ps
+        q["packets_received"] = pr
+        q["total_bytes"] = bs + br
+        matched += ps + pr
+
+    print(f"    Matched {matched:,} packets in {time.time() - t0:.2f}s")
+    total_sent = sum(q["bytes_sent"] for q in queries)
+    total_recv = sum(q["bytes_received"] for q in queries)
+    with_data = sum(1 for q in queries if q["total_bytes"] > 0)
+    print(f"    Total: {total_sent:,} B sent, {total_recv:,} B received")
+    print(f"    Queries with data: {with_data}/{len(queries)}")
+    return matched
+
+
+def write_csv(csv_path: Path, queries: List[Dict], backup: bool = True):
+    if backup and csv_path.exists():
+        bak = csv_path.with_suffix(".csv.bak")
+        if not bak.exists():
+            shutil.copy2(csv_path, bak)
+            print(f"    Backup: {bak.name}")
 
-    # Get fieldnames - filter out any existing bandwidth columns to avoid dupes
     original_fields = [
-        f for f in queries[0]['data'].keys()
+        f for f in queries[0]["data"].keys() if f not in BANDWIDTH_COLUMNS
-        if f not in BANDWIDTH_COLUMNS
     ]
     fieldnames = original_fields + BANDWIDTH_COLUMNS
 
-    with open(csv_path, 'w', encoding='utf-8', newline='') as f:
+    with open(csv_path, "w", encoding="utf-8", newline="") as f:
         writer = csv.DictWriter(f, fieldnames=fieldnames)
         writer.writeheader()
-        
         for q in queries:
-            row = {k: v for k, v in q['data'].items() if k not in BANDWIDTH_COLUMNS}
+            row = {k: q["data"][k] for k in original_fields}
-            for field in BANDWIDTH_COLUMNS:
+            for c in BANDWIDTH_COLUMNS:
-                row[field] = q[field]
+                row[c] = q[c]
             writer.writerow(row)
-    
     print(f"    Written: {csv_path.name}")
 
 
-def process_provider_directory(provider_path: Path):
+def process_provider(provider_path: Path, local_ip_bytes: bytes):
-    """Process all CSV/PCAP pairs in a provider directory."""
+    print(f"\n{'=' * 60}\nProcessing: {provider_path.name.upper()}\n{'=' * 60}")
-    print(f"\n{'='*60}")
-    print(f"Processing: {provider_path.name.upper()}")
-    print(f"{'='*60}")
 
-    csv_files = sorted(provider_path.glob('*.csv'))
+    processed = skipped = 0
-    processed = 0
+    total_time = 0.0
-    skipped = 0
-    total_time = 0
 
-    for csv_path in csv_files:
+    for csv_path in sorted(provider_path.glob("*.csv")):
-        # Skip backup files
+        name = csv_path.name.lower()
-        if '.bak' in csv_path.name:
+        if ".bak" in name or name.endswith((".cpu.csv", ".mem.csv")):
             continue
 
-        pcap_path = csv_path.with_suffix('.pcap')
+        pcap_path = csv_path.with_suffix(".pcap")
-        
         if not pcap_path.exists():
-            print(f"\n  ⚠ Skipping {csv_path.name} - no matching PCAP")
+            print(f"\n  ⚠ {csv_path.name}: no matching PCAP")
             continue
 
-        # Check if already processed
+        if not needs_processing(csv_path):
-        if is_already_processed(csv_path):
+            print(f"\n  ⏭ {csv_path.name}: already processed")
-            print(f"\n  ⏭ Skipping {csv_path.name} - already processed")
             skipped += 1
             continue
 
         print(f"\n  📁 {csv_path.name}")
-        file_start = time.time()
+        t0 = time.time()
 
-        # Load PCAP into memory first
+        packets = load_pcap(pcap_path, local_ip_bytes)
-        packets = load_pcap_into_memory(pcap_path)
         if not packets:
-            print(f"    ⚠ No packets found in PCAP")
+            print("    ⚠ No usable packets in PCAP")
             continue
 
-        # Load CSV queries
         queries = load_csv_queries(csv_path)
         if not queries:
-            print(f"    ⚠ No valid queries found")
+            print("    ⚠ No valid queries in CSV")
             continue
-        
         print(f"    Loaded {len(queries):,} queries")
 
-        # Match packets to queries
+        match_packets(packets, queries)
-        enriched_queries = match_packets_to_queries(packets, queries)
+        write_csv(csv_path, queries)
 
-        # Write enriched CSV
+        dt = time.time() - t0
-        write_enriched_csv(csv_path, enriched_queries)
+        total_time += dt
-        
-        file_time = time.time() - file_start
-        total_time += file_time
         processed += 1
-        print(f"    ✓ Completed in {file_time:.2f}s")
+        print(f"    ✓ Completed in {dt:.2f}s")
-    
-    print(f"\n  {'='*58}")
-    print(f"  {provider_path.name}: {processed} processed, {skipped} skipped")
-    print(f"  Time: {total_time:.2f}s")
-    print(f"  {'='*58}")
 
+    print(
+        f"\n  {provider_path.name}: {processed} processed, "
+        f"{skipped} skipped, {total_time:.2f}s"
+    )
 
 def main():
-    """Main preprocessing pipeline."""
+    ap = argparse.ArgumentParser(description=__doc__)
-    overall_start = time.time()
+    ap.add_argument(
+        "--local-ip",
+        default=DEFAULT_LOCAL_IP,
+        help=f"Local (netns veth) IP used to determine direction "
+        f"(default: {DEFAULT_LOCAL_IP})",
+    )
+    ap.add_argument("--results-dir", default="results", type=Path)
+    ap.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS)
+    args = ap.parse_args()
 
-    print("\n" + "="*60)
+    local_ip_bytes = socket.inet_aton(args.local_ip)
-    print("DNS PCAP PREPROCESSOR - Memory-Optimized Edition")
-    print("="*60)
 
-    results_dir = Path('results')
+    print(f"\n{'=' * 60}\nDNS PCAP PREPROCESSOR\n{'=' * 60}")
+    print(f"Local IP: {args.local_ip}")
+    print(f"Results:  {args.results_dir}")
 
-    if not results_dir.exists():
+    if not args.results_dir.exists():
-        print(f"\n❌ Error: '{results_dir}' directory not found")
+        print(f"\n❌ Directory not found: {args.results_dir}")
         return
 
-    providers = ['adguard', 'cloudflare', 'google', 'quad9']
+    t0 = time.time()
-    
+    for provider in args.providers:
-    for provider in providers:
+        path = args.results_dir / provider
-        provider_path = results_dir / provider
+        if path.exists():
-        if provider_path.exists():
+            process_provider(path, local_ip_bytes)
-            process_provider_directory(provider_path)
         else:
-            print(f"\n⚠ Warning: Provider directory not found: {provider}")
+            print(f"\n⚠ Missing provider directory: {provider}")
 
-    overall_time = time.time() - overall_start
+    total = time.time() - t0
-    
+    print(f"\n{'=' * 60}")
-    print("\n" + "="*60)
+    print(f"✓ DONE in {total:.2f}s ({total / 60:.1f} min)")
-    print(f"✓ PREPROCESSING COMPLETE")
+    print(f"{'=' * 60}\n")
-    print(f"  Total time: {overall_time:.2f}s ({overall_time/60:.1f} minutes)")
-    print("="*60 + "\n")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

scripts/post_processing/merge_files.py

@@ -74,6 +74,10 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
     
     csv_files = find_csv_files(input_dir)
 
+    print(f"Files to process ({len(csv_files)}):")
+    for f in csv_files:
+        print(f"  {f}")
+    
     if not csv_files:
         print("No CSV files found")
         return
@@ -96,11 +100,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
         'duration_ms',
         'request_size_bytes',
         'response_size_bytes',
-        'bytes_sent',
-        'bytes_received',
-        'packets_sent',
-        'packets_received',
-        'total_bytes',
         'response_code',
         'error',
     ]
@@ -146,11 +145,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
                         'duration_ms': ns_to_ms(row.get('duration_ns', '')),
                         'request_size_bytes': row.get('request_size_bytes', ''),
                         'response_size_bytes': row.get('response_size_bytes', ''),
-                        'bytes_sent': row.get('bytes_sent', ''),
-                        'bytes_received': row.get('bytes_received', ''),
-                        'packets_sent': row.get('packets_sent', ''),
-                        'packets_received': row.get('packets_received', ''),
-                        'total_bytes': row.get('total_bytes', ''),
                         'response_code': row.get('response_code', ''),
                         'error': row.get('error', ''),
                     }
@@ -173,7 +167,7 @@ def main():
     parser.add_argument(
         'input_dir',
         nargs='?',
-        default='.',
+        default='results',
         help='Input directory containing provider folders (default: .)'
     )
     parser.add_argument(

scripts/post_processing/merge_mem.py

@@ -50,7 +50,7 @@ def merge_mem_files(input_dir: Path, output_path: Path):
     print(f"Found {len(mem_files)} Memory metric files")
     
     output_columns = [
-        'id',' provider', 'protocol', 'dnssec_mode', 'keep_alive',
+        'id','provider', 'protocol', 'dnssec_mode', 'keep_alive',
         'timestamp', 'total_alloc_bytes', 'mallocs', 'gc_cycles',
         'alloc_delta', 'mallocs_delta', 'gc_delta'
     ]

scripts/post_processing/merge_pcap.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+Summarize all .pcap files into a single unified PCAP metrics CSV.
+Each row represents one scenario (one PCAP file): total bytes/packets sent and received.
+Adds provider, protocol, dnssec_mode, keep_alive columns parsed from filenames.
+"""
+
+import argparse
+import socket
+import time
+import csv
+from pathlib import Path
+from typing import List, NamedTuple
+
+import dpkt
+
+
+DEFAULT_LOCAL_IP = "192.168.100.2"  # netns veth1 address
+DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"]
+
+
+class PacketSummary(NamedTuple):
+    bytes_sent: int
+    bytes_received: int
+    packets_sent: int
+    packets_received: int
+    total_bytes: int
+    total_packets: int
+
+
+def parse_config_from_filename(filename: str) -> dict:
+    """Parse protocol, dnssec_mode, keep_alive from filename like 'dot-trust-persist.pcap'"""
+    base = filename.replace(".pcap", "").replace(".PCAP", "")
+    parts = base.split("-")
+
+    protocol = parts[0]
+    dnssec_mode = "off"
+    keep_alive = 0
+
+    for part in parts[1:]:
+        if part in ("auth", "trust"):
+            dnssec_mode = part
+        elif part == "persist":
+            keep_alive = 1
+
+    return {
+        "protocol": protocol,
+        "dnssec_mode": dnssec_mode,
+        "keep_alive": keep_alive,
+    }
+
+
+def find_pcap_files(input_dir: Path, providers: List[str]) -> List[Path]:
+    files: List[Path] = []
+    for provider in providers:
+        provider_path = input_dir / provider
+        if not provider_path.exists():
+            print(f"  ⚠ Missing provider directory: {provider}")
+            continue
+        for p in sorted(provider_path.glob("*.pcap")):
+            if ".bak" not in p.name:
+                files.append(p)
+    return files
+
+
+def summarize_pcap(pcap_path: Path, local_ip_bytes: bytes) -> PacketSummary:
+    """Read a PCAP file and return aggregated byte/packet counts by direction."""
+    bytes_sent = bytes_received = packets_sent = packets_received = 0
+
+    with open(pcap_path, "rb") as f:
+        try:
+            reader = dpkt.pcap.Reader(f)
+        except ValueError:
+            f.seek(0)
+            reader = dpkt.pcapng.Reader(f)
+
+        for _ts, buf in reader:
+            try:
+                eth = dpkt.ethernet.Ethernet(buf)
+                ip = eth.data
+                if not isinstance(ip, dpkt.ip.IP):
+                    continue
+                size = len(buf)
+                if ip.src == local_ip_bytes:
+                    bytes_sent += size
+                    packets_sent += 1
+                elif ip.dst == local_ip_bytes:
+                    bytes_received += size
+                    packets_received += 1
+            except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError):
+                continue
+
+    return PacketSummary(
+        bytes_sent=bytes_sent,
+        bytes_received=bytes_received,
+        packets_sent=packets_sent,
+        packets_received=packets_received,
+        total_bytes=bytes_sent + bytes_received,
+        total_packets=packets_sent + packets_received,
+    )
+
+
+def merge_pcap_files(input_dir: Path, output_path: Path, local_ip: str, providers: List[str]):
+    local_ip_bytes = socket.inet_aton(local_ip)
+
+    pcap_files = find_pcap_files(input_dir, providers)
+    if not pcap_files:
+        print("No .pcap files found.")
+        return
+
+    print(f"Found {len(pcap_files)} PCAP files")
+    print(f"Local IP: {local_ip}\n")
+
+    output_columns = [
+        "id", "provider", "protocol", "dnssec_mode", "keep_alive",
+        "bytes_sent", "bytes_received", "packets_sent", "packets_received",
+        "total_bytes", "total_packets",
+    ]
+
+    total_rows = 0
+    t_total = time.time()
+
+    with open(output_path, "w", newline="", encoding="utf-8") as outfile:
+        writer = csv.DictWriter(outfile, fieldnames=output_columns)
+        writer.writeheader()
+
+        for pcap_path in pcap_files:
+            provider = pcap_path.parent.name.lower()
+            config = parse_config_from_filename(pcap_path.name)
+
+            print(f"  {provider}/{pcap_path.name} "
+                  f"({config['protocol']}, {config['dnssec_mode']}, persist={config['keep_alive']})")
+
+            t0 = time.time()
+            summary = summarize_pcap(pcap_path, local_ip_bytes)
+            dt = time.time() - t0
+
+            total_rows += 1
+            writer.writerow({
+                "id": total_rows,
+                "provider": provider,
+                "protocol": config["protocol"],
+                "dnssec_mode": config["dnssec_mode"],
+                "keep_alive": config["keep_alive"],
+                "bytes_sent": summary.bytes_sent,
+                "bytes_received": summary.bytes_received,
+                "packets_sent": summary.packets_sent,
+                "packets_received": summary.packets_received,
+                "total_bytes": summary.total_bytes,
+                "total_packets": summary.total_packets,
+            })
+
+            print(f"    ✓ {summary.packets_sent:,} sent / {summary.packets_received:,} received "
+                  f"| {summary.total_bytes:,} B total  ({dt:.2f}s)")
+
+    elapsed = time.time() - t_total
+    print(f"\n{'=' * 60}")
+    print(f"PCAP metrics merged → {output_path}")
+    print(f"Total scenarios: {total_rows}")
+    print(f"Completed in {elapsed:.2f}s ({elapsed / 60:.1f} min)")
+    print(f"{'=' * 60}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Summarize all .pcap files into a unified CSV.")
+    parser.add_argument("input_dir", nargs="?", default=".", help="Input directory (default: .)")
+    parser.add_argument("-o", "--output", default="dns_results_pcap.csv", help="Output CSV path")
+    parser.add_argument("--local-ip", default=DEFAULT_LOCAL_IP,
+                        help=f"Local netns veth IP for direction detection (default: {DEFAULT_LOCAL_IP})")
+    parser.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS,
+                        help="Provider subdirectory names to scan")
+    args = parser.parse_args()
+
+    merge_pcap_files(Path(args.input_dir), Path(args.output), args.local_ip, args.providers)
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())

scripts/post_processing/minimal.py

@@ -0,0 +1,71 @@
+import csv
+import sys
+from collections import defaultdict
+
+COLUMNS = ["provider", "protocol", "dnssec_mode", "query_type", "keep_alive"]
+
+
+def get_uncovered_values(rows, selected_indices, col_indices, all_values):
+    covered = defaultdict(set)
+    for idx in selected_indices:
+        for col, col_idx in col_indices.items():
+            covered[col].add(rows[idx][col_idx])
+    
+    uncovered = {}
+    for col in COLUMNS:
+        uncovered[col] = all_values[col] - covered[col]
+    return uncovered
+
+
+def main(input_file, output_file):
+    with open(input_file, newline="") as f:
+        reader = csv.reader(f)
+        header = next(reader)
+        rows = list(reader)
+
+    col_indices = {col: header.index(col) for col in COLUMNS}
+
+    # Collect all unique values per column
+    all_values = defaultdict(set)
+    for row in rows:
+        for col, idx in col_indices.items():
+            all_values[col].add(row[idx])
+
+    # Greedy set cover
+    selected = []
+    uncovered = get_uncovered_values(rows, selected, col_indices, all_values)
+
+    while any(uncovered.values()):
+        best_row = None
+        best_score = 0
+
+        for i, row in enumerate(rows):
+            if i in selected:
+                continue
+            score = sum(
+                1 for col, idx in col_indices.items() if row[idx] in uncovered[col]
+            )
+            if score > best_score:
+                best_score = score
+                best_row = i
+
+        if best_row is None:
+            break
+
+        selected.append(best_row)
+        uncovered = get_uncovered_values(rows, selected, col_indices, all_values)
+
+    with open(output_file, "w", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow(header)
+        for idx in selected:
+            writer.writerow(rows[idx])
+
+    print(f"Selected {len(selected)} rows out of {len(rows)}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python minimize_csv.py input.csv output.csv")
+        sys.exit(1)
+    main(sys.argv[1], sys.argv[2])

scripts/shell/run.sh

@@ -10,6 +10,7 @@ OUTPUT_DIR="./results"
 INTERFACE="veth1"
 TIMEOUT="5s"
 SLEEP_TIME="1"
+DRY_RUN="false"
 
 # Parse arguments
 while [[ $# -gt 0 ]]; do
@@ -38,6 +39,10 @@ while [[ $# -gt 0 ]]; do
       SLEEP_TIME="$2"
       shift 2
       ;;
+    -n|--dry-run)
+      DRY_RUN="true"
+      shift
+      ;;
     --help)
       echo "Usage: $0 [OPTIONS]"
       echo ""
@@ -48,6 +53,7 @@ while [[ $# -gt 0 ]]; do
       echo "  -I, --interface NAME      Network interface (default: veth1)"
       echo "  -T, --timeout DURATION    Timeout duration (default: 5s)"
       echo "  -s, --sleep SECONDS       Sleep between runs (default: 1)"
+      echo "  -n, --dry-run             Print the scenarios that would run, then exit"
       echo "  --help                    Show this help"
       exit 0
       ;;
@@ -66,39 +72,102 @@ echo "  Output dir: $OUTPUT_DIR"
 echo "  Interface: $INTERFACE"
 echo "  Timeout: $TIMEOUT"
 echo "  Sleep time: ${SLEEP_TIME}s"
+echo "  Dry run: $DRY_RUN"
 echo ""
 
-# Server definitions as associative arrays (name -> url)
+SC_NAME=()
-declare -A CONN_SERVERS=(
+SC_URL=()
-    ["google-dotcp"]="dotcp://8.8.8.8:53"
+SC_DNSSEC=()
-    ["cloudflare-dotcp"]="dotcp://1.1.1.1:53"
+SC_AUTH=()
-    ["quad9-dotcp"]="dotcp://9.9.9.9:53"
+SC_KEEP=()
-    ["adguard-dotcp"]="dotcp://dns.adguard-dns.com:53"
-    ["google-dot"]="tls://8.8.8.8:853"
-    ["cloudflare-dot"]="tls://1.1.1.1:853"
-    ["quad9-dot"]="tls://9.9.9.9:853"
-    ["adguard-dot"]="tls://dns.adguard-dns.com:853"
-    ["google-doh"]="https://dns.google/dns-query"
-    ["cloudflare-doh"]="https://cloudflare-dns.com/dns-query"
-    ["quad9-doh"]="https://dns10.quad9.net/dns-query"
-    ["adguard-doh"]="https://dns.adguard-dns.com/dns-query"
-)
 
-declare -A QUIC_SERVERS=(
+add() {  # add <name> <url> <dnssec> <auth> <keepalive>
-    ["google-doh3"]="doh3://dns.google/dns-query"
+    SC_NAME+=("$1")
-    ["cloudflare-doh3"]="doh3://cloudflare-dns.com/dns-query"
+    SC_URL+=("$2")
-    ["adguard-doh3"]="doh3://dns.adguard-dns.com/dns-query"
+    SC_DNSSEC+=("$3")
-    ["adguard-doq"]="doq://dns.adguard-dns.com:853"
+    SC_AUTH+=("$4")
-)
+    SC_KEEP+=("$5")
+}
 
-declare -A CONNLESS_SERVERS=(
+# Protocol comparison - DNSSEC off, non-persistent.
-    ["google-doudp"]="udp://8.8.8.8:53"
+# The primary comparison axis: full encryption cost, handshake included.
-    ["cloudflare-doudp"]="udp://1.1.1.1:53"
+# All transports.
-    ["quad9-doudp"]="udp://9.9.9.9:53"
+add google-dotcp      "dotcp://8.8.8.8:53"                      false false false
-    ["adguard-doudp"]="udp://dns.adguard-dns.com:53"
+add cloudflare-dotcp  "dotcp://1.1.1.1:53"                      false false false
-    ["adguard-dnscrypt"]="sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20"
+add quad9-dotcp       "dotcp://9.9.9.9:53"                      false false false
-    ["quad9-dnscrypt"]="sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ"
+add adguard-dotcp     "dotcp://dns.adguard-dns.com:53"          false false false
-)
+
+add google-dot        "tls://8.8.8.8:853"                       false false false
+add cloudflare-dot    "tls://1.1.1.1:853"                       false false false
+add quad9-dot         "tls://9.9.9.9:853"                       false false false
+add adguard-dot       "tls://dns.adguard-dns.com:853"           false false false
+
+add google-doh        "https://dns.google/dns-query"            false false false
+add cloudflare-doh    "https://cloudflare-dns.com/dns-query"    false false false
+add quad9-doh         "https://dns10.quad9.net/dns-query"       false false false
+add adguard-doh       "https://dns.adguard-dns.com/dns-query"   false false false
+
+add google-doh3       "doh3://dns.google/dns-query"             false false false
+add cloudflare-doh3   "doh3://cloudflare-dns.com/dns-query"     false false false
+add adguard-doh3      "doh3://dns.adguard-dns.com/dns-query"    false false false
+add adguard-doq       "doq://dns.adguard-dns.com:853"           false false false
+
+add google-doudp      "udp://8.8.8.8:53"                        false false false
+add cloudflare-doudp  "udp://1.1.1.1:53"                        false false false
+add quad9-doudp       "udp://9.9.9.9:53"                        false false false
+add adguard-doudp     "udp://dns.adguard-dns.com:53"            false false false
+add adguard-dnscrypt  "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" false false false
+add quad9-dnscrypt    "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" false false false
+
+# Persistence contrast - DNSSEC off, persistent.
+# Teste for TCP based protocols.
+# QUIC makes no sense cause of 0-RTT.
+add google-dotcp      "dotcp://8.8.8.8:53"                      false false true
+add cloudflare-dotcp  "dotcp://1.1.1.1:53"                      false false true
+add quad9-dotcp       "dotcp://9.9.9.9:53"                      false false true
+add adguard-dotcp     "dotcp://dns.adguard-dns.com:53"          false false true
+
+add google-dot        "tls://8.8.8.8:853"                       false false true
+add cloudflare-dot    "tls://1.1.1.1:853"                       false false true
+add quad9-dot         "tls://9.9.9.9:853"                       false false true
+add adguard-dot       "tls://dns.adguard-dns.com:853"           false false true
+
+add google-doh        "https://dns.google/dns-query"            false false true
+add cloudflare-doh    "https://cloudflare-dns.com/dns-query"    false false true
+add quad9-doh         "https://dns10.quad9.net/dns-query"       false false true
+add adguard-doh       "https://dns.adguard-dns.com/dns-query"   false false true
+
+# DNSSEC trust - non-persistent.
+add google-dotcp      "dotcp://8.8.8.8:53"                      true false false
+add cloudflare-dotcp  "dotcp://1.1.1.1:53"                      true false false
+add quad9-dotcp       "dotcp://9.9.9.9:53"                      true false false
+add adguard-dotcp     "dotcp://dns.adguard-dns.com:53"          true false false
+
+add google-dot        "tls://8.8.8.8:853"                       true false false
+add cloudflare-dot    "tls://1.1.1.1:853"                       true false false
+add quad9-dot         "tls://9.9.9.9:853"                       true false false
+add adguard-dot       "tls://dns.adguard-dns.com:853"           true false false
+
+add google-doh        "https://dns.google/dns-query"            true false false
+add cloudflare-doh    "https://cloudflare-dns.com/dns-query"    true false false
+add quad9-doh         "https://dns10.quad9.net/dns-query"       true false false
+add adguard-doh       "https://dns.adguard-dns.com/dns-query"   true false false
+
+add google-doh3       "doh3://dns.google/dns-query"             true false false
+add cloudflare-doh3   "doh3://cloudflare-dns.com/dns-query"     true false false
+add adguard-doh3      "doh3://dns.adguard-dns.com/dns-query"    true false false
+add adguard-doq       "doq://dns.adguard-dns.com:853"           true false false
+
+add google-doudp      "udp://8.8.8.8:53"                        true false false
+add cloudflare-doudp  "udp://1.1.1.1:53"                        true false false
+add quad9-doudp       "udp://9.9.9.9:53"                        true false false
+add adguard-doudp     "udp://dns.adguard-dns.com:53"            true false false
+add adguard-dnscrypt  "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" true false false
+add quad9-dnscrypt    "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" true false false
+
+# DNSSEC auth - DoUDP only.
+# auth walks the chain of trust over plain DoUDP regardless of the protocol.
+add adguard-doudp     "udp://dns.adguard-dns.com:53"            true true false
 
 # Function to get flags suffix for filename
 get_flags_suffix() {
@@ -133,10 +202,6 @@ run_with_perf() {
     local keepalive="$5"
 
     local suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
-    local base_name="${name}"
-    if [[ -n "$suffix" ]]; then
-        base_name="${name}-${suffix}"
-    fi
     local provider="${name%%-*}"
     local protocol="${name#*-}"
     local base_name="${protocol}"
@@ -194,64 +259,39 @@ run_with_perf() {
     # Cleanup
     rm -f "$perf_tmp"
 
-    echo "  -> CPU metrics saved to ${base_name}.cpu.csv"
+    echo "  -> CPU metrics saved to ${provider}/${base_name}.cpu.csv"
 }
 
-# Function to run servers with given flags
+echo "Total scenarios: ${#SC_NAME[@]}"
-run_server_group() {
+echo ""
-    local -n servers=$1
-    local dnssec="$2"
-    local auth="$3"
-    local keepalive="$4"
-    local desc="$5"
 
-    echo "Running: $desc"
+for i in "${!SC_NAME[@]}"; do
+    name="${SC_NAME[$i]}"
+    url="${SC_URL[$i]}"
+    dnssec="${SC_DNSSEC[$i]}"
+    auth="${SC_AUTH[$i]}"
+    keepalive="${SC_KEEP[$i]}"
 
-    for name in "${!servers[@]}"; do
+    suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
-        local url="${servers[$name]}"
+    provider="${name%%-*}"
-        echo "  Processing: $name ($url)"
+    protocol="${name#*-}"
+    label="${protocol}"
+    [[ -n "$suffix" ]] && label="${protocol}-${suffix}"
+
+    if [[ "$DRY_RUN" == "true" ]]; then
+        printf '[dry] %-28s dnssec=%-5s auth=%-5s keepalive=%-5s -> %s/%s\n' \
+            "$name" "$dnssec" "$auth" "$keepalive" "$provider" "$label"
+        continue
+    fi
+
+    echo "Processing: $name ($url)  [dnssec=$dnssec auth=$auth keepalive=$keepalive]"
     run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive"
     sleep "$SLEEP_TIME"
-    done
+done
-}
-
-echo "=== Running TCP-based protocols (TLS/HTTPS) ==="
-
-# DNSSEC off, Keep off
-run_server_group CONN_SERVERS "false" "false" "false" "no-dnssec, no-keepalive"
-
-# DNSSEC off, Keep on
-run_server_group CONN_SERVERS "false" "false" "true" "no-dnssec, keepalive"
-
-# DNSSEC on (trust), Keep on
-run_server_group CONN_SERVERS "true" "false" "true" "dnssec-trust, keepalive"
-
-# DNSSEC on (auth), Keep on
-run_server_group CONN_SERVERS "true" "true" "true" "dnssec-auth, keepalive"
 
 echo ""
-echo "=== Running QUIC-based protocols (DoH3/DoQ) ==="
+if [[ "$DRY_RUN" == "true" ]]; then
-
+    echo "Dry run complete - nothing was executed."
-# DNSSEC off
+else
-run_server_group QUIC_SERVERS "false" "false" "false" "no-dnssec"
+    echo "All scenarios completed!"
-
+fi
-# DNSSEC on (trust)
-run_server_group QUIC_SERVERS "true" "false" "false" "dnssec-trust"
-
-# DNSSEC on (auth)
-run_server_group QUIC_SERVERS "true" "true" "false" "dnssec-auth"
-
-echo ""
-echo "=== Running connectionless protocols (UDP) ==="
-
-# DNSSEC off
-run_server_group CONNLESS_SERVERS "false" "false" "false" "no-dnssec"
-
-# DNSSEC on (trust)
-run_server_group CONNLESS_SERVERS "true" "false" "false" "dnssec-trust"
-
-# DNSSEC on (auth)
-run_server_group CONNLESS_SERVERS "true" "true" "false" "dnssec-auth"
-
-echo ""
-echo "All combinations completed!"