From cc5e0ab00a69956335993b3c18c3400363868120 Mon Sep 17 00:00:00 2001 From: afonsofrancof Date: Wed, 8 Jul 2026 08:42:39 +0100 Subject: [PATCH] feat(everything): fix doh and increase min TLS to 1.3 --- .gitignore | 1 + Makefile | 10 + common/protocols/doh/doh.go | 76 ++- common/protocols/doq/doq.go | 4 +- common/protocols/dot/dot.go | 1 + go.mod | 32 +- go.sum | 92 ++-- .../add_extra_metrics_to_csv.py | 499 ++++++------------ scripts/post_processing/merge_files.py | 16 +- scripts/post_processing/merge_mem.py | 2 +- scripts/post_processing/merge_pcap.py | 179 +++++++ scripts/post_processing/minimal.py | 71 +++ scripts/shell/run.sh | 238 +++++---- 13 files changed, 691 insertions(+), 530 deletions(-) create mode 100644 Makefile create mode 100644 scripts/post_processing/merge_pcap.py create mode 100644 scripts/post_processing/minimal.py diff --git a/.gitignore b/.gitignore index aa3ee76..90bf36b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ /results /results.bak /results_merged +dns_results* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5792227 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +.PHONY: all clean + +all: + python3 scripts/post_processing/merge_files.py + python3 scripts/post_processing/merge_cpu.py + python3 scripts/post_processing/merge_mem.py + python3 scripts/post_processing/merge_pcap.py + +clean: + rm -f ./dns_results.csv ./dns_results_cpu.csv ./dns_results_mem.csv ./dns_results_pcap.csv diff --git a/common/protocols/doh/doh.go b/common/protocols/doh/doh.go index 5f1a6a8..0966003 100644 --- a/common/protocols/doh/doh.go +++ b/common/protocols/doh/doh.go @@ -27,7 +27,6 @@ type Config struct { Path string DNSSEC bool HTTP3 bool - HTTP2 bool KeepAlive bool } @@ -62,6 +61,7 @@ func New(config Config) (*Client, error) { tlsConfig := &tls.Config{ ServerName: config.Host, + MinVersion: tls.VersionTLS13, ClientSessionCache: tls.NewLRUClientSessionCache(100), } @@ -88,14 +88,7 @@ func New(config Config) (*Client, error) { } var transportType string - if config.HTTP2 { - http2Transport := &http2.Transport{ - TLSClientConfig: tlsConfig, - AllowHTTP: true, - } - httpClient.Transport = http2Transport - transportType = "HTTP/2" - } else if config.HTTP3 { + if config.HTTP3 { quicTlsConfig := http3.ConfigureTLSConfig(tlsConfig) http3Transport := &http3.Transport{ TLSClientConfig: quicTlsConfig, @@ -104,7 +97,12 @@ func New(config Config) (*Client, error) { httpClient.Transport = http3Transport transportType = "HTTP/3" } else { - transportType = "HTTP/1.1" + http2Transport := &http2.Transport{ + TLSClientConfig: tlsConfig, + AllowHTTP: true, + } + httpClient.Transport = http2Transport + transportType = "HTTP/2" } logger.Debug("DoH client created: %s (%s, DNSSEC: %v, KeepAlive: %v)", rawURL, transportType, config.DNSSEC, config.KeepAlive) @@ -116,6 +114,43 @@ func New(config Config) (*Client, error) { }, nil } +// newSingleUseClient builds a throwaway *http.Client whose transport is torn +// down by the returned closeFn. Used for non-persistent (non-keep-alive) mode +// so that each query establishes and tears down its own connection, and thus +// pays its own TCP+TLS (or QUIC) handshake. Without this, the pooled +// HTTP/2 transport silently reuses a single connection across all queries +// (HTTP/2 multiplexes and ignores the Connection: close header), which would +// make the non-persistent measurement indistinguishable from persistent. +func (c *Client) newSingleUseClient() (*http.Client, func()) { + tlsConfig := &tls.Config{ + ServerName: c.config.Host, + MinVersion: tls.VersionTLS13, + ClientSessionCache: nil, // no resumption: force a full handshake each query + } + + if c.config.HTTP3 { + quicConf := &quic.Config{ + MaxIdleTimeout: 30 * time.Second, + DisablePathMTUDiscovery: true, + } + quicTLS := http3.ConfigureTLSConfig(tlsConfig) + tr := &http3.Transport{ + TLSClientConfig: quicTLS, + QUICConfig: quicConf, + } + client := &http.Client{Transport: tr, Timeout: 30 * time.Second} + return client, func() { tr.Close() } + } + + tr := &http2.Transport{ + TLSClientConfig: tlsConfig, + AllowHTTP: true, + DisableCompression: true, + } + client := &http.Client{Transport: tr, Timeout: 30 * time.Second} + return client, func() { tr.CloseIdleConnections() } +} + func (c *Client) Close() { logger.Debug("Closing DoH client") if t, ok := c.httpClient.Transport.(*http.Transport); ok { @@ -143,6 +178,18 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) { return nil, fmt.Errorf("doh: failed to pack DNS message: %w", err) } + // Select the HTTP client. In persistent mode, reuse the pooled client built + // in New(). In non-persistent mode, build a fresh client (and transport) + // per query and tear it down afterwards, so every query pays its own + // handshake — otherwise HTTP/2 would reuse one connection and the + // non-persistent measurement would be wrong. + httpClient := c.httpClient + if !c.config.KeepAlive { + freshClient, closeFn := c.newSingleUseClient() + defer closeFn() + httpClient = freshClient + } + httpReq, err := http.NewRequest(http.MethodPost, c.upstreamURL.String(), bytes.NewReader(packedMsg)) if err != nil { logger.Error("DoH failed to create HTTP request: %v", err) @@ -153,14 +200,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) { httpReq.Header.Set("Content-Type", dnsMessageContentType) httpReq.Header.Set("Accept", dnsMessageContentType) - // Set Connection header based on KeepAlive setting - if c.config.KeepAlive { - httpReq.Header.Set("Connection", "keep-alive") - } else { - httpReq.Header.Set("Connection", "close") - } - - httpResp, err := c.httpClient.Do(httpReq) + httpResp, err := httpClient.Do(httpReq) if err != nil { logger.Error("DoH request failed to %s: %v", c.upstreamURL.Host, err) return nil, fmt.Errorf("doh: failed executing HTTP request to %s: %w", c.upstreamURL.Host, err) diff --git a/common/protocols/doq/doq.go b/common/protocols/doq/doq.go index fb4a41e..83d9063 100644 --- a/common/protocols/doq/doq.go +++ b/common/protocols/doq/doq.go @@ -26,7 +26,7 @@ type Client struct { targetAddr *net.UDPAddr tlsConfig *tls.Config udpConn *net.UDPConn - quicConn quic.Connection + quicConn *quic.Conn quicTransport *quic.Transport quicConfig *quic.Config config Config @@ -119,7 +119,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) { return nil, fmt.Errorf("doq: failed to pack message: %w", err) } - var quicStream quic.Stream + var quicStream *quic.Stream quicStream, err = c.quicConn.OpenStream() if err != nil { logger.Debug("DoQ stream failed, reconnecting: %v", err) diff --git a/common/protocols/dot/dot.go b/common/protocols/dot/dot.go index 88b3682..1c2c309 100644 --- a/common/protocols/dot/dot.go +++ b/common/protocols/dot/dot.go @@ -49,6 +49,7 @@ func New(config Config) (*Client, error) { tlsConfig := &tls.Config{ ServerName: config.Host, + MinVersion: tls.VersionTLS13, } client := &Client{ diff --git a/go.mod b/go.mod index d461476..3b6d09d 100644 --- a/go.mod +++ b/go.mod @@ -1,29 +1,25 @@ module github.com/afonsofrancof/sdns-proxy -go 1.24.1 +go 1.26.4 require ( - github.com/alecthomas/kong v1.8.1 + github.com/alecthomas/kong v1.15.0 github.com/ameshkov/dnscrypt/v2 v2.4.0 github.com/google/gopacket v1.1.19 - github.com/miekg/dns v1.1.65 - github.com/quic-go/quic-go v0.50.0 - golang.org/x/net v0.38.0 + github.com/miekg/dns v1.1.72 + github.com/quic-go/quic-go v0.60.0 + golang.org/x/net v0.56.0 ) require ( - github.com/AdguardTeam/golibs v0.32.7 // indirect + github.com/AdguardTeam/golibs v0.35.14 // indirect github.com/ameshkov/dnsstamps v1.0.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect - github.com/onsi/ginkgo/v2 v2.9.5 // indirect - github.com/quic-go/qpack v0.5.1 // indirect - go.uber.org/mock v0.5.0 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect - golang.org/x/tools v0.31.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.39.0 // indirect + golang.org/x/tools v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 3d0c331..89694af 100644 --- a/go.sum +++ b/go.sum @@ -1,88 +1,64 @@ -github.com/AdguardTeam/golibs v0.32.7 h1:3dmGlAVgmvquCCwHsvEl58KKcRAK3z1UnjMnwSIeDH4= -github.com/AdguardTeam/golibs v0.32.7/go.mod h1:bE8KV1zqTzgZjmjFyBJ9f9O5DEKO717r7e57j1HclJA= +github.com/AdguardTeam/golibs v0.35.14 h1:LGqoRMbuxUgZyhOjlkmPxy5t1/XPuFpDcG5RBIDcskA= +github.com/AdguardTeam/golibs v0.35.14/go.mod h1:FL470ALCuUs7cwk88ltBgTZ6JYCUG2NpNzp3EOMirxo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/kong v1.8.1 h1:6aamvWBE/REnR/BCq10EcozmcpUPc5aGI1lPAWdB0EE= -github.com/alecthomas/kong v1.8.1/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU= -github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= -github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI= +github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/ameshkov/dnscrypt/v2 v2.4.0 h1:if6ZG2cuQmcP2TwSY+D0+8+xbPfoatufGlOQTMNkI9o= github.com/ameshkov/dnscrypt/v2 v2.4.0/go.mod h1:WpEFV2uhebXb8Jhes/5/fSdpmhGV8TL22RDaeWwV6hI= github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo= github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= -github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= -github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo= -github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= +github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= -golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/post_processing/add_extra_metrics_to_csv.py b/scripts/post_processing/add_extra_metrics_to_csv.py index 3f6fdb8..8742513 100644 --- a/scripts/post_processing/add_extra_metrics_to_csv.py +++ b/scripts/post_processing/add_extra_metrics_to_csv.py @@ -1,405 +1,258 @@ #!/usr/bin/env python3 """ -Fast PCAP Preprocessor for DNS QoS Analysis -Loads PCAP into memory first, then uses binary search for matching. -Uses LAN IP to determine direction (LAN = sent, non-LAN = received). +Fast PCAP Preprocessor for DNS QoS Analysis. + +Loads PCAP into memory, then uses binary search to match packets to query +windows. Direction is determined by a configured local IP (the netns veth IP). """ +import argparse +import bisect import csv import shutil +import socket +import time from pathlib import Path from typing import Dict, List, NamedTuple -import time import dpkt from dateutil import parser as date_parser - BANDWIDTH_COLUMNS = [ - 'bytes_sent', - 'bytes_received', - 'packets_sent', - 'packets_received', - 'total_bytes', + "bytes_sent", + "bytes_received", + "packets_sent", + "packets_received", + "total_bytes", ] +DEFAULT_LOCAL_IP = "192.168.100.2" # netns veth1 address +DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"] + class Packet(NamedTuple): - """Lightweight packet representation.""" timestamp: float size: int - is_outbound: bool # True if from LAN, False if from internet + is_outbound: bool -class QueryWindow: - """Efficient query window representation.""" - __slots__ = ['index', 'start', 'end', 'sent', 'received', 'pkts_sent', 'pkts_received'] - - def __init__(self, index: int, start: float, end: float): - self.index = index - self.start = start - self.end = end - self.sent = 0 - self.received = 0 - self.pkts_sent = 0 - self.pkts_received = 0 - - -def is_already_processed(csv_path: Path) -> bool: - """ - Check if CSV has already been processed. - Returns True if bandwidth columns exist AND at least one row has non-zero data. - """ +def needs_processing(csv_path: Path) -> bool: + """True if file lacks bandwidth columns OR all values are empty/zero.""" try: - with open(csv_path, 'r', encoding='utf-8') as f: + with open(csv_path, "r", encoding="utf-8") as f: reader = csv.DictReader(f) - - # Check if columns exist if not reader.fieldnames: - return False - - if not all(col in reader.fieldnames for col in BANDWIDTH_COLUMNS): - return False - - # Check if any row has non-zero bandwidth data + return True + if not all(c in reader.fieldnames for c in BANDWIDTH_COLUMNS): + return True for row in reader: for col in BANDWIDTH_COLUMNS: - val = row.get(col, '').strip() - if val and val != '0': - return True - - # All rows have zero/empty values - not truly processed - return False - + val = (row.get(col) or "").strip() + if val and val != "0": + return False # has real data + return True # columns exist but all empty/zero except Exception: - return False + return True def parse_csv_timestamp(ts_str: str) -> float: - """Convert RFC3339Nano timestamp to Unix epoch (seconds).""" - dt = date_parser.isoparse(ts_str) - return dt.timestamp() + return date_parser.isoparse(ts_str).timestamp() -def is_lan_ip(ip_bytes: bytes) -> bool: - """Check if IP is a private/LAN address.""" - if len(ip_bytes) != 4: - return False - - first = ip_bytes[0] - second = ip_bytes[1] - - # 10.0.0.0/8 - if first == 10: - return True - - # 172.16.0.0/12 - if first == 172 and 16 <= second <= 31: - return True - - # 192.168.0.0/16 - if first == 192 and second == 168: - return True - - # 127.0.0.0/8 (localhost) - if first == 127: - return True - - return False +def load_pcap(pcap_path: Path, local_ip_bytes: bytes) -> List[Packet]: + """Load PCAP into a list of Packets sorted by timestamp.""" + print(" Loading PCAP...") + t0 = time.time() + packets: List[Packet] = [] + with open(pcap_path, "rb") as f: + try: + reader = dpkt.pcap.Reader(f) + except ValueError: + f.seek(0) + reader = dpkt.pcapng.Reader(f) -def load_pcap_into_memory(pcap_path: Path) -> List[Packet]: - """Load all packets from PCAP into memory with minimal data.""" - packets = [] - - print(f" Loading PCAP into memory...") - start_time = time.time() - - try: - with open(pcap_path, 'rb') as f: + for ts, buf in reader: try: - pcap = dpkt.pcap.Reader(f) - except: - # Try pcapng format - f.seek(0) - pcap = dpkt.pcapng.Reader(f) - - for ts, buf in pcap: - try: - packet_time = float(ts) - packet_size = len(buf) - - # Parse to get source IP - eth = dpkt.ethernet.Ethernet(buf) - - # Default to outbound if we can't determine - is_outbound = True - - if isinstance(eth.data, dpkt.ip.IP): - ip = eth.data - src_ip = ip.src - is_outbound = is_lan_ip(src_ip) - - packets.append(Packet( - timestamp=packet_time, - size=packet_size, - is_outbound=is_outbound - )) - - except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError): + eth = dpkt.ethernet.Ethernet(buf) + ip = eth.data + if not isinstance(ip, dpkt.ip.IP): continue - - except Exception as e: - print(f" Error reading PCAP: {e}") - return [] - - elapsed = time.time() - start_time - print(f" Loaded {len(packets):,} packets in {elapsed:.2f}s") - - # Sort by timestamp for binary search + if ip.src == local_ip_bytes: + is_outbound = True + elif ip.dst == local_ip_bytes: + is_outbound = False + else: + continue # not our traffic + packets.append(Packet(float(ts), len(buf), is_outbound)) + except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError): + continue + packets.sort(key=lambda p: p.timestamp) - + print(f" Loaded {len(packets):,} packets in {time.time() - t0:.2f}s") return packets -def find_packets_in_window( - packets: List[Packet], - start_time: float, - end_time: float, - left_hint: int = 0 -) -> tuple[List[Packet], int]: - """ - Binary search to find all packets within time window. - Returns (matching_packets, left_index_hint_for_next_search). - """ - if not packets: - return [], 0 - - # Binary search for first packet >= start_time - left, right = left_hint, len(packets) - 1 - first_idx = len(packets) - - while left <= right: - mid = (left + right) // 2 - if packets[mid].timestamp >= start_time: - first_idx = mid - right = mid - 1 - else: - left = mid + 1 - - # No packets in range - if first_idx >= len(packets) or packets[first_idx].timestamp > end_time: - return [], first_idx - - # Collect all packets in window - matching = [] - idx = first_idx - while idx < len(packets) and packets[idx].timestamp <= end_time: - matching.append(packets[idx]) - idx += 1 - - return matching, first_idx - - def load_csv_queries(csv_path: Path) -> List[Dict]: - """Load CSV and create query data structures.""" queries = [] - with open(csv_path, 'r', encoding='utf-8') as f: - reader = csv.DictReader(f) - for row in reader: + with open(csv_path, "r", encoding="utf-8") as f: + for row in csv.DictReader(f): try: - ts_epoch = parse_csv_timestamp(row['timestamp']) - duration_s = float(row['duration_ns']) / 1e9 - queries.append({ - 'data': row, - 'start_time': ts_epoch, - 'end_time': ts_epoch + duration_s, - }) + start = parse_csv_timestamp(row["timestamp"]) + duration = float(row["duration_ns"]) / 1e9 + queries.append( + {"data": row, "start_time": start, "end_time": start + duration} + ) except Exception as e: - print(f" Warning: Skipping row - {e}") - continue + print(f" Warning: skipping row - {e}") return queries -def match_packets_to_queries( - packets: List[Packet], - queries: List[Dict] -) -> List[Dict]: - """Match packets to query windows using binary search.""" - if not queries or not packets: - return queries - - print(f" Matching packets to queries...") - start_time = time.time() - - # Initialize metrics +def match_packets(packets: List[Packet], queries: List[Dict]) -> int: + """Assign bandwidth metrics to each query. Returns total matched packets.""" + if not packets or not queries: + for q in queries: + q.update({c: 0 for c in BANDWIDTH_COLUMNS}) + return 0 + + print(" Matching packets to queries...") + t0 = time.time() + timestamps = [p.timestamp for p in packets] + matched = 0 + for q in queries: - q['bytes_sent'] = 0 - q['bytes_received'] = 0 - q['packets_sent'] = 0 - q['packets_received'] = 0 - q['total_bytes'] = 0 - - # Sort queries by start time for sequential processing - queries_sorted = sorted(enumerate(queries), key=lambda x: x[1]['start_time']) - - matched_packets = 0 - left_hint = 0 # Optimization: start next search from here - - for original_idx, q in queries_sorted: - matching, left_hint = find_packets_in_window( - packets, - q['start_time'], - q['end_time'], - left_hint - ) - - for pkt in matching: - matched_packets += 1 + lo = bisect.bisect_left(timestamps, q["start_time"]) + hi = bisect.bisect_right(timestamps, q["end_time"]) + bs = br = ps = pr = 0 + for pkt in packets[lo:hi]: if pkt.is_outbound: - q['bytes_sent'] += pkt.size - q['packets_sent'] += 1 + bs += pkt.size + ps += 1 else: - q['bytes_received'] += pkt.size - q['packets_received'] += 1 - - q['total_bytes'] = q['bytes_sent'] + q['bytes_received'] - - elapsed = time.time() - start_time - print(f" Matched {matched_packets:,} packets in {elapsed:.2f}s") - - # Statistics - total_sent = sum(q['bytes_sent'] for q in queries) - total_recv = sum(q['bytes_received'] for q in queries) - queries_with_data = sum(1 for q in queries if q['total_bytes'] > 0) - print(f" Total: {total_sent:,} bytes sent, {total_recv:,} bytes received") - print(f" Queries with data: {queries_with_data}/{len(queries)}") - - return queries + br += pkt.size + pr += 1 + q["bytes_sent"] = bs + q["bytes_received"] = br + q["packets_sent"] = ps + q["packets_received"] = pr + q["total_bytes"] = bs + br + matched += ps + pr + + print(f" Matched {matched:,} packets in {time.time() - t0:.2f}s") + total_sent = sum(q["bytes_sent"] for q in queries) + total_recv = sum(q["bytes_received"] for q in queries) + with_data = sum(1 for q in queries if q["total_bytes"] > 0) + print(f" Total: {total_sent:,} B sent, {total_recv:,} B received") + print(f" Queries with data: {with_data}/{len(queries)}") + return matched -def write_enriched_csv( - csv_path: Path, queries: List[Dict], backup: bool = True -): - """Write enriched CSV with bandwidth columns.""" +def write_csv(csv_path: Path, queries: List[Dict], backup: bool = True): if backup and csv_path.exists(): - backup_path = csv_path.with_suffix('.csv.bak') - if not backup_path.exists(): # Don't overwrite existing backup - shutil.copy2(csv_path, backup_path) - print(f" Backup: {backup_path.name}") - - # Get fieldnames - filter out any existing bandwidth columns to avoid dupes + bak = csv_path.with_suffix(".csv.bak") + if not bak.exists(): + shutil.copy2(csv_path, bak) + print(f" Backup: {bak.name}") + original_fields = [ - f for f in queries[0]['data'].keys() - if f not in BANDWIDTH_COLUMNS + f for f in queries[0]["data"].keys() if f not in BANDWIDTH_COLUMNS ] fieldnames = original_fields + BANDWIDTH_COLUMNS - - with open(csv_path, 'w', encoding='utf-8', newline='') as f: + + with open(csv_path, "w", encoding="utf-8", newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() - for q in queries: - row = {k: v for k, v in q['data'].items() if k not in BANDWIDTH_COLUMNS} - for field in BANDWIDTH_COLUMNS: - row[field] = q[field] + row = {k: q["data"][k] for k in original_fields} + for c in BANDWIDTH_COLUMNS: + row[c] = q[c] writer.writerow(row) - - print(f" Written: {csv_path.name}") + print(f" Written: {csv_path.name}") -def process_provider_directory(provider_path: Path): - """Process all CSV/PCAP pairs in a provider directory.""" - print(f"\n{'='*60}") - print(f"Processing: {provider_path.name.upper()}") - print(f"{'='*60}") - - csv_files = sorted(provider_path.glob('*.csv')) - processed = 0 - skipped = 0 - total_time = 0 - - for csv_path in csv_files: - # Skip backup files - if '.bak' in csv_path.name: +def process_provider(provider_path: Path, local_ip_bytes: bytes): + print(f"\n{'=' * 60}\nProcessing: {provider_path.name.upper()}\n{'=' * 60}") + + processed = skipped = 0 + total_time = 0.0 + + for csv_path in sorted(provider_path.glob("*.csv")): + name = csv_path.name.lower() + if ".bak" in name or name.endswith((".cpu.csv", ".mem.csv")): continue - - pcap_path = csv_path.with_suffix('.pcap') - + + pcap_path = csv_path.with_suffix(".pcap") if not pcap_path.exists(): - print(f"\n ⚠ Skipping {csv_path.name} - no matching PCAP") + print(f"\n ⚠ {csv_path.name}: no matching PCAP") continue - - # Check if already processed - if is_already_processed(csv_path): - print(f"\n ⏭ Skipping {csv_path.name} - already processed") + + if not needs_processing(csv_path): + print(f"\n ⏭ {csv_path.name}: already processed") skipped += 1 continue - + print(f"\n 📁 {csv_path.name}") - file_start = time.time() - - # Load PCAP into memory first - packets = load_pcap_into_memory(pcap_path) + t0 = time.time() + + packets = load_pcap(pcap_path, local_ip_bytes) if not packets: - print(f" ⚠ No packets found in PCAP") + print(" ⚠ No usable packets in PCAP") continue - - # Load CSV queries + queries = load_csv_queries(csv_path) if not queries: - print(f" ⚠ No valid queries found") + print(" ⚠ No valid queries in CSV") continue - print(f" Loaded {len(queries):,} queries") - - # Match packets to queries - enriched_queries = match_packets_to_queries(packets, queries) - - # Write enriched CSV - write_enriched_csv(csv_path, enriched_queries) - - file_time = time.time() - file_start - total_time += file_time - processed += 1 - print(f" ✓ Completed in {file_time:.2f}s") - - print(f"\n {'='*58}") - print(f" {provider_path.name}: {processed} processed, {skipped} skipped") - print(f" Time: {total_time:.2f}s") - print(f" {'='*58}") + match_packets(packets, queries) + write_csv(csv_path, queries) + + dt = time.time() - t0 + total_time += dt + processed += 1 + print(f" ✓ Completed in {dt:.2f}s") + + print( + f"\n {provider_path.name}: {processed} processed, " + f"{skipped} skipped, {total_time:.2f}s" + ) def main(): - """Main preprocessing pipeline.""" - overall_start = time.time() - - print("\n" + "="*60) - print("DNS PCAP PREPROCESSOR - Memory-Optimized Edition") - print("="*60) - - results_dir = Path('results') - - if not results_dir.exists(): - print(f"\n❌ Error: '{results_dir}' directory not found") + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--local-ip", + default=DEFAULT_LOCAL_IP, + help=f"Local (netns veth) IP used to determine direction " + f"(default: {DEFAULT_LOCAL_IP})", + ) + ap.add_argument("--results-dir", default="results", type=Path) + ap.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS) + args = ap.parse_args() + + local_ip_bytes = socket.inet_aton(args.local_ip) + + print(f"\n{'=' * 60}\nDNS PCAP PREPROCESSOR\n{'=' * 60}") + print(f"Local IP: {args.local_ip}") + print(f"Results: {args.results_dir}") + + if not args.results_dir.exists(): + print(f"\n❌ Directory not found: {args.results_dir}") return - - providers = ['adguard', 'cloudflare', 'google', 'quad9'] - - for provider in providers: - provider_path = results_dir / provider - if provider_path.exists(): - process_provider_directory(provider_path) + + t0 = time.time() + for provider in args.providers: + path = args.results_dir / provider + if path.exists(): + process_provider(path, local_ip_bytes) else: - print(f"\n⚠ Warning: Provider directory not found: {provider}") - - overall_time = time.time() - overall_start - - print("\n" + "="*60) - print(f"✓ PREPROCESSING COMPLETE") - print(f" Total time: {overall_time:.2f}s ({overall_time/60:.1f} minutes)") - print("="*60 + "\n") + print(f"\n⚠ Missing provider directory: {provider}") + + total = time.time() - t0 + print(f"\n{'=' * 60}") + print(f"✓ DONE in {total:.2f}s ({total / 60:.1f} min)") + print(f"{'=' * 60}\n") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/post_processing/merge_files.py b/scripts/post_processing/merge_files.py index 8db08f0..a6f4294 100644 --- a/scripts/post_processing/merge_files.py +++ b/scripts/post_processing/merge_files.py @@ -73,6 +73,10 @@ def merge_all_csvs(input_dir: Path, output_path: Path): """Merge all CSVs into a single file.""" csv_files = find_csv_files(input_dir) + + print(f"Files to process ({len(csv_files)}):") + for f in csv_files: + print(f" {f}") if not csv_files: print("No CSV files found") @@ -96,11 +100,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path): 'duration_ms', 'request_size_bytes', 'response_size_bytes', - 'bytes_sent', - 'bytes_received', - 'packets_sent', - 'packets_received', - 'total_bytes', 'response_code', 'error', ] @@ -146,11 +145,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path): 'duration_ms': ns_to_ms(row.get('duration_ns', '')), 'request_size_bytes': row.get('request_size_bytes', ''), 'response_size_bytes': row.get('response_size_bytes', ''), - 'bytes_sent': row.get('bytes_sent', ''), - 'bytes_received': row.get('bytes_received', ''), - 'packets_sent': row.get('packets_sent', ''), - 'packets_received': row.get('packets_received', ''), - 'total_bytes': row.get('total_bytes', ''), 'response_code': row.get('response_code', ''), 'error': row.get('error', ''), } @@ -173,7 +167,7 @@ def main(): parser.add_argument( 'input_dir', nargs='?', - default='.', + default='results', help='Input directory containing provider folders (default: .)' ) parser.add_argument( diff --git a/scripts/post_processing/merge_mem.py b/scripts/post_processing/merge_mem.py index 0cd6963..9c6c195 100644 --- a/scripts/post_processing/merge_mem.py +++ b/scripts/post_processing/merge_mem.py @@ -50,7 +50,7 @@ def merge_mem_files(input_dir: Path, output_path: Path): print(f"Found {len(mem_files)} Memory metric files") output_columns = [ - 'id',' provider', 'protocol', 'dnssec_mode', 'keep_alive', + 'id','provider', 'protocol', 'dnssec_mode', 'keep_alive', 'timestamp', 'total_alloc_bytes', 'mallocs', 'gc_cycles', 'alloc_delta', 'mallocs_delta', 'gc_delta' ] diff --git a/scripts/post_processing/merge_pcap.py b/scripts/post_processing/merge_pcap.py new file mode 100644 index 0000000..52b715d --- /dev/null +++ b/scripts/post_processing/merge_pcap.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +Summarize all .pcap files into a single unified PCAP metrics CSV. +Each row represents one scenario (one PCAP file): total bytes/packets sent and received. +Adds provider, protocol, dnssec_mode, keep_alive columns parsed from filenames. +""" + +import argparse +import socket +import time +import csv +from pathlib import Path +from typing import List, NamedTuple + +import dpkt + + +DEFAULT_LOCAL_IP = "192.168.100.2" # netns veth1 address +DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"] + + +class PacketSummary(NamedTuple): + bytes_sent: int + bytes_received: int + packets_sent: int + packets_received: int + total_bytes: int + total_packets: int + + +def parse_config_from_filename(filename: str) -> dict: + """Parse protocol, dnssec_mode, keep_alive from filename like 'dot-trust-persist.pcap'""" + base = filename.replace(".pcap", "").replace(".PCAP", "") + parts = base.split("-") + + protocol = parts[0] + dnssec_mode = "off" + keep_alive = 0 + + for part in parts[1:]: + if part in ("auth", "trust"): + dnssec_mode = part + elif part == "persist": + keep_alive = 1 + + return { + "protocol": protocol, + "dnssec_mode": dnssec_mode, + "keep_alive": keep_alive, + } + + +def find_pcap_files(input_dir: Path, providers: List[str]) -> List[Path]: + files: List[Path] = [] + for provider in providers: + provider_path = input_dir / provider + if not provider_path.exists(): + print(f" ⚠ Missing provider directory: {provider}") + continue + for p in sorted(provider_path.glob("*.pcap")): + if ".bak" not in p.name: + files.append(p) + return files + + +def summarize_pcap(pcap_path: Path, local_ip_bytes: bytes) -> PacketSummary: + """Read a PCAP file and return aggregated byte/packet counts by direction.""" + bytes_sent = bytes_received = packets_sent = packets_received = 0 + + with open(pcap_path, "rb") as f: + try: + reader = dpkt.pcap.Reader(f) + except ValueError: + f.seek(0) + reader = dpkt.pcapng.Reader(f) + + for _ts, buf in reader: + try: + eth = dpkt.ethernet.Ethernet(buf) + ip = eth.data + if not isinstance(ip, dpkt.ip.IP): + continue + size = len(buf) + if ip.src == local_ip_bytes: + bytes_sent += size + packets_sent += 1 + elif ip.dst == local_ip_bytes: + bytes_received += size + packets_received += 1 + except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError): + continue + + return PacketSummary( + bytes_sent=bytes_sent, + bytes_received=bytes_received, + packets_sent=packets_sent, + packets_received=packets_received, + total_bytes=bytes_sent + bytes_received, + total_packets=packets_sent + packets_received, + ) + + +def merge_pcap_files(input_dir: Path, output_path: Path, local_ip: str, providers: List[str]): + local_ip_bytes = socket.inet_aton(local_ip) + + pcap_files = find_pcap_files(input_dir, providers) + if not pcap_files: + print("No .pcap files found.") + return + + print(f"Found {len(pcap_files)} PCAP files") + print(f"Local IP: {local_ip}\n") + + output_columns = [ + "id", "provider", "protocol", "dnssec_mode", "keep_alive", + "bytes_sent", "bytes_received", "packets_sent", "packets_received", + "total_bytes", "total_packets", + ] + + total_rows = 0 + t_total = time.time() + + with open(output_path, "w", newline="", encoding="utf-8") as outfile: + writer = csv.DictWriter(outfile, fieldnames=output_columns) + writer.writeheader() + + for pcap_path in pcap_files: + provider = pcap_path.parent.name.lower() + config = parse_config_from_filename(pcap_path.name) + + print(f" {provider}/{pcap_path.name} " + f"({config['protocol']}, {config['dnssec_mode']}, persist={config['keep_alive']})") + + t0 = time.time() + summary = summarize_pcap(pcap_path, local_ip_bytes) + dt = time.time() - t0 + + total_rows += 1 + writer.writerow({ + "id": total_rows, + "provider": provider, + "protocol": config["protocol"], + "dnssec_mode": config["dnssec_mode"], + "keep_alive": config["keep_alive"], + "bytes_sent": summary.bytes_sent, + "bytes_received": summary.bytes_received, + "packets_sent": summary.packets_sent, + "packets_received": summary.packets_received, + "total_bytes": summary.total_bytes, + "total_packets": summary.total_packets, + }) + + print(f" ✓ {summary.packets_sent:,} sent / {summary.packets_received:,} received " + f"| {summary.total_bytes:,} B total ({dt:.2f}s)") + + elapsed = time.time() - t_total + print(f"\n{'=' * 60}") + print(f"PCAP metrics merged → {output_path}") + print(f"Total scenarios: {total_rows}") + print(f"Completed in {elapsed:.2f}s ({elapsed / 60:.1f} min)") + print(f"{'=' * 60}") + + +def main(): + parser = argparse.ArgumentParser(description="Summarize all .pcap files into a unified CSV.") + parser.add_argument("input_dir", nargs="?", default=".", help="Input directory (default: .)") + parser.add_argument("-o", "--output", default="dns_results_pcap.csv", help="Output CSV path") + parser.add_argument("--local-ip", default=DEFAULT_LOCAL_IP, + help=f"Local netns veth IP for direction detection (default: {DEFAULT_LOCAL_IP})") + parser.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS, + help="Provider subdirectory names to scan") + args = parser.parse_args() + + merge_pcap_files(Path(args.input_dir), Path(args.output), args.local_ip, args.providers) + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/post_processing/minimal.py b/scripts/post_processing/minimal.py new file mode 100644 index 0000000..bd914be --- /dev/null +++ b/scripts/post_processing/minimal.py @@ -0,0 +1,71 @@ +import csv +import sys +from collections import defaultdict + +COLUMNS = ["provider", "protocol", "dnssec_mode", "query_type", "keep_alive"] + + +def get_uncovered_values(rows, selected_indices, col_indices, all_values): + covered = defaultdict(set) + for idx in selected_indices: + for col, col_idx in col_indices.items(): + covered[col].add(rows[idx][col_idx]) + + uncovered = {} + for col in COLUMNS: + uncovered[col] = all_values[col] - covered[col] + return uncovered + + +def main(input_file, output_file): + with open(input_file, newline="") as f: + reader = csv.reader(f) + header = next(reader) + rows = list(reader) + + col_indices = {col: header.index(col) for col in COLUMNS} + + # Collect all unique values per column + all_values = defaultdict(set) + for row in rows: + for col, idx in col_indices.items(): + all_values[col].add(row[idx]) + + # Greedy set cover + selected = [] + uncovered = get_uncovered_values(rows, selected, col_indices, all_values) + + while any(uncovered.values()): + best_row = None + best_score = 0 + + for i, row in enumerate(rows): + if i in selected: + continue + score = sum( + 1 for col, idx in col_indices.items() if row[idx] in uncovered[col] + ) + if score > best_score: + best_score = score + best_row = i + + if best_row is None: + break + + selected.append(best_row) + uncovered = get_uncovered_values(rows, selected, col_indices, all_values) + + with open(output_file, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(header) + for idx in selected: + writer.writerow(rows[idx]) + + print(f"Selected {len(selected)} rows out of {len(rows)}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python minimize_csv.py input.csv output.csv") + sys.exit(1) + main(sys.argv[1], sys.argv[2]) diff --git a/scripts/shell/run.sh b/scripts/shell/run.sh index 85b7e23..c128420 100755 --- a/scripts/shell/run.sh +++ b/scripts/shell/run.sh @@ -10,6 +10,7 @@ OUTPUT_DIR="./results" INTERFACE="veth1" TIMEOUT="5s" SLEEP_TIME="1" +DRY_RUN="false" # Parse arguments while [[ $# -gt 0 ]]; do @@ -38,6 +39,10 @@ while [[ $# -gt 0 ]]; do SLEEP_TIME="$2" shift 2 ;; + -n|--dry-run) + DRY_RUN="true" + shift + ;; --help) echo "Usage: $0 [OPTIONS]" echo "" @@ -48,6 +53,7 @@ while [[ $# -gt 0 ]]; do echo " -I, --interface NAME Network interface (default: veth1)" echo " -T, --timeout DURATION Timeout duration (default: 5s)" echo " -s, --sleep SECONDS Sleep between runs (default: 1)" + echo " -n, --dry-run Print the scenarios that would run, then exit" echo " --help Show this help" exit 0 ;; @@ -66,46 +72,109 @@ echo " Output dir: $OUTPUT_DIR" echo " Interface: $INTERFACE" echo " Timeout: $TIMEOUT" echo " Sleep time: ${SLEEP_TIME}s" +echo " Dry run: $DRY_RUN" echo "" -# Server definitions as associative arrays (name -> url) -declare -A CONN_SERVERS=( - ["google-dotcp"]="dotcp://8.8.8.8:53" - ["cloudflare-dotcp"]="dotcp://1.1.1.1:53" - ["quad9-dotcp"]="dotcp://9.9.9.9:53" - ["adguard-dotcp"]="dotcp://dns.adguard-dns.com:53" - ["google-dot"]="tls://8.8.8.8:853" - ["cloudflare-dot"]="tls://1.1.1.1:853" - ["quad9-dot"]="tls://9.9.9.9:853" - ["adguard-dot"]="tls://dns.adguard-dns.com:853" - ["google-doh"]="https://dns.google/dns-query" - ["cloudflare-doh"]="https://cloudflare-dns.com/dns-query" - ["quad9-doh"]="https://dns10.quad9.net/dns-query" - ["adguard-doh"]="https://dns.adguard-dns.com/dns-query" -) +SC_NAME=() +SC_URL=() +SC_DNSSEC=() +SC_AUTH=() +SC_KEEP=() -declare -A QUIC_SERVERS=( - ["google-doh3"]="doh3://dns.google/dns-query" - ["cloudflare-doh3"]="doh3://cloudflare-dns.com/dns-query" - ["adguard-doh3"]="doh3://dns.adguard-dns.com/dns-query" - ["adguard-doq"]="doq://dns.adguard-dns.com:853" -) +add() { # add + SC_NAME+=("$1") + SC_URL+=("$2") + SC_DNSSEC+=("$3") + SC_AUTH+=("$4") + SC_KEEP+=("$5") +} -declare -A CONNLESS_SERVERS=( - ["google-doudp"]="udp://8.8.8.8:53" - ["cloudflare-doudp"]="udp://1.1.1.1:53" - ["quad9-doudp"]="udp://9.9.9.9:53" - ["adguard-doudp"]="udp://dns.adguard-dns.com:53" - ["adguard-dnscrypt"]="sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" - ["quad9-dnscrypt"]="sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" -) +# Protocol comparison - DNSSEC off, non-persistent. +# The primary comparison axis: full encryption cost, handshake included. +# All transports. +add google-dotcp "dotcp://8.8.8.8:53" false false false +add cloudflare-dotcp "dotcp://1.1.1.1:53" false false false +add quad9-dotcp "dotcp://9.9.9.9:53" false false false +add adguard-dotcp "dotcp://dns.adguard-dns.com:53" false false false + +add google-dot "tls://8.8.8.8:853" false false false +add cloudflare-dot "tls://1.1.1.1:853" false false false +add quad9-dot "tls://9.9.9.9:853" false false false +add adguard-dot "tls://dns.adguard-dns.com:853" false false false + +add google-doh "https://dns.google/dns-query" false false false +add cloudflare-doh "https://cloudflare-dns.com/dns-query" false false false +add quad9-doh "https://dns10.quad9.net/dns-query" false false false +add adguard-doh "https://dns.adguard-dns.com/dns-query" false false false + +add google-doh3 "doh3://dns.google/dns-query" false false false +add cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" false false false +add adguard-doh3 "doh3://dns.adguard-dns.com/dns-query" false false false +add adguard-doq "doq://dns.adguard-dns.com:853" false false false + +add google-doudp "udp://8.8.8.8:53" false false false +add cloudflare-doudp "udp://1.1.1.1:53" false false false +add quad9-doudp "udp://9.9.9.9:53" false false false +add adguard-doudp "udp://dns.adguard-dns.com:53" false false false +add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" false false false +add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" false false false + +# Persistence contrast - DNSSEC off, persistent. +# Teste for TCP based protocols. +# QUIC makes no sense cause of 0-RTT. +add google-dotcp "dotcp://8.8.8.8:53" false false true +add cloudflare-dotcp "dotcp://1.1.1.1:53" false false true +add quad9-dotcp "dotcp://9.9.9.9:53" false false true +add adguard-dotcp "dotcp://dns.adguard-dns.com:53" false false true + +add google-dot "tls://8.8.8.8:853" false false true +add cloudflare-dot "tls://1.1.1.1:853" false false true +add quad9-dot "tls://9.9.9.9:853" false false true +add adguard-dot "tls://dns.adguard-dns.com:853" false false true + +add google-doh "https://dns.google/dns-query" false false true +add cloudflare-doh "https://cloudflare-dns.com/dns-query" false false true +add quad9-doh "https://dns10.quad9.net/dns-query" false false true +add adguard-doh "https://dns.adguard-dns.com/dns-query" false false true + +# DNSSEC trust - non-persistent. +add google-dotcp "dotcp://8.8.8.8:53" true false false +add cloudflare-dotcp "dotcp://1.1.1.1:53" true false false +add quad9-dotcp "dotcp://9.9.9.9:53" true false false +add adguard-dotcp "dotcp://dns.adguard-dns.com:53" true false false + +add google-dot "tls://8.8.8.8:853" true false false +add cloudflare-dot "tls://1.1.1.1:853" true false false +add quad9-dot "tls://9.9.9.9:853" true false false +add adguard-dot "tls://dns.adguard-dns.com:853" true false false + +add google-doh "https://dns.google/dns-query" true false false +add cloudflare-doh "https://cloudflare-dns.com/dns-query" true false false +add quad9-doh "https://dns10.quad9.net/dns-query" true false false +add adguard-doh "https://dns.adguard-dns.com/dns-query" true false false + +add google-doh3 "doh3://dns.google/dns-query" true false false +add cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" true false false +add adguard-doh3 "doh3://dns.adguard-dns.com/dns-query" true false false +add adguard-doq "doq://dns.adguard-dns.com:853" true false false + +add google-doudp "udp://8.8.8.8:53" true false false +add cloudflare-doudp "udp://1.1.1.1:53" true false false +add quad9-doudp "udp://9.9.9.9:53" true false false +add adguard-doudp "udp://dns.adguard-dns.com:53" true false false +add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" true false false +add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" true false false + +# DNSSEC auth - DoUDP only. +# auth walks the chain of trust over plain DoUDP regardless of the protocol. +add adguard-doudp "udp://dns.adguard-dns.com:53" true true false # Function to get flags suffix for filename get_flags_suffix() { local dnssec="$1" local auth="$2" local keepalive="$3" - + local suffix="" if [[ "$dnssec" == "true" ]]; then if [[ "$auth" == "true" ]]; then @@ -131,12 +200,8 @@ run_with_perf() { local dnssec="$3" local auth="$4" local keepalive="$5" - + local suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive") - local base_name="${name}" - if [[ -n "$suffix" ]]; then - base_name="${name}-${suffix}" - fi local provider="${name%%-*}" local protocol="${name#*-}" local base_name="${protocol}" @@ -147,12 +212,12 @@ run_with_perf() { # Create directory if needed mkdir -p "$(dirname "$cpu_csv_file")" - + # Write header if needed if [[ ! -f "$cpu_csv_file" ]]; then echo "timestamp,wall_time_seconds,instructions,cycles" > "$cpu_csv_file" fi - + # Build command arguments local cmd_args=( "$DOMAINS_FILE" @@ -161,97 +226,72 @@ run_with_perf() { --timeout "$TIMEOUT" -s "$url" ) - + if [[ "$dnssec" == "true" ]]; then cmd_args+=(--dnssec) if [[ "$auth" == "true" ]]; then cmd_args+=(--authoritative-dnssec) fi fi - + if [[ "$keepalive" == "true" ]]; then cmd_args+=(--keep-alive) fi - + # Create temp file for perf local perf_tmp=$(mktemp) - + # Run with perf stat local timestamp=$(date -Iseconds) - + perf stat -e instructions,cycles \ -o "$perf_tmp" \ "$TOOL_PATH" run "${cmd_args[@]}" || true - + # Parse perf output local instructions=$(grep -oP '\d[\d,]*(?=\s+instructions)' "$perf_tmp" 2>/dev/null | tr -d ',' || echo "0") local cycles=$(grep -oP '\d[\d,]*(?=\s+cycles)' "$perf_tmp" 2>/dev/null | tr -d ',' || echo "0") local wall_time=$(grep -oP '\d+\.\d+(?= seconds time elapsed)' "$perf_tmp" 2>/dev/null || echo "0") - + # Append to CPU CSV echo "${timestamp},${wall_time},${instructions},${cycles}" >> "$cpu_csv_file" - + # Cleanup rm -f "$perf_tmp" - - echo " -> CPU metrics saved to ${base_name}.cpu.csv" + + echo " -> CPU metrics saved to ${provider}/${base_name}.cpu.csv" } -# Function to run servers with given flags -run_server_group() { - local -n servers=$1 - local dnssec="$2" - local auth="$3" - local keepalive="$4" - local desc="$5" - - echo "Running: $desc" - - for name in "${!servers[@]}"; do - local url="${servers[$name]}" - echo " Processing: $name ($url)" - run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive" - sleep "$SLEEP_TIME" - done -} +echo "Total scenarios: ${#SC_NAME[@]}" +echo "" -echo "=== Running TCP-based protocols (TLS/HTTPS) ===" +for i in "${!SC_NAME[@]}"; do + name="${SC_NAME[$i]}" + url="${SC_URL[$i]}" + dnssec="${SC_DNSSEC[$i]}" + auth="${SC_AUTH[$i]}" + keepalive="${SC_KEEP[$i]}" -# DNSSEC off, Keep off -run_server_group CONN_SERVERS "false" "false" "false" "no-dnssec, no-keepalive" + suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive") + provider="${name%%-*}" + protocol="${name#*-}" + label="${protocol}" + [[ -n "$suffix" ]] && label="${protocol}-${suffix}" -# DNSSEC off, Keep on -run_server_group CONN_SERVERS "false" "false" "true" "no-dnssec, keepalive" + if [[ "$DRY_RUN" == "true" ]]; then + printf '[dry] %-28s dnssec=%-5s auth=%-5s keepalive=%-5s -> %s/%s\n' \ + "$name" "$dnssec" "$auth" "$keepalive" "$provider" "$label" + continue + fi -# DNSSEC on (trust), Keep on -run_server_group CONN_SERVERS "true" "false" "true" "dnssec-trust, keepalive" - -# DNSSEC on (auth), Keep on -run_server_group CONN_SERVERS "true" "true" "true" "dnssec-auth, keepalive" + echo "Processing: $name ($url) [dnssec=$dnssec auth=$auth keepalive=$keepalive]" + run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive" + sleep "$SLEEP_TIME" +done echo "" -echo "=== Running QUIC-based protocols (DoH3/DoQ) ===" - -# DNSSEC off -run_server_group QUIC_SERVERS "false" "false" "false" "no-dnssec" - -# DNSSEC on (trust) -run_server_group QUIC_SERVERS "true" "false" "false" "dnssec-trust" - -# DNSSEC on (auth) -run_server_group QUIC_SERVERS "true" "true" "false" "dnssec-auth" - -echo "" -echo "=== Running connectionless protocols (UDP) ===" - -# DNSSEC off -run_server_group CONNLESS_SERVERS "false" "false" "false" "no-dnssec" - -# DNSSEC on (trust) -run_server_group CONNLESS_SERVERS "true" "false" "false" "dnssec-trust" - -# DNSSEC on (auth) -run_server_group CONNLESS_SERVERS "true" "true" "false" "dnssec-auth" - -echo "" -echo "All combinations completed!" +if [[ "$DRY_RUN" == "true" ]]; then + echo "Dry run complete - nothing was executed." +else + echo "All scenarios completed!" +fi