feat(everything): fix doh and increase min TLS to 1.3
This commit is contained in:
@@ -18,3 +18,4 @@
|
||||
/results
|
||||
/results.bak
|
||||
/results_merged
|
||||
dns_results*
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
.PHONY: all clean
|
||||
|
||||
all:
|
||||
python3 scripts/post_processing/merge_files.py
|
||||
python3 scripts/post_processing/merge_cpu.py
|
||||
python3 scripts/post_processing/merge_mem.py
|
||||
python3 scripts/post_processing/merge_pcap.py
|
||||
|
||||
clean:
|
||||
rm -f ./dns_results.csv ./dns_results_cpu.csv ./dns_results_mem.csv ./dns_results_pcap.csv
|
||||
+58
-18
@@ -27,7 +27,6 @@ type Config struct {
|
||||
Path string
|
||||
DNSSEC bool
|
||||
HTTP3 bool
|
||||
HTTP2 bool
|
||||
KeepAlive bool
|
||||
}
|
||||
|
||||
@@ -62,6 +61,7 @@ func New(config Config) (*Client, error) {
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: config.Host,
|
||||
MinVersion: tls.VersionTLS13,
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(100),
|
||||
}
|
||||
|
||||
@@ -88,14 +88,7 @@ func New(config Config) (*Client, error) {
|
||||
}
|
||||
|
||||
var transportType string
|
||||
if config.HTTP2 {
|
||||
http2Transport := &http2.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
AllowHTTP: true,
|
||||
}
|
||||
httpClient.Transport = http2Transport
|
||||
transportType = "HTTP/2"
|
||||
} else if config.HTTP3 {
|
||||
if config.HTTP3 {
|
||||
quicTlsConfig := http3.ConfigureTLSConfig(tlsConfig)
|
||||
http3Transport := &http3.Transport{
|
||||
TLSClientConfig: quicTlsConfig,
|
||||
@@ -104,7 +97,12 @@ func New(config Config) (*Client, error) {
|
||||
httpClient.Transport = http3Transport
|
||||
transportType = "HTTP/3"
|
||||
} else {
|
||||
transportType = "HTTP/1.1"
|
||||
http2Transport := &http2.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
AllowHTTP: true,
|
||||
}
|
||||
httpClient.Transport = http2Transport
|
||||
transportType = "HTTP/2"
|
||||
}
|
||||
|
||||
logger.Debug("DoH client created: %s (%s, DNSSEC: %v, KeepAlive: %v)", rawURL, transportType, config.DNSSEC, config.KeepAlive)
|
||||
@@ -116,6 +114,43 @@ func New(config Config) (*Client, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newSingleUseClient builds a throwaway *http.Client whose transport is torn
|
||||
// down by the returned closeFn. Used for non-persistent (non-keep-alive) mode
|
||||
// so that each query establishes and tears down its own connection, and thus
|
||||
// pays its own TCP+TLS (or QUIC) handshake. Without this, the pooled
|
||||
// HTTP/2 transport silently reuses a single connection across all queries
|
||||
// (HTTP/2 multiplexes and ignores the Connection: close header), which would
|
||||
// make the non-persistent measurement indistinguishable from persistent.
|
||||
func (c *Client) newSingleUseClient() (*http.Client, func()) {
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: c.config.Host,
|
||||
MinVersion: tls.VersionTLS13,
|
||||
ClientSessionCache: nil, // no resumption: force a full handshake each query
|
||||
}
|
||||
|
||||
if c.config.HTTP3 {
|
||||
quicConf := &quic.Config{
|
||||
MaxIdleTimeout: 30 * time.Second,
|
||||
DisablePathMTUDiscovery: true,
|
||||
}
|
||||
quicTLS := http3.ConfigureTLSConfig(tlsConfig)
|
||||
tr := &http3.Transport{
|
||||
TLSClientConfig: quicTLS,
|
||||
QUICConfig: quicConf,
|
||||
}
|
||||
client := &http.Client{Transport: tr, Timeout: 30 * time.Second}
|
||||
return client, func() { tr.Close() }
|
||||
}
|
||||
|
||||
tr := &http2.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
AllowHTTP: true,
|
||||
DisableCompression: true,
|
||||
}
|
||||
client := &http.Client{Transport: tr, Timeout: 30 * time.Second}
|
||||
return client, func() { tr.CloseIdleConnections() }
|
||||
}
|
||||
|
||||
func (c *Client) Close() {
|
||||
logger.Debug("Closing DoH client")
|
||||
if t, ok := c.httpClient.Transport.(*http.Transport); ok {
|
||||
@@ -143,6 +178,18 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
|
||||
return nil, fmt.Errorf("doh: failed to pack DNS message: %w", err)
|
||||
}
|
||||
|
||||
// Select the HTTP client. In persistent mode, reuse the pooled client built
|
||||
// in New(). In non-persistent mode, build a fresh client (and transport)
|
||||
// per query and tear it down afterwards, so every query pays its own
|
||||
// handshake — otherwise HTTP/2 would reuse one connection and the
|
||||
// non-persistent measurement would be wrong.
|
||||
httpClient := c.httpClient
|
||||
if !c.config.KeepAlive {
|
||||
freshClient, closeFn := c.newSingleUseClient()
|
||||
defer closeFn()
|
||||
httpClient = freshClient
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequest(http.MethodPost, c.upstreamURL.String(), bytes.NewReader(packedMsg))
|
||||
if err != nil {
|
||||
logger.Error("DoH failed to create HTTP request: %v", err)
|
||||
@@ -153,14 +200,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
|
||||
httpReq.Header.Set("Content-Type", dnsMessageContentType)
|
||||
httpReq.Header.Set("Accept", dnsMessageContentType)
|
||||
|
||||
// Set Connection header based on KeepAlive setting
|
||||
if c.config.KeepAlive {
|
||||
httpReq.Header.Set("Connection", "keep-alive")
|
||||
} else {
|
||||
httpReq.Header.Set("Connection", "close")
|
||||
}
|
||||
|
||||
httpResp, err := c.httpClient.Do(httpReq)
|
||||
httpResp, err := httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
logger.Error("DoH request failed to %s: %v", c.upstreamURL.Host, err)
|
||||
return nil, fmt.Errorf("doh: failed executing HTTP request to %s: %w", c.upstreamURL.Host, err)
|
||||
|
||||
@@ -26,7 +26,7 @@ type Client struct {
|
||||
targetAddr *net.UDPAddr
|
||||
tlsConfig *tls.Config
|
||||
udpConn *net.UDPConn
|
||||
quicConn quic.Connection
|
||||
quicConn *quic.Conn
|
||||
quicTransport *quic.Transport
|
||||
quicConfig *quic.Config
|
||||
config Config
|
||||
@@ -119,7 +119,7 @@ func (c *Client) Query(msg *dns.Msg) (*dns.Msg, error) {
|
||||
return nil, fmt.Errorf("doq: failed to pack message: %w", err)
|
||||
}
|
||||
|
||||
var quicStream quic.Stream
|
||||
var quicStream *quic.Stream
|
||||
quicStream, err = c.quicConn.OpenStream()
|
||||
if err != nil {
|
||||
logger.Debug("DoQ stream failed, reconnecting: %v", err)
|
||||
|
||||
@@ -49,6 +49,7 @@ func New(config Config) (*Client, error) {
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: config.Host,
|
||||
MinVersion: tls.VersionTLS13,
|
||||
}
|
||||
|
||||
client := &Client{
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
module github.com/afonsofrancof/sdns-proxy
|
||||
|
||||
go 1.24.1
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/alecthomas/kong v1.8.1
|
||||
github.com/alecthomas/kong v1.15.0
|
||||
github.com/ameshkov/dnscrypt/v2 v2.4.0
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/miekg/dns v1.1.65
|
||||
github.com/quic-go/quic-go v0.50.0
|
||||
golang.org/x/net v0.38.0
|
||||
github.com/miekg/dns v1.1.72
|
||||
github.com/quic-go/quic-go v0.60.0
|
||||
golang.org/x/net v0.56.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/AdguardTeam/golibs v0.32.7 // indirect
|
||||
github.com/AdguardTeam/golibs v0.35.14 // indirect
|
||||
github.com/ameshkov/dnsstamps v1.0.3 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
golang.org/x/crypto v0.37.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
|
||||
golang.org/x/mod v0.24.0 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
golang.org/x/tools v0.31.0 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/text v0.39.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,88 +1,64 @@
|
||||
github.com/AdguardTeam/golibs v0.32.7 h1:3dmGlAVgmvquCCwHsvEl58KKcRAK3z1UnjMnwSIeDH4=
|
||||
github.com/AdguardTeam/golibs v0.32.7/go.mod h1:bE8KV1zqTzgZjmjFyBJ9f9O5DEKO717r7e57j1HclJA=
|
||||
github.com/AdguardTeam/golibs v0.35.14 h1:LGqoRMbuxUgZyhOjlkmPxy5t1/XPuFpDcG5RBIDcskA=
|
||||
github.com/AdguardTeam/golibs v0.35.14/go.mod h1:FL470ALCuUs7cwk88ltBgTZ6JYCUG2NpNzp3EOMirxo=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/kong v1.8.1 h1:6aamvWBE/REnR/BCq10EcozmcpUPc5aGI1lPAWdB0EE=
|
||||
github.com/alecthomas/kong v1.8.1/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI=
|
||||
github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/ameshkov/dnscrypt/v2 v2.4.0 h1:if6ZG2cuQmcP2TwSY+D0+8+xbPfoatufGlOQTMNkI9o=
|
||||
github.com/ameshkov/dnscrypt/v2 v2.4.0/go.mod h1:WpEFV2uhebXb8Jhes/5/fSdpmhGV8TL22RDaeWwV6hI=
|
||||
github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo=
|
||||
github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
|
||||
github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
|
||||
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
|
||||
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo=
|
||||
github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0=
|
||||
github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
|
||||
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
|
||||
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
|
||||
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
|
||||
golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,405 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fast PCAP Preprocessor for DNS QoS Analysis
|
||||
Loads PCAP into memory first, then uses binary search for matching.
|
||||
Uses LAN IP to determine direction (LAN = sent, non-LAN = received).
|
||||
Fast PCAP Preprocessor for DNS QoS Analysis.
|
||||
|
||||
Loads PCAP into memory, then uses binary search to match packets to query
|
||||
windows. Direction is determined by a configured local IP (the netns veth IP).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import csv
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, NamedTuple
|
||||
import time
|
||||
|
||||
import dpkt
|
||||
from dateutil import parser as date_parser
|
||||
|
||||
|
||||
BANDWIDTH_COLUMNS = [
|
||||
'bytes_sent',
|
||||
'bytes_received',
|
||||
'packets_sent',
|
||||
'packets_received',
|
||||
'total_bytes',
|
||||
"bytes_sent",
|
||||
"bytes_received",
|
||||
"packets_sent",
|
||||
"packets_received",
|
||||
"total_bytes",
|
||||
]
|
||||
|
||||
DEFAULT_LOCAL_IP = "192.168.100.2" # netns veth1 address
|
||||
DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"]
|
||||
|
||||
|
||||
class Packet(NamedTuple):
|
||||
"""Lightweight packet representation."""
|
||||
timestamp: float
|
||||
size: int
|
||||
is_outbound: bool # True if from LAN, False if from internet
|
||||
is_outbound: bool
|
||||
|
||||
|
||||
class QueryWindow:
|
||||
"""Efficient query window representation."""
|
||||
__slots__ = ['index', 'start', 'end', 'sent', 'received', 'pkts_sent', 'pkts_received']
|
||||
|
||||
def __init__(self, index: int, start: float, end: float):
|
||||
self.index = index
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.sent = 0
|
||||
self.received = 0
|
||||
self.pkts_sent = 0
|
||||
self.pkts_received = 0
|
||||
|
||||
|
||||
def is_already_processed(csv_path: Path) -> bool:
|
||||
"""
|
||||
Check if CSV has already been processed.
|
||||
Returns True if bandwidth columns exist AND at least one row has non-zero data.
|
||||
"""
|
||||
def needs_processing(csv_path: Path) -> bool:
|
||||
"""True if file lacks bandwidth columns OR all values are empty/zero."""
|
||||
try:
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
with open(csv_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
|
||||
# Check if columns exist
|
||||
if not reader.fieldnames:
|
||||
return False
|
||||
|
||||
if not all(col in reader.fieldnames for col in BANDWIDTH_COLUMNS):
|
||||
return False
|
||||
|
||||
# Check if any row has non-zero bandwidth data
|
||||
return True
|
||||
if not all(c in reader.fieldnames for c in BANDWIDTH_COLUMNS):
|
||||
return True
|
||||
for row in reader:
|
||||
for col in BANDWIDTH_COLUMNS:
|
||||
val = row.get(col, '').strip()
|
||||
if val and val != '0':
|
||||
return True
|
||||
|
||||
# All rows have zero/empty values - not truly processed
|
||||
return False
|
||||
|
||||
val = (row.get(col) or "").strip()
|
||||
if val and val != "0":
|
||||
return False # has real data
|
||||
return True # columns exist but all empty/zero
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_csv_timestamp(ts_str: str) -> float:
|
||||
"""Convert RFC3339Nano timestamp to Unix epoch (seconds)."""
|
||||
dt = date_parser.isoparse(ts_str)
|
||||
return dt.timestamp()
|
||||
return date_parser.isoparse(ts_str).timestamp()
|
||||
|
||||
|
||||
def is_lan_ip(ip_bytes: bytes) -> bool:
|
||||
"""Check if IP is a private/LAN address."""
|
||||
if len(ip_bytes) != 4:
|
||||
return False
|
||||
|
||||
first = ip_bytes[0]
|
||||
second = ip_bytes[1]
|
||||
|
||||
# 10.0.0.0/8
|
||||
if first == 10:
|
||||
return True
|
||||
|
||||
# 172.16.0.0/12
|
||||
if first == 172 and 16 <= second <= 31:
|
||||
return True
|
||||
|
||||
# 192.168.0.0/16
|
||||
if first == 192 and second == 168:
|
||||
return True
|
||||
|
||||
# 127.0.0.0/8 (localhost)
|
||||
if first == 127:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def load_pcap_into_memory(pcap_path: Path) -> List[Packet]:
|
||||
"""Load all packets from PCAP into memory with minimal data."""
|
||||
packets = []
|
||||
|
||||
print(f" Loading PCAP into memory...")
|
||||
start_time = time.time()
|
||||
def load_pcap(pcap_path: Path, local_ip_bytes: bytes) -> List[Packet]:
|
||||
"""Load PCAP into a list of Packets sorted by timestamp."""
|
||||
print(" Loading PCAP...")
|
||||
t0 = time.time()
|
||||
packets: List[Packet] = []
|
||||
|
||||
with open(pcap_path, "rb") as f:
|
||||
try:
|
||||
with open(pcap_path, 'rb') as f:
|
||||
try:
|
||||
pcap = dpkt.pcap.Reader(f)
|
||||
except:
|
||||
# Try pcapng format
|
||||
reader = dpkt.pcap.Reader(f)
|
||||
except ValueError:
|
||||
f.seek(0)
|
||||
pcap = dpkt.pcapng.Reader(f)
|
||||
reader = dpkt.pcapng.Reader(f)
|
||||
|
||||
for ts, buf in pcap:
|
||||
for ts, buf in reader:
|
||||
try:
|
||||
packet_time = float(ts)
|
||||
packet_size = len(buf)
|
||||
|
||||
# Parse to get source IP
|
||||
eth = dpkt.ethernet.Ethernet(buf)
|
||||
|
||||
# Default to outbound if we can't determine
|
||||
is_outbound = True
|
||||
|
||||
if isinstance(eth.data, dpkt.ip.IP):
|
||||
ip = eth.data
|
||||
src_ip = ip.src
|
||||
is_outbound = is_lan_ip(src_ip)
|
||||
|
||||
packets.append(Packet(
|
||||
timestamp=packet_time,
|
||||
size=packet_size,
|
||||
is_outbound=is_outbound
|
||||
))
|
||||
|
||||
if not isinstance(ip, dpkt.ip.IP):
|
||||
continue
|
||||
if ip.src == local_ip_bytes:
|
||||
is_outbound = True
|
||||
elif ip.dst == local_ip_bytes:
|
||||
is_outbound = False
|
||||
else:
|
||||
continue # not our traffic
|
||||
packets.append(Packet(float(ts), len(buf), is_outbound))
|
||||
except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError):
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error reading PCAP: {e}")
|
||||
return []
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f" Loaded {len(packets):,} packets in {elapsed:.2f}s")
|
||||
|
||||
# Sort by timestamp for binary search
|
||||
packets.sort(key=lambda p: p.timestamp)
|
||||
|
||||
print(f" Loaded {len(packets):,} packets in {time.time() - t0:.2f}s")
|
||||
return packets
|
||||
|
||||
|
||||
def find_packets_in_window(
|
||||
packets: List[Packet],
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
left_hint: int = 0
|
||||
) -> tuple[List[Packet], int]:
|
||||
"""
|
||||
Binary search to find all packets within time window.
|
||||
Returns (matching_packets, left_index_hint_for_next_search).
|
||||
"""
|
||||
if not packets:
|
||||
return [], 0
|
||||
|
||||
# Binary search for first packet >= start_time
|
||||
left, right = left_hint, len(packets) - 1
|
||||
first_idx = len(packets)
|
||||
|
||||
while left <= right:
|
||||
mid = (left + right) // 2
|
||||
if packets[mid].timestamp >= start_time:
|
||||
first_idx = mid
|
||||
right = mid - 1
|
||||
else:
|
||||
left = mid + 1
|
||||
|
||||
# No packets in range
|
||||
if first_idx >= len(packets) or packets[first_idx].timestamp > end_time:
|
||||
return [], first_idx
|
||||
|
||||
# Collect all packets in window
|
||||
matching = []
|
||||
idx = first_idx
|
||||
while idx < len(packets) and packets[idx].timestamp <= end_time:
|
||||
matching.append(packets[idx])
|
||||
idx += 1
|
||||
|
||||
return matching, first_idx
|
||||
|
||||
|
||||
def load_csv_queries(csv_path: Path) -> List[Dict]:
|
||||
"""Load CSV and create query data structures."""
|
||||
queries = []
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
with open(csv_path, "r", encoding="utf-8") as f:
|
||||
for row in csv.DictReader(f):
|
||||
try:
|
||||
ts_epoch = parse_csv_timestamp(row['timestamp'])
|
||||
duration_s = float(row['duration_ns']) / 1e9
|
||||
queries.append({
|
||||
'data': row,
|
||||
'start_time': ts_epoch,
|
||||
'end_time': ts_epoch + duration_s,
|
||||
})
|
||||
except Exception as e:
|
||||
print(f" Warning: Skipping row - {e}")
|
||||
continue
|
||||
return queries
|
||||
|
||||
|
||||
def match_packets_to_queries(
|
||||
packets: List[Packet],
|
||||
queries: List[Dict]
|
||||
) -> List[Dict]:
|
||||
"""Match packets to query windows using binary search."""
|
||||
if not queries or not packets:
|
||||
return queries
|
||||
|
||||
print(f" Matching packets to queries...")
|
||||
start_time = time.time()
|
||||
|
||||
# Initialize metrics
|
||||
for q in queries:
|
||||
q['bytes_sent'] = 0
|
||||
q['bytes_received'] = 0
|
||||
q['packets_sent'] = 0
|
||||
q['packets_received'] = 0
|
||||
q['total_bytes'] = 0
|
||||
|
||||
# Sort queries by start time for sequential processing
|
||||
queries_sorted = sorted(enumerate(queries), key=lambda x: x[1]['start_time'])
|
||||
|
||||
matched_packets = 0
|
||||
left_hint = 0 # Optimization: start next search from here
|
||||
|
||||
for original_idx, q in queries_sorted:
|
||||
matching, left_hint = find_packets_in_window(
|
||||
packets,
|
||||
q['start_time'],
|
||||
q['end_time'],
|
||||
left_hint
|
||||
start = parse_csv_timestamp(row["timestamp"])
|
||||
duration = float(row["duration_ns"]) / 1e9
|
||||
queries.append(
|
||||
{"data": row, "start_time": start, "end_time": start + duration}
|
||||
)
|
||||
|
||||
for pkt in matching:
|
||||
matched_packets += 1
|
||||
if pkt.is_outbound:
|
||||
q['bytes_sent'] += pkt.size
|
||||
q['packets_sent'] += 1
|
||||
else:
|
||||
q['bytes_received'] += pkt.size
|
||||
q['packets_received'] += 1
|
||||
|
||||
q['total_bytes'] = q['bytes_sent'] + q['bytes_received']
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f" Matched {matched_packets:,} packets in {elapsed:.2f}s")
|
||||
|
||||
# Statistics
|
||||
total_sent = sum(q['bytes_sent'] for q in queries)
|
||||
total_recv = sum(q['bytes_received'] for q in queries)
|
||||
queries_with_data = sum(1 for q in queries if q['total_bytes'] > 0)
|
||||
print(f" Total: {total_sent:,} bytes sent, {total_recv:,} bytes received")
|
||||
print(f" Queries with data: {queries_with_data}/{len(queries)}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Warning: skipping row - {e}")
|
||||
return queries
|
||||
|
||||
|
||||
def write_enriched_csv(
|
||||
csv_path: Path, queries: List[Dict], backup: bool = True
|
||||
):
|
||||
"""Write enriched CSV with bandwidth columns."""
|
||||
if backup and csv_path.exists():
|
||||
backup_path = csv_path.with_suffix('.csv.bak')
|
||||
if not backup_path.exists(): # Don't overwrite existing backup
|
||||
shutil.copy2(csv_path, backup_path)
|
||||
print(f" Backup: {backup_path.name}")
|
||||
def match_packets(packets: List[Packet], queries: List[Dict]) -> int:
|
||||
"""Assign bandwidth metrics to each query. Returns total matched packets."""
|
||||
if not packets or not queries:
|
||||
for q in queries:
|
||||
q.update({c: 0 for c in BANDWIDTH_COLUMNS})
|
||||
return 0
|
||||
|
||||
print(" Matching packets to queries...")
|
||||
t0 = time.time()
|
||||
timestamps = [p.timestamp for p in packets]
|
||||
matched = 0
|
||||
|
||||
for q in queries:
|
||||
lo = bisect.bisect_left(timestamps, q["start_time"])
|
||||
hi = bisect.bisect_right(timestamps, q["end_time"])
|
||||
bs = br = ps = pr = 0
|
||||
for pkt in packets[lo:hi]:
|
||||
if pkt.is_outbound:
|
||||
bs += pkt.size
|
||||
ps += 1
|
||||
else:
|
||||
br += pkt.size
|
||||
pr += 1
|
||||
q["bytes_sent"] = bs
|
||||
q["bytes_received"] = br
|
||||
q["packets_sent"] = ps
|
||||
q["packets_received"] = pr
|
||||
q["total_bytes"] = bs + br
|
||||
matched += ps + pr
|
||||
|
||||
print(f" Matched {matched:,} packets in {time.time() - t0:.2f}s")
|
||||
total_sent = sum(q["bytes_sent"] for q in queries)
|
||||
total_recv = sum(q["bytes_received"] for q in queries)
|
||||
with_data = sum(1 for q in queries if q["total_bytes"] > 0)
|
||||
print(f" Total: {total_sent:,} B sent, {total_recv:,} B received")
|
||||
print(f" Queries with data: {with_data}/{len(queries)}")
|
||||
return matched
|
||||
|
||||
|
||||
def write_csv(csv_path: Path, queries: List[Dict], backup: bool = True):
|
||||
if backup and csv_path.exists():
|
||||
bak = csv_path.with_suffix(".csv.bak")
|
||||
if not bak.exists():
|
||||
shutil.copy2(csv_path, bak)
|
||||
print(f" Backup: {bak.name}")
|
||||
|
||||
# Get fieldnames - filter out any existing bandwidth columns to avoid dupes
|
||||
original_fields = [
|
||||
f for f in queries[0]['data'].keys()
|
||||
if f not in BANDWIDTH_COLUMNS
|
||||
f for f in queries[0]["data"].keys() if f not in BANDWIDTH_COLUMNS
|
||||
]
|
||||
fieldnames = original_fields + BANDWIDTH_COLUMNS
|
||||
|
||||
with open(csv_path, 'w', encoding='utf-8', newline='') as f:
|
||||
with open(csv_path, "w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for q in queries:
|
||||
row = {k: v for k, v in q['data'].items() if k not in BANDWIDTH_COLUMNS}
|
||||
for field in BANDWIDTH_COLUMNS:
|
||||
row[field] = q[field]
|
||||
row = {k: q["data"][k] for k in original_fields}
|
||||
for c in BANDWIDTH_COLUMNS:
|
||||
row[c] = q[c]
|
||||
writer.writerow(row)
|
||||
|
||||
print(f" Written: {csv_path.name}")
|
||||
|
||||
|
||||
def process_provider_directory(provider_path: Path):
|
||||
"""Process all CSV/PCAP pairs in a provider directory."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Processing: {provider_path.name.upper()}")
|
||||
print(f"{'='*60}")
|
||||
def process_provider(provider_path: Path, local_ip_bytes: bytes):
|
||||
print(f"\n{'=' * 60}\nProcessing: {provider_path.name.upper()}\n{'=' * 60}")
|
||||
|
||||
csv_files = sorted(provider_path.glob('*.csv'))
|
||||
processed = 0
|
||||
skipped = 0
|
||||
total_time = 0
|
||||
processed = skipped = 0
|
||||
total_time = 0.0
|
||||
|
||||
for csv_path in csv_files:
|
||||
# Skip backup files
|
||||
if '.bak' in csv_path.name:
|
||||
for csv_path in sorted(provider_path.glob("*.csv")):
|
||||
name = csv_path.name.lower()
|
||||
if ".bak" in name or name.endswith((".cpu.csv", ".mem.csv")):
|
||||
continue
|
||||
|
||||
pcap_path = csv_path.with_suffix('.pcap')
|
||||
|
||||
pcap_path = csv_path.with_suffix(".pcap")
|
||||
if not pcap_path.exists():
|
||||
print(f"\n ⚠ Skipping {csv_path.name} - no matching PCAP")
|
||||
print(f"\n ⚠ {csv_path.name}: no matching PCAP")
|
||||
continue
|
||||
|
||||
# Check if already processed
|
||||
if is_already_processed(csv_path):
|
||||
print(f"\n ⏭ Skipping {csv_path.name} - already processed")
|
||||
if not needs_processing(csv_path):
|
||||
print(f"\n ⏭ {csv_path.name}: already processed")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
print(f"\n 📁 {csv_path.name}")
|
||||
file_start = time.time()
|
||||
t0 = time.time()
|
||||
|
||||
# Load PCAP into memory first
|
||||
packets = load_pcap_into_memory(pcap_path)
|
||||
packets = load_pcap(pcap_path, local_ip_bytes)
|
||||
if not packets:
|
||||
print(f" ⚠ No packets found in PCAP")
|
||||
print(" ⚠ No usable packets in PCAP")
|
||||
continue
|
||||
|
||||
# Load CSV queries
|
||||
queries = load_csv_queries(csv_path)
|
||||
if not queries:
|
||||
print(f" ⚠ No valid queries found")
|
||||
print(" ⚠ No valid queries in CSV")
|
||||
continue
|
||||
|
||||
print(f" Loaded {len(queries):,} queries")
|
||||
|
||||
# Match packets to queries
|
||||
enriched_queries = match_packets_to_queries(packets, queries)
|
||||
match_packets(packets, queries)
|
||||
write_csv(csv_path, queries)
|
||||
|
||||
# Write enriched CSV
|
||||
write_enriched_csv(csv_path, enriched_queries)
|
||||
|
||||
file_time = time.time() - file_start
|
||||
total_time += file_time
|
||||
dt = time.time() - t0
|
||||
total_time += dt
|
||||
processed += 1
|
||||
print(f" ✓ Completed in {file_time:.2f}s")
|
||||
|
||||
print(f"\n {'='*58}")
|
||||
print(f" {provider_path.name}: {processed} processed, {skipped} skipped")
|
||||
print(f" Time: {total_time:.2f}s")
|
||||
print(f" {'='*58}")
|
||||
print(f" ✓ Completed in {dt:.2f}s")
|
||||
|
||||
print(
|
||||
f"\n {provider_path.name}: {processed} processed, "
|
||||
f"{skipped} skipped, {total_time:.2f}s"
|
||||
)
|
||||
|
||||
def main():
|
||||
"""Main preprocessing pipeline."""
|
||||
overall_start = time.time()
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument(
|
||||
"--local-ip",
|
||||
default=DEFAULT_LOCAL_IP,
|
||||
help=f"Local (netns veth) IP used to determine direction "
|
||||
f"(default: {DEFAULT_LOCAL_IP})",
|
||||
)
|
||||
ap.add_argument("--results-dir", default="results", type=Path)
|
||||
ap.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS)
|
||||
args = ap.parse_args()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("DNS PCAP PREPROCESSOR - Memory-Optimized Edition")
|
||||
print("="*60)
|
||||
local_ip_bytes = socket.inet_aton(args.local_ip)
|
||||
|
||||
results_dir = Path('results')
|
||||
print(f"\n{'=' * 60}\nDNS PCAP PREPROCESSOR\n{'=' * 60}")
|
||||
print(f"Local IP: {args.local_ip}")
|
||||
print(f"Results: {args.results_dir}")
|
||||
|
||||
if not results_dir.exists():
|
||||
print(f"\n❌ Error: '{results_dir}' directory not found")
|
||||
if not args.results_dir.exists():
|
||||
print(f"\n❌ Directory not found: {args.results_dir}")
|
||||
return
|
||||
|
||||
providers = ['adguard', 'cloudflare', 'google', 'quad9']
|
||||
|
||||
for provider in providers:
|
||||
provider_path = results_dir / provider
|
||||
if provider_path.exists():
|
||||
process_provider_directory(provider_path)
|
||||
t0 = time.time()
|
||||
for provider in args.providers:
|
||||
path = args.results_dir / provider
|
||||
if path.exists():
|
||||
process_provider(path, local_ip_bytes)
|
||||
else:
|
||||
print(f"\n⚠ Warning: Provider directory not found: {provider}")
|
||||
print(f"\n⚠ Missing provider directory: {provider}")
|
||||
|
||||
overall_time = time.time() - overall_start
|
||||
|
||||
print("\n" + "="*60)
|
||||
print(f"✓ PREPROCESSING COMPLETE")
|
||||
print(f" Total time: {overall_time:.2f}s ({overall_time/60:.1f} minutes)")
|
||||
print("="*60 + "\n")
|
||||
total = time.time() - t0
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"✓ DONE in {total:.2f}s ({total / 60:.1f} min)")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -74,6 +74,10 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
|
||||
|
||||
csv_files = find_csv_files(input_dir)
|
||||
|
||||
print(f"Files to process ({len(csv_files)}):")
|
||||
for f in csv_files:
|
||||
print(f" {f}")
|
||||
|
||||
if not csv_files:
|
||||
print("No CSV files found")
|
||||
return
|
||||
@@ -96,11 +100,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
|
||||
'duration_ms',
|
||||
'request_size_bytes',
|
||||
'response_size_bytes',
|
||||
'bytes_sent',
|
||||
'bytes_received',
|
||||
'packets_sent',
|
||||
'packets_received',
|
||||
'total_bytes',
|
||||
'response_code',
|
||||
'error',
|
||||
]
|
||||
@@ -146,11 +145,6 @@ def merge_all_csvs(input_dir: Path, output_path: Path):
|
||||
'duration_ms': ns_to_ms(row.get('duration_ns', '')),
|
||||
'request_size_bytes': row.get('request_size_bytes', ''),
|
||||
'response_size_bytes': row.get('response_size_bytes', ''),
|
||||
'bytes_sent': row.get('bytes_sent', ''),
|
||||
'bytes_received': row.get('bytes_received', ''),
|
||||
'packets_sent': row.get('packets_sent', ''),
|
||||
'packets_received': row.get('packets_received', ''),
|
||||
'total_bytes': row.get('total_bytes', ''),
|
||||
'response_code': row.get('response_code', ''),
|
||||
'error': row.get('error', ''),
|
||||
}
|
||||
@@ -173,7 +167,7 @@ def main():
|
||||
parser.add_argument(
|
||||
'input_dir',
|
||||
nargs='?',
|
||||
default='.',
|
||||
default='results',
|
||||
help='Input directory containing provider folders (default: .)'
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@@ -50,7 +50,7 @@ def merge_mem_files(input_dir: Path, output_path: Path):
|
||||
print(f"Found {len(mem_files)} Memory metric files")
|
||||
|
||||
output_columns = [
|
||||
'id',' provider', 'protocol', 'dnssec_mode', 'keep_alive',
|
||||
'id','provider', 'protocol', 'dnssec_mode', 'keep_alive',
|
||||
'timestamp', 'total_alloc_bytes', 'mallocs', 'gc_cycles',
|
||||
'alloc_delta', 'mallocs_delta', 'gc_delta'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Summarize all .pcap files into a single unified PCAP metrics CSV.
|
||||
Each row represents one scenario (one PCAP file): total bytes/packets sent and received.
|
||||
Adds provider, protocol, dnssec_mode, keep_alive columns parsed from filenames.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import socket
|
||||
import time
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import List, NamedTuple
|
||||
|
||||
import dpkt
|
||||
|
||||
|
||||
DEFAULT_LOCAL_IP = "192.168.100.2" # netns veth1 address
|
||||
DEFAULT_PROVIDERS = ["adguard", "cloudflare", "google", "quad9"]
|
||||
|
||||
|
||||
class PacketSummary(NamedTuple):
|
||||
bytes_sent: int
|
||||
bytes_received: int
|
||||
packets_sent: int
|
||||
packets_received: int
|
||||
total_bytes: int
|
||||
total_packets: int
|
||||
|
||||
|
||||
def parse_config_from_filename(filename: str) -> dict:
|
||||
"""Parse protocol, dnssec_mode, keep_alive from filename like 'dot-trust-persist.pcap'"""
|
||||
base = filename.replace(".pcap", "").replace(".PCAP", "")
|
||||
parts = base.split("-")
|
||||
|
||||
protocol = parts[0]
|
||||
dnssec_mode = "off"
|
||||
keep_alive = 0
|
||||
|
||||
for part in parts[1:]:
|
||||
if part in ("auth", "trust"):
|
||||
dnssec_mode = part
|
||||
elif part == "persist":
|
||||
keep_alive = 1
|
||||
|
||||
return {
|
||||
"protocol": protocol,
|
||||
"dnssec_mode": dnssec_mode,
|
||||
"keep_alive": keep_alive,
|
||||
}
|
||||
|
||||
|
||||
def find_pcap_files(input_dir: Path, providers: List[str]) -> List[Path]:
|
||||
files: List[Path] = []
|
||||
for provider in providers:
|
||||
provider_path = input_dir / provider
|
||||
if not provider_path.exists():
|
||||
print(f" ⚠ Missing provider directory: {provider}")
|
||||
continue
|
||||
for p in sorted(provider_path.glob("*.pcap")):
|
||||
if ".bak" not in p.name:
|
||||
files.append(p)
|
||||
return files
|
||||
|
||||
|
||||
def summarize_pcap(pcap_path: Path, local_ip_bytes: bytes) -> PacketSummary:
|
||||
"""Read a PCAP file and return aggregated byte/packet counts by direction."""
|
||||
bytes_sent = bytes_received = packets_sent = packets_received = 0
|
||||
|
||||
with open(pcap_path, "rb") as f:
|
||||
try:
|
||||
reader = dpkt.pcap.Reader(f)
|
||||
except ValueError:
|
||||
f.seek(0)
|
||||
reader = dpkt.pcapng.Reader(f)
|
||||
|
||||
for _ts, buf in reader:
|
||||
try:
|
||||
eth = dpkt.ethernet.Ethernet(buf)
|
||||
ip = eth.data
|
||||
if not isinstance(ip, dpkt.ip.IP):
|
||||
continue
|
||||
size = len(buf)
|
||||
if ip.src == local_ip_bytes:
|
||||
bytes_sent += size
|
||||
packets_sent += 1
|
||||
elif ip.dst == local_ip_bytes:
|
||||
bytes_received += size
|
||||
packets_received += 1
|
||||
except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError, AttributeError):
|
||||
continue
|
||||
|
||||
return PacketSummary(
|
||||
bytes_sent=bytes_sent,
|
||||
bytes_received=bytes_received,
|
||||
packets_sent=packets_sent,
|
||||
packets_received=packets_received,
|
||||
total_bytes=bytes_sent + bytes_received,
|
||||
total_packets=packets_sent + packets_received,
|
||||
)
|
||||
|
||||
|
||||
def merge_pcap_files(input_dir: Path, output_path: Path, local_ip: str, providers: List[str]):
|
||||
local_ip_bytes = socket.inet_aton(local_ip)
|
||||
|
||||
pcap_files = find_pcap_files(input_dir, providers)
|
||||
if not pcap_files:
|
||||
print("No .pcap files found.")
|
||||
return
|
||||
|
||||
print(f"Found {len(pcap_files)} PCAP files")
|
||||
print(f"Local IP: {local_ip}\n")
|
||||
|
||||
output_columns = [
|
||||
"id", "provider", "protocol", "dnssec_mode", "keep_alive",
|
||||
"bytes_sent", "bytes_received", "packets_sent", "packets_received",
|
||||
"total_bytes", "total_packets",
|
||||
]
|
||||
|
||||
total_rows = 0
|
||||
t_total = time.time()
|
||||
|
||||
with open(output_path, "w", newline="", encoding="utf-8") as outfile:
|
||||
writer = csv.DictWriter(outfile, fieldnames=output_columns)
|
||||
writer.writeheader()
|
||||
|
||||
for pcap_path in pcap_files:
|
||||
provider = pcap_path.parent.name.lower()
|
||||
config = parse_config_from_filename(pcap_path.name)
|
||||
|
||||
print(f" {provider}/{pcap_path.name} "
|
||||
f"({config['protocol']}, {config['dnssec_mode']}, persist={config['keep_alive']})")
|
||||
|
||||
t0 = time.time()
|
||||
summary = summarize_pcap(pcap_path, local_ip_bytes)
|
||||
dt = time.time() - t0
|
||||
|
||||
total_rows += 1
|
||||
writer.writerow({
|
||||
"id": total_rows,
|
||||
"provider": provider,
|
||||
"protocol": config["protocol"],
|
||||
"dnssec_mode": config["dnssec_mode"],
|
||||
"keep_alive": config["keep_alive"],
|
||||
"bytes_sent": summary.bytes_sent,
|
||||
"bytes_received": summary.bytes_received,
|
||||
"packets_sent": summary.packets_sent,
|
||||
"packets_received": summary.packets_received,
|
||||
"total_bytes": summary.total_bytes,
|
||||
"total_packets": summary.total_packets,
|
||||
})
|
||||
|
||||
print(f" ✓ {summary.packets_sent:,} sent / {summary.packets_received:,} received "
|
||||
f"| {summary.total_bytes:,} B total ({dt:.2f}s)")
|
||||
|
||||
elapsed = time.time() - t_total
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"PCAP metrics merged → {output_path}")
|
||||
print(f"Total scenarios: {total_rows}")
|
||||
print(f"Completed in {elapsed:.2f}s ({elapsed / 60:.1f} min)")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Summarize all .pcap files into a unified CSV.")
|
||||
parser.add_argument("input_dir", nargs="?", default=".", help="Input directory (default: .)")
|
||||
parser.add_argument("-o", "--output", default="dns_results_pcap.csv", help="Output CSV path")
|
||||
parser.add_argument("--local-ip", default=DEFAULT_LOCAL_IP,
|
||||
help=f"Local netns veth IP for direction detection (default: {DEFAULT_LOCAL_IP})")
|
||||
parser.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS,
|
||||
help="Provider subdirectory names to scan")
|
||||
args = parser.parse_args()
|
||||
|
||||
merge_pcap_files(Path(args.input_dir), Path(args.output), args.local_ip, args.providers)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
import csv
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
COLUMNS = ["provider", "protocol", "dnssec_mode", "query_type", "keep_alive"]
|
||||
|
||||
|
||||
def get_uncovered_values(rows, selected_indices, col_indices, all_values):
|
||||
covered = defaultdict(set)
|
||||
for idx in selected_indices:
|
||||
for col, col_idx in col_indices.items():
|
||||
covered[col].add(rows[idx][col_idx])
|
||||
|
||||
uncovered = {}
|
||||
for col in COLUMNS:
|
||||
uncovered[col] = all_values[col] - covered[col]
|
||||
return uncovered
|
||||
|
||||
|
||||
def main(input_file, output_file):
|
||||
with open(input_file, newline="") as f:
|
||||
reader = csv.reader(f)
|
||||
header = next(reader)
|
||||
rows = list(reader)
|
||||
|
||||
col_indices = {col: header.index(col) for col in COLUMNS}
|
||||
|
||||
# Collect all unique values per column
|
||||
all_values = defaultdict(set)
|
||||
for row in rows:
|
||||
for col, idx in col_indices.items():
|
||||
all_values[col].add(row[idx])
|
||||
|
||||
# Greedy set cover
|
||||
selected = []
|
||||
uncovered = get_uncovered_values(rows, selected, col_indices, all_values)
|
||||
|
||||
while any(uncovered.values()):
|
||||
best_row = None
|
||||
best_score = 0
|
||||
|
||||
for i, row in enumerate(rows):
|
||||
if i in selected:
|
||||
continue
|
||||
score = sum(
|
||||
1 for col, idx in col_indices.items() if row[idx] in uncovered[col]
|
||||
)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_row = i
|
||||
|
||||
if best_row is None:
|
||||
break
|
||||
|
||||
selected.append(best_row)
|
||||
uncovered = get_uncovered_values(rows, selected, col_indices, all_values)
|
||||
|
||||
with open(output_file, "w", newline="") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(header)
|
||||
for idx in selected:
|
||||
writer.writerow(rows[idx])
|
||||
|
||||
print(f"Selected {len(selected)} rows out of {len(rows)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python minimize_csv.py input.csv output.csv")
|
||||
sys.exit(1)
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
+126
-86
@@ -10,6 +10,7 @@ OUTPUT_DIR="./results"
|
||||
INTERFACE="veth1"
|
||||
TIMEOUT="5s"
|
||||
SLEEP_TIME="1"
|
||||
DRY_RUN="false"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -38,6 +39,10 @@ while [[ $# -gt 0 ]]; do
|
||||
SLEEP_TIME="$2"
|
||||
shift 2
|
||||
;;
|
||||
-n|--dry-run)
|
||||
DRY_RUN="true"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
@@ -48,6 +53,7 @@ while [[ $# -gt 0 ]]; do
|
||||
echo " -I, --interface NAME Network interface (default: veth1)"
|
||||
echo " -T, --timeout DURATION Timeout duration (default: 5s)"
|
||||
echo " -s, --sleep SECONDS Sleep between runs (default: 1)"
|
||||
echo " -n, --dry-run Print the scenarios that would run, then exit"
|
||||
echo " --help Show this help"
|
||||
exit 0
|
||||
;;
|
||||
@@ -66,39 +72,102 @@ echo " Output dir: $OUTPUT_DIR"
|
||||
echo " Interface: $INTERFACE"
|
||||
echo " Timeout: $TIMEOUT"
|
||||
echo " Sleep time: ${SLEEP_TIME}s"
|
||||
echo " Dry run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
# Server definitions as associative arrays (name -> url)
|
||||
declare -A CONN_SERVERS=(
|
||||
["google-dotcp"]="dotcp://8.8.8.8:53"
|
||||
["cloudflare-dotcp"]="dotcp://1.1.1.1:53"
|
||||
["quad9-dotcp"]="dotcp://9.9.9.9:53"
|
||||
["adguard-dotcp"]="dotcp://dns.adguard-dns.com:53"
|
||||
["google-dot"]="tls://8.8.8.8:853"
|
||||
["cloudflare-dot"]="tls://1.1.1.1:853"
|
||||
["quad9-dot"]="tls://9.9.9.9:853"
|
||||
["adguard-dot"]="tls://dns.adguard-dns.com:853"
|
||||
["google-doh"]="https://dns.google/dns-query"
|
||||
["cloudflare-doh"]="https://cloudflare-dns.com/dns-query"
|
||||
["quad9-doh"]="https://dns10.quad9.net/dns-query"
|
||||
["adguard-doh"]="https://dns.adguard-dns.com/dns-query"
|
||||
)
|
||||
SC_NAME=()
|
||||
SC_URL=()
|
||||
SC_DNSSEC=()
|
||||
SC_AUTH=()
|
||||
SC_KEEP=()
|
||||
|
||||
declare -A QUIC_SERVERS=(
|
||||
["google-doh3"]="doh3://dns.google/dns-query"
|
||||
["cloudflare-doh3"]="doh3://cloudflare-dns.com/dns-query"
|
||||
["adguard-doh3"]="doh3://dns.adguard-dns.com/dns-query"
|
||||
["adguard-doq"]="doq://dns.adguard-dns.com:853"
|
||||
)
|
||||
add() { # add <name> <url> <dnssec> <auth> <keepalive>
|
||||
SC_NAME+=("$1")
|
||||
SC_URL+=("$2")
|
||||
SC_DNSSEC+=("$3")
|
||||
SC_AUTH+=("$4")
|
||||
SC_KEEP+=("$5")
|
||||
}
|
||||
|
||||
declare -A CONNLESS_SERVERS=(
|
||||
["google-doudp"]="udp://8.8.8.8:53"
|
||||
["cloudflare-doudp"]="udp://1.1.1.1:53"
|
||||
["quad9-doudp"]="udp://9.9.9.9:53"
|
||||
["adguard-doudp"]="udp://dns.adguard-dns.com:53"
|
||||
["adguard-dnscrypt"]="sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20"
|
||||
["quad9-dnscrypt"]="sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ"
|
||||
)
|
||||
# Protocol comparison - DNSSEC off, non-persistent.
|
||||
# The primary comparison axis: full encryption cost, handshake included.
|
||||
# All transports.
|
||||
add google-dotcp "dotcp://8.8.8.8:53" false false false
|
||||
add cloudflare-dotcp "dotcp://1.1.1.1:53" false false false
|
||||
add quad9-dotcp "dotcp://9.9.9.9:53" false false false
|
||||
add adguard-dotcp "dotcp://dns.adguard-dns.com:53" false false false
|
||||
|
||||
add google-dot "tls://8.8.8.8:853" false false false
|
||||
add cloudflare-dot "tls://1.1.1.1:853" false false false
|
||||
add quad9-dot "tls://9.9.9.9:853" false false false
|
||||
add adguard-dot "tls://dns.adguard-dns.com:853" false false false
|
||||
|
||||
add google-doh "https://dns.google/dns-query" false false false
|
||||
add cloudflare-doh "https://cloudflare-dns.com/dns-query" false false false
|
||||
add quad9-doh "https://dns10.quad9.net/dns-query" false false false
|
||||
add adguard-doh "https://dns.adguard-dns.com/dns-query" false false false
|
||||
|
||||
add google-doh3 "doh3://dns.google/dns-query" false false false
|
||||
add cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" false false false
|
||||
add adguard-doh3 "doh3://dns.adguard-dns.com/dns-query" false false false
|
||||
add adguard-doq "doq://dns.adguard-dns.com:853" false false false
|
||||
|
||||
add google-doudp "udp://8.8.8.8:53" false false false
|
||||
add cloudflare-doudp "udp://1.1.1.1:53" false false false
|
||||
add quad9-doudp "udp://9.9.9.9:53" false false false
|
||||
add adguard-doudp "udp://dns.adguard-dns.com:53" false false false
|
||||
add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" false false false
|
||||
add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" false false false
|
||||
|
||||
# Persistence contrast - DNSSEC off, persistent.
|
||||
# Teste for TCP based protocols.
|
||||
# QUIC makes no sense cause of 0-RTT.
|
||||
add google-dotcp "dotcp://8.8.8.8:53" false false true
|
||||
add cloudflare-dotcp "dotcp://1.1.1.1:53" false false true
|
||||
add quad9-dotcp "dotcp://9.9.9.9:53" false false true
|
||||
add adguard-dotcp "dotcp://dns.adguard-dns.com:53" false false true
|
||||
|
||||
add google-dot "tls://8.8.8.8:853" false false true
|
||||
add cloudflare-dot "tls://1.1.1.1:853" false false true
|
||||
add quad9-dot "tls://9.9.9.9:853" false false true
|
||||
add adguard-dot "tls://dns.adguard-dns.com:853" false false true
|
||||
|
||||
add google-doh "https://dns.google/dns-query" false false true
|
||||
add cloudflare-doh "https://cloudflare-dns.com/dns-query" false false true
|
||||
add quad9-doh "https://dns10.quad9.net/dns-query" false false true
|
||||
add adguard-doh "https://dns.adguard-dns.com/dns-query" false false true
|
||||
|
||||
# DNSSEC trust - non-persistent.
|
||||
add google-dotcp "dotcp://8.8.8.8:53" true false false
|
||||
add cloudflare-dotcp "dotcp://1.1.1.1:53" true false false
|
||||
add quad9-dotcp "dotcp://9.9.9.9:53" true false false
|
||||
add adguard-dotcp "dotcp://dns.adguard-dns.com:53" true false false
|
||||
|
||||
add google-dot "tls://8.8.8.8:853" true false false
|
||||
add cloudflare-dot "tls://1.1.1.1:853" true false false
|
||||
add quad9-dot "tls://9.9.9.9:853" true false false
|
||||
add adguard-dot "tls://dns.adguard-dns.com:853" true false false
|
||||
|
||||
add google-doh "https://dns.google/dns-query" true false false
|
||||
add cloudflare-doh "https://cloudflare-dns.com/dns-query" true false false
|
||||
add quad9-doh "https://dns10.quad9.net/dns-query" true false false
|
||||
add adguard-doh "https://dns.adguard-dns.com/dns-query" true false false
|
||||
|
||||
add google-doh3 "doh3://dns.google/dns-query" true false false
|
||||
add cloudflare-doh3 "doh3://cloudflare-dns.com/dns-query" true false false
|
||||
add adguard-doh3 "doh3://dns.adguard-dns.com/dns-query" true false false
|
||||
add adguard-doq "doq://dns.adguard-dns.com:853" true false false
|
||||
|
||||
add google-doudp "udp://8.8.8.8:53" true false false
|
||||
add cloudflare-doudp "udp://1.1.1.1:53" true false false
|
||||
add quad9-doudp "udp://9.9.9.9:53" true false false
|
||||
add adguard-doudp "udp://dns.adguard-dns.com:53" true false false
|
||||
add adguard-dnscrypt "sdns://AQMAAAAAAAAAETk0LjE0MC4xNC4xNDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20" true false false
|
||||
add quad9-dnscrypt "sdns://AQMAAAAAAAAAFDE0OS4xMTIuMTEyLjExMjo4NDQzIGfIR7jIdYzRICRVQ751Z0bfNN8dhMALjEcDaN-CHYY-GTIuZG5zY3J5cHQtY2VydC5xdWFkOS5uZXQ" true false false
|
||||
|
||||
# DNSSEC auth - DoUDP only.
|
||||
# auth walks the chain of trust over plain DoUDP regardless of the protocol.
|
||||
add adguard-doudp "udp://dns.adguard-dns.com:53" true true false
|
||||
|
||||
# Function to get flags suffix for filename
|
||||
get_flags_suffix() {
|
||||
@@ -133,10 +202,6 @@ run_with_perf() {
|
||||
local keepalive="$5"
|
||||
|
||||
local suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
|
||||
local base_name="${name}"
|
||||
if [[ -n "$suffix" ]]; then
|
||||
base_name="${name}-${suffix}"
|
||||
fi
|
||||
local provider="${name%%-*}"
|
||||
local protocol="${name#*-}"
|
||||
local base_name="${protocol}"
|
||||
@@ -194,64 +259,39 @@ run_with_perf() {
|
||||
# Cleanup
|
||||
rm -f "$perf_tmp"
|
||||
|
||||
echo " -> CPU metrics saved to ${base_name}.cpu.csv"
|
||||
echo " -> CPU metrics saved to ${provider}/${base_name}.cpu.csv"
|
||||
}
|
||||
|
||||
# Function to run servers with given flags
|
||||
run_server_group() {
|
||||
local -n servers=$1
|
||||
local dnssec="$2"
|
||||
local auth="$3"
|
||||
local keepalive="$4"
|
||||
local desc="$5"
|
||||
echo "Total scenarios: ${#SC_NAME[@]}"
|
||||
echo ""
|
||||
|
||||
echo "Running: $desc"
|
||||
for i in "${!SC_NAME[@]}"; do
|
||||
name="${SC_NAME[$i]}"
|
||||
url="${SC_URL[$i]}"
|
||||
dnssec="${SC_DNSSEC[$i]}"
|
||||
auth="${SC_AUTH[$i]}"
|
||||
keepalive="${SC_KEEP[$i]}"
|
||||
|
||||
for name in "${!servers[@]}"; do
|
||||
local url="${servers[$name]}"
|
||||
echo " Processing: $name ($url)"
|
||||
suffix=$(get_flags_suffix "$dnssec" "$auth" "$keepalive")
|
||||
provider="${name%%-*}"
|
||||
protocol="${name#*-}"
|
||||
label="${protocol}"
|
||||
[[ -n "$suffix" ]] && label="${protocol}-${suffix}"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[dry] %-28s dnssec=%-5s auth=%-5s keepalive=%-5s -> %s/%s\n' \
|
||||
"$name" "$dnssec" "$auth" "$keepalive" "$provider" "$label"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Processing: $name ($url) [dnssec=$dnssec auth=$auth keepalive=$keepalive]"
|
||||
run_with_perf "$name" "$url" "$dnssec" "$auth" "$keepalive"
|
||||
sleep "$SLEEP_TIME"
|
||||
done
|
||||
}
|
||||
|
||||
echo "=== Running TCP-based protocols (TLS/HTTPS) ==="
|
||||
|
||||
# DNSSEC off, Keep off
|
||||
run_server_group CONN_SERVERS "false" "false" "false" "no-dnssec, no-keepalive"
|
||||
|
||||
# DNSSEC off, Keep on
|
||||
run_server_group CONN_SERVERS "false" "false" "true" "no-dnssec, keepalive"
|
||||
|
||||
# DNSSEC on (trust), Keep on
|
||||
run_server_group CONN_SERVERS "true" "false" "true" "dnssec-trust, keepalive"
|
||||
|
||||
# DNSSEC on (auth), Keep on
|
||||
run_server_group CONN_SERVERS "true" "true" "true" "dnssec-auth, keepalive"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Running QUIC-based protocols (DoH3/DoQ) ==="
|
||||
|
||||
# DNSSEC off
|
||||
run_server_group QUIC_SERVERS "false" "false" "false" "no-dnssec"
|
||||
|
||||
# DNSSEC on (trust)
|
||||
run_server_group QUIC_SERVERS "true" "false" "false" "dnssec-trust"
|
||||
|
||||
# DNSSEC on (auth)
|
||||
run_server_group QUIC_SERVERS "true" "true" "false" "dnssec-auth"
|
||||
|
||||
echo ""
|
||||
echo "=== Running connectionless protocols (UDP) ==="
|
||||
|
||||
# DNSSEC off
|
||||
run_server_group CONNLESS_SERVERS "false" "false" "false" "no-dnssec"
|
||||
|
||||
# DNSSEC on (trust)
|
||||
run_server_group CONNLESS_SERVERS "true" "false" "false" "dnssec-trust"
|
||||
|
||||
# DNSSEC on (auth)
|
||||
run_server_group CONNLESS_SERVERS "true" "true" "false" "dnssec-auth"
|
||||
|
||||
echo ""
|
||||
echo "All combinations completed!"
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo "Dry run complete - nothing was executed."
|
||||
else
|
||||
echo "All scenarios completed!"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user